From 3af8c886ef60d5e90dd61e04d6b91a397f7a13d5 Mon Sep 17 00:00:00 2001
From: Matthias <Matthias.Wiesenberger@uibk.ac.at>
Date: Tue, 5 Apr 2016 15:49:20 +0200
Subject: [PATCH 001/453] interpolation in ds

---
 inc/dg/ds.h                     | 157 ++++++++++++++++++++------------
 inc/dg/ds_b.cu                  |   5 +-
 inc/dg/geometry/fieldaligned.h  |   2 +
 inc/geometries/guenther_ds_b.cu |  31 ++++---
 4 files changed, 120 insertions(+), 75 deletions(-)

diff --git a/inc/dg/ds.h b/inc/dg/ds.h
index 7e8dd475c..a1e8784f5 100644
--- a/inc/dg/ds.h
+++ b/inc/dg/ds.h
@@ -43,8 +43,8 @@ struct DS
     * @param no norm or not_normed affects the behaviour of the symv function
     * @param dir the direction affects both the operator() and the symv function
     */
-    template<class InvB>
-    DS(const FA& field, InvB invB, dg::norm no=dg::normed, dg::direction dir = dg::centered, bool jumpX = true);
+    template<class InvB, class Geometry>
+    DS(const FA& field, Geometry, InvB invB, dg::norm no=dg::normed, dg::direction dir = dg::centered, bool jumpX = true);
 
     /**
     * @brief Apply the derivative on a 3d vector
@@ -214,6 +214,7 @@ struct DS
     private:
     FA f_;
     Matrix jumpX, jumpY;
+    typename FA::InterpolationMatrix f2c, c2f, f2cT, c2fT;
     container tempP, temp0, tempM;
     container vol3d, inv3d;
     container invB;
@@ -227,16 +228,24 @@ struct DS
 ////////////////////////////////////DEFINITIONS////////////////////////////////////////
 
 template<class FA, class M, class container>
-template <class Field>
-DS<FA, M,container>::DS(const FA& field, Field inverseB, dg::norm no, dg::direction dir, bool jumpX):
+template <class Field, class Geometry>
+DS<FA, M,container>::DS(const FA& field, Geometry gridc, Field inverseB, dg::norm no, dg::direction dir, bool jumpX):
         f_(field),
-        jumpX( dg::create::jumpX( field.grid())),
-        jumpY( dg::create::jumpY( field.grid())),
+        jumpX( dg::create::jumpX( gridc)),
+        jumpY( dg::create::jumpY( gridc)),
         tempP( dg::evaluate( dg::zero, field.grid())), temp0( tempP), tempM( tempP), 
-        vol3d( dg::create::volume( field.grid())), inv3d( dg::create::inv_volume( field.grid())),
+        vol3d( dg::create::volume( gridc)), inv3d( dg::create::inv_volume( gridc)),
         invB(dg::pullback(inverseB,field.grid())), //R_(dg::evaluate(dg::coo1,grid)), 
         no_(no), dir_(dir), apply_jumpX_(jumpX)
-{ }
+{
+
+    f2c = dg::create::interpolation( gridc, field.grid());
+    c2f = dg::create::interpolation( field.grid(), gridc);
+    cusp::transpose( f2c, f2cT);
+    cusp::transpose( c2f, c2fT);     
+
+
+}
 
 template<class F, class M, class container>
 inline void DS<F,M,container>::operator()( const container& f, container& dsf) { 
@@ -250,10 +259,12 @@ inline void DS<F,M,container>::operator()( const container& f, container& dsf) {
 
 
 template<class F, class M, class container>
-void DS<F,M,container>::centered( const container& f, container& dsf)
+void DS<F,M,container>::centered( const container& fc, container& dsfc)
 {
     //direct discretisation
-    assert( &f != &dsf);
+    assert( &fc != &dsfc);
+    container f(tempP), dsf( tempP);
+    cusp::multiply( c2f, fc, f);
     f_.einsPlus( f, tempP);
     f_.einsMinus( f, tempM);
     dg::blas1::axpby( 1., tempP, -1., tempM);
@@ -270,21 +281,26 @@ void DS<F,M,container>::centered( const container& f, container& dsf)
     dg::blas1::axpby( 1., tempM, -1., tempP);
     dg::blas1::pointwiseDot( inv3d, tempP, dsf);
     dg::blas1::pointwiseDot( dsf, invB, dsf);  */  
+    cusp::multiply( f2c, dsf, dsfc);
 
 }
 
 template<class F, class M, class container>
-void DS<F,M,container>::centeredT( const container& f, container& dsf)
+void DS<F,M,container>::centeredT( const container& fc, container& dsfc)
 {               
 //     //adjoint discretisation
-    assert( &f != &dsf);    
-    dg::blas1::pointwiseDot( vol3d, f, dsf);
+    assert( &fc != &dsfc);    
+    container f(tempP), dsf( tempP);
+    dg::blas1::pointwiseDot( vol3d, fc, dsfc);
+    cusp::multiply( f2cT, dsfc, dsf);
 
     dg::blas1::pointwiseDivide( dsf, f_.hz(), dsf);
     f_.einsPlusT( dsf, tempP);
     f_.einsMinusT( dsf, tempM);
     dg::blas1::axpby( 1., tempM, -1., tempP);        
-    dg::blas1::pointwiseDot( inv3d, tempP, dsf); 
+
+    cusp::multiply( c2fT, tempP, dsfc);
+    dg::blas1::pointwiseDot( inv3d, dsfc, dsfc); 
 
 //       dg::blas1::pointwiseDot( inv3d, tempP,tempP); //make it symmetric
         //stegmeir weights
@@ -316,10 +332,12 @@ void DS<F,M,container>::centeredTD( const container& f, container& dsf)
 }
 
 template<class F, class M, class container>
-void DS<F,M,container>::forward( const container& f, container& dsf)
+void DS<F,M,container>::forward( const container& fc, container& dsfc)
 {
     //direct
-    assert( &f != &dsf);
+    assert( &fc != &dsfc);
+    container f(tempP), dsf( tempP);
+    cusp::multiply( c2f, fc, f);
     f_.einsPlus( f, tempP);
     dg::blas1::axpby( 1., tempP, -1., f, tempP);
     dg::blas1::pointwiseDivide( tempP, f_.hp(), dsf);
@@ -332,38 +350,50 @@ void DS<F,M,container>::forward( const container& f, container& dsf)
 //     dg::blas1::axpby( 1., tempP,-1.,dsf,dsf);
 //     dg::blas1::pointwiseDot( inv3d, dsf, dsf);
 //     dg::blas1::pointwiseDot( dsf, invB, dsf);
+    cusp::multiply( f2c, dsf, dsfc);
 }
 
 template<class F, class M, class container>
-void DS<F,M,container>::forwardT( const container& f, container& dsf)
+void DS<F,M,container>::forwardT( const container& fc, container& dsfc)
 {    
     //adjoint discretisation
-    assert( &f != &dsf);
-    dg::blas1::pointwiseDot( vol3d, f, dsf);   
+    assert( &fc != &dsfc);
+    container f(tempP), dsf( tempP);
+    dg::blas1::pointwiseDot( vol3d, fc, dsfc);
+    cusp::multiply( f2cT, dsfc, dsf);
     dg::blas1::pointwiseDivide( dsf, f_.hp(), dsf);
     f_.einsPlusT( dsf, tempP);
     dg::blas1::axpby( -1., tempP, 1., dsf, dsf);
-    dg::blas1::pointwiseDot( inv3d, dsf, dsf);
+
+    cusp::multiply( c2fT, dsf, dsfc);
+    dg::blas1::pointwiseDot( inv3d, dsfc, dsfc); 
 }
 
 template<class F, class M, class container>
-void DS<F,M,container>::forwardTD( const container& f, container& dsf)
+void DS<F,M,container>::forwardTD( const container& fc, container& dsfc)
 {
     //direct discretisation
-    assert( &f != &dsf);    
+    assert( &fc != &dsfc);
+    container f(tempP), dsf( tempP);
+    cusp::multiply( c2f, fc, f);
+
     dg::blas1::pointwiseDot( f, invB, dsf);
     f_.einsMinus( dsf, tempP);
     dg::blas1::axpby( -1., tempP, 1., dsf, dsf);
     dg::blas1::pointwiseDivide( dsf, f_.hm(), dsf);        
     dg::blas1::pointwiseDivide( dsf, invB, dsf);
 
+    cusp::multiply( f2c, dsf, dsfc);
+
 
 }
 template<class F, class M, class container>
-void DS<F,M,container>::backward( const container& f, container& dsf)
+void DS<F,M,container>::backward( const container& fc, container& dsfc)
 {
     //direct
-    assert( &f != &dsf);
+    assert( &fc != &dsfc);
+    container f(tempP), dsf( tempP);
+    cusp::multiply( c2f, fc, f);
     f_.einsMinus( f, tempM);
     dg::blas1::axpby( 1., tempM, -1., f, tempM);
     dg::blas1::pointwiseDivide( tempM, f_.hm(), dsf);
@@ -377,80 +407,91 @@ void DS<F,M,container>::backward( const container& f, container& dsf)
 //     dg::blas1::axpby( 1., tempM, -1.,dsf,dsf);
 //     dg::blas1::pointwiseDot( inv3d,dsf, dsf);
 //     dg::blas1::pointwiseDot( dsf, invB, dsf);
+    cusp::multiply( f2c, dsf, dsfc);
 }
 template<class F, class M, class container>
-void DS<F,M,container>::backwardT( const container& f, container& dsf)
+void DS<F,M,container>::backwardT( const container& fc, container& dsfc)
 {    
     //adjoint discretisation
-    assert( &f != &dsf);
-    dg::blas1::pointwiseDot( vol3d, f, dsf);
+    assert( &fc != &dsfc);
+    dg::blas1::pointwiseDot( vol3d, fc, dsfc);
+    container f(tempP), dsf( tempP);
+    cusp::multiply( f2cT, dsfc, dsf);
+
     dg::blas1::pointwiseDivide( dsf, f_.hm(), dsf);
     f_.einsMinusT( dsf, tempM);
     dg::blas1::axpby( -1., tempM, 1., dsf, dsf);
-    dg::blas1::pointwiseDot( inv3d, dsf, dsf);   
+
+    cusp::multiply( c2fT, dsf, dsfc);
+    dg::blas1::pointwiseDot( inv3d, dsfc, dsfc); 
 }
 
 template<class F, class M, class container>
-void DS<F,M,container>::backwardTD( const container& f, container& dsf)
+void DS<F,M,container>::backwardTD( const container& fc, container& dsfc)
 {
     //direct
-    assert( &f != &dsf);    
+    assert( &fc != &dsfc);
+    container f(tempP), dsf( tempP);
+    cusp::multiply( c2f, fc, f);
     dg::blas1::pointwiseDot( f, invB, dsf);
     f_.einsPlus( dsf, tempM);
     dg::blas1::axpby( -1., tempM, 1., dsf, dsf);
     dg::blas1::pointwiseDivide( dsf, f_.hp(), dsf);        
     dg::blas1::pointwiseDivide( dsf, invB, dsf);
+
+    cusp::multiply( f2c, dsf, dsfc);
 }
 
 template< class F, class M, class container >
 void DS<F,M,container>::symv( const container& f, container& dsTdsf)
 {
+    container tempPc(f), tempMc(f), temp0c(f);
     if(dir_ == dg::centered)
     {
-        centered( f, tempP);
-        centeredT( tempP, dsTdsf);
+        centered( f, tempPc);
+        centeredT( tempPc, dsTdsf);
     }
     else 
     {
-        forward( f, tempP);
-        forwardT( tempP, dsTdsf);
-        backward( f, tempM);
-        backwardT( tempM, temp0);
-        dg::blas1::axpby(0.5,temp0,0.5,dsTdsf,dsTdsf);
+        forward( f, tempPc);
+        forwardT( tempPc, dsTdsf);
+        backward( f, tempMc);
+        backwardT( tempMc, temp0c);
+        dg::blas1::axpby(0.5,temp0c,0.5,dsTdsf,dsTdsf);
     }
 //     add jump term 
 
     if(apply_jumpX_)
     {
-        dg::blas2::symv( jumpX, f, temp0);
-        dg::geo::divideVolume( temp0, f_.grid());
-        dg::blas1::axpby( -1., temp0, 1., dsTdsf, dsTdsf);
+        dg::blas2::symv( jumpX, f, temp0c);
+        dg::geo::divideVolume( temp0c, f_.grid());
+        dg::blas1::axpby( -1., temp0c, 1., dsTdsf, dsTdsf);
     }
-    dg::blas2::symv( jumpY, f, temp0);
-    dg::geo::divideVolume( temp0, f_.grid());
+    dg::blas2::symv( jumpY, f, temp0c);
+    dg::geo::divideVolume( temp0c, f_.grid());
     //dg::blas1::pointwiseDivide( temp0, R_, temp0);
-    dg::blas1::axpby( -1., temp0, 1., dsTdsf, dsTdsf);
+    dg::blas1::axpby( -1., temp0c, 1., dsTdsf, dsTdsf);
     if( no_ == not_normed)
     {
         dg::blas1::pointwiseDot( vol3d, dsTdsf, dsTdsf); //make it symmetric
     }
 }
 
-template< class F, class M, class container >
-void DS<F,M,container>::dss( const container& f, container& dssf)
-{
-    assert( &f != &dssf);
-    f_.einsPlus(  f, tempP);
-    f_.einsMinus( f, tempM);
-    dg::blas1::pointwiseDivide( tempP, f_.hp(), tempP);
-    dg::blas1::pointwiseDivide( tempP, f_.hz(), tempP);
-    dg::blas1::pointwiseDivide( f, f_.hp(), temp0);
-    dg::blas1::pointwiseDivide( temp0, f_.hm(), temp0);
-    dg::blas1::pointwiseDivide( tempM, f_.hm(), tempM);
-    dg::blas1::pointwiseDivide( tempM, f_.hz(), tempM);
-    dg::blas1::axpby(  2., tempP, +2., tempM); //fp+fm
-    dg::blas1::axpby( -2., temp0, +1., tempM, dssf); 
-}
+//template< class F, class M, class container >
+//void DS<F,M,container>::dss( const container& f, container& dssf)
+//{
+//    assert( &f != &dssf);
+//    f_.einsPlus(  f, tempP);
+//    f_.einsMinus( f, tempM);
+//    dg::blas1::pointwiseDivide( tempP, f_.hp(), tempP);
+//    dg::blas1::pointwiseDivide( tempP, f_.hz(), tempP);
+//    dg::blas1::pointwiseDivide( f, f_.hp(), temp0);
+//    dg::blas1::pointwiseDivide( temp0, f_.hm(), temp0);
+//    dg::blas1::pointwiseDivide( tempM, f_.hm(), tempM);
+//    dg::blas1::pointwiseDivide( tempM, f_.hz(), tempM);
+//    dg::blas1::axpby(  2., tempP, +2., tempM); //fp+fm
+//    dg::blas1::axpby( -2., temp0, +1., tempM, dssf); 
+//}
 
 
 //enables the use of the dg::blas2::symv function 
diff --git a/inc/dg/ds_b.cu b/inc/dg/ds_b.cu
index 602b00e6e..4512e0d92 100644
--- a/inc/dg/ds_b.cu
+++ b/inc/dg/ds_b.cu
@@ -61,12 +61,13 @@ int main()
     std::cin >> n>> Nx>>Ny>>Nz;
     std::cout << "You typed "<<n<<" "<<Nx<<" "<<Ny<<" "<<Nz<<std::endl;
     dg::CylindricalGrid<dg::DVec> g3d( R_0 - 1, R_0+1, -1, 1, 0, 2.*M_PI, n, Nx, Ny, Nz, dg::NEU, dg::NEU, dg::PER);
+    dg::CylindricalGrid<dg::DVec> g3d_fine( R_0 - 1, R_0+1, -1, 1, 0, 2.*M_PI, 3*n, 2*Nx, 2*Ny, Nz, dg::NEU, dg::NEU, dg::PER);
     const dg::DVec w3d = dg::create::volume( g3d);
     dg::Timer t;
     t.tic();
-    dg::DDS::FieldAligned dsFA( field, g3d, 1e-10, dg::DefaultLimiter(), dg::NEU);
+    dg::DDS::FieldAligned dsFA( field, g3d_fine, 1e-10, dg::DefaultLimiter(), dg::NEU);
 
-    dg::DDS ds ( dsFA, field, dg::not_normed, dg::centered);
+    dg::DDS ds ( dsFA, g3d, field, dg::not_normed, dg::centered);
     t.toc();
     std::cout << "Creation of parallel Derivative took     "<<t.diff()<<"s\n";
 
diff --git a/inc/dg/geometry/fieldaligned.h b/inc/dg/geometry/fieldaligned.h
index 5f1873a96..d374228fa 100644
--- a/inc/dg/geometry/fieldaligned.h
+++ b/inc/dg/geometry/fieldaligned.h
@@ -273,6 +273,8 @@ template< class Geometry, class Matrix, class container >
 struct FieldAligned
 {
 
+    typedef Matrix InterpolationMatrix;
+
     /**
     * @brief Construct from a field and a grid
     *
diff --git a/inc/geometries/guenther_ds_b.cu b/inc/geometries/guenther_ds_b.cu
index c36ec0340..1c49e4509 100644
--- a/inc/geometries/guenther_ds_b.cu
+++ b/inc/geometries/guenther_ds_b.cu
@@ -80,6 +80,7 @@ int main( )
 
 
         dg::CylindricalGrid<dg::DVec> g3d( Rmin,Rmax, Zmin,Zmax, z0, z1,  n,Nxn ,Nyn, Nzn,dg::DIR, dg::DIR, dg::PER);
+        dg::CylindricalGrid<dg::DVec> g3d_fein( Rmin,Rmax, Zmin,Zmax, z0, z1,  3*n,Nxn ,Nyn, Nzn,dg::DIR, dg::DIR, dg::PER);
         dg::Grid2d<double> g2d( Rmin,Rmax, Zmin,Zmax,  n, Nxn ,Nyn);
 
         std::cout << "NR = " << Nxn << std::endl;
@@ -95,12 +96,12 @@ int main( )
     const dg::DVec v3d = dg::create::inv_volume( g3d);
 
     std::cout << "computing dsDIR" << std::endl;
-    dg::DDS::FieldAligned dsFA( field, g3d, rk4eps, dg::DefaultLimiter(), dg::DIR);
+    dg::DDS::FieldAligned dsFA( field, g3d_fein, rk4eps, dg::DefaultLimiter(), dg::DIR);
     std::cout << "computing dsNEU" << std::endl;
-    dg::DDS::FieldAligned dsNUFA( field, g3d, rk4eps, dg::DefaultLimiter(), dg::NEU);
+    dg::DDS::FieldAligned dsNUFA( field, g3d_fein, rk4eps, dg::DefaultLimiter(), dg::NEU);
 
-    dg::DDS ds ( dsFA, field, dg::not_normed, dg::centered), 
-        dsNU ( dsNUFA, field, dg::not_normed, dg::centered);
+    dg::DDS ds ( dsFA, g3d, field, dg::not_normed, dg::centered), 
+        dsNU ( dsNUFA, g3d, field, dg::not_normed, dg::centered);
 
 //     dg::DS<dg::DMatrix, dg::DVec> dsNEU( field, g3d, g3d.hz(), rk4eps, dg::DefaultLimiter(), dg::NEU);
     
@@ -255,13 +256,13 @@ int main( )
         //dg::blas1::pointwiseDivide( dsTdsfb, inverseB, dsTdsfb);
 //     ds.centeredT( derivative2, dsTds2); //dsT(ds(f))
 //     dg::blas1::pointwiseDivide(ones,  inverseB, temp2); //B
-//     ds.centeredT( ones, divbT);
+     ds.centeredT( ones, divbT);
 //     
 //     double normdsds =dg::blas2::dot(derivative2, w3d,derivative2);
 //     double normds1ds =dg::blas2::dot(derivativeones, w3d,derivative2);
 //     double normdivBT =dg::blas2::dot(divBT, w3d,divBT);
-//     double normdivbT =dg::blas2::dot(divbT, w3d,divbT);
-//     double normdivb =dg::blas2::dot(divbsol, w3d,divbsol); 
+     double normdivbT =dg::blas2::dot(divbT, w3d,divbT);
+     double normdivb =dg::blas2::dot(divbsol, w3d,divbsol); 
 //     double normdsTf = dg::blas2::dot(derivativeT2, w3d, function2);
 //     double normdsT_1 = dg::blas2::dot(derivativeT2, w3d, ones);
 //     double normdsT1 = dg::blas2::dot(derivativeTones, w3d, function2);
@@ -291,14 +292,14 @@ int main( )
 //     errRZPhi =dg::blas2::dot( w3d, derivativeRZPhi);    
 //     std::cout << "Relative Difference in DS is "<< sqrt( errRZPhi/norm )<<"\n"; 
 //     
-//     std::cout << "--------------------testing dsT" << std::endl;
-//     std::cout << "|| divbsol ||  "<<sqrt( normdivb)<<"\n";
-//     std::cout << "|| divbT  ||   "<<sqrt( normdivbT)<<"\n";
-//     dg::blas1::axpby( 1., divbsol, -1., divbT);
-//     normdivbT =dg::blas2::dot(divbT, w3d,divbT);
-//     std::cout << "Relative Difference in DST is   "<<sqrt( normdivbT)<<"\n";
-//     std::cout << "-------------------- " << std::endl;
-//     std::cout << "|| divB || "<<sqrt( normdivBT)<<"\n";
+     std::cout << "--------------------testing dsT" << std::endl;
+     std::cout << "|| divbsol ||  "<<sqrt( normdivb)<<"\n";
+     std::cout << "|| divbT  ||   "<<sqrt( normdivbT)<<"\n";
+     dg::blas1::axpby( 1., divbsol, -1., divbT);
+     normdivbT =dg::blas2::dot(divbT, w3d,divbT);
+     std::cout << "Relative Difference in DST is   "<<sqrt( normdivbT)<<"\n";
+     std::cout << "-------------------- " << std::endl;
+     //std::cout << "|| divB || "<<sqrt( normdivBT)<<"\n";
 // 
 //     
 //     std::cout << "-------------------- " << std::endl;
-- 
GitLab


From bd18ba63bfaca140835e1c915c6970107f3f1015 Mon Sep 17 00:00:00 2001
From: Matthias <Matthias.Wiesenberger@uibk.ac.at>
Date: Tue, 5 Apr 2016 17:17:03 +0200
Subject: [PATCH 002/453] corrected bug in ds.h

---
 inc/dg/ds.h                     | 23 ++++++++++++-----------
 inc/geometries/guenther_ds_b.cu | 31 ++++++++++++++++---------------
 2 files changed, 28 insertions(+), 26 deletions(-)

diff --git a/inc/dg/ds.h b/inc/dg/ds.h
index a1e8784f5..39899c0f4 100644
--- a/inc/dg/ds.h
+++ b/inc/dg/ds.h
@@ -216,12 +216,16 @@ struct DS
     Matrix jumpX, jumpY;
     typename FA::InterpolationMatrix f2c, c2f, f2cT, c2fT;
     container tempP, temp0, tempM;
+    container tempPc, tempMc, temp0c;
+    container f, dsf;
     container vol3d, inv3d;
     container invB;
     //container R_;
     dg::norm no_;
     dg::direction dir_;
     bool apply_jumpX_;
+
+    container volume_;
 };
 
 ///@cond
@@ -234,6 +238,8 @@ DS<FA, M,container>::DS(const FA& field, Geometry gridc, Field inverseB, dg::nor
         jumpX( dg::create::jumpX( gridc)),
         jumpY( dg::create::jumpY( gridc)),
         tempP( dg::evaluate( dg::zero, field.grid())), temp0( tempP), tempM( tempP), 
+        tempPc( dg::evaluate( dg::zero, gridc)), temp0c( tempPc), tempMc( tempPc), 
+        f(tempP), dsf(tempP),
         vol3d( dg::create::volume( gridc)), inv3d( dg::create::inv_volume( gridc)),
         invB(dg::pullback(inverseB,field.grid())), //R_(dg::evaluate(dg::coo1,grid)), 
         no_(no), dir_(dir), apply_jumpX_(jumpX)
@@ -244,6 +250,9 @@ DS<FA, M,container>::DS(const FA& field, Geometry gridc, Field inverseB, dg::nor
     cusp::transpose( f2c, f2cT);
     cusp::transpose( c2f, c2fT);     
 
+    volume_ = dg::evaluate( dg::one, gridc);
+    dg::geo::multiplyVolume( volume_, gridc);
+
 
 }
 
@@ -263,7 +272,6 @@ void DS<F,M,container>::centered( const container& fc, container& dsfc)
 {
     //direct discretisation
     assert( &fc != &dsfc);
-    container f(tempP), dsf( tempP);
     cusp::multiply( c2f, fc, f);
     f_.einsPlus( f, tempP);
     f_.einsMinus( f, tempM);
@@ -290,7 +298,6 @@ void DS<F,M,container>::centeredT( const container& fc, container& dsfc)
 {               
 //     //adjoint discretisation
     assert( &fc != &dsfc);    
-    container f(tempP), dsf( tempP);
     dg::blas1::pointwiseDot( vol3d, fc, dsfc);
     cusp::multiply( f2cT, dsfc, dsf);
 
@@ -336,7 +343,6 @@ void DS<F,M,container>::forward( const container& fc, container& dsfc)
 {
     //direct
     assert( &fc != &dsfc);
-    container f(tempP), dsf( tempP);
     cusp::multiply( c2f, fc, f);
     f_.einsPlus( f, tempP);
     dg::blas1::axpby( 1., tempP, -1., f, tempP);
@@ -358,7 +364,6 @@ void DS<F,M,container>::forwardT( const container& fc, container& dsfc)
 {    
     //adjoint discretisation
     assert( &fc != &dsfc);
-    container f(tempP), dsf( tempP);
     dg::blas1::pointwiseDot( vol3d, fc, dsfc);
     cusp::multiply( f2cT, dsfc, dsf);
     dg::blas1::pointwiseDivide( dsf, f_.hp(), dsf);
@@ -374,7 +379,6 @@ void DS<F,M,container>::forwardTD( const container& fc, container& dsfc)
 {
     //direct discretisation
     assert( &fc != &dsfc);
-    container f(tempP), dsf( tempP);
     cusp::multiply( c2f, fc, f);
 
     dg::blas1::pointwiseDot( f, invB, dsf);
@@ -392,7 +396,6 @@ void DS<F,M,container>::backward( const container& fc, container& dsfc)
 {
     //direct
     assert( &fc != &dsfc);
-    container f(tempP), dsf( tempP);
     cusp::multiply( c2f, fc, f);
     f_.einsMinus( f, tempM);
     dg::blas1::axpby( 1., tempM, -1., f, tempM);
@@ -415,7 +418,6 @@ void DS<F,M,container>::backwardT( const container& fc, container& dsfc)
     //adjoint discretisation
     assert( &fc != &dsfc);
     dg::blas1::pointwiseDot( vol3d, fc, dsfc);
-    container f(tempP), dsf( tempP);
     cusp::multiply( f2cT, dsfc, dsf);
 
     dg::blas1::pointwiseDivide( dsf, f_.hm(), dsf);
@@ -431,7 +433,6 @@ void DS<F,M,container>::backwardTD( const container& fc, container& dsfc)
 {
     //direct
     assert( &fc != &dsfc);
-    container f(tempP), dsf( tempP);
     cusp::multiply( c2f, fc, f);
     dg::blas1::pointwiseDot( f, invB, dsf);
     f_.einsPlus( dsf, tempM);
@@ -445,7 +446,6 @@ void DS<F,M,container>::backwardTD( const container& fc, container& dsfc)
 template< class F, class M, class container >
 void DS<F,M,container>::symv( const container& f, container& dsTdsf)
 {
-    container tempPc(f), tempMc(f), temp0c(f);
     if(dir_ == dg::centered)
     {
         centered( f, tempPc);
@@ -464,11 +464,12 @@ void DS<F,M,container>::symv( const container& f, container& dsTdsf)
     if(apply_jumpX_)
     {
         dg::blas2::symv( jumpX, f, temp0c);
-        dg::geo::divideVolume( temp0c, f_.grid());
+        dg::blas1::pointwiseDivide( temp0c, volume_, temp0c);
         dg::blas1::axpby( -1., temp0c, 1., dsTdsf, dsTdsf);
     }
     dg::blas2::symv( jumpY, f, temp0c);
-    dg::geo::divideVolume( temp0c, f_.grid());
+    //dg::geo::divideVolume( temp0c, gridc);
+    dg::blas1::pointwiseDivide( temp0c, volume_, temp0c);
     //dg::blas1::pointwiseDivide( temp0, R_, temp0);
     dg::blas1::axpby( -1., temp0c, 1., dsTdsf, dsTdsf);
     if( no_ == not_normed)
diff --git a/inc/geometries/guenther_ds_b.cu b/inc/geometries/guenther_ds_b.cu
index 1c49e4509..3236e91d6 100644
--- a/inc/geometries/guenther_ds_b.cu
+++ b/inc/geometries/guenther_ds_b.cu
@@ -59,10 +59,11 @@ int main( )
     guenther::Divb divb(gp.R_0,gp.I_0);
     guenther::B Bfield(gp);
     
-    std::cout << "Type n, Nx, Ny, Nz\n";
+    std::cout << "Type fn, fN\n";
     //std::cout << "Note, that function is resolved exactly in R,Z for n > 2\n";
     unsigned n=3, Nx=5, Ny=5, Nz=5;
-    //std::cin >> n>> Nx>>Ny>>Nz;
+    unsigned fn, fN;
+    std::cin >> fn>> fN;
     unsigned Nxn = Nx;
     unsigned Nyn = Ny;
     unsigned Nzn = Nz;
@@ -71,7 +72,7 @@ int main( )
     //std::cout << "Type RK4 eps (1e-8)\n";
     //std::cin >> rk4eps;
     double z0 = 0, z1 = 2.*M_PI;
-    for (unsigned i=1;i<4;i+=2) { 
+    for (unsigned i=0;i<4;i+=1) { 
 
         Nzn = unsigned(Nz*pow(2,i));
         Nxn = (unsigned)ceil(Nx*pow(2,(double)(i*2./n)));
@@ -80,7 +81,7 @@ int main( )
 
 
         dg::CylindricalGrid<dg::DVec> g3d( Rmin,Rmax, Zmin,Zmax, z0, z1,  n,Nxn ,Nyn, Nzn,dg::DIR, dg::DIR, dg::PER);
-        dg::CylindricalGrid<dg::DVec> g3d_fein( Rmin,Rmax, Zmin,Zmax, z0, z1,  3*n,Nxn ,Nyn, Nzn,dg::DIR, dg::DIR, dg::PER);
+        dg::CylindricalGrid<dg::DVec> g3d_fein( Rmin,Rmax, Zmin,Zmax, z0, z1,  fn,fN*Nxn ,fN*Nyn, Nzn,dg::DIR, dg::DIR, dg::PER);
         dg::Grid2d<double> g2d( Rmin,Rmax, Zmin,Zmax,  n, Nxn ,Nyn);
 
         std::cout << "NR = " << Nxn << std::endl;
@@ -277,9 +278,9 @@ int main( )
 // 
     std::cout << "--------------------testing ds" << std::endl;
     double norm = dg::blas2::dot( w3d, solution);
-    std::cout << "|| Solution ||   "<<sqrt( norm)<<"\n";
+    //std::cout << "|| Solution ||   "<<sqrt( norm)<<"\n";
     double err =dg::blas2::dot( w3d, derivative);
-    std::cout << "|| Derivative || "<<sqrt( err)<<"\n";
+    //std::cout << "|| Derivative || "<<sqrt( err)<<"\n";
     dg::blas1::axpby( 1., solution, -1., derivative);
     err =dg::blas2::dot( w3d, derivative);
     std::cout << "Relative Difference in DS is "<< sqrt( err/norm )<<"\n"; 
@@ -292,13 +293,13 @@ int main( )
 //     errRZPhi =dg::blas2::dot( w3d, derivativeRZPhi);    
 //     std::cout << "Relative Difference in DS is "<< sqrt( errRZPhi/norm )<<"\n"; 
 //     
-     std::cout << "--------------------testing dsT" << std::endl;
-     std::cout << "|| divbsol ||  "<<sqrt( normdivb)<<"\n";
-     std::cout << "|| divbT  ||   "<<sqrt( normdivbT)<<"\n";
+     std::cout << "--------------------testing divb" << std::endl;
+     //std::cout << "|| divbsol ||  "<<sqrt( normdivb)<<"\n";
+     //std::cout << "|| divbT  ||   "<<sqrt( normdivbT)<<"\n";
      dg::blas1::axpby( 1., divbsol, -1., divbT);
      normdivbT =dg::blas2::dot(divbT, w3d,divbT);
-     std::cout << "Relative Difference in DST is   "<<sqrt( normdivbT)<<"\n";
-     std::cout << "-------------------- " << std::endl;
+     std::cout << "Relative Difference in divb is   "<<sqrt( normdivbT)<<"\n";
+     //std::cout << "-------------------- " << std::endl;
      //std::cout << "|| divB || "<<sqrt( normdivBT)<<"\n";
 // 
 //     
@@ -340,11 +341,11 @@ int main( )
 //     std::cout << "Relative Difference in DST is "<< sqrt( errdsTds/normdsTds )<<"\n";   
     
     std::cout << "--------------------testing dsTdsfb " << std::endl;
-    std::cout << "|| SolutionT ||      "<<sqrt( normdsTds)<<"\n";
+    //std::cout << "|| SolutionT ||      "<<sqrt( normdsTds)<<"\n";
     double remainder =dg::blas1::dot( w3d,dsTdsfb);
     double errdsTdsfb =dg::blas2::dot( w3d,dsTdsfb);
-    std::cout << "|| DerivativeTds ||  "<<sqrt( errdsTdsfb)<<"\n";
-    std::cout << "   Integral          "<<remainder<<"\n";
+    //std::cout << "|| DerivativeTds ||  "<<sqrt( errdsTdsfb)<<"\n";
+    //std::cout << "   Integral          "<<remainder<<"\n";
     dg::blas1::axpby( 1., solutiondsTds, -1., dsTdsfb);
     errdsTdsfb =dg::blas2::dot( w3d, dsTdsfb);
     std::cout << "Relative Difference in DST is "<< sqrt( errdsTdsfb/normdsTds )<<"\n";
@@ -425,7 +426,7 @@ int main( )
 //     errinvT =dg::blas2::dot( w3d, functionTinv);
 //     std::cout << "Relative Difference is  "<< sqrt( errinvT/normf )<<"\n";
 //     
-    std::cout << "--------------------testing dsT" << std::endl; 
+    std::cout << "--------------------testing inversion dsTds" << std::endl; 
     std::cout << " # of iterations "<< invert( dsNU, functionTinv2,solutiondsTds ) << std::endl; //is dsTds
     std::cout << "Norm analytic Solution  "<<sqrt( normf)<<"\n";
     double errinvT2 =dg::blas2::dot( w3d, functionTinv2);
-- 
GitLab


From a7445282339bc00060e06a11d1b0243980889ce2 Mon Sep 17 00:00:00 2001
From: Matthias <Matthias.Wiesenberger@uibk.ac.at>
Date: Tue, 5 Apr 2016 17:21:28 +0200
Subject: [PATCH 003/453] added relative difference in divb

---
 inc/geometries/guenther_ds_b.cu | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/inc/geometries/guenther_ds_b.cu b/inc/geometries/guenther_ds_b.cu
index 3236e91d6..5122724dc 100644
--- a/inc/geometries/guenther_ds_b.cu
+++ b/inc/geometries/guenther_ds_b.cu
@@ -298,7 +298,7 @@ int main( )
      //std::cout << "|| divbT  ||   "<<sqrt( normdivbT)<<"\n";
      dg::blas1::axpby( 1., divbsol, -1., divbT);
      normdivbT =dg::blas2::dot(divbT, w3d,divbT);
-     std::cout << "Relative Difference in divb is   "<<sqrt( normdivbT)<<"\n";
+     std::cout << "Relative Difference in divb is   "<<sqrt( normdivbT/normdivb)<<"\n";
      //std::cout << "-------------------- " << std::endl;
      //std::cout << "|| divB || "<<sqrt( normdivBT)<<"\n";
 // 
-- 
GitLab


From c8a4a643f735900efdf4e397bf9dacfb45e85e69 Mon Sep 17 00:00:00 2001
From: mheld <markus.held@uibk.ac.at>
Date: Wed, 6 Apr 2016 16:25:35 +0200
Subject: [PATCH 004/453] tried some stuff with newds

---
 diag/feltorSHdiag.cu            | 50 ++++++++++++++++++---------------
 inc/dg/backend/projection_t.cu  |  5 ++--
 inc/dg/ds.h                     | 41 ++++++++++++++++++---------
 inc/geometries/guenther_ds_b.cu | 18 +++++++-----
 4 files changed, 69 insertions(+), 45 deletions(-)

diff --git a/diag/feltorSHdiag.cu b/diag/feltorSHdiag.cu
index a46654a7a..ee53ce97c 100644
--- a/diag/feltorSHdiag.cu
+++ b/diag/feltorSHdiag.cu
@@ -23,8 +23,8 @@ double Y( double x, double y) {return y;}
 
 struct Heaviside2d
 {
-    Heaviside2d( double sigma):sigma2_(sigma*sigma/4.), x_(0), y_(0){}
-//     Heaviside2d( double sigma):sigma2_(sigma*sigma), x_(0), y_(0){}
+//     Heaviside2d( double sigma):sigma2_(sigma*sigma/2.), x_(0), y_(0){} //for 2sigma^2
+    Heaviside2d( double sigma):sigma2_(sigma*sigma), x_(0), y_(0){} //for 4 sigma^2
     void set_origin( double x0, double y0){ x_=x0, y_=y0;}
     double operator()(double x, double y)
     {
@@ -143,7 +143,7 @@ int main( int argc, char* argv[])
     const double hy = g2d.hy()/(double)g2d.n();
     unsigned Nx = p.Nx*p.n; 
     //routiens to compute ti
-    dg::Invert<dg::DVec> invert_invgamma2( helper, helper.size(), 1e-3);
+    dg::Invert<dg::DVec> invert_invgamma2( helper, helper.size(), 1e-6);
     dg::Helmholtz2< dg::DMatrix, dg::DVec, dg::DVec > invgamma2( g2d,g2d.bcx(), g2d.bcy(), -0.5*p.tau[1]*p.mu[1],dg::centered);
     dg::DVec binv(dg::evaluate( dg::LinearX( p.mcv, 1.), g2d) );
     dg::DVec B2(dg::evaluate( dg::one, g2d));
@@ -164,7 +164,7 @@ int main( int argc, char* argv[])
     double posX_max = 0.0,posY_max = 0.0,posX_max_old = 0.0,posY_max_old = 0.0,velX_max=0.0, velY_max=0.0,posX_max_hs=0.0,posY_max_hs=0.0,velCOM=0.0;
     double compactness_ne=0.0;
     //-----------------Start timestepping
-    for( unsigned i=0; i<p.maxout; i++)
+    for( unsigned i=0; i<2; i++)
     {
         start2d[0] = i;
         start1d[0] = i;
@@ -289,7 +289,7 @@ int main( int argc, char* argv[])
 //         err_out = nc_put_vara_double( ncid_out, names2dID[2], start2d, count2d, transfer2d.data());
 // 
 //      
-// //         //Compute ion temperature with phi=0
+//         //Compute ion temperature with phi=0
 //         dg::blas1::pointwiseDivide(B2,tpe[1],helper);        //helper  = B^2/Ti
 //         invgamma2.set_chi(helper);                           //helmholtz2 chi = B^2/Ti
 // 
@@ -322,34 +322,40 @@ int main( int argc, char* argv[])
 //         dg::blas2::symv(polti,phi,helper3); //-nabla(P_i/B^2 (nabla_perp phi));
 //         dg::blas1::axpby(1.0, target, -2.0*p.mu[1], helper3,helper2);//pi  =  2 nabla(P_i/B^2 (nabla_perp phi)) + p_i_bar
 //         dg::blas1::pointwiseDivide(helper2,npe[0],helper2); //ti=(pi-amp^2)/ne
-// //         
-// //          //compute Omega_d
-//                 dg::blas1::transform(helper2,helper, dg::PLUS<>(-(p.bgprofamp + p.nprofileamp))); 
-// //                 
+//         
+        //compute Omega_d //therm
+//         dg::blas1::transform(helper2,helper, dg::PLUS<>(-(p.bgprofamp + p.nprofileamp))); //                 
 //         polti.set_chi(one);
 //         dg::blas2::symv(polti,helper,helper3); //nabla(nabla_perp p_i/B^2 );
 //         dg::blas1::scal(helper3,-1.0*p.tau[1]);
-// //         
-// //         //write t_i into 2dnetcdf
-//         transfer2d = helper2;
-//         err_out = nc_put_vara_double( ncid_out, names2dID[0], start2d, count2d, transfer2d.data());
-//         //write Omega_d into 2dnetcdf
-//         transfer2d = helper3;
-//         err_out = nc_put_vara_double( ncid_out, names2dID[1], start2d, count2d, transfer2d.data());
-//               
+        //compute Omega_d //iso
+        dg::blas1::transform(npe[0],helper, dg::PLUS<>(-(p.bgprofamp + p.nprofileamp))); //       
+       dg::blas1::pointwiseDot( helper, binv, helper);
+        dg::blas1::pointwiseDot( helper, binv, helper);    //helper2 = P_i/B^2  
+        polti.set_chi(one);
+        dg::blas2::symv(polti,helper,helper3); //nabla(nabla_perp p_i/B^2 );
+        dg::blas1::scal(helper3,-1.0*p.tau[1]);
 //         
+//         //write t_i into 2dnetcdf
+        transfer2d = helper2;
+        err_out = nc_put_vara_double( ncid_out, names2dID[0], start2d, count2d, transfer2d.data());
+        //write Omega_d into 2dnetcdf
+        transfer2d = helper3;
+        err_out = nc_put_vara_double( ncid_out, names2dID[1], start2d, count2d, transfer2d.data());
+              
+        
         
-        //compute Omega_E and write Omega_E into 2dnetcdf
-        dg::blas1::pointwiseDot( npe[1],one,helper3); 
+//      compute Omega_E and write Omega_E into 2dnetcdf
+        dg::blas1::pointwiseDot( npe[0],one,helper3); 
         dg::blas1::pointwiseDot( helper3, binv, helper2);
         dg::blas1::pointwiseDot( helper2, binv, helper2);   
         polti.set_chi(helper2);
         dg::blas2::symv(polti,phi,helper3); 
         dg::blas1::scal(helper3,-1.0*p.mu[1]);
         polti.set_chi(one);
-        dg::blas2::symv(polti,phi,helper); 
-        dg::blas1::pointwiseDot(helper,helper2,helper);
-        dg::blas1::axpby(1.0,helper3,p.mu[1],helper,helper3);
+//         dg::blas2::symv(polti,phi,helper); 
+//         dg::blas1::pointwiseDot(helper,helper2,helper);
+//         dg::blas1::axpby(1.0,helper3,p.mu[1],helper,helper3);
         transfer2d = helper3;
         err_out = nc_put_vara_double( ncid_out, names2dID[3], start2d, count2d, transfer2d.data());
 
diff --git a/inc/dg/backend/projection_t.cu b/inc/dg/backend/projection_t.cu
index 838c3b6ed..ffa3b738b 100644
--- a/inc/dg/backend/projection_t.cu
+++ b/inc/dg/backend/projection_t.cu
@@ -33,9 +33,10 @@ int main()
 
     std::cout << "TEST 2D\n";
     n_old = 7, n_new = 3, N = 4, Nf = 3;
-    std::cout << "Type n, N, Nf "<< std::endl;
+    std::cout << "Type n_old,n_new, N, Nf "<< std::endl;
     std::cin >>n_old;
-    n_new = n_old;
+//     n_new = n_old;
+    std::cin >> n_new;
     std::cin >> N;
     std::cin >> Nf;
     
diff --git a/inc/dg/ds.h b/inc/dg/ds.h
index 39899c0f4..1ae636a4a 100644
--- a/inc/dg/ds.h
+++ b/inc/dg/ds.h
@@ -219,6 +219,7 @@ struct DS
     container tempPc, tempMc, temp0c;
     container f, dsf;
     container vol3d, inv3d;
+    container vol3df, inv3df;
     container invB;
     //container R_;
     dg::norm no_;
@@ -241,6 +242,7 @@ DS<FA, M,container>::DS(const FA& field, Geometry gridc, Field inverseB, dg::nor
         tempPc( dg::evaluate( dg::zero, gridc)), temp0c( tempPc), tempMc( tempPc), 
         f(tempP), dsf(tempP),
         vol3d( dg::create::volume( gridc)), inv3d( dg::create::inv_volume( gridc)),
+        vol3df( dg::create::volume( field.grid())), inv3df( dg::create::inv_volume( field.grid())),
         invB(dg::pullback(inverseB,field.grid())), //R_(dg::evaluate(dg::coo1,grid)), 
         no_(no), dir_(dir), apply_jumpX_(jumpX)
 {
@@ -277,26 +279,27 @@ void DS<F,M,container>::centered( const container& fc, container& dsfc)
     f_.einsMinus( f, tempM);
     dg::blas1::axpby( 1., tempP, -1., tempM);
     dg::blas1::pointwiseDivide( tempM, f_.hz(), dsf);
+      cusp::multiply( f2c, dsf, dsfc);  
+
     
     ////adjoint discretisation
-/*    assert( &f != &dsf);    
-    dg::blas1::pointwiseDot( vol3d, f, dsf);
-    dg::blas1::pointwiseDivide( dsf, f_.hz(), dsf);
-    dg::blas1::pointwiseDivide( dsf, invB, dsf);
-
-    einsPlusT( dsf, tempP);
-    einsMinusT( dsf, tempM);
-    dg::blas1::axpby( 1., tempM, -1., tempP);
-    dg::blas1::pointwiseDot( inv3d, tempP, dsf);
-    dg::blas1::pointwiseDot( dsf, invB, dsf);  */  
-    cusp::multiply( f2c, dsf, dsfc);
-
+//     assert( &fc != &dsfc);
+//     dg::blas1::pointwiseDot( vol3d, fc, dsfc);
+//     cusp::multiply( f2cT, dsfc, dsf);
+//     dg::blas1::pointwiseDivide( dsf, f_.hz(), dsf);
+//     dg::blas1::pointwiseDivide( dsf, invB, dsf);
+//     f_.einsPlusT( dsf, tempP);
+//     f_.einsMinusT( dsf, tempM);
+//     dg::blas1::axpby( 1., tempM, -1., tempP);
+//     dg::blas1::pointwiseDot( tempP, invB, tempP);
+//     cusp::multiply( c2fT, tempP, dsfc);
+//     dg::blas1::pointwiseDot( inv3d, dsfc, dsfc); 
 }
 
 template<class F, class M, class container>
 void DS<F,M,container>::centeredT( const container& fc, container& dsfc)
 {               
-//     //adjoint discretisation
+    //adjoint discretisation
     assert( &fc != &dsfc);    
     dg::blas1::pointwiseDot( vol3d, fc, dsfc);
     cusp::multiply( f2cT, dsfc, dsf);
@@ -308,7 +311,17 @@ void DS<F,M,container>::centeredT( const container& fc, container& dsfc)
 
     cusp::multiply( c2fT, tempP, dsfc);
     dg::blas1::pointwiseDot( inv3d, dsfc, dsfc); 
-
+    
+    //non adjoint direct 
+//     assert( &fc != &dsfc);     
+//     cusp::multiply( c2f, fc, f);
+//     dg::blas1::pointwiseDot( f, invB, dsf);
+//     f_.einsPlus( dsf, tempP);
+//     f_.einsMinus( dsf, tempM);
+//     dg::blas1::axpby( 1., tempP, -1., tempM);
+//     dg::blas1::pointwiseDivide( tempM, f_.hz(), dsf);        
+//     dg::blas1::pointwiseDivide( dsf, invB, dsf);
+//     cusp::multiply( f2c, dsf, dsfc);  
 //       dg::blas1::pointwiseDot( inv3d, tempP,tempP); //make it symmetric
         //stegmeir weights
 //         dg::blas1::pointwiseDot( f_.hz()h, f, dsf);
diff --git a/inc/geometries/guenther_ds_b.cu b/inc/geometries/guenther_ds_b.cu
index 5122724dc..7b99b7c1e 100644
--- a/inc/geometries/guenther_ds_b.cu
+++ b/inc/geometries/guenther_ds_b.cu
@@ -58,21 +58,25 @@ int main( )
     guenther::DeriNeuT deriNEUT(gp.R_0,gp.I_0);
     guenther::Divb divb(gp.R_0,gp.I_0);
     guenther::B Bfield(gp);
-    
-    std::cout << "Type fn, fN\n";
+        
     //std::cout << "Note, that function is resolved exactly in R,Z for n > 2\n";
     unsigned n=3, Nx=5, Ny=5, Nz=5;
-    unsigned fn, fN;
+    unsigned fn;
+    double fN;
+    std::cout << "Type n\n";
+    std::cin >> n;
+    std::cout << "Type fn, fN\n";
+
     std::cin >> fn>> fN;
     unsigned Nxn = Nx;
     unsigned Nyn = Ny;
     unsigned Nzn = Nz;
 
     double rk4eps = 1e-8;
-    //std::cout << "Type RK4 eps (1e-8)\n";
-    //std::cin >> rk4eps;
+//     std::cout << "Type RK4 eps (1e-8)\n";
+//     std::cin >> rk4eps;
     double z0 = 0, z1 = 2.*M_PI;
-    for (unsigned i=0;i<4;i+=1) { 
+    for (unsigned i=0;i<6;i+=1) { 
 
         Nzn = unsigned(Nz*pow(2,i));
         Nxn = (unsigned)ceil(Nx*pow(2,(double)(i*2./n)));
@@ -408,7 +412,7 @@ int main( )
 //     elliptic.set_z(bhatPhi);
     
     
-    double eps =1e-8;   
+    double eps =1e-7;   
     dg::Invert< dg::DVec> invert( dg::evaluate(dg::zero,g3d), w3d.size(), eps );  
     std::cout << "MAX # iterations = " << w3d.size() << std::endl;
 // 
-- 
GitLab


From 30a5f039b2ec8aae3593a7175478732037e2c7a8 Mon Sep 17 00:00:00 2001
From: mheld <markus.held@uibk.ac.at>
Date: Thu, 14 Apr 2016 10:43:01 +0200
Subject: [PATCH 005/453] fixed feltorSH and played around with newds

---
 inc/dg/helmholtz.h      |  4 ++--
 src/feltorSH/feltor.cu  |  4 +++-
 src/feltorSH/feltor.cuh | 48 +++++++++++++++++++++++++----------------
 3 files changed, 35 insertions(+), 21 deletions(-)

diff --git a/inc/dg/helmholtz.h b/inc/dg/helmholtz.h
index 9791da59c..d7ed5defb 100644
--- a/inc/dg/helmholtz.h
+++ b/inc/dg/helmholtz.h
@@ -189,7 +189,7 @@ struct Helmholtz2
             blas1::pointwiseDot( chi_, x, temp_); //temp = chi*x
         if( alpha_ != 0)
         {
-            blas2::symv( laplaceM_, x, y); // y = W nabla_perp^2 x
+            blas2::symv( laplaceM_, x, y); // y =- W nabla_perp^2 x
             blas1::pointwiseDivide(y, chi_, temp2_); //temp2_ = (chi^-1)*W*nabla_perp^2 x
             blas2::symv( laplaceM_.precond(), temp2_, temp3_); //temp3_ = V*(chi^-1)*W*nabla_perp^2 x
             blas2::symv( laplaceM_, temp3_, temp2_);//temp2_ = W * nabla_perp^2 *(chi^-1)*nabla_perp^2 x            
@@ -198,7 +198,7 @@ struct Helmholtz2
         {
             //y = W temp - alpha*y = W(chi + alpha nabla_perp^2)x
             blas2::symv( 1., laplaceM_.weights(), temp_, -2.*alpha_, y); 
-            //y = W(chi + alpha nabla_perp^2+  nabla_perp^2 *(chi^-1)*nabla_perp^2 )x
+            //y = W(chi + alpha nabla_perp^2+  alpha*alpha*nabla_perp^2 *(chi^-1)*nabla_perp^2 )x
             blas1::axpby( alpha_*alpha_, temp2_, 1.0, y, y);
         }
         else
diff --git a/src/feltorSH/feltor.cu b/src/feltorSH/feltor.cu
index 09b99c01c..dc72902ff 100644
--- a/src/feltorSH/feltor.cu
+++ b/src/feltorSH/feltor.cu
@@ -109,8 +109,10 @@ int main( int argc, char* argv[])
         dg::blas1::pointwiseDivide(y0[2],y0[0],y0[2]);
         
 
-        if( p.init != 0)
+        if( p.init != 0){
+            dg::blas1::axpby(0., y0[3], 1., y1[2],y0[3]); //constant temperature
             dg::blas1::axpby( 1., y0[3], 0., y0[2], y0[2]); //for Omega*=0
+        }
 
         dg::blas1::transform(y0[2], y0[2], dg::PLUS<>(-(p.bgprofamp + p.nprofileamp)));
         dg::blas1::transform(y0[0], y0[0], dg::PLUS<>(-(p.bgprofamp + p.nprofileamp))); // =ne-bg
diff --git a/src/feltorSH/feltor.cuh b/src/feltorSH/feltor.cuh
index de28a474e..96e121e56 100644
--- a/src/feltorSH/feltor.cuh
+++ b/src/feltorSH/feltor.cuh
@@ -363,20 +363,26 @@ void Feltor<G, Matrix, container>::operator()( std::vector<container>& y, std::v
         dg::blas1::pointwiseDot( yp[i+2], binv, yp[i+2]);  //dt T = 1/B [T-1,psi]_xy
     }
     //add 2nd order FLR terms to ExB dynamics 
-    //[chi,Ni] and [2 chi,Ti] terms
-    poisson( y[1], chii, omega);  //omega  = [Ni-1,chi_i]_xy
-    dg::blas1::pointwiseDot(omega, binv, omega); //omega = 1/B[Ni-1,chii]_xy
-    dg::blas1::axpby(1.,omega,1.0,yp[1]); //dt N_i += 1/B[Ni-1,chii]_xy
+    //[2 chi,Ti] terms 
     poisson( y[3], chii, omega);  //omega = [T-1,chi_i]_xy
     dg::blas1::pointwiseDot(omega, binv, omega); //omega = 1/B [T-1,chii]_xy
     dg::blas1::axpby(2.,omega,1.0,yp[3]); //dt T_i += 1/B [T-1, 2 chii]_xy
     
     //Moment mixing terms
-    //[lnTi,Ni chii] term
-    dg::blas1::pointwiseDot(ype[1],chii,lambda); // lambda = N_i chii
-    poisson( logype[3], lambda, omega);  //omega = [ln(Ti),Ni*chii]_xy
-    dg::blas1::pointwiseDot(omega, binv, omega); //omega = 1/B [ln(Ti),Ni*chii]_xy
-    dg::blas1::axpby(1.,omega,1.0,yp[1]); //dt N_i += 1/B [ln(Ti),Ni*chii]_xy
+    //[lnTi,Ni chii] term version 1
+//     dg::blas1::pointwiseDot(ype[1],chii,lambda); // lambda = N_i chii
+//     poisson( logype[3], lambda, omega);  //omega = [ln(Ti),Ni*chii]_xy
+//     dg::blas1::pointwiseDot(omega, binv, omega); //omega = 1/B [ln(Ti),Ni*chii]_xy
+//     dg::blas1::axpby(1.,omega,1.0,yp[1]); //dt N_i += 1/B [ln(Ti),Ni*chii]_xy
+        //[lnTi,Ni chii] term version 2
+    poisson( logype[3], chii, omega);  //omega = [ln(Ti),chii]_xy
+    dg::blas1::pointwiseDot(omega, binv, omega); //omega = 1/B [ln(Ti),chii]_xy
+    dg::blas1::pointwiseDot(omega, ype[1], omega); //omega = Ni/B [ln(Ti),chii]_xy
+    dg::blas1::axpby(1.,omega,1.0,yp[1]); //dt N_i += Ni/B [ln(Ti),chii]_xy
+    poisson( logype[3],  y[1], omega);  //omega = [ln(Ti),Ni]_xy
+    dg::blas1::pointwiseDot(omega, binv, omega); //omega = 1/B [ln(Ti),Ni]_xy
+    dg::blas1::pointwiseDot(omega, chii, omega); //omega = chii/B [ln(Ti),Ni]_xy
+    dg::blas1::axpby(1.,omega,1.0,yp[1]); //dt N_i += chii/B [ln(Ti),Ni]_xy
     //Ti chii [ln(chii)- ln(Ti),ln(Ni)] term
     poisson( logype[1],chii, omega);  //omega = [ln(Ni),chii]_xy
     dg::blas1::pointwiseDot(omega, binv, omega); //omega = 1/B [ln(Ni),chii]_xy
@@ -390,12 +396,13 @@ void Feltor<G, Matrix, container>::operator()( std::vector<container>& y, std::v
     //curvature dynamics
     for(unsigned i=0; i<2; i++)
     {
-        //N*K(psi) T*K(psi)  terms
+        //N*K(psi) and T*K(psi)  terms
         dg::blas2::gemv( poisson.dyrhs(), phi[i], lambda); //lambda = dy psi
         dg::blas1::pointwiseDot(lambda,ype[i],omega); //omega =  n dy psi
         dg::blas1::axpby(p.mcv,omega,1.0,yp[i]);   // dtN +=  mcv* N dy psi
         dg::blas1::pointwiseDot(lambda,ype[i+2],omega); // T dy psi
         dg::blas1::axpby(p.mcv,omega,1.0,yp[i+2]);   // dtT +=  mcv* T dy psi
+        
         // K(T N) terms
         dg::blas2::gemv( poisson.dyrhs(), y[i], lambda); //lambda = dy (N-1)
         dg::blas1::pointwiseDot(lambda,ype[i+2],omega); //omega =  T dy (N-1)
@@ -403,8 +410,8 @@ void Feltor<G, Matrix, container>::operator()( std::vector<container>& y, std::v
         dg::blas2::gemv( poisson.dyrhs(), y[i+2], lambda); //lambda = dy (T-1)
         dg::blas1::pointwiseDot(lambda,ype[i],omega); // omega = N dy (T-1)
         dg::blas1::axpby(p.tau[i]*p.mcv,omega,1.0,yp[i]);  //dt N += tau*mcv*N dy (T-1)
-// 
-//      //3 T*K(T) terms
+        
+        //3 T*K(T) terms
         dg::blas2::gemv( poisson.dyrhs(), y[i+2], lambda); //lambda = dy (T-1)
         dg::blas1::pointwiseDot(lambda,ype[i+2],omega); // omega = T dy (T-1)
         dg::blas1::axpby(3.*p.tau[i]*p.mcv,omega,1.0,yp[i+2]); //dt T +=  3 tau*mcv* T dy (T-1)
@@ -412,8 +419,7 @@ void Feltor<G, Matrix, container>::operator()( std::vector<container>& y, std::v
         dg::blas2::gemv( poisson.dyrhs(), logype[i], lambda); //lambda = dy (ln(N))
         dg::blas1::pointwiseDot(lambda,ype[i+2],omega); //omega = T dy (ln(N))
         dg::blas1::pointwiseDot(omega,ype[i+2],omega); //omega =  T^2 dy (ln(N))
-        dg::blas1::axpby(p.tau[i]*p.mcv,omega,1.0,yp[i+2]); //dt T += tau mcv T^2 dy (ln(N)) 
-        
+        dg::blas1::axpby(p.tau[i]*p.mcv,omega,1.0,yp[i+2]); //dt T += tau mcv T^2 dy (ln(N))         
     }   
     //add FLR correction to curvature dynamics
     //Ni K(chii) and Ti K(3 chii) term
@@ -422,17 +428,23 @@ void Feltor<G, Matrix, container>::operator()( std::vector<container>& y, std::v
     dg::blas1::axpby(p.mcv,omega,1.0,yp[1]);   // dtNi +=  mcv* Ni dy chii
     dg::blas1::pointwiseDot(lambda,ype[3],omega); // omega = Ti dy chii
     dg::blas1::axpby(3.*p.mcv,omega,1.0,yp[3]);   // dtTi += 3.* mcv* Ti dy chii
-    //Ni chii K(lnTi + lnNi) term
+    
+//     //Ni chii K(lnTi - lnNi) term
     dg::blas1::axpby(1.,logype[1],-1.0,logype[3],omega); //omega = -ln(Ti)+ln(Ni)
     dg::blas2::gemv( poisson.dyrhs(), omega, lambda); //lambda = dy(-ln(Ti)+ln(Ni))
     dg::blas1::pointwiseDot(lambda,ype[1],omega); // omega = Ni dy(-ln(Ti)+ln(Ni))
     dg::blas1::pointwiseDot(omega,chii,omega); //omega =  Ni  chii dy(-ln(Ti)+ln(Ni))
     dg::blas1::axpby(p.mcv,omega,1.0,yp[1]);   // dtNi +=  mcv* Ni  chii dy(-ln(Ti)+ln(Ni))
-    //chii K(Ti) term
+
+    
+//     //chii K(Ti) term
+//     dg::blas2::gemv( poisson.dyrhs(), y[3], lambda); //lambda = dy (Ti-1)
+//     dg::blas1::pointwiseDot(lambda,chii,omega); //omega =  chii dy (Ti-1)
+//     dg::blas1::axpby(p.mcv,omega,1.0,yp[3]);   // dtTi +=  mcv*  chii dy (Ti-1)
     dg::blas2::gemv( poisson.dyrhs(), y[3], lambda); //lambda = dy (Ti-1)
     dg::blas1::pointwiseDot(lambda,chii,omega); //omega =  chii dy (Ti-1)
-    dg::blas1::axpby(p.mcv,omega,1.0,yp[3]);   // dtTi +=  mcv*  chii dy (Ti-1)
-    //Ti chii K(lnNi)) term
+    dg::blas1::axpby(-p.mcv,omega,1.0,yp[3]);   // dtTi +=-  mcv*  chii dy (Ti-1)
+//     //Ti chii K(lnNi)) term
     dg::blas2::gemv( poisson.dyrhs(), logype[1], lambda); //lambda = dy (ln(Ni))
     dg::blas1::pointwiseDot(lambda,chii,omega); // omega = chii dy (ln(Ni))
     dg::blas1::pointwiseDot(omega,ype[3],omega); // omega =Ti chii dy (ln(Ni))
-- 
GitLab


From a3b2a159107cca0afae50266534ac097f2966494 Mon Sep 17 00:00:00 2001
From: Matthias <Matthias.Wiesenberger@uibk.ac.at>
Date: Tue, 19 Apr 2016 15:48:54 +0200
Subject: [PATCH 006/453] tried out new interpolation scheme

---
 inc/dg/backend/interpolation.cuh  | 134 ++++++++++++++++++++++++++++++
 inc/dg/backend/interpolation_t.cu |  51 ++++++++++--
 inc/dg/ds.h                       |   2 +-
 3 files changed, 180 insertions(+), 7 deletions(-)

diff --git a/inc/dg/backend/interpolation.cuh b/inc/dg/backend/interpolation.cuh
index 882bb608b..ab6c419b8 100644
--- a/inc/dg/backend/interpolation.cuh
+++ b/inc/dg/backend/interpolation.cuh
@@ -372,6 +372,140 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const Grid3d<dou
     return interpolation( pointsX, pointsY, pointsZ, g_old);
 
 }
+
+cusp::coo_matrix<int, double, cusp::host_memory> scalar_interpolation( const Grid2d<double>& g_coarse, const Grid2d<double>& g_fine)
+{
+
+    assert( g_coarse.x0() >= g_fine.x0());
+    assert( g_coarse.x1() <= g_fine.x1());
+    assert( g_coarse.y0() >= g_fine.y0());
+    assert( g_coarse.y1() <= g_fine.y1());
+    assert( g_coarse.n()*g_coarse.Nx() == g_fine.Nx());
+    assert( g_coarse.n()*g_coarse.Ny() == g_fine.Ny());
+
+    thrust::host_vector<double> x = dg::evaluate( dg::coo1, g_coarse);
+    thrust::host_vector<double> y = dg::evaluate( dg::coo2, g_coarse);
+    thrust::host_vector<double> wcoarse = dg::create::weights( g_coarse);
+    cusp::coo_matrix<int, double, cusp::host_memory> A( x.size(), g_fine.size(), x.size()*g_fine.n()*g_fine.n());
+
+    int number = 0;
+    for( unsigned i=0; i<x.size(); i++)
+    {
+        if (!(x[i] >= g_fine.x0() && x[i] <= g_fine.x1())) {
+            std::cerr << g_fine.x0()<<"< xi = " << x[i] <<" < "<<g_fine.x1()<<std::endl;
+        }
+        
+        assert(x[i] >= g_fine.x0() && x[i] <= g_fine.x1());
+        
+        if (!(y[i] >= g_fine.y0() && y[i] <= g_fine.y1())) {
+            std::cerr << g_fine.y0()<<"< yi = " << y[i] <<" < "<<g_fine.y1()<<std::endl;
+        }
+        assert( y[i] >= g_fine.y0() && y[i] <= g_fine.y1());
+
+        //determine which cell (x,y) lies in 
+
+        double xnn = (x[i]-g_fine.x0())/g_fine.hx();
+        double ynn = (y[i]-g_fine.y0())/g_fine.hy();
+        unsigned n = (unsigned)floor(xnn);
+        unsigned m = (unsigned)floor(ynn);
+        //determine normalized coordinates
+
+        //double xn =  2.*xnn - (double)(2*n+1); 
+        //double yn =  2.*ynn - (double)(2*m+1); 
+        ////interval correction
+        //if (n==g_fine.Nx()) {
+        //    n-=1;
+        //    xn = 1.;
+        //}
+        //if (m==g_fine.Ny()) {
+        //    m-=1;
+        //    yn =1.;
+        //}
+
+
+        std::vector<double> pxy( g_fine.n()*g_fine.n());
+        for( int k=0; k<g_fine.n(); k++)
+            for( int l=0; l<g_fine.n(); l++)
+            {
+                pxy[k*g_fine.n()+l] = g_fine.hx()*g_fine.hy()/4.*
+                            g_fine.dlt().weights()[ k]*
+                            g_fine.dlt().weights()[ l];
+                pxy[k*g_fine.n()+l] /= wcoarse[i];
+            }
+
+        unsigned col_begin = (m)*g_fine.Nx()*g_fine.n()*g_fine.n() + (n)*g_fine.n();
+        detail::add_line( A, number, i,  col_begin, g_fine.n(), g_fine.Nx(), pxy); 
+    }
+    return A;
+}
+
+cusp::coo_matrix<int, double, cusp::host_memory> scalar_interpolation( const Grid3d<double>& g_coarse, const Grid3d<double>& g_fine)
+{
+    assert( g_coarse.x0() >= g_fine.x0());
+    assert( g_coarse.x1() <= g_fine.x1());
+    assert( g_coarse.y0() >= g_fine.y0());
+    assert( g_coarse.y1() <= g_fine.y1());
+    assert( g_coarse.z0() >= g_fine.z0());
+    assert( g_coarse.z1() <= g_fine.z1());
+    assert( g_coarse.n()*g_coarse.Nx() == g_fine.Nx());
+    assert( g_coarse.n()*g_coarse.Ny() == g_fine.Ny());
+    assert( g_coarse.Nz() == g_fine.Nz());
+
+    thrust::host_vector<double> x = dg::evaluate( dg::coo1, g_coarse);
+    thrust::host_vector<double> y = dg::evaluate( dg::coo2, g_coarse);
+    thrust::host_vector<double> wcoarse = dg::create::weights( g_coarse);
+    cusp::coo_matrix<int, double, cusp::host_memory> A( x.size(), g_fine.size(), x.size()*g_fine.n()*g_fine.n());
+
+    int number = 0;
+    for( unsigned i=0; i<x.size(); i++)
+    {
+        if (!(x[i] >= g_fine.x0() && x[i] <= g_fine.x1())) {
+            std::cerr << g_fine.x0()<<"< xi = " << x[i] <<" < "<<g_fine.x1()<<std::endl;
+        }
+        
+        assert(x[i] >= g_fine.x0() && x[i] <= g_fine.x1());
+        
+        if (!(y[i] >= g_fine.y0() && y[i] <= g_fine.y1())) {
+            std::cerr << g_fine.y0()<<"< yi = " << y[i] <<" < "<<g_fine.y1()<<std::endl;
+        }
+        assert( y[i] >= g_fine.y0() && y[i] <= g_fine.y1());
+
+        //determine which cell (x,y) lies in 
+
+        double xnn = (x[i]-g_fine.x0())/g_fine.hx();
+        double ynn = (y[i]-g_fine.y0())/g_fine.hy();
+        unsigned n = (unsigned)floor(xnn);
+        unsigned m = (unsigned)floor(ynn);
+        //determine normalized coordinates
+
+        //double xn =  2.*xnn - (double)(2*n+1); 
+        //double yn =  2.*ynn - (double)(2*m+1); 
+        ////interval correction
+        //if (n==g_fine.Nx()) {
+        //    n-=1;
+        //    xn = 1.;
+        //}
+        //if (m==g_fine.Ny()) {
+        //    m-=1;
+        //    yn =1.;
+        //}
+
+
+        std::vector<double> pxy( g_fine.n()*g_fine.n());
+        for( int k=0; k<g_fine.n(); k++)
+            for( int l=0; l<g_fine.n(); l++)
+            {
+                pxy[k*g_fine.n()+l] = g_fine.hz()*g_fine.hx()*g_fine.hy()/4.*
+                            g_fine.dlt().weights()[ k]*
+                            g_fine.dlt().weights()[ l];
+                pxy[k*g_fine.n()+l] /= wcoarse[i];
+            }
+
+        unsigned col_begin = (m)*g_fine.Nx()*g_fine.n()*g_fine.n() + (n)*g_fine.n();
+        detail::add_line( A, number, i,  col_begin, g_fine.n(), g_fine.Nx(), pxy); 
+    }
+    return A;
+}
 ///@}
 
 
diff --git a/inc/dg/backend/interpolation_t.cu b/inc/dg/backend/interpolation_t.cu
index 40b580027..c211a7b54 100644
--- a/inc/dg/backend/interpolation_t.cu
+++ b/inc/dg/backend/interpolation_t.cu
@@ -1,18 +1,25 @@
 #include <iostream>
 
 #include <cusp/print.h>
+#include "../blas.h"
 #include "xspacelib.cuh"
 #include "interpolation.cuh"
 
-const unsigned n = 3;
-const unsigned Nx = 3; 
-const unsigned Ny = 5; 
-const unsigned Nz = 2; 
+ unsigned n = 3;
+ unsigned Nx = 30; 
+ unsigned Ny = 50; 
+ unsigned Nz = 2; 
 
 typedef cusp::coo_matrix<int, double, cusp::host_memory> Matrix;
 
+double sinex( double x, double y) {return sin(x)*sin(x)*sin(y)*sin(y)*x*x*y*y;}
+double sinez( double x, double y, double z) {return sin(x)*sin(x)*sin(y)*sin(y)*x*x*y*y;}
+
 int main()
 {
+    std::cout << "type n, Nx, Ny, Nz\n";
+    std::cin >> n >> Nx >> Ny >> Nz;
+
 
     {
     dg::Grid2d<double> g( -10, 10, -5, 5, n, Nx, Ny);
@@ -35,7 +42,7 @@ int main()
     //ATTENTION: backscatter might delete zeroes in matrices
     for( unsigned i=0; i<A.values.size(); i++)
     {
-        if( (A.values[i] - B.values[i]) > 1e-14)
+        if( (A.values[i] - B.values[i]) > 1e-10)
         {
             std::cerr << "NOT EQUAL "<<A.row_indices[i] <<" "<<A.column_indices[i]<<" "<<A.values[i] << "\t "<<B.row_indices[i]<<" "<<B.column_indices[i]<<" "<<B.values[i]<<"\n";
             passed = false;
@@ -72,6 +79,25 @@ int main()
     }
     if( passed)
         std::cout << "2D INTERPOLATE TEST PASSED!\n";
+
+    dg::Grid2d<double> gfine( -10, 10, -5, 5, n, n*Nx, n*Ny);
+    thrust::host_vector<double> xfine = dg::evaluate( sinex, gfine);
+    thrust::host_vector<double> xcoarse = dg::evaluate( sinex, g);
+    thrust::host_vector<double> wfine = dg::create::weights( gfine);
+    thrust::host_vector<double> wcoarse = dg::create::weights( g);
+    double coarse = dg::blas2::dot( xcoarse, wcoarse, xcoarse);
+    std::cout << "coar integral: "<<coarse<<"\n";
+
+    Matrix f2c = dg::create::interpolation( g, gfine); 
+    dg::blas2::symv( f2c, xfine, xcoarse);
+    double fine = dg::blas2::dot( xfine, wfine, xfine);
+    coarse = dg::blas2::dot( xcoarse, wcoarse, xcoarse);
+    //double fine = dg::blas1::dot( wfine, xfine);
+    //coarse = dg::blas1::dot( wcoarse, xcoarse);
+    std::cout << "Fine integral: "<<fine<<" \n";
+    std::cout << "coar integral: "<<coarse<<"\n";
+    std::cout << "Rel Difference "<<fabs(fine-coarse)/fabs(fine)<<"\n";
+
     }
     ////////////////////////////////////////////////////////////////////////////
     {
@@ -95,7 +121,7 @@ int main()
     bool passed = true;
     for( unsigned i=0; i<A.values.size(); i++)
     {
-        if( (A.values[i] - B.values[i]) > 1e-14)
+        if( (A.values[i] - B.values[i]) > 1e-10)
         {
             std::cerr << "NOT EQUAL "<<A.row_indices[i] <<" "<<A.column_indices[i]<<" "<<A.values[i] << "\t "<<B.row_indices[i]<<" "<<B.column_indices[i]<<" "<<B.values[i]<<"\n";
             passed = false;
@@ -103,6 +129,19 @@ int main()
     }
     if( passed)
         std::cout << "3D TEST PASSED!\n";
+    dg::Grid3d<double> gfine( -10, 10, -5, 5, -7, -3,  n, n*Nx, n*Ny, Nz);
+    thrust::host_vector<double> xfine = dg::evaluate( sinez, gfine);
+    thrust::host_vector<double> xcoarse = dg::evaluate( sinez, g);
+    thrust::host_vector<double> wfine = dg::create::weights( gfine);
+    thrust::host_vector<double> wcoarse = dg::create::weights( g);
+
+    Matrix f2c = dg::create::scalar_interpolation( g, gfine); 
+    dg::blas2::symv( f2c, xfine, xcoarse);
+    double fine = dg::blas1::dot( xfine, wfine);
+    double coarse = dg::blas1::dot( xcoarse, wcoarse);
+    std::cout << "Fine integral: "<<fine<<" \n";
+    std::cout << "coar integral: "<<coarse<<"\n";
+    std::cout << "Difference   : "<<fine-coarse<<"\n";
     }
 
     return 0;
diff --git a/inc/dg/ds.h b/inc/dg/ds.h
index 39899c0f4..400c38963 100644
--- a/inc/dg/ds.h
+++ b/inc/dg/ds.h
@@ -245,7 +245,7 @@ DS<FA, M,container>::DS(const FA& field, Geometry gridc, Field inverseB, dg::nor
         no_(no), dir_(dir), apply_jumpX_(jumpX)
 {
 
-    f2c = dg::create::interpolation( gridc, field.grid());
+    f2c = dg::create::scalar_interpolation( gridc, field.grid());
     c2f = dg::create::interpolation( field.grid(), gridc);
     cusp::transpose( f2c, f2cT);
     cusp::transpose( c2f, c2fT);     
-- 
GitLab


From 477db998c1312d1eed690732838f592e57b63977 Mon Sep 17 00:00:00 2001
From: Matthias <Matthias.Wiesenberger@uibk.ac.at>
Date: Wed, 20 Apr 2016 16:22:30 +0200
Subject: [PATCH 007/453] new interpolation scheme projection

---
 inc/dg/backend/interpolation.cuh  | 256 +++++++++++++++---------------
 inc/dg/backend/interpolation_t.cu |  89 +++++++----
 inc/dg/ds.h                       |   2 +-
 3 files changed, 190 insertions(+), 157 deletions(-)

diff --git a/inc/dg/backend/interpolation.cuh b/inc/dg/backend/interpolation.cuh
index ab6c419b8..7bdef8c86 100644
--- a/inc/dg/backend/interpolation.cuh
+++ b/inc/dg/backend/interpolation.cuh
@@ -61,6 +61,30 @@ std::vector<double> coefficients( double xn, unsigned n)
     }
     return px;
 }
+    
+/**
+ * @brief The n-th Legendre Polynomial on [-1;1]
+ */
+struct Legendre
+{
+    Legendre( unsigned n): n_(n+1), m_(0){}
+    Legendre( unsigned n, unsigned m): n_(n+1), m_(m+1){}
+    double operator()( double x)
+    {
+        //compute p_i(xn) and return the last value
+        std::vector<double> px = coefficients(x, n_);
+        return px[n_-1];
+    }
+    double operator()( double x, double y)
+    {
+        std::vector<double> px = coefficients(x, n_);
+        std::vector<double> py = coefficients(y, m_);
+        return px[n_-1]*py[m_-1];
+    }
+    private:
+    unsigned n_, m_;
+};
+
 
 }//namespace detail
 ///@endcond
@@ -373,156 +397,138 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const Grid3d<dou
 
 }
 
-cusp::coo_matrix<int, double, cusp::host_memory> scalar_interpolation( const Grid2d<double>& g_coarse, const Grid2d<double>& g_fine)
+thrust::host_vector<double> transform( const Operator<double>& op, const thrust::host_vector<double>& in, const Grid2d<double>& g)
+{
+    assert( op.size() == g.n());
+    thrust::host_vector<double> out(in.size(), 0);
+    for( unsigned i=0; i<g.Ny(); i++)
+    for( unsigned k=0; k<g.n(); k++)
+    for( unsigned j=0; j<g.Nx(); j++)
+    for( unsigned l=0; l<g.n(); l++)
+    for( unsigned o=0; o<g.n(); o++)
+    for( unsigned m=0; m<g.n(); m++)
+        out[((i*g.n() + k)*g.Nx() + j)*g.n() + l] += op(k,o)*op( l, m)*in[((i*g.n() + o)*g.Nx() + j)*g.n() + m];
+    return out;
+}
+thrust::host_vector<double> forward_transform( const thrust::host_vector<double>& in, const Grid2d<double>& g)
+{
+    dg::Operator<double> forward( g.dlt().forward());
+    return transform( forward, in, g);
+}
+
+cusp::coo_matrix<int, double, cusp::host_memory> projection( const Grid2d<double>& g_coarse, const Grid2d<double>& g_fine)
 {
 
     assert( g_coarse.x0() >= g_fine.x0());
     assert( g_coarse.x1() <= g_fine.x1());
     assert( g_coarse.y0() >= g_fine.y0());
     assert( g_coarse.y1() <= g_fine.y1());
-    assert( g_coarse.n()*g_coarse.Nx() == g_fine.Nx());
-    assert( g_coarse.n()*g_coarse.Ny() == g_fine.Ny());
-
-    thrust::host_vector<double> x = dg::evaluate( dg::coo1, g_coarse);
-    thrust::host_vector<double> y = dg::evaluate( dg::coo2, g_coarse);
-    thrust::host_vector<double> wcoarse = dg::create::weights( g_coarse);
-    cusp::coo_matrix<int, double, cusp::host_memory> A( x.size(), g_fine.size(), x.size()*g_fine.n()*g_fine.n());
-
-    int number = 0;
-    for( unsigned i=0; i<x.size(); i++)
+    assert( g_fine.Nx() % g_coarse.Nx() == 0);
+    assert( g_fine.Ny() % g_coarse.Ny() == 0);
+
+    unsigned num_cellsX = g_fine.Nx() / g_coarse.Nx();
+    unsigned num_cellsY = g_fine.Ny() / g_coarse.Ny();
+
+    //construct elemental grid with fine number of cells and polynomials
+    Grid2d<double> g_elemental( -1., 1., -1., 1., g_fine.n(), num_cellsX, num_cellsY);
+    //now evaluate the coarse Legendre polynomials on the fine grid
+    thrust::host_vector<double> coeffsX[g_coarse.n()*g_coarse.n()];
+    thrust::host_vector<double> coeffsL[g_coarse.n()*g_coarse.n()];
+    Operator<double> sisj = dg::create::pipj( g_elemental.n());
+    Operator<double> forward( g_elemental.dlt().forward());
+    for( unsigned p=0; p<g_coarse.n(); p++) //y
+        for( unsigned q=0; q<g_coarse.n(); q++)//x
+        {
+            detail::Legendre legendre( q, p); 
+            coeffsX[p*g_coarse.n()+q] = dg::evaluate( legendre, g_elemental);
+            //forward transform coefficients
+            coeffsL[p*g_coarse.n()+q] = transform( forward, coeffsX[p*g_coarse.n() + q], g_elemental);
+
+            //multiply by S matrix 
+            coeffsX[p*g_coarse.n()+q] = transform( sisj, coeffsL[p*g_coarse.n() + q], g_elemental);
+            //std::cout << "p "<<p<<" q "<<q<<"\n";
+            //for( unsigned i=0; i<g_fine.n(); i++)
+            //{
+            //    for( unsigned j=0; j<g_fine.n(); j++)
+            //        std::cout << coeffsX[p*g_coarse.n()+q][i*g_fine.n()*num_cellsX+j] <<" ";
+            //    std::cout << "\n";
+            //}
+            //std::cout <<std::endl;
+            //multiply by forward transpose
+            coeffsL[p*g_coarse.n()+q] = transform( forward.transpose(), coeffsX[p*g_coarse.n() + q], g_elemental);
+            
+        }
+    Grid2d<double> gc_elemental( -1., 1., -1., 1., g_coarse.n(), 1, 1);
+    Operator<double> backward( gc_elemental.dlt().backward());
+    Operator<double> sisj_inv = dg::create::pipj_inv( gc_elemental.n());
+    Operator<double> left = backward*sisj_inv;    
+    //multiply left over all coarse polynomials
+    for( unsigned k=0; k<g_coarse.n(); k++)
+    for( unsigned q=0; q<g_coarse.n(); q++)
     {
-        if (!(x[i] >= g_fine.x0() && x[i] <= g_fine.x1())) {
-            std::cerr << g_fine.x0()<<"< xi = " << x[i] <<" < "<<g_fine.x1()<<std::endl;
+        for( unsigned i=0; i<g_elemental.size(); i++)
+        {
+                coeffsX[k*g_coarse.n()+q][i] = 0.;
+                for( unsigned m=0; m<g_coarse.n(); m++)
+                for( unsigned l=0; l<g_coarse.n(); l++)
+                {
+                    coeffsX[k*g_coarse.n()+q][i] += left(k,m)*left(q,l)*coeffsL[m*g_coarse.n()+l][i];
+                }
+                coeffsX[k*g_coarse.n()+q][i] *= g_fine.hx()*g_fine.hy()/g_coarse.hx()/g_coarse.hy();
         }
-        
-        assert(x[i] >= g_fine.x0() && x[i] <= g_fine.x1());
-        
-        if (!(y[i] >= g_fine.y0() && y[i] <= g_fine.y1())) {
-            std::cerr << g_fine.y0()<<"< yi = " << y[i] <<" < "<<g_fine.y1()<<std::endl;
+    }
+    cusp::coo_matrix<int, double, cusp::host_memory> A( g_coarse.size(), g_fine.size(), g_coarse.size()*num_cellsX*num_cellsY*g_fine.n()*g_fine.n());
+    int number = 0;
+    for( unsigned i=0; i<g_coarse.Ny(); i++)
+    for( unsigned k=0; k<g_coarse.n(); k++)
+    for( unsigned j=0; j<g_coarse.Nx(); j++)
+    for( unsigned q=0; q<g_coarse.n(); q++)
+    {
+        unsigned line = ((i*g_coarse.n()+k)*g_coarse.Nx()+j)*g_coarse.n()+q;
+        //add correct line to A
+        for( unsigned m=0; m<num_cellsY; m++)
+        for( unsigned n=0; n<num_cellsX; n++)
+        {
+            //column for indices (i,j,m,n) the (0,0) element
+            unsigned col_begin = (((i*num_cellsY+m)*g_fine.n()*g_coarse.Nx()+j)*num_cellsX+n)*g_fine.n();
+            std::vector<double> temp(g_fine.n()*g_fine.n());
+            for( unsigned e=0; e<g_fine.n(); e++)
+                for( unsigned h=0; h<g_fine.n(); h++)
+                    temp[e*g_fine.n()+h] = coeffsX[k*g_coarse.n()+q]
+                        [((m*g_fine.n() + e)*num_cellsX + n)*g_fine.n()+h];
+            detail::add_line( A, number, line,  col_begin, g_fine.n(), g_fine.Nx(), temp); 
         }
-        assert( y[i] >= g_fine.y0() && y[i] <= g_fine.y1());
-
-        //determine which cell (x,y) lies in 
-
-        double xnn = (x[i]-g_fine.x0())/g_fine.hx();
-        double ynn = (y[i]-g_fine.y0())/g_fine.hy();
-        unsigned n = (unsigned)floor(xnn);
-        unsigned m = (unsigned)floor(ynn);
-        //determine normalized coordinates
-
-        //double xn =  2.*xnn - (double)(2*n+1); 
-        //double yn =  2.*ynn - (double)(2*m+1); 
-        ////interval correction
-        //if (n==g_fine.Nx()) {
-        //    n-=1;
-        //    xn = 1.;
-        //}
-        //if (m==g_fine.Ny()) {
-        //    m-=1;
-        //    yn =1.;
-        //}
-
-
-        std::vector<double> pxy( g_fine.n()*g_fine.n());
-        for( int k=0; k<g_fine.n(); k++)
-            for( int l=0; l<g_fine.n(); l++)
-            {
-                pxy[k*g_fine.n()+l] = g_fine.hx()*g_fine.hy()/4.*
-                            g_fine.dlt().weights()[ k]*
-                            g_fine.dlt().weights()[ l];
-                pxy[k*g_fine.n()+l] /= wcoarse[i];
-            }
-
-        unsigned col_begin = (m)*g_fine.Nx()*g_fine.n()*g_fine.n() + (n)*g_fine.n();
-        detail::add_line( A, number, i,  col_begin, g_fine.n(), g_fine.Nx(), pxy); 
     }
+
+    A.sort_by_row_and_column();
     return A;
 }
 
-cusp::coo_matrix<int, double, cusp::host_memory> scalar_interpolation( const Grid3d<double>& g_coarse, const Grid3d<double>& g_fine)
+cusp::coo_matrix<int, double, cusp::host_memory> projection( const Grid3d<double>& g_coarse, const Grid3d<double>& g_fine)
 {
-    assert( g_coarse.x0() >= g_fine.x0());
-    assert( g_coarse.x1() <= g_fine.x1());
-    assert( g_coarse.y0() >= g_fine.y0());
-    assert( g_coarse.y1() <= g_fine.y1());
+
     assert( g_coarse.z0() >= g_fine.z0());
     assert( g_coarse.z1() <= g_fine.z1());
-    assert( g_coarse.n()*g_coarse.Nx() == g_fine.Nx());
-    assert( g_coarse.n()*g_coarse.Ny() == g_fine.Ny());
     assert( g_coarse.Nz() == g_fine.Nz());
+    const unsigned Nz = g_coarse.Nz();
 
-    thrust::host_vector<double> x = dg::evaluate( dg::coo1, g_coarse);
-    thrust::host_vector<double> y = dg::evaluate( dg::coo2, g_coarse);
-    thrust::host_vector<double> wcoarse = dg::create::weights( g_coarse);
-    cusp::coo_matrix<int, double, cusp::host_memory> A( x.size(), g_fine.size(), x.size()*g_fine.n()*g_fine.n());
+    Grid2d<double> g2d_coarse( g_coarse.x0(), g_coarse.x1(), g_coarse.y0(), g_coarse.y1(), g_coarse.n(), g_coarse.Nx(), g_coarse.Ny());
+    Grid2d<double> g2d_fine( g_fine.x0(), g_fine.x1(), g_fine.y0(), g_fine.y1(), g_fine.n(), g_fine.Nx(), g_fine.Ny());
+    cusp::coo_matrix<int, double, cusp::host_memory> A2d = projection( g2d_coarse, g2d_fine);
 
-    int number = 0;
-    for( unsigned i=0; i<x.size(); i++)
-    {
-        if (!(x[i] >= g_fine.x0() && x[i] <= g_fine.x1())) {
-            std::cerr << g_fine.x0()<<"< xi = " << x[i] <<" < "<<g_fine.x1()<<std::endl;
-        }
-        
-        assert(x[i] >= g_fine.x0() && x[i] <= g_fine.x1());
-        
-        if (!(y[i] >= g_fine.y0() && y[i] <= g_fine.y1())) {
-            std::cerr << g_fine.y0()<<"< yi = " << y[i] <<" < "<<g_fine.y1()<<std::endl;
+    cusp::coo_matrix<int, double, cusp::host_memory> A( A2d.num_rows*Nz, A2d.num_cols*Nz, A2d.num_entries*Nz);
+    for( unsigned i=0; i<Nz; i++)
+        for( unsigned j=0; j<A2d.num_entries; j++)
+        {
+            A.column_indices[i*A2d.num_entries+j] = i*A2d.num_cols + A2d.column_indices[j];
+            A.row_indices[i*A2d.num_entries+j] = i*A2d.num_rows + A2d.row_indices[j];
+            A.values[i*A2d.num_entries+j] = A2d.values[j];
         }
-        assert( y[i] >= g_fine.y0() && y[i] <= g_fine.y1());
-
-        //determine which cell (x,y) lies in 
-
-        double xnn = (x[i]-g_fine.x0())/g_fine.hx();
-        double ynn = (y[i]-g_fine.y0())/g_fine.hy();
-        unsigned n = (unsigned)floor(xnn);
-        unsigned m = (unsigned)floor(ynn);
-        //determine normalized coordinates
-
-        //double xn =  2.*xnn - (double)(2*n+1); 
-        //double yn =  2.*ynn - (double)(2*m+1); 
-        ////interval correction
-        //if (n==g_fine.Nx()) {
-        //    n-=1;
-        //    xn = 1.;
-        //}
-        //if (m==g_fine.Ny()) {
-        //    m-=1;
-        //    yn =1.;
-        //}
-
-
-        std::vector<double> pxy( g_fine.n()*g_fine.n());
-        for( int k=0; k<g_fine.n(); k++)
-            for( int l=0; l<g_fine.n(); l++)
-            {
-                pxy[k*g_fine.n()+l] = g_fine.hz()*g_fine.hx()*g_fine.hy()/4.*
-                            g_fine.dlt().weights()[ k]*
-                            g_fine.dlt().weights()[ l];
-                pxy[k*g_fine.n()+l] /= wcoarse[i];
-            }
-
-        unsigned col_begin = (m)*g_fine.Nx()*g_fine.n()*g_fine.n() + (n)*g_fine.n();
-        detail::add_line( A, number, i,  col_begin, g_fine.n(), g_fine.Nx(), pxy); 
-    }
     return A;
 }
 ///@}
 
 
-thrust::host_vector<double> forward_transform( const thrust::host_vector<double>& in, const Grid2d<double>& g)
-{
-    thrust::host_vector<double> out(in.size(), 0);
-    dg::Operator<double> forward( g.dlt().forward());
-    for( unsigned i=0; i<g.Ny(); i++)
-    for( unsigned k=0; k<g.n(); k++)
-    for( unsigned j=0; j<g.Nx(); j++)
-    for( unsigned l=0; l<g.n(); l++)
-    for( unsigned m=0; m<g.n(); m++)
-    for( unsigned o=0; o<g.n(); o++)
-        out[((i*g.n() + k)*g.Nx() + j)*g.n() + l] += forward(k,o)*forward( l, m)*in[((i*g.n() + o)*g.Nx() + j)*g.n() + m];
-    return out;
-
-}
 }//namespace create
 
 /**
diff --git a/inc/dg/backend/interpolation_t.cu b/inc/dg/backend/interpolation_t.cu
index c211a7b54..df6ecad9f 100644
--- a/inc/dg/backend/interpolation_t.cu
+++ b/inc/dg/backend/interpolation_t.cu
@@ -13,12 +13,15 @@
 typedef cusp::coo_matrix<int, double, cusp::host_memory> Matrix;
 
 double sinex( double x, double y) {return sin(x)*sin(x)*sin(y)*sin(y)*x*x*y*y;}
-double sinez( double x, double y, double z) {return sin(x)*sin(x)*sin(y)*sin(y)*x*x*y*y;}
+double sinex( double x, double y, double z) {return sin(x)*sin(x)*sin(y)*sin(y)*x*x*y*y;}
 
 int main()
 {
     std::cout << "type n, Nx, Ny, Nz\n";
     std::cin >> n >> Nx >> Ny >> Nz;
+    std::cout << "type nfine, Nmultiply (fine grid is nf, NfNx, NfNy, Nz)\n";
+    unsigned nf, Nf;
+    std::cin >> nf >> Nf;
 
 
     {
@@ -80,23 +83,33 @@ int main()
     if( passed)
         std::cout << "2D INTERPOLATE TEST PASSED!\n";
 
-    dg::Grid2d<double> gfine( -10, 10, -5, 5, n, n*Nx, n*Ny);
-    thrust::host_vector<double> xfine = dg::evaluate( sinex, gfine);
-    thrust::host_vector<double> xcoarse = dg::evaluate( sinex, g);
-    thrust::host_vector<double> wfine = dg::create::weights( gfine);
-    thrust::host_vector<double> wcoarse = dg::create::weights( g);
-    double coarse = dg::blas2::dot( xcoarse, wcoarse, xcoarse);
-    std::cout << "coar integral: "<<coarse<<"\n";
-
-    Matrix f2c = dg::create::interpolation( g, gfine); 
-    dg::blas2::symv( f2c, xfine, xcoarse);
-    double fine = dg::blas2::dot( xfine, wfine, xfine);
-    coarse = dg::blas2::dot( xcoarse, wcoarse, xcoarse);
-    //double fine = dg::blas1::dot( wfine, xfine);
-    //coarse = dg::blas1::dot( wcoarse, xcoarse);
-    std::cout << "Fine integral: "<<fine<<" \n";
-    std::cout << "coar integral: "<<coarse<<"\n";
-    std::cout << "Rel Difference "<<fabs(fine-coarse)/fabs(fine)<<"\n";
+    dg::Grid2d<double> gfine( -10, 10, -5, 5, nf, Nf*Nx, Nf*Ny);
+    const thrust::host_vector<double> xfine = dg::evaluate( sinex, gfine);
+    thrust::host_vector<double> xcoarseI = dg::evaluate( sinex, g);
+    const thrust::host_vector<double> xcoarse = dg::evaluate( sinex, g);
+    const thrust::host_vector<double> wfine = dg::create::weights( gfine);
+    const thrust::host_vector<double> wcoarse = dg::create::weights( g);
+    double coarseL2 = dg::blas2::dot( xcoarse, wcoarse, xcoarse);
+    double fineL2 =   dg::blas2::dot( xfine, wfine, xfine);
+    std::cout << "coarse L2 norm:       "<<coarseL2<<"\n";
+    std::cout << "Fine L2 norm:         "<<fineL2<<" \n";
+
+    Matrix f2c = dg::create::projection( g, gfine); 
+    dg::blas2::symv( f2c, xfine, xcoarseI);
+    coarseL2 = dg::blas2::dot( xcoarseI, wcoarse, xcoarseI);
+    std::cout << "interpolated L2 norm: "<<coarseL2<<"\n";
+    std::cout << "Difference in L2      "<<fabs(fineL2-coarseL2)/fabs(fineL2)<<"\n";
+    //integrals
+    double coarseI = dg::blas1::dot( wcoarse, xcoarse);
+    double fineI = dg::blas1::dot( wfine, xfine);
+    std::cout << "coarse integral:      "<<coarseI<<"\n";
+    std::cout << "Fine integral:        "<<fineI<<" \n";
+    coarseI = dg::blas1::dot( wcoarse, xcoarseI);
+    std::cout << "interpolated integral "<<coarseI<<"\n";
+    std::cout << "Difference Integral   "<<fabs(fineI-coarseI)/fabs(fineI)<<"\n";
+    dg::blas1::axpby( 1., xcoarseI, -1., xcoarse, xcoarseI);
+    double norm = dg::blas2::dot( xcoarseI, wcoarse, xcoarseI);
+    std::cout << "Difference evaluated to interpolated: "<<norm/coarseL2<<"\n";
 
     }
     ////////////////////////////////////////////////////////////////////////////
@@ -129,19 +142,33 @@ int main()
     }
     if( passed)
         std::cout << "3D TEST PASSED!\n";
-    dg::Grid3d<double> gfine( -10, 10, -5, 5, -7, -3,  n, n*Nx, n*Ny, Nz);
-    thrust::host_vector<double> xfine = dg::evaluate( sinez, gfine);
-    thrust::host_vector<double> xcoarse = dg::evaluate( sinez, g);
-    thrust::host_vector<double> wfine = dg::create::weights( gfine);
-    thrust::host_vector<double> wcoarse = dg::create::weights( g);
-
-    Matrix f2c = dg::create::scalar_interpolation( g, gfine); 
-    dg::blas2::symv( f2c, xfine, xcoarse);
-    double fine = dg::blas1::dot( xfine, wfine);
-    double coarse = dg::blas1::dot( xcoarse, wcoarse);
-    std::cout << "Fine integral: "<<fine<<" \n";
-    std::cout << "coar integral: "<<coarse<<"\n";
-    std::cout << "Difference   : "<<fine-coarse<<"\n";
+    dg::Grid3d<double> gfine( -10, 10, -5, 5, -7, -3,  nf, Nf*Nx, Nf*Ny, Nz);
+    const thrust::host_vector<double> xfine = dg::evaluate( sinex, gfine);
+    thrust::host_vector<double> xcoarseI = dg::evaluate( sinex, g);
+    const thrust::host_vector<double> xcoarse = dg::evaluate( sinex, g);
+    const thrust::host_vector<double> wfine = dg::create::weights( gfine);
+    const thrust::host_vector<double> wcoarse = dg::create::weights( g);
+    double coarseL2 = dg::blas2::dot( xcoarse, wcoarse, xcoarse);
+    double fineL2 =   dg::blas2::dot( xfine, wfine, xfine);
+    std::cout << "coarse L2 norm:       "<<coarseL2<<"\n";
+    std::cout << "Fine L2 norm:         "<<fineL2<<" \n";
+
+    Matrix f2c = dg::create::projection( g, gfine); 
+    dg::blas2::symv( f2c, xfine, xcoarseI);
+    coarseL2 = dg::blas2::dot( xcoarseI, wcoarse, xcoarseI);
+    std::cout << "interpolated L2 norm: "<<coarseL2<<"\n";
+    std::cout << "Difference in L2      "<<fabs(fineL2-coarseL2)/fabs(fineL2)<<"\n";
+    //integrals
+    double coarseI = dg::blas1::dot( wcoarse, xcoarse);
+    double fineI = dg::blas1::dot( wfine, xfine);
+    std::cout << "coarse integral:      "<<coarseI<<"\n";
+    std::cout << "Fine integral:        "<<fineI<<" \n";
+    coarseI = dg::blas1::dot( wcoarse, xcoarseI);
+    std::cout << "interpolated integral "<<coarseI<<"\n";
+    std::cout << "Difference Integral   "<<fabs(fineI-coarseI)/fabs(fineI)<<"\n";
+    dg::blas1::axpby( 1., xcoarseI, -1., xcoarse, xcoarseI);
+    double norm = dg::blas2::dot( xcoarseI, wcoarse, xcoarseI);
+    std::cout << "Difference evaluated to interpolated: "<<norm/coarseL2<<"\n";
     }
 
     return 0;
diff --git a/inc/dg/ds.h b/inc/dg/ds.h
index 400c38963..2b3fb3c69 100644
--- a/inc/dg/ds.h
+++ b/inc/dg/ds.h
@@ -245,7 +245,7 @@ DS<FA, M,container>::DS(const FA& field, Geometry gridc, Field inverseB, dg::nor
         no_(no), dir_(dir), apply_jumpX_(jumpX)
 {
 
-    f2c = dg::create::scalar_interpolation( gridc, field.grid());
+    f2c = dg::create::projection( gridc, field.grid());
     c2f = dg::create::interpolation( field.grid(), gridc);
     cusp::transpose( f2c, f2cT);
     cusp::transpose( c2f, c2fT);     
-- 
GitLab


From 77d137e0f85970813849b61dd0fd22572ee398df Mon Sep 17 00:00:00 2001
From: mheld <markus.held@uibk.ac.at>
Date: Mon, 25 Apr 2016 16:18:41 +0200
Subject: [PATCH 008/453] added timer in newds

---
 inc/geometries/guenther_ds_b.cu | 91 ++++++++++++++++++---------------
 1 file changed, 51 insertions(+), 40 deletions(-)

diff --git a/inc/geometries/guenther_ds_b.cu b/inc/geometries/guenther_ds_b.cu
index 7b99b7c1e..2c2298c4f 100644
--- a/inc/geometries/guenther_ds_b.cu
+++ b/inc/geometries/guenther_ds_b.cu
@@ -100,14 +100,18 @@ int main( )
     const dg::DVec w2d = dg::create::weights( g2d);
     const dg::DVec v3d = dg::create::inv_volume( g3d);
 
-    std::cout << "computing dsDIR" << std::endl;
-    dg::DDS::FieldAligned dsFA( field, g3d_fein, rk4eps, dg::DefaultLimiter(), dg::DIR);
+//     std::cout << "computing dsDIR" << std::endl;
+//     dg::DDS::FieldAligned dsFA( field, g3d_fein, rk4eps, dg::DefaultLimiter(), dg::DIR);
     std::cout << "computing dsNEU" << std::endl;
     dg::DDS::FieldAligned dsNUFA( field, g3d_fein, rk4eps, dg::DefaultLimiter(), dg::NEU);
-
-    dg::DDS ds ( dsFA, g3d, field, dg::not_normed, dg::centered), 
-        dsNU ( dsNUFA, g3d, field, dg::not_normed, dg::centered);
-
+    dg::Timer timer;
+
+    timer.tic();
+    dg::DDS 
+//     ds ( dsFA, g3d, field, dg::not_normed, dg::centered), 
+        dsNU ( dsNUFA, g3d, field, dg::not_normed, dg::forward);
+    timer.toc();
+    std::cout << "Creating took " << timer.diff() << std::endl;
 //     dg::DS<dg::DMatrix, dg::DVec> dsNEU( field, g3d, g3d.hz(), rk4eps, dg::DefaultLimiter(), dg::NEU);
     
 //     dg::Grid3d<double> g3dp( Rmin,Rmax, Zmin,Zmax, z0, z1,  n, Nx, Ny, 1);
@@ -183,7 +187,7 @@ int main( )
 //     
 //     
   
-    dsNU( function, derivative); //ds(f)
+//     dsNU( function, derivative); //ds(f)
 
 //     dsNU.forward( function, derivativef); //ds(f)
 //     dsNU.backward( function, derivativeb); //ds(f)
@@ -242,7 +246,7 @@ int main( )
 //     ellipticsym.symv(function,dsTds);
 //     dg::blas1::scal(dsTds,-1.0);
 // //     ds.centeredT(ones,divbT);
-    ds.forwardT( derivativef, dsTdsf);  //dsT(ds(f))
+//     ds.forwardT( derivativef, dsTdsf);  //dsT(ds(f))
 //     ds.backwardT( derivativeb, dsTdsb); //dsT(ds(f))
 
 //     //centered
@@ -253,21 +257,21 @@ int main( )
 //     //arithmetic average
 //     dg::blas1::axpby(0.5,dsTdsb,0.5,dsTdsf,dsTdsfb);
 //     dg::blas1::axpby(0.5,dsTdsbd,0.5,dsTdsfd,dsTdsfbd); 
-    ds.symv(function,dsTdsfb);
-    dg::blas1::pointwiseDot(v3d,dsTdsfb,dsTdsfb);
+//     ds.symv(function,dsTdsfb);
+//     dg::blas1::pointwiseDot(v3d,dsTdsfb,dsTdsfb);
         //ds( function, temp);
         //dg::blas1::pointwiseDot( temp, inverseB, temp);
         //ds(temp, dsTdsfb);
         //dg::blas1::pointwiseDivide( dsTdsfb, inverseB, dsTdsfb);
 //     ds.centeredT( derivative2, dsTds2); //dsT(ds(f))
 //     dg::blas1::pointwiseDivide(ones,  inverseB, temp2); //B
-     ds.centeredT( ones, divbT);
+//      ds.centeredT( ones, divbT);
 //     
 //     double normdsds =dg::blas2::dot(derivative2, w3d,derivative2);
 //     double normds1ds =dg::blas2::dot(derivativeones, w3d,derivative2);
 //     double normdivBT =dg::blas2::dot(divBT, w3d,divBT);
-     double normdivbT =dg::blas2::dot(divbT, w3d,divbT);
-     double normdivb =dg::blas2::dot(divbsol, w3d,divbsol); 
+//      double normdivbT =dg::blas2::dot(divbT, w3d,divbT);
+//      double normdivb =dg::blas2::dot(divbsol, w3d,divbsol); 
 //     double normdsTf = dg::blas2::dot(derivativeT2, w3d, function2);
 //     double normdsT_1 = dg::blas2::dot(derivativeT2, w3d, ones);
 //     double normdsT1 = dg::blas2::dot(derivativeTones, w3d, function2);
@@ -280,14 +284,14 @@ int main( )
 //     double normBds1 = dg::blas2::dot(temp2, w3d, derivativeones);
 //     double normfds1 = dg::blas2::dot(function2, w3d, derivativeones);
 // 
-    std::cout << "--------------------testing ds" << std::endl;
-    double norm = dg::blas2::dot( w3d, solution);
-    //std::cout << "|| Solution ||   "<<sqrt( norm)<<"\n";
-    double err =dg::blas2::dot( w3d, derivative);
-    //std::cout << "|| Derivative || "<<sqrt( err)<<"\n";
-    dg::blas1::axpby( 1., solution, -1., derivative);
-    err =dg::blas2::dot( w3d, derivative);
-    std::cout << "Relative Difference in DS is "<< sqrt( err/norm )<<"\n"; 
+//     std::cout << "--------------------testing ds" << std::endl;
+//     double norm = dg::blas2::dot( w3d, solution);
+//     //std::cout << "|| Solution ||   "<<sqrt( norm)<<"\n";
+//     double err =dg::blas2::dot( w3d, derivative);
+//     //std::cout << "|| Derivative || "<<sqrt( err)<<"\n";
+//     dg::blas1::axpby( 1., solution, -1., derivative);
+//     err =dg::blas2::dot( w3d, derivative);
+//     std::cout << "Relative Difference in DS is "<< sqrt( err/norm )<<"\n"; 
    
 //     std::cout << "--------------------testing ds with RZPhi method" << std::endl;
 //     std::cout << "|| Solution ||   "<<sqrt( norm)<<"\n";
@@ -297,15 +301,15 @@ int main( )
 //     errRZPhi =dg::blas2::dot( w3d, derivativeRZPhi);    
 //     std::cout << "Relative Difference in DS is "<< sqrt( errRZPhi/norm )<<"\n"; 
 //     
-     std::cout << "--------------------testing divb" << std::endl;
-     //std::cout << "|| divbsol ||  "<<sqrt( normdivb)<<"\n";
-     //std::cout << "|| divbT  ||   "<<sqrt( normdivbT)<<"\n";
-     dg::blas1::axpby( 1., divbsol, -1., divbT);
-     normdivbT =dg::blas2::dot(divbT, w3d,divbT);
-     std::cout << "Relative Difference in divb is   "<<sqrt( normdivbT/normdivb)<<"\n";
-     //std::cout << "-------------------- " << std::endl;
-     //std::cout << "|| divB || "<<sqrt( normdivBT)<<"\n";
-// 
+//      std::cout << "--------------------testing divb" << std::endl;
+//      //std::cout << "|| divbsol ||  "<<sqrt( normdivb)<<"\n";
+//      //std::cout << "|| divbT  ||   "<<sqrt( normdivbT)<<"\n";
+//      dg::blas1::axpby( 1., divbsol, -1., divbT);
+//      normdivbT =dg::blas2::dot(divbT, w3d,divbT);
+//      std::cout << "Relative Difference in divb is   "<<sqrt( normdivbT/normdivb)<<"\n";
+//      //std::cout << "-------------------- " << std::endl;
+//      //std::cout << "|| divB || "<<sqrt( normdivBT)<<"\n";
+// // 
 //     
 //     std::cout << "-------------------- " << std::endl;
 //     double normT = dg::blas2::dot( w3d, solutionT);
@@ -344,15 +348,15 @@ int main( )
 //     errdsTds =dg::blas2::dot( w3d, dsTds);
 //     std::cout << "Relative Difference in DST is "<< sqrt( errdsTds/normdsTds )<<"\n";   
     
-    std::cout << "--------------------testing dsTdsfb " << std::endl;
-    //std::cout << "|| SolutionT ||      "<<sqrt( normdsTds)<<"\n";
-    double remainder =dg::blas1::dot( w3d,dsTdsfb);
-    double errdsTdsfb =dg::blas2::dot( w3d,dsTdsfb);
-    //std::cout << "|| DerivativeTds ||  "<<sqrt( errdsTdsfb)<<"\n";
-    //std::cout << "   Integral          "<<remainder<<"\n";
-    dg::blas1::axpby( 1., solutiondsTds, -1., dsTdsfb);
-    errdsTdsfb =dg::blas2::dot( w3d, dsTdsfb);
-    std::cout << "Relative Difference in DST is "<< sqrt( errdsTdsfb/normdsTds )<<"\n";
+//     std::cout << "--------------------testing dsTdsfb " << std::endl;
+//     //std::cout << "|| SolutionT ||      "<<sqrt( normdsTds)<<"\n";
+//     double remainder =dg::blas1::dot( w3d,dsTdsfb);
+//     double errdsTdsfb =dg::blas2::dot( w3d,dsTdsfb);
+//     //std::cout << "|| DerivativeTds ||  "<<sqrt( errdsTdsfb)<<"\n";
+//     //std::cout << "   Integral          "<<remainder<<"\n";
+//     dg::blas1::axpby( 1., solutiondsTds, -1., dsTdsfb);
+//     errdsTdsfb =dg::blas2::dot( w3d, dsTdsfb);
+//     std::cout << "Relative Difference in DST is "<< sqrt( errdsTdsfb/normdsTds )<<"\n";
 //   
 //     std::cout << "--------------------testing dsTdsfb with direct method" << std::endl;
 //     std::cout << "|| SolutionT ||      "<<sqrt( normdsTds)<<"\n";
@@ -431,7 +435,14 @@ int main( )
 //     std::cout << "Relative Difference is  "<< sqrt( errinvT/normf )<<"\n";
 //     
     std::cout << "--------------------testing inversion dsTds" << std::endl; 
-    std::cout << " # of iterations "<< invert( dsNU, functionTinv2,solutiondsTds ) << std::endl; //is dsTds
+    timer.tic();
+    
+    unsigned iterations;
+    iterations =  invert( dsNU, functionTinv2,solutiondsTds ); //is dsTds
+    timer.toc();
+    std::cout << "inversion took " << timer.diff() << std::endl;
+    std::cout << " # of iterations "<< iterations << std::endl;
+    std::cout << " inversion/# of iterations "<<timer.diff()/iterations << std::endl;
     std::cout << "Norm analytic Solution  "<<sqrt( normf)<<"\n";
     double errinvT2 =dg::blas2::dot( w3d, functionTinv2);
     std::cout << "Norm numerical Solution "<<sqrt( errinvT2)<<"\n";
-- 
GitLab


From 6df8646741f95b0f5d2bcd02085f163bf5c5f19d Mon Sep 17 00:00:00 2001
From: Albert BSC <albert.gutierrez@bsc.es>
Date: Wed, 26 Apr 2017 12:19:33 +0200
Subject: [PATCH 009/453] Add compilation optimization flags for Marconi KNL

---
 config/marconi.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/marconi.mk b/config/marconi.mk
index 6e77e8a57..e00b7bd1f 100644
--- a/config/marconi.mk
+++ b/config/marconi.mk
@@ -5,7 +5,7 @@ INCLUDE += -I$(NETCDF_INC) -I$(HDF5_INC)
 GLFLAGS  = -lm
 CC=icc
 MPICC=mpiicc
-OPT=-O3 -xHost
+OPT=-O3 -xHost -fma -xMIC-AVX512 -finline-functions -align
 #MPICFLAGS+= -DMPICH_IGNORE_CXX_SEEK
 OMPFLAG=-qopenmp
 JSONLIB=-L$(HOME)/include/json/../../src/lib_json -ljsoncpp # json library for input parameters
-- 
GitLab


From 02b080d58efa242a255e14bcfad33fc5ad30ee9c Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Wed, 5 Jul 2017 19:06:30 +0200
Subject: [PATCH 010/453] updated projection.cuh and projection_t

---
 inc/dg/backend/projection.cuh  | 265 ++++++++++++++++++---------------
 inc/dg/backend/projection_t.cu |  52 ++++---
 2 files changed, 175 insertions(+), 142 deletions(-)

diff --git a/inc/dg/backend/projection.cuh b/inc/dg/backend/projection.cuh
index 5829a37bf..b0d95967d 100644
--- a/inc/dg/backend/projection.cuh
+++ b/inc/dg/backend/projection.cuh
@@ -1,174 +1,201 @@
 #pragma once
 #include <vector>
 #include <cusp/coo_matrix.h>
+#include <cusp/transpose.h>
 #include "grid.h"
 #include "interpolation.cuh"
-#include "matrix_traits_thrust.h"
-#include "../blas2.h"
 
 /*!@file 
   
-  contains the Difference Norm class that computes differences between vectors on different grids
+  contains creation of projection matrices
  */
 namespace dg{
 ///@addtogroup utilities
 ///@{
+
+namespace create{
+
+/**
+ * @brief Create the transpose of the interpolation matrix from new to old
+ *
+ * @param g_new The new grid 
+ * @param g_old The old grid
+ *
+ * @return transposed interpolation matrix
+ * @note The boundaries of the old grid must lie within the boundaries of the new grid
+ */
+cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const Grid1d& g_new, const Grid1d& g_old)
+{
+    cusp::coo_matrix<int, double, cusp::host_memory> temp = interpolation( g_old, g_new), A;
+    cusp::transpose( temp, A);
+    return A;
+}
+
 /**
- * @brief Greatest common divisor
+ * @brief Create a projection between two grids
  *
- * @param a First number
- * @param b Second number
+ * The projection matrix is the adjoint of the interpolation matrix
+ * This matrix can be applied to vectors defined on the old (fine) grid to obtain
+ * its values on the new (coarse) grid. 
+ * If the fine grid is a multiple of the coarse grid, the integral value
+ of the projected vector will be conserved and the difference in the L2 norm 
+ between old and new vector small. 
+ * 
+ * @param g_new The new (coarse) grid 
+ * @param g_old The old (fine) grid
  *
- * @return greatest common divisor
+ * @return Projection matrix
+ * @note The boundaries of the old grid must lie within the boundaries of the new grid
+ @attention Projection only works if the number of cells in the
+ fine grid are multiple of the number of cells in the coarse grid
  */
-unsigned gcd( unsigned a, unsigned b)
+cusp::coo_matrix< int, double, cusp::host_memory> projection( const Grid1d& g_new, const Grid1d& g_old)
 {
-    unsigned r2 = std::max(a,b);
-    unsigned r1 = std::min(a,b);
-    while( r1!=0)
+    if( g_old.N() % g_new.N() != 0) std::cerr << "ATTENTION: you project between incompatible grids!!\n";
+    //form the adjoint
+    thrust::host_vector<double> w_f = dg::create::weights( g_old);
+    thrust::host_vector<double> v_c = dg::create::inv_weights( g_new );
+    cusp::coo_matrix<int, double, cusp::host_memory> Wf( w_f.size(), w_f.size(), w_f.size());
+    cusp::coo_matrix<int, double, cusp::host_memory> Vc( v_c.size(), v_c.size(), v_c.size());
+    for( int i =0; i<(int)w_f.size(); i++)
+    {
+        Wf.row_indices[i] = Wf.column_indices[i] = i;
+        Wf.values[i] = w_f[i];
+    }
+    for( int i =0; i<(int)v_c.size(); i++)
     {
-        r2 = r2%r1;
-        std::swap( r1, r2);
+        Vc.row_indices[i] = Vc.column_indices[i] = i;
+        Vc.values[i] = v_c[i];
     }
-    return r2;
+    cusp::coo_matrix<int, double, cusp::host_memory> A = interpolationT( g_new, g_old), temp;
+    cusp::multiply( A, Wf, temp);
+    cusp::multiply( Vc, temp, A);
+    A.sort_by_row_and_column();
+    return A;
 }
 
 /**
- * @brief Least common multiple
+ * @brief Create the transpose of the interpolation matrix from new to old
  *
- * @param a Fist number
- * @param b Second number 
+ * @param g_new The new grid 
+ * @param g_old The old grid
  *
- * @return Least common multiple
+ * @return transposed interpolation matrix
+ * @note The boundaries of the old grid must lie within the boundaries of the new grid
  */
-unsigned lcm( unsigned a, unsigned b)
+cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const Grid2d& g_new, const Grid2d& g_old)
 {
-    unsigned g = gcd( a,b);
-    return a/g*b;
+    cusp::coo_matrix<int, double, cusp::host_memory> temp = interpolation( g_old, g_new), A;
+    cusp::transpose( temp, A);
+    return A;
 }
 
-namespace create{
-
 /**
- * @brief Create a 1D projection matrix onto a finer grid
+ * @brief Create a projection between two grids
  *
- * Grid space must be equal. Nx of the second grid must be a multiple of 
- * Nx of the first grid.
- * @param g1 Grid of the original vector
- * @param g2 Grid of the target vector
+ * The projection matrix is the adjoint of the interpolation matrix
+ * This matrix can be applied to vectors defined on the old (fine) grid to obtain
+ * its values on the new (coarse) grid. 
+ * If the fine grid is a multiple of the coarse grid, the integral value
+ of the projected vector will be conserved and the difference in the L2 norm 
+ between old and new vector small. 
+ * 
+ * @param g_new The new grid 
+ * @param g_old The old grid
  *
- * @return Projection matrix
+ * @return transposed interpolation matrix
+ * @note The boundaries of the old grid must lie within the boundaries of the new grid
+ @attention Projection only works if the number of cells in the
+ fine grid are multiple of the number of cells in the coarse grid
  */
-cusp::coo_matrix< int, double, cusp::host_memory> projection( const Grid1d& g1, const Grid1d& g2)
+cusp::coo_matrix< int, double, cusp::host_memory> projection( const Grid2d& g_new, const Grid2d& g_old)
 {
-    assert( g1.x0() == g2.x0()); assert( g1.x1() == g2.x1());
-    assert( g2.N() % g1.N() == 0);
-    return dg::create::interpolation( g2, g1);
+    if( g_old.Nx() % g_new.Nx() != 0) std::cerr << "ATTENTION: you project between incompatible grids in x!!\n";
+    if( g_old.Ny() % g_new.Ny() != 0) std::cerr << "ATTENTION: you project between incompatible grids in y!!\n";
+    //form the adjoint
+    thrust::host_vector<double> w_f = dg::create::weights( g_old);
+    thrust::host_vector<double> v_c = dg::create::inv_weights( g_new );
+    cusp::coo_matrix<int, double, cusp::host_memory> Wf( w_f.size(), w_f.size(), w_f.size());
+    cusp::coo_matrix<int, double, cusp::host_memory> Vc( v_c.size(), v_c.size(), v_c.size());
+    for( int i =0; i<(int)w_f.size(); i++)
+    {
+        Wf.row_indices[i] = Wf.column_indices[i] = i;
+        Wf.values[i] = w_f[i];
+    }
+    for( int i =0; i<(int)v_c.size(); i++)
+    {
+        Vc.row_indices[i] = Vc.column_indices[i] = i;
+        Vc.values[i] = v_c[i];
+    }
+    cusp::coo_matrix<int, double, cusp::host_memory> A = interpolationT( g_new, g_old), temp;
+    cusp::multiply( A, Wf, temp);
+    cusp::multiply( Vc, temp, A);
+    A.sort_by_row_and_column();
+    return A;
 }
-
 /**
- * @brief Create a 2D projection matrix onto a finer grid
+ * @brief Create the transpose of the interpolation matrix from new to old
  *
- * Grid space must be equal. Nx and Ny of the second grid must be multiples of 
- * Nx and Ny of the first grid.
- * @param g1 Grid of the original vector
- * @param g2 Grid of the target vector
+ * @param g_new The new grid 
+ * @param g_old The old grid
  *
- * @return Projection matrix
+ * @return transposed interpolation matrix
+ * @note The boundaries of the old grid must lie within the boundaries of the new grid
  */
-cusp::coo_matrix< int, double, cusp::host_memory> projection( const Grid2d& g1, const Grid2d& g2)
+cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const Grid3d& g_new, const Grid3d& g_old)
 {
-    //TODO: projection in y direction needs permutation
-    assert( g1.x0() == g2.x0()); assert( g1.x1() == g2.x1());
-    assert( g1.y0() == g2.y0()); assert( g1.y1() == g2.y1());
-    //assert( g2.Nx() % g1.Nx() == 0);
-    //assert( g2.Ny() % g1.Ny() == 0);
-    return dg::create::interpolation( g2, g1);
+    cusp::coo_matrix<int, double, cusp::host_memory> temp = interpolation( g_old, g_new), A;
+    cusp::transpose( temp, A);
+    return A;
 }
 
-
-}//namespace create
-
-
-//eventuell könnte man zwei Projektionsmatrizen malnehmen um eine kleinere zu erhalten
 /**
- * @brief Class to perform comparison of dG vectors on different grids
+ * @brief Create a projection between two grids
  *
- * it basically interpolates values from the rougher grid to values on the finer grid and then uses the existing methods to compute the norm
- * @tparam container
+ * The projection matrix is the adjoint of the interpolation matrix
+ * This matrix can be applied to vectors defined on the old (fine) grid to obtain
+ * its values on the new (coarse) grid. 
+ * If the fine grid is a multiple of the coarse grid, the integral value
+ of the projected vector will be conserved and the difference in the L2 norm 
+ between old and new vector small. 
+ * 
+ * @param g_new The new grid 
+ * @param g_old The old grid
+ *
+ * @return transposed interpolation matrix
+ * @note The boundaries of the old grid must lie within the boundaries of the new grid
+ @attention Projection only works if the number of cells in the
+ fine grid are multiple of the number of cells in the coarse grid
  */
-template <typename container>
-struct DifferenceNorm
+cusp::coo_matrix< int, double, cusp::host_memory> projection( const Grid3d& g_new, const Grid3d& g_old)
 {
-    /**
-     * @brief Construct from two different grids
-     *
-     * @param g1 first grid
-     * @param g2 second grid
-     */
-    DifferenceNorm( const Grid2d& g1, const Grid2d& g2)
+    if( g_old.Nx() % g_new.Nx() != 0) std::cerr << "ATTENTION: you project between incompatible grids in x!!\n";
+    if( g_old.Ny() % g_new.Ny() != 0) std::cerr << "ATTENTION: you project between incompatible grids in y!!\n";
+    //form the adjoint
+    thrust::host_vector<double> w_f = dg::create::weights( g_old);
+    thrust::host_vector<double> v_c = dg::create::inv_weights( g_new );
+    cusp::coo_matrix<int, double, cusp::host_memory> Wf( w_f.size(), w_f.size(), w_f.size());
+    cusp::coo_matrix<int, double, cusp::host_memory> Vc( v_c.size(), v_c.size(), v_c.size());
+    for( int i =0; i<(int)w_f.size(); i++)
     {
-        //find common grid
-        Grid2d gC(    g1.x0(), g1.x1(), g1.y0(), g1.y1(), 
-                            std::min( g1.n(), g2.n()), 
-                            lcm( g1.Nx(), g2.Nx()), 
-                            lcm( g1.Ny(), g2.Ny()) );
-        p1 = dg::create::interpolation( gC, g1);
-        p2 = dg::create::interpolation( gC, g2);
-        w2d = dg::create::weights( gC); v11 = w2d, v22 = w2d;
-        wg1 = dg::create::weights( g1); 
-        wg2 = dg::create::weights( g2); 
+        Wf.row_indices[i] = Wf.column_indices[i] = i;
+        Wf.values[i] = w_f[i];
     }
-    /**
-     * @brief Compute difference of two vectors
-     *
-     * \f[ ||v_1 - v_2|| = \sqrt{ \int (v_1-v_2)^2 dV} \f]
-     * @param v1
-     * @param v2
-     *
-     * @return 
-     */
-    double operator()( const container& v1, const container& v2)
+    for( int i =0; i<(int)v_c.size(); i++)
     {
-        double f2, g2, fg;
-        f2 = blas2::dot( wg1, v1);
-        g2 = blas2::dot( wg2, v2);
-
-        blas2::gemv( p1, v1, v11);
-        blas2::gemv( p2, v2, v22);
-        fg = blas2::dot( v11, w2d, v22);
-        return sqrt( f2 - 2.*fg + g2);
+        Vc.row_indices[i] = Vc.column_indices[i] = i;
+        Vc.values[i] = v_c[i];
     }
+    cusp::coo_matrix<int, double, cusp::host_memory> A = interpolationT( g_new, g_old), temp;
+    cusp::multiply( A, Wf, temp);
+    cusp::multiply( Vc, temp, A);
+    A.sort_by_row_and_column();
+    return A;
+}
 
-    /**
-     * @brief Compute the sum of two vectors
-     *
-     * \f[ ||v_1 + v_2|| = \sqrt{ \int (v_1+v_2)^2 dV} \f]
-     * @param v1
-     * @param v2
-     *
-     * @return 
-     */
-    double sum( const container& v1, const container& v2)
-    {
-        double f2, g2, fg;
-        f2 = blas2::dot( wg1, v1);
-        g2 = blas2::dot( wg2, v2);
 
-        blas2::gemv( p1, v1, v11);
-        blas2::gemv( p2, v2, v22);
-        fg = blas2::dot( v11, w2d, v22);
-        return sqrt( f2 + 2.*fg + g2);
-    }
-  private:
-    typedef typename container::value_type value_type;
-    typedef typename thrust::iterator_system<typename container::iterator>::type MemorySpace;
-    typedef cusp::csr_matrix<int, double, MemorySpace> Matrix;
+}//namespace create
 
-    container wg1, wg2, w2d;
-    container v11, v22;
-    Matrix p1, p2;
-};
 ///@}
 
 
diff --git a/inc/dg/backend/projection_t.cu b/inc/dg/backend/projection_t.cu
index 878745e23..7c7e6b811 100644
--- a/inc/dg/backend/projection_t.cu
+++ b/inc/dg/backend/projection_t.cu
@@ -10,12 +10,16 @@ double sine( double x, double y){return sin(x)*sin(y);}
 
 int main()
 {
-    //Projection might not be correct any more due to layout change
     std::cout << "TEST 1D\n";
-    unsigned n_old = 4, n_new = 3, N = 10, Nf = 1;
-    dg::Grid1d go ( 0, M_PI, n_old, N);
-    dg::Grid1d gn ( 0, M_PI, n_new, N*Nf);
-    cusp::coo_matrix<int, double, cusp::host_memory> proj = dg::create::projection( go, gn);
+    unsigned n_old = 4, n_new = 3, N_old = 10, N_new = 1;
+    std::cout << "Type n and N of old (find) grid!\n";
+    std::cin >> n_old >> N_old;
+    std::cout << "Type n and N of new (coarser) grid!\n";
+    std::cin >> n_new >> N_new;
+    dg::Grid1d go ( 0, M_PI, n_old, N_old);
+    dg::Grid1d gn ( 0, M_PI, n_new, N_new);
+    cusp::coo_matrix<int, double, cusp::host_memory> proj = dg::create::projection( gn, go);
+    cusp::coo_matrix<int, double, cusp::host_memory> inte = dg::create::interpolation( gn, go);
     thrust::host_vector<double> v = dg::evaluate( sine, go);
     thrust::host_vector<double> w1do = dg::create::weights( go);
     thrust::host_vector<double> w1dn = dg::create::weights( gn);
@@ -26,23 +30,23 @@ int main()
     std::cout << "Original vector  "<<dg::blas2::dot( oneo, w1do, v) << "\n";
     std::cout << "Projected vector "<<dg::blas2::dot( onen, w1dn, w) << "\n";
     std::cout << "Difference       "<<dg::blas2::dot( oneo, w1do, v) - dg::blas2::dot( onen, w1dn, w) << "\n"<<std::endl;
-
-    std::cout << "TEST GCD AND LCM\n";
-    std::cout << "gcd of 1071 and 462 is "<<dg::gcd( 1071, 462)<<" (21)\n";
-    std::cout << "lcm of 1071 and 462 is "<<dg::lcm( 1071, 462)<<" (23562)\n"<<std::endl;
+    dg::blas2::gemv( inte, v, w);
+    std::cout << "Original vector  "<<dg::blas2::dot( oneo, w1do, v) << "\n";
+    std::cout << "Interpolated vec "<<dg::blas2::dot( onen, w1dn, w) << "\n";
+    std::cout << "Difference       "<<dg::blas2::dot( oneo, w1do, v) - dg::blas2::dot( onen, w1dn, w) << "\n"<<std::endl;
 
     std::cout << "TEST 2D\n";
-    n_old = 7, n_new = 3, N = 4, Nf = 3;
-    std::cout << "Type n, N, Nf "<< std::endl;
-    std::cin >>n_old;
-    n_new = n_old;
-    std::cin >> N;
-    std::cin >> Nf;
+    //n_old = 7, n_new = 3, N_old = 4, N_new = 3;
+    //std::cout << "Type n and N of old (find) grid!\n";
+    //std::cin >> n_old >> N_old;
+    //std::cout << "Type n and N of new (coarser) grid!\n";
+    //std::cin >> n_new >> N_new;
     
     //old grid is larger than the new grid
-    dg::Grid2d g2o (0, M_PI, 0, M_PI, n_old, N*Nf, N*Nf);
-    dg::Grid2d g2n (0, M_PI, 0, M_PI, n_new, N, N);
-    cusp::coo_matrix<int, double, cusp::host_memory> proj2d = dg::create::projection( g2o, g2n);
+    dg::Grid2d g2o (0, M_PI, 0, M_PI, n_old, N_old, N_old);
+    dg::Grid2d g2n (0, M_PI, 0, M_PI, n_new, N_new, N_new);
+    cusp::coo_matrix<int, double, cusp::host_memory> proj2d = dg::create::projection( g2n, g2o);
+    cusp::coo_matrix<int, double, cusp::host_memory> inte2d = dg::create::interpolation( g2n, g2o);
     const dg::HVec sinO = dg::evaluate( sine, g2o), 
                    sinN = dg::evaluate( sine, g2n);
     dg::HVec w2do = dg::create::weights( g2o);
@@ -51,21 +55,23 @@ int main()
     dg::blas2::gemv( proj2d, sinO, sinP);
     std::cout << "Original vector     "<<sqrt(dg::blas2::dot( sinO, w2do, sinO)) << "\n";
     std::cout << "Projected vector    "<<sqrt(dg::blas2::dot( sinP, w2dn, sinP)) << "\n";
-    //std::cout << "Evaluated vector "<<dg::blas2::dot(sinN, w2dn, sinN) << "\n";
     std::cout << "Difference in Norms "<<sqrt(dg::blas2::dot( sinO, w2do, sinO)) - sqrt(dg::blas2::dot( sinP, w2dn, sinP)) << "\n" << std::endl;
     dg::HVec temp(sinP);
     dg::blas1::axpby(1.,sinN,-1.,sinP,temp);
     std::cout << "Rel Difference in L2 norm is " << sqrt(dg::blas2::dot( temp, w2dn, temp)/dg::blas2::dot( sinN, w2dn, sinN))<< std::endl;
-    std::cout << "TEST OF DIFFERENCE\n";
-    dg::DifferenceNorm<dg::HVec> diff( g2o, g2n);
-    std::cout << "Difference between original and projection:\n";
-    std::cout << diff( sinO, sinP)<<" (should converge to zero) \n";
     //std::cout << "Difference between two grid evaluations:\n";
     //std::cout << diff( sinO, sinN)<<" (should converge to zero!) \n";
     std::cout << "Difference between projection and evaluation      \n";
     dg::blas1::axpby( 1., sinN, -1., sinP);
     std::cout << dg::blas2::dot( sinP, w2dn, sinP)<<" (smaller than above)\n";
+    dg::blas2::gemv( inte2d, sinO, sinP);
+    std::cout << "Original vector     "<<sqrt(dg::blas2::dot( sinO, w2do, sinO)) << "\n";
+    std::cout << "Interpolated vec    "<<sqrt(dg::blas2::dot( sinP, w2dn, sinP)) << "\n";
+    std::cout << "Difference in Norms "<<sqrt(dg::blas2::dot( sinO, w2do, sinO)) - sqrt(dg::blas2::dot( sinP, w2dn, sinP)) << "\n" << std::endl;
 
+    std::cout << "What you should observe is that the projection does only work if the coarse grid is a division of the fine grid!\n"
+    << "If it works it is better than interpolation in that it conserves the integral value and has smaller L2 errors\n";
+    std::cout << "Interpolation always works!\n";
 
     return 0;
 }
-- 
GitLab


From eb3308849524c5ba437e583a51cb962e3c4d4382 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Wed, 5 Jul 2017 19:34:43 +0200
Subject: [PATCH 011/453] update documentation of projection and interpolation

---
 inc/dg/backend/interpolation.cuh  |  3 +-
 inc/dg/backend/interpolationX.cuh |  2 +-
 inc/dg/backend/projection.cuh     | 12 ++++----
 inc/dg/backend/timer.cuh          | 46 ++++++++++++++++---------------
 inc/dg/dg_doc.h                   | 17 ++++++++----
 5 files changed, 44 insertions(+), 36 deletions(-)

diff --git a/inc/dg/backend/interpolation.cuh b/inc/dg/backend/interpolation.cuh
index 54c6cad7c..b4b4174bb 100644
--- a/inc/dg/backend/interpolation.cuh
+++ b/inc/dg/backend/interpolation.cuh
@@ -65,7 +65,7 @@ std::vector<double> coefficients( double xn, unsigned n)
 
 }//namespace detail
 ///@endcond
-///@addtogroup utilities
+///@addtogroup interpolation
 ///@{
 /**
  * @brief Create interpolation matrix
@@ -510,6 +510,7 @@ thrust::host_vector<double> forward_transform( const thrust::host_vector<double>
  * @param v The vector to interpolate in LSPACE
  * @param g The Grid on which to operate
  *
+ * @ingroup interpolation
  * @return interpolated point
  */
 double interpolate( double x, double y,  const thrust::host_vector<double>& v, const Grid2d& g )
diff --git a/inc/dg/backend/interpolationX.cuh b/inc/dg/backend/interpolationX.cuh
index 31f1320e2..7c49150ff 100644
--- a/inc/dg/backend/interpolationX.cuh
+++ b/inc/dg/backend/interpolationX.cuh
@@ -11,7 +11,7 @@
 namespace dg{
 
 namespace create{
-///@addtogroup utilities
+///@addtogroup interpolation
 ///@{
 /**
  * @brief Create interpolation matrix
diff --git a/inc/dg/backend/projection.cuh b/inc/dg/backend/projection.cuh
index b0d95967d..95ca65f41 100644
--- a/inc/dg/backend/projection.cuh
+++ b/inc/dg/backend/projection.cuh
@@ -10,7 +10,7 @@
   contains creation of projection matrices
  */
 namespace dg{
-///@addtogroup utilities
+///@addtogroup interpolation
 ///@{
 
 namespace create{
@@ -100,8 +100,8 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const Grid2d& g
  of the projected vector will be conserved and the difference in the L2 norm 
  between old and new vector small. 
  * 
- * @param g_new The new grid 
- * @param g_old The old grid
+ * @param g_new The new (coarse) grid 
+ * @param g_old The old (fine) grid
  *
  * @return transposed interpolation matrix
  * @note The boundaries of the old grid must lie within the boundaries of the new grid
@@ -159,8 +159,8 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const Grid3d& g
  of the projected vector will be conserved and the difference in the L2 norm 
  between old and new vector small. 
  * 
- * @param g_new The new grid 
- * @param g_old The old grid
+ * @param g_new The new (coarse) grid 
+ * @param g_old The old (fine) grid
  *
  * @return transposed interpolation matrix
  * @note The boundaries of the old grid must lie within the boundaries of the new grid
@@ -193,10 +193,10 @@ cusp::coo_matrix< int, double, cusp::host_memory> projection( const Grid3d& g_ne
     return A;
 }
 
+///@}
 
 }//namespace create
 
-///@}
 
 
 
diff --git a/inc/dg/backend/timer.cuh b/inc/dg/backend/timer.cuh
index d5d0450b9..4780c0884 100644
--- a/inc/dg/backend/timer.cuh
+++ b/inc/dg/backend/timer.cuh
@@ -5,9 +5,7 @@ namespace dg
 {
 #if (THRUST_DEVICE_SYSTEM!=THRUST_DEVICE_SYSTEM_CUDA) //if we don't use a GPU
 #ifdef MPI_VERSION //(mpi.h is included)
-/*! @brief Very simple tool for performance measuring
- * @ingroup utilities
- */
+///@cond
 class Timer
 {
   public:
@@ -25,26 +23,36 @@ class Timer
     * @note uses MPI_Barrier(comm)
     */
     void toc( MPI_Comm comm = MPI_COMM_WORLD ){ MPI_Barrier(comm); stop = MPI_Wtime(); }
-    /*! \brief Return time elapsed between tic and toc
-     *
-     * \return Time in seconds between calls of tic and toc*/
     double diff(){ return stop - start; }
   private:
     double start, stop;
 };
+///@endcond
+
 #elif THRUST_DEVICE_SYSTEM==THRUST_DEVICE_SYSTEM_OMP //MPI_VERSION
 #include "omp.h"
+
+/*! @brief Very simple tool for performance measuring
+ * 
+ * @code
+   dg::Timer t;
+   t.tic();
+   some_function_to_benchmark();
+   t.toc();
+   std::cout << "Function took "<<t.diff()<<"s\n";
+ * @endcode
+ * @ingroup timer
+ * @note The Timer knows what hardware you are on!
+ */
 class Timer
 {
   public:
     /**
-    * @brief Start timer using omp_get_wtime
+    * @brief Start timer 
     */
     void tic( ){ start = omp_get_wtime();}
     /**
-    * @brief Stop timer using omp_get_wtime
-    *
-    * @note uses MPI_Barrier(comm)
+    * @brief Stop timer 
     */
     void toc( ){ stop = omp_get_wtime(); }
     /*! \brief Return time elapsed between tic and toc
@@ -56,10 +64,8 @@ class Timer
 };
 #else
 
+///@cond
 #include <sys/time.h>
-/*! @brief Very simple tool for performance measuring
- * @ingroup utilities
- */
 class Timer
 {
     timeval start;
@@ -74,10 +80,12 @@ class Timer
      * \return Time in seconds between calls of tic and toc*/
     double diff(){ return ((stop.tv_sec - start.tv_sec)*1000000u + (stop.tv_usec - start.tv_usec))/1e6;}
 };
+///@endcond
 #endif //MPI_VERSION
 #else //THRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_CUDA
 #ifdef MPI_VERSION
 
+///@cond
 class Timer
 {
   public:
@@ -104,21 +112,17 @@ class Timer
     cudaEventSynchronize( cu_sync); //sync cpu  on event
     MPI_Barrier(comm); //sync other cpus on event
     stop = MPI_Wtime(); }
-    /*! \brief Return time elapsed between tic and toc
-     *
-     * \return Time in seconds between calls of tic and toc*/
     double diff(){ return stop - start; }
   private:
     double start, stop;
     cudaEvent_t cu_sync;
 };
+///@endcond
 
 
 #else //MPI_VERSION
 
-/*! @brief Very simple tool for performance measurements using CUDA-API 
- * @ingroup utilities
- */
+///@cond
 class Timer
 {
   public:
@@ -141,9 +145,6 @@ class Timer
         cudaEventRecord( stop, stream);
         cudaEventSynchronize( stop);
     }
-    /*! \brief Return time elapsed between tic and toc
-     *
-     * \return Time in seconds between calls of tic and toc*/
     float diff(){ 
         float time; 
         cudaEventElapsedTime( &time, start, stop);
@@ -152,6 +153,7 @@ class Timer
   private:
     cudaEvent_t start, stop;
 };
+///@endcond
 #endif //MPI_VERSION
 #endif //THRUST
 
diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index d9e2f3d7b..11fe1a7cc 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -52,17 +52,13 @@
  *             use the product space. We choose x to be the contiguous direction.
  *             The first elements of the resulting vector lie in the cell at (x0,y0) and the last
  *             in (x1, y1).
- *         @defgroup functions Functions and Functors
- *
- *             The functions are useful mainly in the constructor of Operator objects. 
- *             The functors are useful for either vector transformations or
- *             as init functions in the evaluate routines.
  *         @defgroup lowlevel Lowlevel helper functions and classes
  *             Low level helper routines.
  *         @defgroup highlevel Weight functions
  *         @defgroup creation Discrete derivatives 
  *
  *             High level matrix creation functions
+           @defgroup interpolation Interpolation and projection
  *         @defgroup scatter Scatter
  *     @}
  *     @defgroup geometry Geometric grids and operations
@@ -72,7 +68,7 @@
       @{
           @defgroup basicgrids Basic grids
  *        @defgroup utilities Fieldalignment and Averaging
- *            Utilities that might come in handy at some place or the other.
+ *            The classes to perform field line integration for DS and averaging classes
       @}
  * @}
  * @defgroup numerical1 Level 4: Advanced numerical schemes
@@ -85,6 +81,15 @@
  * @}
  * @defgroup templates Level 99: Template models
    Documentation for template models
+ * @defgroup misc Level 00: Miscellaneous additions
+   @{
+ *     @defgroup functions Functions and Functors
+ * 
+ *         The functions are useful mainly in the constructor of Operator objects. 
+ *         The functors are useful for either vector transformations or
+ *         as init functions in the evaluate routines.
+       @defgroup timer Timer class
+   @}
  * 
  */
 /*! @mainpage
-- 
GitLab


From bc58d4938190271daf56699e1c396f570d5a20c8 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Wed, 5 Jul 2017 21:35:54 +0200
Subject: [PATCH 012/453] succesfully made MPI_FieldAligned a specialization of
 FieldAligned

---
 inc/dg/ds.h                        |  4 ++--
 inc/dg/geometry/mpi_fieldaligned.h | 18 +++++++++---------
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/inc/dg/ds.h b/inc/dg/ds.h
index 6f054d46d..082d1fac1 100644
--- a/inc/dg/ds.h
+++ b/inc/dg/ds.h
@@ -467,8 +467,8 @@ struct MatrixTraits< DS<F,M, V> >
 typedef dg::DS<dg::FieldAligned<dg::CylindricalGrid3d<dg::DVec>, dg::IDMatrix, dg::DVec>, dg::DMatrix, dg::DVec> DDS;//!< device DS type
 typedef dg::DS<dg::FieldAligned<dg::CylindricalGrid3d<dg::HVec>, dg::IHMatrix, dg::HVec>, dg::HMatrix, dg::HVec> HDS; //!< host DS type
 #ifdef MPI_VERSION
-typedef dg::DS< dg::MPI_FieldAligned<dg::CylindricalMPIGrid3d<dg::MDVec>, dg::IDMatrix, dg::BijectiveComm< dg::iDVec, dg::DVec >, dg::DVec>, dg::MDMatrix, dg::MDVec > MDDS; //!< MPI device DS type
-typedef dg::DS< dg::MPI_FieldAligned<dg::CylindricalMPIGrid3d<dg::MHVec>, dg::IHMatrix, dg::BijectiveComm< dg::iHVec, dg::HVec >, dg::HVec>, dg::MHMatrix, dg::MHVec > MHDS; //!< MPI host DS type
+typedef dg::DS< dg::FieldAligned<dg::CylindricalMPIGrid3d<dg::MDVec>, dg::RowDistMat<dg::IDMatrix, dg::BijectiveComm< dg::iDVec, dg::DVec > >, dg::MPI_Vector<dg::DVec> >, dg::MDMatrix, dg::MDVec > MDDS; //!< MPI device DS type
+typedef dg::DS< dg::FieldAligned<dg::CylindricalMPIGrid3d<dg::MHVec>, dg::RowDistMat<dg::IHMatrix, dg::BijectiveComm< dg::iHVec, dg::HVec > >, dg::MPI_Vector<dg::HVec> >, dg::MHMatrix, dg::MPI_Vector<dg::MHVec>  > MHDS; //!< MPI host DS type
 #endif //MPI_VERSION
 ///@}
 
diff --git a/inc/dg/geometry/mpi_fieldaligned.h b/inc/dg/geometry/mpi_fieldaligned.h
index 0aedc7d0e..e065774bc 100644
--- a/inc/dg/geometry/mpi_fieldaligned.h
+++ b/inc/dg/geometry/mpi_fieldaligned.h
@@ -94,7 +94,7 @@ struct ZShifter
  * @tparam LocalContainer The container-class to on which the interpolation matrix operates on (does not need to be dg::HVec)
  */
 template <class Geometry, class LocalMatrix, class Communicator, class LocalContainer>
-struct MPI_FieldAligned
+struct FieldAligned< Geometry, RowDistMat<LocalMatrix, Communicator>, MPI_Vector<LocalContainer> > 
 {
     /**
     * @brief Construct from a field and a grid
@@ -111,7 +111,7 @@ struct MPI_FieldAligned
     * @note If there is a limiter, the boundary condition is set by the bcz variable from the grid and can be changed by the set_boundaries function. If there is no limiter the boundary condition is periodic.
     */
     template <class Field, class Limiter>
-    MPI_FieldAligned(Field field, Geometry grid, double eps = 1e-4, Limiter limit = DefaultLimiter(), dg::bc globalbcz = dg::DIR, double deltaPhi = -1 );
+    FieldAligned(Field field, Geometry grid, double eps = 1e-4, Limiter limit = DefaultLimiter(), dg::bc globalbcz = dg::DIR, double deltaPhi = -1 );
 
     /**
      * @brief Set boundary conditions
@@ -281,7 +281,7 @@ struct MPI_FieldAligned
 //////////////////////////////////////DEFINITIONS/////////////////////////////////////
 template<class MPIGeometry, class LocalMatrix, class CommunicatorXY, class LocalContainer>
 template <class Field, class Limiter>
-MPI_FieldAligned<MPIGeometry, LocalMatrix, CommunicatorXY, LocalContainer>::MPI_FieldAligned(Field field, MPIGeometry grid, double eps, Limiter limit, dg::bc globalbcz, double deltaPhi ): 
+FieldAligned<MPIGeometry, RowDistMat<LocalMatrix, CommunicatorXY>, MPI_Vector<LocalContainer> >::FieldAligned(Field field, MPIGeometry grid, double eps, Limiter limit, dg::bc globalbcz, double deltaPhi ): 
     hz_( dg::evaluate( dg::zero, grid)), hp_( hz_), hm_( hz_), 
     g_(grid), bcz_(grid.bcz()), 
     tempXYplus_(g_.Nz()), tempXYminus_(g_.Nz()), temp_(g_.Nz())
@@ -377,14 +377,14 @@ MPI_FieldAligned<MPIGeometry, LocalMatrix, CommunicatorXY, LocalContainer>::MPI_
 
 template<class G, class M, class C, class container>
 template< class BinaryOp>
-MPI_Vector<container> MPI_FieldAligned<G,M,C,container>::evaluate( BinaryOp binary, unsigned p0) const
+MPI_Vector<container> FieldAligned<G,RowDistMat<M,C>,MPI_Vector<container> >::evaluate( BinaryOp binary, unsigned p0) const
 {
     return evaluate( binary, dg::CONSTANT(1), p0, 0);
 }
 
 template<class G, class M, class C, class container>
 template< class BinaryOp, class UnaryOp>
-MPI_Vector<container> MPI_FieldAligned<G,M,C, container>::evaluate( BinaryOp binary, UnaryOp unary, unsigned p0, unsigned rounds) const
+MPI_Vector<container> FieldAligned<G,RowDistMat<M,C>, MPI_Vector<container> >::evaluate( BinaryOp binary, UnaryOp unary, unsigned p0, unsigned rounds) const
 {
     //idea: simply apply I+/I- enough times on the init2d vector to get the result in each plane
     //unary function is always such that the p0 plane is at x=0
@@ -463,7 +463,7 @@ MPI_Vector<container> MPI_FieldAligned<G,M,C, container>::evaluate( BinaryOp bin
 }
 
 template<class G, class M, class C, class container>
-void MPI_FieldAligned<G,M,C, container>::einsPlus( const MPI_Vector<container>& f, MPI_Vector<container>& fplus ) 
+void FieldAligned<G,RowDistMat<M,C>, MPI_Vector<container> >::einsPlus( const MPI_Vector<container>& f, MPI_Vector<container>& fplus ) 
 {
     //dg::blas2::detail::doSymv( plus, f, fplus, MPIMatrixTag(), MPIVectorTag(), MPIVectorTag());
     const container& in = f.data();
@@ -539,7 +539,7 @@ void MPI_FieldAligned<G,M,C, container>::einsPlus( const MPI_Vector<container>&
 }
 
 template<class G,class M, class C, class container>
-void MPI_FieldAligned<G,M,C,container>::einsMinus( const MPI_Vector<container>& f, MPI_Vector<container>& fminus ) 
+void FieldAligned<G,RowDistMat<M,C>,MPI_Vector<container> >::einsMinus( const MPI_Vector<container>& f, MPI_Vector<container>& fminus ) 
 {
     const container& in = f.data();
     container& out = fminus.data();
@@ -610,7 +610,7 @@ void MPI_FieldAligned<G,M,C,container>::einsMinus( const MPI_Vector<container>&
     }
 }
 template< class G, class M, class C, class container>
-void MPI_FieldAligned<G,M,C,container>::einsMinusT( const MPI_Vector<container>& f, MPI_Vector<container>& fpe)
+void FieldAligned<G,RowDistMat<M,C>,MPI_Vector<container> >::einsMinusT( const MPI_Vector<container>& f, MPI_Vector<container>& fpe)
 {
     //dg::blas2::detail::doSymv( minusT, f, fpe, MPIMatrixTag(), MPIVectorTag(), MPIVectorTag());
     const container& in = f.data();
@@ -684,7 +684,7 @@ void MPI_FieldAligned<G,M,C,container>::einsMinusT( const MPI_Vector<container>&
     }
 }
 template< class G,class M, class C, class container>
-void MPI_FieldAligned<G,M,C,container>::einsPlusT( const MPI_Vector<container>& f, MPI_Vector<container>& fme)
+void FieldAligned<G,RowDistMat<M,C>,MPI_Vector<container> >::einsPlusT( const MPI_Vector<container>& f, MPI_Vector<container>& fme)
 {
     //dg::blas2::detail::doSymv( plusT, f, fme, MPIMatrixTag(), MPIVectorTag(), MPIVectorTag());
     const container& in = f.data();
-- 
GitLab


From d8811ba6145b0fe331cb876e65d98c826638539c Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Wed, 5 Jul 2017 23:00:20 +0200
Subject: [PATCH 013/453] update README and documentation a bit

---
 inc/dg/backend/README           | 33 ++++++---------------------------
 inc/dg/backend/mpi_collective.h |  3 +++
 inc/dg/backend/mpi_vector.h     |  1 +
 inc/dg/dg_doc.h                 |  4 +++-
 4 files changed, 13 insertions(+), 28 deletions(-)

diff --git a/inc/dg/backend/README b/inc/dg/backend/README
index 9cda80ed9..50f666e84 100644
--- a/inc/dg/backend/README
+++ b/inc/dg/backend/README
@@ -4,11 +4,6 @@ well as additional utility functions
 Performance hints
 ===========================================
 FIRST: (Small things to do)
-0. Remove the permanent ghostcells in the MPI_Vector class and replace it by copying 
-   values to a vector with ghostcells on demand (e.g. in a matrix - vector multiplication)
-
-   ATTENTION: This might affect *all* current mpi files but only slightly
-   AIM: vectors are easier to handle, less bookkeeping, less error prone
 1. Implement transposition of mpi interpolation for dz.h (explanation further below)
    - transpose interpolation matrices locally
    - first apply collective.gather()
@@ -16,25 +11,9 @@ FIRST: (Small things to do)
 
 2. Implement an mpi version of create::dz and the application of it in mpi_matrix.h (if needed in general_elliptic.h)
 
-SECOND: (memory efficient gpu sparse matrix format)
-0. Change gpu matrix format to something self-made like the current mpi_matrix format (i.e. store blocks only once)
-    AIM: huge memory advantage, should be at least as fast as the current cusp formats, if not faster
-    - it might pay off to improve the data structure and algorithm in mpi_matrix.h a bit first,
-        especially the handling of the boundary terms is a bit messy and the long term approach should be 
-        to separate the  mpi communication from the local matrix-vector multiplication
-    - the mpi_matrix.h and mpi_derivatives.h show the ideas to implement
-        the format and how to then actually create the matrices,
-        also have a look at ell_interpolation.cuh to get an idea of how to implement the gpu kernels
-
 THIRD: (road to MPI + X, where X is all we already have, i.e. OpenMP, MIC, or GPU)
 The general idea is to separate global communication from local parallelization and thus 
 readily reuse the existing, optimized library for the local part
-0. make mpi_vector a template accepting a container class as parameter and generalize the mpi_vector_blas functions falling back to existing local code.
-    AIM: this way e.g. also a thrust::device_vector can be used
-1. replace mpi_precon by mpi_vector 
-    IDEA: the performance gain is not worth the effort, also for gpus manually written kernels would have to be written
-    - rewrite the create::weights function using existing create::weights functions
-    - rewrite the blas2 preconditioner function using existing code
 
 2. implement a general sparse mpi matrix format (unifying the existing mpi_matrix and the mpi-interpolation )
 The idea is that the mpi matrix takes care of the communication and then defers the actual computation 
@@ -44,13 +23,13 @@ to an existing local matrix format.
     column distributed (CD). In the first each node gets the same rows as the vector it holds, in the
     second it gets the same columns as the vector it holds.
     IDEA how to then implement gemv(input, output) algorithm:
-    RD: 1. create a local send buffer and locally gather values from input vector (c) into a send buffer (am Besten nach PID geordnet)
-        2. globally scatter these values into recv buffer (b) 
+    RD: 1. create a local send buffer and locally gather values from input vector (c) into a send buffer (am Besten nach PID geordnet, note that a given value can be sent to several processes -> that's why it's a gather)
+        2. globally scatter these values into recv buffer (b)  (The first two points are a global gather)
         3. then apply the local matrix to that buffer and store result in output vector (a)
 
     CD: 1. apply the local matrix on the input vector and store result in a local send buffer,  (a)
-        2. globally scatter the values in this buffer to a recv buffer and  (b)
-        3. then permute and reduce the recv buffer on double indices and store result in output vector (c)
+        2. globally scatter the values in this buffer to a recv buffer (b) (every value in the result belongs to exactly one line/process)
+        3. then permute and reduce the recv buffer on double indices and store result in output vector (c) 
     
         Transposition is easy: if a RD matrix is transposed you get a CD matrix, also transpose the Collective object (swap scatter and gather maps)
 
@@ -65,8 +44,8 @@ to an existing local matrix format.
     c) A local index map, to map the local buffer indices to local row/col indices
     
     IDEA: how to create such a matrix:
-    If you can, manually create (a), (b), (c) and construct the matrix. This shouldn't be too difficult for 
-    the existing interpolation in dz.h. The current mpi matrix is a row distributed matrix and the communication 
+    If you can, manually create (a), (b), (c) and construct the matrix. 
+    The current mpi matrix is a row distributed matrix and the communication 
     is done by manual MPI_SendRecv calls. This has to (maybe even should) be replaced by a Collective object, which
     internally uses MPI_Alltoallv (If need be, the MPI_Alltoallv might be replaced by MPI_Neighbor_alltoallv in MPI 3)
 
diff --git a/inc/dg/backend/mpi_collective.h b/inc/dg/backend/mpi_collective.h
index 7aac60ecf..677608e22 100644
--- a/inc/dg/backend/mpi_collective.h
+++ b/inc/dg/backend/mpi_collective.h
@@ -161,6 +161,9 @@ void Collective::gather_( const thrust::host_vector<double>& gatherFrom, thrust:
  * @ingroup mpi_structures
  * @brief Struct that performs collective scatter and gather operations across processes
  * on distributed vectors using mpi
+ * In the bijective operations every element in a vector belongs to 
+ exactly one MPI process, i.e. one value is not sent to more than 
+ one process and the reduction in the send_and_reduce is not necessary
  *
  * @code
  int i = myrank;
diff --git a/inc/dg/backend/mpi_vector.h b/inc/dg/backend/mpi_vector.h
index dbceecca5..9ea095873 100644
--- a/inc/dg/backend/mpi_vector.h
+++ b/inc/dg/backend/mpi_vector.h
@@ -18,6 +18,7 @@ namespace dg
  communicate (e.g. boundary points in matrix-vector multiplications) and use
  the existing blas functions for the local computations. 
  * (At the blas level 1 level communication is needed for scalar products)
+ * @note Don't start looking for ghostcells, there aren't any
  * @tparam container underlying local container class
  */
 template<class container>
diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index 11fe1a7cc..6a80b7192 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -25,8 +25,10 @@
  *     @}
  *     @defgroup sparsematrix Sparse matrix formats
  *     @defgroup mpi_structures MPI backend functionality
+ *             The general idea is to separate global communication from local parallelization and thus 
+ *             readily reuse the existing, optimized library for the local part
  *     @defgroup typedefs Typedefs
-       Useful type definitions for easy programming
+ *          Useful type definitions for easy programming
  * @}
  * @defgroup numerical0 Level 2: Basic numerical algorithms
  * These algorithms make use only of blas level 1 and 2 functions
-- 
GitLab


From 2d42d29d80fe88c9fc2ac739354543bdec2720b6 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 6 Jul 2017 12:44:28 +0200
Subject: [PATCH 014/453] renamed collect and send_and_reduce to global_gather
 and global_scatter_reduce

---
 inc/dg/backend/mpi_collective.h |  6 ++--
 inc/dg/backend/mpi_matrix.h     | 27 +++++++--------
 inc/dg/backend/mpi_vector.h     |  8 ++---
 inc/dg/dg_doc.h                 | 61 +++++++++++++++++++++++++--------
 4 files changed, 66 insertions(+), 36 deletions(-)

diff --git a/inc/dg/backend/mpi_collective.h b/inc/dg/backend/mpi_collective.h
index 677608e22..26da83447 100644
--- a/inc/dg/backend/mpi_collective.h
+++ b/inc/dg/backend/mpi_collective.h
@@ -163,7 +163,7 @@ void Collective::gather_( const thrust::host_vector<double>& gatherFrom, thrust:
  * on distributed vectors using mpi
  * In the bijective operations every element in a vector belongs to 
  exactly one MPI process, i.e. one value is not sent to more than 
- one process and the reduction in the send_and_reduce is not necessary
+ one process and the reduction in the global_scatter_reduce is not necessary
  *
  * @code
  int i = myrank;
@@ -229,7 +229,7 @@ struct BijectiveComm
      * @return received data from other processes of size recv_size()
      * @note a scatter followed by a gather of the received values restores the original array
      */
-     Vector collect( const Vector& values)const
+     Vector global_gather( const Vector& values)const
     {
         assert( values.size() == idx_.size());
         Vector values_(values);
@@ -249,7 +249,7 @@ struct BijectiveComm
      * @param values contains values from other processes sent back to the origin (must have the size of the map given in the constructor, or send_size())
      * @note a scatter followed by a gather of the received values restores the original array
      */
-    void send_and_reduce( const Vector& gatherFrom, Vector& values) const
+    void global_scatter_reduce( const Vector& gatherFrom, Vector& values) const
     {
         Vector values_(values.size());
         //sammeln
diff --git a/inc/dg/backend/mpi_matrix.h b/inc/dg/backend/mpi_matrix.h
index f5bcfad6a..f4dd1b6ad 100644
--- a/inc/dg/backend/mpi_matrix.h
+++ b/inc/dg/backend/mpi_matrix.h
@@ -26,11 +26,10 @@ namespace dg
  doSymv(m,x,y) needs to be callable on the container class of the MPI_Vector
 * @tparam LocalMatrixOuter The class of the matrix for local computations of the outer points. 
  doSymv(1,m,x,1,y) needs to be callable on the container class of the MPI_Vector
-* @tparam Collective models aCommunicator The Communication class needs to gather values across processes. 
-container collect( const container& input);
+* @tparam Collective models aCommunicator The Communication class needs to gather values across processes. Only the global_gather and size
+member functions are needed. 
 Gather points from other processes that are necessary for the outer computations.
-int size(); 
-should give the size of the vector that collect returns. If size()==0 the collect() function won't be called and
+ If size()==0 the global_gather() function won't be called and
 only the inner matrix is applied.
 */
 template<class LocalMatrixInner, class LocalMatrixOuter, class Collective >
@@ -86,7 +85,7 @@ struct RowColDistMat
     * @brief Matrix Vector product
     *
     * First the inner elements are computed with a call to doSymv then 
-    * the collect function of the communication object is called. 
+    * the global_gather function of the communication object is called. 
     * Finally the outer elements are added with a call to doSymv for the outer matrix
     * @tparam container container class of the vector elements
     * @param x input
@@ -119,7 +118,7 @@ struct RowColDistMat
         //if(rank==0)std::cout << "Inner points took "<<t.diff()<<"s\n";
         //2. communicate outer points
         //t.tic();
-        const container& temp = c_.collect( x.data());
+        const container& temp = c_.global_gather( x.data());
         //t.toc();
         //if(rank==0)std::cout << "Collect      took "<<t.diff()<<"s\n";
         //3. compute and add outer points
@@ -147,11 +146,11 @@ struct RowColDistMat
 * @tparam LocalMatrix The class of the matrix for local computations. 
  symv needs to be callable on the container class of the MPI_Vector
 * @tparam Collective models aCommunicator The Communication class needs to scatter and gather values across processes. 
-container collect( const container& input);
+container global_gather( const container& input);
 Gather all points (including the ones that the process already has) necessary for the local matrix-vector
 product into one vector, such that the local matrix can be applied.
 int size(); 
-should give the size of the vector that collect returns. If size()==0 the collect() function won't be called and
+should give the size of the vector that global_gather returns. If size()==0 the global_gather() function won't be called and
 only the inner matrix is applied.
 */
 template<class LocalMatrix, class Collective >
@@ -215,9 +214,9 @@ struct RowDistMat
             return;
 
         }
-        container temp = c_.collect( x.data());
+        container temp = c_.global_gather( x.data());
         //t.toc();
-        //if(rank==0)std::cout << "collect took "<<t.diff()<<"s\n";
+        //if(rank==0)std::cout << "global_gather took "<<t.diff()<<"s\n";
         //t.tic();
         dg::blas2::detail::doSymv( m_, temp, y.data(), 
                        typename dg::MatrixTraits<LocalMatrix>::matrix_category(), 
@@ -241,12 +240,12 @@ struct RowDistMat
 * This class assumes that the matrix and vector elements are distributed columnwise among mpi processes.
 * @tparam LocalMatrix The class of the matrix for local computations. 
  symv needs to be callable on the container class of the MPI_Vector
-* @tparam Collective models aCommunicator The Communication class needs to scatter and gather values across processes. 
-void send_and_reduce( const container& input, container& output);
+* @tparam Collective models aCommunicator 
+void global_scatter_reduce( const container& input, container& output);
 Sends the results of the local computations to the processes they belong to. 
 After that the results of the same lines need to be reduced.
 int size(); 
-should give the size of the vector that send_and_reduce needs. If size()==0 the send_and_reduce() function won't be called and
+should give the size of the vector that global_scatter_reduce needs. If size()==0 the global_scatter_reduce() function won't be called and
 only the inner matrix is applied.
 */
 template<class LocalMatrix, class Collective >
@@ -300,7 +299,7 @@ struct ColDistMat
                        typename dg::MatrixTraits<LocalMatrix>::matrix_category(), 
                        typename dg::VectorTraits<container>::vector_category(),
                        typename dg::VectorTraits<container>::vector_category() );
-        c_.send_and_reduce( temp, y.data());
+        c_.global_scatter_reduce( temp, y.data());
     }
     private:
     LocalMatrix m_;
diff --git a/inc/dg/backend/mpi_vector.h b/inc/dg/backend/mpi_vector.h
index 9ea095873..b2077c074 100644
--- a/inc/dg/backend/mpi_vector.h
+++ b/inc/dg/backend/mpi_vector.h
@@ -218,9 +218,9 @@ struct NearestNeighborComm
     *
     * @return new container
     */
-    const Vector& collect( const Vector& input)const;
+    const Vector& global_gather( const Vector& input)const;
     /**
-    * @brief Size of the output of collect
+    * @brief Size of the output of global_gather
     *
     * @return size
     */
@@ -258,7 +258,7 @@ struct NearestNeighborComm
     int direction_;
     bool silent_;
     Index gather_map1, gather_map2, scatter_map1, scatter_map2;
-    //dynamically allocate buffer so that collect can be const
+    //dynamically allocate buffer so that global_gather can be const
     Buffer<Vector> values, buffer1, buffer2, rb1, rb2; 
 
     void sendrecv( Vector&, Vector&, Vector& , Vector&)const;
@@ -360,7 +360,7 @@ int NearestNeighborComm<I,V>::buffer_size() const
 }
 
 template<class I, class V>
-const V& NearestNeighborComm<I,V>::collect( const V& input) const
+const V& NearestNeighborComm<I,V>::global_gather( const V& input) const
 {
     if( silent_) return *values.data();
         //int rank;
diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index 6a80b7192..e780d13b0 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -116,9 +116,42 @@
  * @brief Struct that performs collective scatter and gather operations across processes
  * on distributed vectors using mpi
  *
+ * In order to understand the issue you must first really(!) understand what 
+ gather and scatter operations are, so grab pen and paper: 
+
+ Gather: imagine a vector v and a map that gives to every element in this vector v
+ an index into a source vector w where the value of this element should be taken from
+ i.e. \f$ v[i] = w[idx[i]] \f$ 
+ Note that an index in the source vector can appear several times or not at all. 
+ This is why the source vector w can have any size and even be smaller than v. 
+ If we throw away all unused elements in w the source vector is always equal 
+ or smaller in size than v. 
+
+ Scatter: imagine a vector w and a map that gives to every element in the vector w an
+ index in a target vector v where this element should go to, 
+ i.e. \f$ w[idx[i]] = v[i] \f$. 
+ Note again that a target index can appear several times. Then in our case we 
+ perform a reduction operation (we sum up all elements). If we throw away all
+ unused elements in w the, then w is always equal or smaller in size than v. 
+
+Think of the index map as establishing fixed connections between two vectors. 
+When you apply scatter and gather operations you send data back and forth
+between these two vectors along these connections.
+ However, only if the index map is bijective, the scatter operation is actually the inverse of 
+ the gather operation. 
+
+ Now, consider that both vectors v and w are distributed across processes.
+ That means when you send data with MPI you will probably need a communication buffers.
+    There are three types of indices that you need to consider: 
+
+    a) the global vector index is the index of an element if there was only one vector that lay contiguously in memory. 
+
+    b) the local vector index is the index of the local chunk a process has. 
+
+    c) the buffer index is the index into the communication buffer. 
+
  * @ingroup templates
  @attention this is not a real class it's there for documentation only
- @attention parameter names can be different
  *
  * @code
  int i = myrank;
@@ -138,32 +171,30 @@ struct aCommunicator
 {
 
     /**
-     * @brief Scatters data according to a specific scheme given in the Constructor
+     * @brief Gather data across processes
      *
-     * The order of the received elements is according to their original array index (i.e. a[0] appears before a[1]) and their process rank of origin ( i.e. values from rank 0 appear before values from rank 1)
-     * @param values data to send (must have the size given 
-     * by the map in the constructor, s.a. send_size())
+        1. create a local send buffer and locally gather values from input vector (c) into a send buffer (order with PID, note that a given value can be sent to several processes -> that's why it's a gather)
+        2. globally scatter these values into recv buffer (b)  
+     * @param values data to send (s.a. send_size())
      * @tparam LocalContainer a container on a shared memory system
      *
      * @return received data from other processes of size recv_size()
-     * @note a scatter followed by a gather of the received values restores the original array
-     * @note this function is only needed in the RowDistMat matrix format
      */
     template< class LocalContainer>
-    LocalContainer collect( const LocalContainer& values)const;
+    LocalContainer global_gather( const LocalContainer& values)const;
 
     /**
-     * @brief Gather data according to the map given in the constructor 
+     * @brief Scatters data accross processes and reduces on double indices
      *
-     * This method is the inverse of scatter 
+     * The order of the received elements is according to their original array index (i.e. a[0] appears before a[1]) and their process rank of origin ( i.e. values from rank 0 appear before values from rank 1)
+        2. globally scatter the values in this buffer to a recv buffer (b) (every value in the result belongs to exactly one line/process)  
+        3. then permute and reduce the recv buffer on double indices and store result in output vector (c) 
      * @tparam LocalContainer a container on a shared memory system
      * @param gatherFrom other processes collect data from this vector (has to be of size given by recv_size())
-     * @param values contains values from other processes sent back to the origin (must have the size of the map given in the constructor, or send_size())
-     * @note a scatter followed by a gather of the received values restores the original array
-     * @note this format is only needed in the ColDistMat matrix format
+     * @param values contains values from other processes sent back to the origin (or send_size())
      */
     template< class LocalContainer>
-    void send_and_reduce( const LocalContainer& gatherFrom, LocalContainer& values) const;
+    void global_scatter_reduce( const LocalContainer& toScatter, LocalContainer& values) const;
 
     /**
      * @brief compute total # of elements the calling process receives in the scatter process (or sends in the gather process)
@@ -193,5 +224,5 @@ struct aCommunicator
     * used to assert that communicators of matrix and vector are the same
     * @return MPI Communicator
     */
-    MPI_Comm communicator() const {return p_.communicator();}
+    MPI_Comm communicator() const;
 };
-- 
GitLab


From 4665fe6b9620e613b86fa911e1c30707f5712cbd Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 6 Jul 2017 15:55:46 +0200
Subject: [PATCH 015/453] reduce difference to develop branch

---
 diag/feltorSHdiag.cu            | 42 +++++++--------
 inc/dg/backend/projection_t.cu  |  5 +-
 inc/geometries/guenther_ds_b.cu | 90 +++++++++++++--------------------
 3 files changed, 56 insertions(+), 81 deletions(-)

diff --git a/diag/feltorSHdiag.cu b/diag/feltorSHdiag.cu
index 1baa39e1d..187edf193 100644
--- a/diag/feltorSHdiag.cu
+++ b/diag/feltorSHdiag.cu
@@ -22,8 +22,8 @@ double Y( double x, double y) {return y;}
 
 struct Heaviside2d
 {
-//     Heaviside2d( double sigma):sigma2_(sigma*sigma/2.), x_(0), y_(0){} //for 2sigma^2
-    Heaviside2d( double sigma):sigma2_(sigma*sigma), x_(0), y_(0){} //for 4 sigma^2
+    Heaviside2d( double sigma):sigma2_(sigma*sigma/4.), x_(0), y_(0){}
+//     Heaviside2d( double sigma):sigma2_(sigma*sigma), x_(0), y_(0){}
     void set_origin( double x0, double y0){ x_=x0, y_=y0;}
     double operator()(double x, double y)
     {
@@ -166,7 +166,7 @@ int main( int argc, char* argv[])
     double posX_max = 0.0,posY_max = 0.0,posX_max_old = 0.0,posY_max_old = 0.0,velX_max=0.0, velY_max=0.0,posX_max_hs=0.0,posY_max_hs=0.0,velCOM=0.0;
     double compactness_ne=0.0;
     //-----------------Start timestepping
-    for( unsigned i=0; i<2; i++)
+    for( unsigned i=0; i<p.maxout; i++)
     {
         start2d[0] = i;
         start1d[0] = i;
@@ -289,7 +289,7 @@ int main( int argc, char* argv[])
 //         err_out = nc_put_vara_double( ncid_out, names2dID[2], start2d, count2d, transfer2d.data());
 // 
 //      
-//         //Compute ion temperature with phi=0
+// //         //Compute ion temperature with phi=0
 //         dg::blas1::pointwiseDivide(B2,tpe[1],helper);        //helper  = B^2/Ti
 //         invgamma2.set_chi(helper);                           //helmholtz2 chi = B^2/Ti
 // 
@@ -322,31 +322,25 @@ int main( int argc, char* argv[])
 //         dg::blas2::symv(polti,phi,helper3); //-nabla(P_i/B^2 (nabla_perp phi));
 //         dg::blas1::axpby(1.0, target, -2.0*p.mu[1], helper3,helper2);//pi  =  2 nabla(P_i/B^2 (nabla_perp phi)) + p_i_bar
 //         dg::blas1::pointwiseDivide(helper2,npe[0],helper2); //ti=(pi-amp^2)/ne
-//         
-        //compute Omega_d //therm
-//         dg::blas1::transform(helper2,helper, dg::PLUS<>(-(p.bgprofamp + p.nprofileamp))); //                 
+// //         
+// //          //compute Omega_d
+//                 dg::blas1::transform(helper2,helper, dg::PLUS<>(-(p.bgprofamp + p.nprofileamp))); 
+// //                 
 //         polti.set_chi(one);
 //         dg::blas2::symv(polti,helper,helper3); //nabla(nabla_perp p_i/B^2 );
 //         dg::blas1::scal(helper3,-1.0*p.tau[1]);
-        //compute Omega_d //iso
-        dg::blas1::transform(npe[0],helper, dg::PLUS<>(-(p.bgprofamp + p.nprofileamp))); //       
-       dg::blas1::pointwiseDot( helper, binv, helper);
-        dg::blas1::pointwiseDot( helper, binv, helper);    //helper2 = P_i/B^2  
-        polti.set_chi(one);
-        dg::blas2::symv(polti,helper,helper3); //nabla(nabla_perp p_i/B^2 );
-        dg::blas1::scal(helper3,-1.0*p.tau[1]);
+// //         
+// //         //write t_i into 2dnetcdf
+//         transfer2d = helper2;
+//         err_out = nc_put_vara_double( ncid_out, names2dID[0], start2d, count2d, transfer2d.data());
+//         //write Omega_d into 2dnetcdf
+//         transfer2d = helper3;
+//         err_out = nc_put_vara_double( ncid_out, names2dID[1], start2d, count2d, transfer2d.data());
+//               
 //         
-//         //write t_i into 2dnetcdf
-        transfer2d = helper2;
-        err_out = nc_put_vara_double( ncid_out, names2dID[0], start2d, count2d, transfer2d.data());
-        //write Omega_d into 2dnetcdf
-        transfer2d = helper3;
-        err_out = nc_put_vara_double( ncid_out, names2dID[1], start2d, count2d, transfer2d.data());
-              
-        
         
-//      compute Omega_E and write Omega_E into 2dnetcdf
-        dg::blas1::pointwiseDot( npe[0],one,helper3); 
+        //compute Omega_E and write Omega_E into 2dnetcdf
+        dg::blas1::pointwiseDot( npe[1],one,helper3); 
         dg::blas1::pointwiseDot( helper3, binv, helper2);
         dg::blas1::pointwiseDot( helper2, binv, helper2);   
         polti.set_chi(helper2);
diff --git a/inc/dg/backend/projection_t.cu b/inc/dg/backend/projection_t.cu
index 3aa65205d..878745e23 100644
--- a/inc/dg/backend/projection_t.cu
+++ b/inc/dg/backend/projection_t.cu
@@ -33,10 +33,9 @@ int main()
 
     std::cout << "TEST 2D\n";
     n_old = 7, n_new = 3, N = 4, Nf = 3;
-    std::cout << "Type n_old,n_new, N, Nf "<< std::endl;
+    std::cout << "Type n, N, Nf "<< std::endl;
     std::cin >>n_old;
-//     n_new = n_old;
-    std::cin >> n_new;
+    n_new = n_old;
     std::cin >> N;
     std::cin >> Nf;
     
diff --git a/inc/geometries/guenther_ds_b.cu b/inc/geometries/guenther_ds_b.cu
index 0fb9516e3..eb28a3378 100644
--- a/inc/geometries/guenther_ds_b.cu
+++ b/inc/geometries/guenther_ds_b.cu
@@ -57,22 +57,16 @@ int main( )
     std::cout << "Type n, Nx, Ny, Nz\n";
     //std::cout << "Note, that function is resolved exactly in R,Z for n > 2\n";
     unsigned n=3, Nx=5, Ny=5, Nz=5;
-    unsigned fn;
-    double fN;
-    std::cout << "Type n\n";
-    std::cin >> n;
-    std::cout << "Type fn, fN\n";
-
-    std::cin >> fn>> fN;
+    //std::cin >> n>> Nx>>Ny>>Nz;
     unsigned Nxn = Nx;
     unsigned Nyn = Ny;
     unsigned Nzn = Nz;
 
     double rk4eps = 1e-8;
-//     std::cout << "Type RK4 eps (1e-8)\n";
-//     std::cin >> rk4eps;
+    //std::cout << "Type RK4 eps (1e-8)\n";
+    //std::cin >> rk4eps;
     double z0 = 0, z1 = 2.*M_PI;
-    for (unsigned i=0;i<6;i+=1) { 
+    for (unsigned i=1;i<4;i+=2) { 
 
         Nzn = unsigned(Nz*pow(2,i));
         Nxn = (unsigned)ceil(Nx*pow(2,(double)(i*2./n)));
@@ -80,8 +74,7 @@ int main( )
 
 
 
-        dg::CylindricalGrid<dg::DVec> g3d( Rmin,Rmax, Zmin,Zmax, z0, z1,  n,Nxn ,Nyn, Nzn,dg::DIR, dg::DIR, dg::PER);
-        dg::CylindricalGrid<dg::DVec> g3d_fein( Rmin,Rmax, Zmin,Zmax, z0, z1,  fn,fN*Nxn ,fN*Nyn, Nzn,dg::DIR, dg::DIR, dg::PER);
+        dg::CylindricalGrid3d<dg::DVec> g3d( Rmin,Rmax, Zmin,Zmax, z0, z1,  n,Nxn ,Nyn, Nzn,dg::DIR, dg::DIR, dg::PER);
         dg::Grid2d g2d( Rmin,Rmax, Zmin,Zmax,  n, Nxn ,Nyn);
 
         std::cout << "NR = " << Nxn << std::endl;
@@ -96,18 +89,14 @@ int main( )
     const dg::DVec w2d = dg::create::weights( g2d);
     const dg::DVec v3d = dg::create::inv_volume( g3d);
 
-//     std::cout << "computing dsDIR" << std::endl;
-//     dg::DDS::FieldAligned dsFA( field, g3d_fein, rk4eps, dg::DefaultLimiter(), dg::DIR);
+    std::cout << "computing dsDIR" << std::endl;
+    dg::DDS::FieldAligned dsFA( field, g3d, rk4eps, dg::DefaultLimiter(), dg::DIR);
     std::cout << "computing dsNEU" << std::endl;
-    dg::DDS::FieldAligned dsNUFA( field, g3d_fein, rk4eps, dg::DefaultLimiter(), dg::NEU);
-    dg::Timer timer;
-
-    timer.tic();
-    dg::DDS 
-//     ds ( dsFA, g3d, field, dg::not_normed, dg::centered), 
-        dsNU ( dsNUFA, g3d, field, dg::not_normed, dg::forward);
-    timer.toc();
-    std::cout << "Creating took " << timer.diff() << std::endl;
+    dg::DDS::FieldAligned dsNUFA( field, g3d, rk4eps, dg::DefaultLimiter(), dg::NEU);
+
+    dg::DDS ds ( dsFA, field, dg::not_normed, dg::centered), 
+        dsNU ( dsNUFA, field, dg::not_normed, dg::centered);
+
 //     dg::DS<dg::DMatrix, dg::DVec> dsNEU( field, g3d, g3d.hz(), rk4eps, dg::DefaultLimiter(), dg::NEU);
     
 //     dg::Grid3d g3dp( Rmin,Rmax, Zmin,Zmax, z0, z1,  n, Nx, Ny, 1);
@@ -183,7 +172,7 @@ int main( )
 //     
 //     
   
-//     dsNU( function, derivative); //ds(f)
+    dsNU( function, derivative); //ds(f)
 
 //     dsNU.forward( function, derivativef); //ds(f)
 //     dsNU.backward( function, derivativeb); //ds(f)
@@ -242,7 +231,7 @@ int main( )
 //     ellipticsym.symv(function,dsTds);
 //     dg::blas1::scal(dsTds,-1.0);
 // //     ds.centeredT(ones,divbT);
-//     ds.forwardT( derivativef, dsTdsf);  //dsT(ds(f))
+    ds.forwardT( derivativef, dsTdsf);  //dsT(ds(f))
 //     ds.backwardT( derivativeb, dsTdsb); //dsT(ds(f))
 
 //     //centered
@@ -253,8 +242,8 @@ int main( )
 //     //arithmetic average
 //     dg::blas1::axpby(0.5,dsTdsb,0.5,dsTdsf,dsTdsfb);
 //     dg::blas1::axpby(0.5,dsTdsbd,0.5,dsTdsfd,dsTdsfbd); 
-//     ds.symv(function,dsTdsfb);
-//     dg::blas1::pointwiseDot(v3d,dsTdsfb,dsTdsfb);
+    ds.symv(function,dsTdsfb);
+    dg::blas1::pointwiseDot(v3d,dsTdsfb,dsTdsfb);
         //ds( function, temp);
         //dg::blas1::pointwiseDot( temp, inverseB, temp);
         //ds(temp, dsTdsfb);
@@ -280,14 +269,14 @@ int main( )
 //     double normBds1 = dg::blas2::dot(temp2, w3d, derivativeones);
 //     double normfds1 = dg::blas2::dot(function2, w3d, derivativeones);
 // 
-//     std::cout << "--------------------testing ds" << std::endl;
-//     double norm = dg::blas2::dot( w3d, solution);
-//     //std::cout << "|| Solution ||   "<<sqrt( norm)<<"\n";
-//     double err =dg::blas2::dot( w3d, derivative);
-//     //std::cout << "|| Derivative || "<<sqrt( err)<<"\n";
-//     dg::blas1::axpby( 1., solution, -1., derivative);
-//     err =dg::blas2::dot( w3d, derivative);
-//     std::cout << "Relative Difference in DS is "<< sqrt( err/norm )<<"\n"; 
+    std::cout << "--------------------testing ds" << std::endl;
+    double norm = dg::blas2::dot( w3d, solution);
+    std::cout << "|| Solution ||   "<<sqrt( norm)<<"\n";
+    double err =dg::blas2::dot( w3d, derivative);
+    std::cout << "|| Derivative || "<<sqrt( err)<<"\n";
+    dg::blas1::axpby( 1., solution, -1., derivative);
+    err =dg::blas2::dot( w3d, derivative);
+    std::cout << "Relative Difference in DS is "<< sqrt( err/norm )<<"\n"; 
    
 //     std::cout << "--------------------testing ds with RZPhi method" << std::endl;
 //     std::cout << "|| Solution ||   "<<sqrt( norm)<<"\n";
@@ -344,15 +333,15 @@ int main( )
 //     errdsTds =dg::blas2::dot( w3d, dsTds);
 //     std::cout << "Relative Difference in DST is "<< sqrt( errdsTds/normdsTds )<<"\n";   
     
-//     std::cout << "--------------------testing dsTdsfb " << std::endl;
-//     //std::cout << "|| SolutionT ||      "<<sqrt( normdsTds)<<"\n";
-//     double remainder =dg::blas1::dot( w3d,dsTdsfb);
-//     double errdsTdsfb =dg::blas2::dot( w3d,dsTdsfb);
-//     //std::cout << "|| DerivativeTds ||  "<<sqrt( errdsTdsfb)<<"\n";
-//     //std::cout << "   Integral          "<<remainder<<"\n";
-//     dg::blas1::axpby( 1., solutiondsTds, -1., dsTdsfb);
-//     errdsTdsfb =dg::blas2::dot( w3d, dsTdsfb);
-//     std::cout << "Relative Difference in DST is "<< sqrt( errdsTdsfb/normdsTds )<<"\n";
+    std::cout << "--------------------testing dsTdsfb " << std::endl;
+    std::cout << "|| SolutionT ||      "<<sqrt( normdsTds)<<"\n";
+    double remainder =dg::blas1::dot( w3d,dsTdsfb);
+    double errdsTdsfb =dg::blas2::dot( w3d,dsTdsfb);
+    std::cout << "|| DerivativeTds ||  "<<sqrt( errdsTdsfb)<<"\n";
+    std::cout << "   Integral          "<<remainder<<"\n";
+    dg::blas1::axpby( 1., solutiondsTds, -1., dsTdsfb);
+    errdsTdsfb =dg::blas2::dot( w3d, dsTdsfb);
+    std::cout << "Relative Difference in DST is "<< sqrt( errdsTdsfb/normdsTds )<<"\n";
 //   
 //     std::cout << "--------------------testing dsTdsfb with direct method" << std::endl;
 //     std::cout << "|| SolutionT ||      "<<sqrt( normdsTds)<<"\n";
@@ -412,7 +401,7 @@ int main( )
 //     elliptic.set_z(bhatPhi);
     
     
-    double eps =1e-7;   
+    double eps =1e-8;   
     dg::Invert< dg::DVec> invert( dg::evaluate(dg::zero,g3d), w3d.size(), eps );  
     std::cout << "MAX # iterations = " << w3d.size() << std::endl;
 // 
@@ -430,15 +419,8 @@ int main( )
 //     errinvT =dg::blas2::dot( w3d, functionTinv);
 //     std::cout << "Relative Difference is  "<< sqrt( errinvT/normf )<<"\n";
 //     
-    std::cout << "--------------------testing inversion dsTds" << std::endl; 
-    timer.tic();
-    
-    unsigned iterations;
-    iterations =  invert( dsNU, functionTinv2,solutiondsTds ); //is dsTds
-    timer.toc();
-    std::cout << "inversion took " << timer.diff() << std::endl;
-    std::cout << " # of iterations "<< iterations << std::endl;
-    std::cout << " inversion/# of iterations "<<timer.diff()/iterations << std::endl;
+    std::cout << "--------------------testing dsT" << std::endl; 
+    std::cout << " # of iterations "<< invert( dsNU, functionTinv2,solutiondsTds ) << std::endl; //is dsTds
     std::cout << "Norm analytic Solution  "<<sqrt( normf)<<"\n";
     double errinvT2 =dg::blas2::dot( w3d, functionTinv2);
     std::cout << "Norm numerical Solution "<<sqrt( errinvT2)<<"\n";
-- 
GitLab


From e36f71c35ac2d4825bd0d8696c92c8018d541b5e Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 6 Jul 2017 16:17:17 +0200
Subject: [PATCH 016/453] interpolation_t recovered

---
 inc/dg/backend/interpolation.cuh  |   8 +-
 inc/dg/backend/interpolation_t.cu | 133 +++++++++---------------------
 2 files changed, 44 insertions(+), 97 deletions(-)

diff --git a/inc/dg/backend/interpolation.cuh b/inc/dg/backend/interpolation.cuh
index 1b451cf30..de1ca28a3 100644
--- a/inc/dg/backend/interpolation.cuh
+++ b/inc/dg/backend/interpolation.cuh
@@ -541,7 +541,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> projection( const Grid2d& g_coa
     unsigned num_cellsY = g_fine.Ny() / g_coarse.Ny();
 
     //construct elemental grid with fine number of cells and polynomials
-    Grid2d<double> g_elemental( -1., 1., -1., 1., g_fine.n(), num_cellsX, num_cellsY);
+    Grid2d g_elemental( -1., 1., -1., 1., g_fine.n(), num_cellsX, num_cellsY);
     //now evaluate the coarse Legendre polynomials on the fine grid
     thrust::host_vector<double> coeffsX[g_coarse.n()*g_coarse.n()];
     thrust::host_vector<double> coeffsL[g_coarse.n()*g_coarse.n()];
@@ -569,7 +569,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> projection( const Grid2d& g_coa
             coeffsL[p*g_coarse.n()+q] = transform( forward.transpose(), coeffsX[p*g_coarse.n() + q], g_elemental);
             
         }
-    Grid2d<double> gc_elemental( -1., 1., -1., 1., g_coarse.n(), 1, 1);
+    Grid2d gc_elemental( -1., 1., -1., 1., g_coarse.n(), 1, 1);
     Operator<double> backward( gc_elemental.dlt().backward());
     Operator<double> sisj_inv = dg::create::pipj_inv( gc_elemental.n());
     Operator<double> left = backward*sisj_inv;    
@@ -623,8 +623,8 @@ cusp::coo_matrix<int, double, cusp::host_memory> projection( const Grid3d& g_coa
     assert( g_coarse.Nz() == g_fine.Nz());
     const unsigned Nz = g_coarse.Nz();
 
-    Grid2d<double> g2d_coarse( g_coarse.x0(), g_coarse.x1(), g_coarse.y0(), g_coarse.y1(), g_coarse.n(), g_coarse.Nx(), g_coarse.Ny());
-    Grid2d<double> g2d_fine( g_fine.x0(), g_fine.x1(), g_fine.y0(), g_fine.y1(), g_fine.n(), g_fine.Nx(), g_fine.Ny());
+    Grid2d g2d_coarse( g_coarse.x0(), g_coarse.x1(), g_coarse.y0(), g_coarse.y1(), g_coarse.n(), g_coarse.Nx(), g_coarse.Ny());
+    Grid2d g2d_fine( g_fine.x0(), g_fine.x1(), g_fine.y0(), g_fine.y1(), g_fine.n(), g_fine.Nx(), g_fine.Ny());
     cusp::coo_matrix<int, double, cusp::host_memory> A2d = projection( g2d_coarse, g2d_fine);
 
     cusp::coo_matrix<int, double, cusp::host_memory> A( A2d.num_rows*Nz, A2d.num_cols*Nz, A2d.num_entries*Nz);
diff --git a/inc/dg/backend/interpolation_t.cu b/inc/dg/backend/interpolation_t.cu
index db82721a2..3fe691134 100644
--- a/inc/dg/backend/interpolation_t.cu
+++ b/inc/dg/backend/interpolation_t.cu
@@ -1,7 +1,6 @@
 #include <iostream>
 
 #include <cusp/print.h>
-#include "../blas.h"
 #include "xspacelib.cuh"
 #include "interpolation.cuh"
 #include "../blas.h"
@@ -10,24 +9,15 @@
 double function( double x, double y){return sin(x)*sin(y);}
 double function( double x, double y, double z){return sin(x)*sin(y)*sin(z);}
 
- unsigned n = 3;
- unsigned Nx = 30; 
- unsigned Ny = 50; 
- unsigned Nz = 2; 
+const unsigned n = 3;
+const unsigned Nx = 3; 
+const unsigned Ny = 5; 
+const unsigned Nz = 2; 
 
 typedef cusp::coo_matrix<int, double, cusp::host_memory> Matrix;
 
-double sinex( double x, double y) {return sin(x)*sin(x)*sin(y)*sin(y)*x*x*y*y;}
-double sinex( double x, double y, double z) {return sin(x)*sin(x)*sin(y)*sin(y)*x*x*y*y;}
-
 int main()
 {
-    std::cout << "type n, Nx, Ny, Nz\n";
-    std::cin >> n >> Nx >> Ny >> Nz;
-    std::cout << "type nfine, Nmultiply (fine grid is nf, NfNx, NfNy, Nz)\n";
-    unsigned nf, Nf;
-    std::cin >> nf >> Nf;
-
 
     {
     dg::Grid2d g( -10, 10, -5, 5, n, Nx, Ny);
@@ -53,20 +43,22 @@ int main()
     //cusp::print(A);
     //cusp::print(B);
     //ATTENTION: backscatter might delete zeroes in matrices
-    for( unsigned i=0; i<A.values.size(); i++)
-    {
-        if( (A.values[i] - B.values[i]) > 1e-10)
-        {
-            std::cerr << "NOT EQUAL "<<A.row_indices[i] <<" "<<A.column_indices[i]<<" "<<A.values[i] << "\t "<<B.row_indices[i]<<" "<<B.column_indices[i]<<" "<<B.values[i]<<"\n";
-            passed = false;
-        }
-    }
-    if( A.num_entries != B.num_entries)
-    {
-        std::cerr << "Number of entries not equal!\n";
-        passed = false;
-    }
-    if( passed)
+    //for( unsigned i=0; i<A.values.size(); i++)
+    //{
+    //    if( (A.values[i] - B.values[i]) > 1e-14)
+    //    {
+    //        std::cerr << "NOT EQUAL "<<A.row_indices[i] <<" "<<A.column_indices[i]<<" "<<A.values[i] << "\t "<<B.row_indices[i]<<" "<<B.column_indices[i]<<" "<<B.values[i]<<"\n";
+    //        passed = false;
+    //    }
+    //}
+    //if( A.num_entries != B.num_entries)
+    //{
+    //    std::cerr << "Number of entries not equal!\n";
+    //    passed = false;
+    //}
+    if( error > 1e-14) 
+        std::cout<< "2D TEST FAILED!\n";
+    else
         std::cout << "2D TEST PASSED!\n";
 
 
@@ -92,35 +84,6 @@ int main()
     }
     if( passed)
         std::cout << "2D INTERPOLATE TEST PASSED!\n";
-
-    dg::Grid2d gfine( -10, 10, -5, 5, nf, Nf*Nx, Nf*Ny);
-    const thrust::host_vector<double> xfine = dg::evaluate( sinex, gfine);
-    thrust::host_vector<double> xcoarseI = dg::evaluate( sinex, g);
-    const thrust::host_vector<double> xcoarse = dg::evaluate( sinex, g);
-    const thrust::host_vector<double> wfine = dg::create::weights( gfine);
-    const thrust::host_vector<double> wcoarse = dg::create::weights( g);
-    double coarseL2 = dg::blas2::dot( xcoarse, wcoarse, xcoarse);
-    double fineL2 =   dg::blas2::dot( xfine, wfine, xfine);
-    std::cout << "coarse L2 norm:       "<<coarseL2<<"\n";
-    std::cout << "Fine L2 norm:         "<<fineL2<<" \n";
-
-    Matrix f2c = dg::create::projection( g, gfine); 
-    dg::blas2::symv( f2c, xfine, xcoarseI);
-    coarseL2 = dg::blas2::dot( xcoarseI, wcoarse, xcoarseI);
-    std::cout << "interpolated L2 norm: "<<coarseL2<<"\n";
-    std::cout << "Difference in L2      "<<fabs(fineL2-coarseL2)/fabs(fineL2)<<"\n";
-    //integrals
-    double coarseI = dg::blas1::dot( wcoarse, xcoarse);
-    double fineI = dg::blas1::dot( wfine, xfine);
-    std::cout << "coarse integral:      "<<coarseI<<"\n";
-    std::cout << "Fine integral:        "<<fineI<<" \n";
-    coarseI = dg::blas1::dot( wcoarse, xcoarseI);
-    std::cout << "interpolated integral "<<coarseI<<"\n";
-    std::cout << "Difference Integral   "<<fabs(fineI-coarseI)/fabs(fineI)<<"\n";
-    dg::blas1::axpby( 1., xcoarseI, -1., xcoarse, xcoarseI);
-    double norm = dg::blas2::dot( xcoarseI, wcoarse, xcoarseI);
-    std::cout << "Difference evaluated to interpolated: "<<norm/coarseL2<<"\n";
-
     }
     ////////////////////////////////////////////////////////////////////////////
     {
@@ -141,44 +104,28 @@ int main()
                         g.z0() + (k+0.5)*g.hz();
             }
     Matrix B = dg::create::interpolation( x, y, z, g);
-    bool passed = true;
-    for( unsigned i=0; i<A.values.size(); i++)
-    {
-        if( (A.values[i] - B.values[i]) > 1e-10)
-        {
-            std::cerr << "NOT EQUAL "<<A.row_indices[i] <<" "<<A.column_indices[i]<<" "<<A.values[i] << "\t "<<B.row_indices[i]<<" "<<B.column_indices[i]<<" "<<B.values[i]<<"\n";
-            passed = false;
-        }
-    }
-    if( passed)
+    thrust::host_vector<double> vec = dg::evaluate( function, g), inter1(vec), inter2(vec);
+    dg::blas2::symv( A, vec, inter1);
+    dg::blas2::symv( B, vec, inter2);
+    dg::blas1::axpby( 1., inter1, -1., inter2, vec);
+    double error = dg::blas1::dot( vec, vec);
+    std::cout << "Error is "<<error<<" (should be small)!\n";
+    if( error > 1e-14) 
+        std::cout<< "3D TEST FAILED!\n";
+    else
         std::cout << "3D TEST PASSED!\n";
-    dg::Grid3d gfine( -10, 10, -5, 5, -7, -3,  nf, Nf*Nx, Nf*Ny, Nz);
-    const thrust::host_vector<double> xfine = dg::evaluate( sinex, gfine);
-    thrust::host_vector<double> xcoarseI = dg::evaluate( sinex, g);
-    const thrust::host_vector<double> xcoarse = dg::evaluate( sinex, g);
-    const thrust::host_vector<double> wfine = dg::create::weights( gfine);
-    const thrust::host_vector<double> wcoarse = dg::create::weights( g);
-    double coarseL2 = dg::blas2::dot( xcoarse, wcoarse, xcoarse);
-    double fineL2 =   dg::blas2::dot( xfine, wfine, xfine);
-    std::cout << "coarse L2 norm:       "<<coarseL2<<"\n";
-    std::cout << "Fine L2 norm:         "<<fineL2<<" \n";
 
-    Matrix f2c = dg::create::projection( g, gfine); 
-    dg::blas2::symv( f2c, xfine, xcoarseI);
-    coarseL2 = dg::blas2::dot( xcoarseI, wcoarse, xcoarseI);
-    std::cout << "interpolated L2 norm: "<<coarseL2<<"\n";
-    std::cout << "Difference in L2      "<<fabs(fineL2-coarseL2)/fabs(fineL2)<<"\n";
-    //integrals
-    double coarseI = dg::blas1::dot( wcoarse, xcoarse);
-    double fineI = dg::blas1::dot( wfine, xfine);
-    std::cout << "coarse integral:      "<<coarseI<<"\n";
-    std::cout << "Fine integral:        "<<fineI<<" \n";
-    coarseI = dg::blas1::dot( wcoarse, xcoarseI);
-    std::cout << "interpolated integral "<<coarseI<<"\n";
-    std::cout << "Difference Integral   "<<fabs(fineI-coarseI)/fabs(fineI)<<"\n";
-    dg::blas1::axpby( 1., xcoarseI, -1., xcoarse, xcoarseI);
-    double norm = dg::blas2::dot( xcoarseI, wcoarse, xcoarseI);
-    std::cout << "Difference evaluated to interpolated: "<<norm/coarseL2<<"\n";
+    //bool passed = true;
+    //for( unsigned i=0; i<A.values.size(); i++)
+    //{
+    //    if( (A.values[i] - B.values[i]) > 1e-14)
+    //    {
+    //        std::cerr << "NOT EQUAL "<<A.row_indices[i] <<" "<<A.column_indices[i]<<" "<<A.values[i] << "\t "<<B.row_indices[i]<<" "<<B.column_indices[i]<<" "<<B.values[i]<<"\n";
+    //        passed = false;
+    //    }
+    //}
+    //if( passed)
+    //    std::cout << "3D TEST PASSED!\n";
     }
 
     return 0;
-- 
GitLab


From fef282e51af624004d8fc56c3dd7453c2ab58a7c Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 6 Jul 2017 16:38:03 +0200
Subject: [PATCH 017/453] tested hand made projection for equality to adjoint
 projection

---
 inc/dg/backend/interpolation.cuh |  4 ++--
 inc/dg/backend/projection.cuh    |  4 ++--
 inc/dg/backend/projection_t.cu   | 41 +++++++++++++++++++++++++++-----
 3 files changed, 39 insertions(+), 10 deletions(-)

diff --git a/inc/dg/backend/interpolation.cuh b/inc/dg/backend/interpolation.cuh
index de1ca28a3..f5a6a2aa1 100644
--- a/inc/dg/backend/interpolation.cuh
+++ b/inc/dg/backend/interpolation.cuh
@@ -534,8 +534,8 @@ cusp::coo_matrix<int, double, cusp::host_memory> projection( const Grid2d& g_coa
     assert( g_coarse.x1() <= g_fine.x1());
     assert( g_coarse.y0() >= g_fine.y0());
     assert( g_coarse.y1() <= g_fine.y1());
-    assert( g_fine.Nx() % g_coarse.Nx() == 0);
-    assert( g_fine.Ny() % g_coarse.Ny() == 0);
+    //assert( g_fine.Nx() % g_coarse.Nx() == 0);
+    //assert( g_fine.Ny() % g_coarse.Ny() == 0);
 
     unsigned num_cellsX = g_fine.Nx() / g_coarse.Nx();
     unsigned num_cellsY = g_fine.Ny() / g_coarse.Ny();
diff --git a/inc/dg/backend/projection.cuh b/inc/dg/backend/projection.cuh
index 95ca65f41..50cd69901 100644
--- a/inc/dg/backend/projection.cuh
+++ b/inc/dg/backend/projection.cuh
@@ -108,7 +108,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const Grid2d& g
  @attention Projection only works if the number of cells in the
  fine grid are multiple of the number of cells in the coarse grid
  */
-cusp::coo_matrix< int, double, cusp::host_memory> projection( const Grid2d& g_new, const Grid2d& g_old)
+cusp::coo_matrix< int, double, cusp::host_memory> projection_( const Grid2d& g_new, const Grid2d& g_old)
 {
     if( g_old.Nx() % g_new.Nx() != 0) std::cerr << "ATTENTION: you project between incompatible grids in x!!\n";
     if( g_old.Ny() % g_new.Ny() != 0) std::cerr << "ATTENTION: you project between incompatible grids in y!!\n";
@@ -167,7 +167,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const Grid3d& g
  @attention Projection only works if the number of cells in the
  fine grid are multiple of the number of cells in the coarse grid
  */
-cusp::coo_matrix< int, double, cusp::host_memory> projection( const Grid3d& g_new, const Grid3d& g_old)
+cusp::coo_matrix< int, double, cusp::host_memory> projection_( const Grid3d& g_new, const Grid3d& g_old)
 {
     if( g_old.Nx() % g_new.Nx() != 0) std::cerr << "ATTENTION: you project between incompatible grids in x!!\n";
     if( g_old.Ny() % g_new.Ny() != 0) std::cerr << "ATTENTION: you project between incompatible grids in y!!\n";
diff --git a/inc/dg/backend/projection_t.cu b/inc/dg/backend/projection_t.cu
index 7c7e6b811..878fa6fde 100644
--- a/inc/dg/backend/projection_t.cu
+++ b/inc/dg/backend/projection_t.cu
@@ -7,19 +7,20 @@
 
 double sine( double x){ return sin(x);}
 double sine( double x, double y){return sin(x)*sin(y);}
+typedef cusp::coo_matrix<int, double, cusp::host_memory> Matrix;
 
 int main()
 {
     std::cout << "TEST 1D\n";
     unsigned n_old = 4, n_new = 3, N_old = 10, N_new = 1;
-    std::cout << "Type n and N of old (find) grid!\n";
+    std::cout << "Type n and N of old (fine) grid!\n";
     std::cin >> n_old >> N_old;
     std::cout << "Type n and N of new (coarser) grid!\n";
     std::cin >> n_new >> N_new;
     dg::Grid1d go ( 0, M_PI, n_old, N_old);
     dg::Grid1d gn ( 0, M_PI, n_new, N_new);
-    cusp::coo_matrix<int, double, cusp::host_memory> proj = dg::create::projection( gn, go);
-    cusp::coo_matrix<int, double, cusp::host_memory> inte = dg::create::interpolation( gn, go);
+    Matrix proj = dg::create::projection( gn, go);
+    Matrix inte = dg::create::interpolation( gn, go);
     thrust::host_vector<double> v = dg::evaluate( sine, go);
     thrust::host_vector<double> w1do = dg::create::weights( go);
     thrust::host_vector<double> w1dn = dg::create::weights( gn);
@@ -45,8 +46,8 @@ int main()
     //old grid is larger than the new grid
     dg::Grid2d g2o (0, M_PI, 0, M_PI, n_old, N_old, N_old);
     dg::Grid2d g2n (0, M_PI, 0, M_PI, n_new, N_new, N_new);
-    cusp::coo_matrix<int, double, cusp::host_memory> proj2d = dg::create::projection( g2n, g2o);
-    cusp::coo_matrix<int, double, cusp::host_memory> inte2d = dg::create::interpolation( g2n, g2o);
+    Matrix proj2d = dg::create::projection_( g2n, g2o);
+    Matrix inte2d = dg::create::interpolation( g2n, g2o);
     const dg::HVec sinO = dg::evaluate( sine, g2o), 
                    sinN = dg::evaluate( sine, g2n);
     dg::HVec w2do = dg::create::weights( g2o);
@@ -61,7 +62,7 @@ int main()
     std::cout << "Rel Difference in L2 norm is " << sqrt(dg::blas2::dot( temp, w2dn, temp)/dg::blas2::dot( sinN, w2dn, sinN))<< std::endl;
     //std::cout << "Difference between two grid evaluations:\n";
     //std::cout << diff( sinO, sinN)<<" (should converge to zero!) \n";
-    std::cout << "Difference between projection and evaluation      \n";
+    std::cout << "Difference between projection and evaluation      ";
     dg::blas1::axpby( 1., sinN, -1., sinP);
     std::cout << dg::blas2::dot( sinP, w2dn, sinP)<<" (smaller than above)\n";
     dg::blas2::gemv( inte2d, sinO, sinP);
@@ -73,5 +74,33 @@ int main()
     << "If it works it is better than interpolation in that it conserves the integral value and has smaller L2 errors\n";
     std::cout << "Interpolation always works!\n";
 
+    std::cout << "TEST hand made projection\n";
+    const thrust::host_vector<double> xfine = dg::evaluate( sine, g2o);
+    thrust::host_vector<double> xcoarseI = dg::evaluate( sine, g2n);
+    const thrust::host_vector<double> xcoarse = dg::evaluate( sine, g2n);
+    const thrust::host_vector<double> wfine = dg::create::weights( g2o);
+    const thrust::host_vector<double> wcoarse = dg::create::weights( g2n);
+    double coarseL2 = sqrt(dg::blas2::dot( xcoarse, wcoarse, xcoarse));
+    double fineL2 =   sqrt(dg::blas2::dot( xfine, wfine, xfine));
+    std::cout << "coarse L2 norm:       "<<coarseL2<<"\n";
+    std::cout << "Fine L2 norm:         "<<fineL2<<" \n";
+
+    Matrix f2c = dg::create::projection( g2n, g2o); 
+    dg::blas2::symv( f2c, xfine, xcoarseI);
+    coarseL2 = sqrt(dg::blas2::dot( xcoarseI, wcoarse, xcoarseI));
+    std::cout << "projected    L2 norm: "<<coarseL2<<"\n";
+    std::cout << "Difference in L2      "<<fabs(fineL2-coarseL2)<<"\n";
+    //integrals
+    double coarseI = dg::blas1::dot( wcoarse, xcoarse);
+    double fineI = dg::blas1::dot( wfine, xfine);
+    std::cout << "coarse integral:      "<<coarseI<<"\n";
+    std::cout << "Fine integral:        "<<fineI<<" \n";
+    coarseI = dg::blas1::dot( wcoarse, xcoarseI);
+    std::cout << "projected    integral "<<coarseI<<"\n";
+    std::cout << "Difference Integral   "<<fabs(fineI-coarseI)/fabs(fineI)<<"\n";
+    dg::blas1::axpby( 1., xcoarseI, -1., xcoarse, xcoarseI);
+    double norm = dg::blas2::dot( xcoarseI, wcoarse, xcoarseI);
+    std::cout << "Difference evaluated to interpolated: "<<norm/coarseL2<<"\n";
+
     return 0;
 }
-- 
GitLab


From f9350ff261eec3bfaf359ea1258534cf11340a94 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 6 Jul 2017 16:47:35 +0200
Subject: [PATCH 018/453] we take the adjoint projection as standard

---
 inc/dg/backend/interpolation.cuh | 161 +++----------------------------
 inc/dg/backend/projection.cuh    |   4 +-
 inc/dg/backend/projection_t.cu   |  41 ++------
 3 files changed, 19 insertions(+), 187 deletions(-)

diff --git a/inc/dg/backend/interpolation.cuh b/inc/dg/backend/interpolation.cuh
index f5a6a2aa1..b4b4174bb 100644
--- a/inc/dg/backend/interpolation.cuh
+++ b/inc/dg/backend/interpolation.cuh
@@ -62,30 +62,6 @@ std::vector<double> coefficients( double xn, unsigned n)
     }
     return px;
 }
-    
-/**
- * @brief The n-th Legendre Polynomial on [-1;1]
- */
-struct Legendre
-{
-    Legendre( unsigned n): n_(n+1), m_(0){}
-    Legendre( unsigned n, unsigned m): n_(n+1), m_(m+1){}
-    double operator()( double x)
-    {
-        //compute p_i(xn) and return the last value
-        std::vector<double> px = coefficients(x, n_);
-        return px[n_-1];
-    }
-    double operator()( double x, double y)
-    {
-        std::vector<double> px = coefficients(x, n_);
-        std::vector<double> py = coefficients(y, m_);
-        return px[n_-1]*py[m_-1];
-    }
-    private:
-    unsigned n_, m_;
-};
-
 
 }//namespace detail
 ///@endcond
@@ -499,20 +475,9 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const Grid3d& g_
     return interpolation( pointsX, pointsY, pointsZ, g_old);
 
 }
+///@}
+
 
-thrust::host_vector<double> transform( const Operator<double>& op, const thrust::host_vector<double>& in, const Grid2d& g)
-{
-    assert( op.size() == g.n());
-    thrust::host_vector<double> out(in.size(), 0);
-    for( unsigned i=0; i<g.Ny(); i++)
-    for( unsigned k=0; k<g.n(); k++)
-    for( unsigned j=0; j<g.Nx(); j++)
-    for( unsigned l=0; l<g.n(); l++)
-    for( unsigned o=0; o<g.n(); o++)
-    for( unsigned m=0; m<g.n(); m++)
-        out[((i*g.n() + k)*g.Nx() + j)*g.n() + l] += op(k,o)*op( l, m)*in[((i*g.n() + o)*g.Nx() + j)*g.n() + m];
-    return out;
-}
 /**
  * @brief Transform a vector from XSPACE to LSPACE
  *
@@ -523,121 +488,17 @@ thrust::host_vector<double> transform( const Operator<double>& op, const thrust:
  */
 thrust::host_vector<double> forward_transform( const thrust::host_vector<double>& in, const Grid2d& g)
 {
+    thrust::host_vector<double> out(in.size(), 0);
     dg::Operator<double> forward( g.dlt().forward());
-    return transform( forward, in, g);
-}
-
-cusp::coo_matrix<int, double, cusp::host_memory> projection( const Grid2d& g_coarse, const Grid2d& g_fine)
-{
-
-    assert( g_coarse.x0() >= g_fine.x0());
-    assert( g_coarse.x1() <= g_fine.x1());
-    assert( g_coarse.y0() >= g_fine.y0());
-    assert( g_coarse.y1() <= g_fine.y1());
-    //assert( g_fine.Nx() % g_coarse.Nx() == 0);
-    //assert( g_fine.Ny() % g_coarse.Ny() == 0);
-
-    unsigned num_cellsX = g_fine.Nx() / g_coarse.Nx();
-    unsigned num_cellsY = g_fine.Ny() / g_coarse.Ny();
-
-    //construct elemental grid with fine number of cells and polynomials
-    Grid2d g_elemental( -1., 1., -1., 1., g_fine.n(), num_cellsX, num_cellsY);
-    //now evaluate the coarse Legendre polynomials on the fine grid
-    thrust::host_vector<double> coeffsX[g_coarse.n()*g_coarse.n()];
-    thrust::host_vector<double> coeffsL[g_coarse.n()*g_coarse.n()];
-    Operator<double> sisj = dg::create::pipj( g_elemental.n());
-    Operator<double> forward( g_elemental.dlt().forward());
-    for( unsigned p=0; p<g_coarse.n(); p++) //y
-        for( unsigned q=0; q<g_coarse.n(); q++)//x
-        {
-            detail::Legendre legendre( q, p); 
-            coeffsX[p*g_coarse.n()+q] = dg::evaluate( legendre, g_elemental);
-            //forward transform coefficients
-            coeffsL[p*g_coarse.n()+q] = transform( forward, coeffsX[p*g_coarse.n() + q], g_elemental);
-
-            //multiply by S matrix 
-            coeffsX[p*g_coarse.n()+q] = transform( sisj, coeffsL[p*g_coarse.n() + q], g_elemental);
-            //std::cout << "p "<<p<<" q "<<q<<"\n";
-            //for( unsigned i=0; i<g_fine.n(); i++)
-            //{
-            //    for( unsigned j=0; j<g_fine.n(); j++)
-            //        std::cout << coeffsX[p*g_coarse.n()+q][i*g_fine.n()*num_cellsX+j] <<" ";
-            //    std::cout << "\n";
-            //}
-            //std::cout <<std::endl;
-            //multiply by forward transpose
-            coeffsL[p*g_coarse.n()+q] = transform( forward.transpose(), coeffsX[p*g_coarse.n() + q], g_elemental);
-            
-        }
-    Grid2d gc_elemental( -1., 1., -1., 1., g_coarse.n(), 1, 1);
-    Operator<double> backward( gc_elemental.dlt().backward());
-    Operator<double> sisj_inv = dg::create::pipj_inv( gc_elemental.n());
-    Operator<double> left = backward*sisj_inv;    
-    //multiply left over all coarse polynomials
-    for( unsigned k=0; k<g_coarse.n(); k++)
-    for( unsigned q=0; q<g_coarse.n(); q++)
-    {
-        for( unsigned i=0; i<g_elemental.size(); i++)
-        {
-                coeffsX[k*g_coarse.n()+q][i] = 0.;
-                for( unsigned m=0; m<g_coarse.n(); m++)
-                for( unsigned l=0; l<g_coarse.n(); l++)
-                {
-                    coeffsX[k*g_coarse.n()+q][i] += left(k,m)*left(q,l)*coeffsL[m*g_coarse.n()+l][i];
-                }
-                coeffsX[k*g_coarse.n()+q][i] *= g_fine.hx()*g_fine.hy()/g_coarse.hx()/g_coarse.hy();
-        }
-    }
-    cusp::coo_matrix<int, double, cusp::host_memory> A( g_coarse.size(), g_fine.size(), g_coarse.size()*num_cellsX*num_cellsY*g_fine.n()*g_fine.n());
-    int number = 0;
-    for( unsigned i=0; i<g_coarse.Ny(); i++)
-    for( unsigned k=0; k<g_coarse.n(); k++)
-    for( unsigned j=0; j<g_coarse.Nx(); j++)
-    for( unsigned q=0; q<g_coarse.n(); q++)
-    {
-        unsigned line = ((i*g_coarse.n()+k)*g_coarse.Nx()+j)*g_coarse.n()+q;
-        //add correct line to A
-        for( unsigned m=0; m<num_cellsY; m++)
-        for( unsigned n=0; n<num_cellsX; n++)
-        {
-            //column for indices (i,j,m,n) the (0,0) element
-            unsigned col_begin = (((i*num_cellsY+m)*g_fine.n()*g_coarse.Nx()+j)*num_cellsX+n)*g_fine.n();
-            std::vector<double> temp(g_fine.n()*g_fine.n());
-            for( unsigned e=0; e<g_fine.n(); e++)
-                for( unsigned h=0; h<g_fine.n(); h++)
-                    temp[e*g_fine.n()+h] = coeffsX[k*g_coarse.n()+q]
-                        [((m*g_fine.n() + e)*num_cellsX + n)*g_fine.n()+h];
-            detail::add_line( A, number, line,  col_begin, g_fine.n(), g_fine.Nx(), temp); 
-        }
-    }
-
-    A.sort_by_row_and_column();
-    return A;
-}
-
-cusp::coo_matrix<int, double, cusp::host_memory> projection( const Grid3d& g_coarse, const Grid3d& g_fine)
-{
-
-    assert( g_coarse.z0() >= g_fine.z0());
-    assert( g_coarse.z1() <= g_fine.z1());
-    assert( g_coarse.Nz() == g_fine.Nz());
-    const unsigned Nz = g_coarse.Nz();
-
-    Grid2d g2d_coarse( g_coarse.x0(), g_coarse.x1(), g_coarse.y0(), g_coarse.y1(), g_coarse.n(), g_coarse.Nx(), g_coarse.Ny());
-    Grid2d g2d_fine( g_fine.x0(), g_fine.x1(), g_fine.y0(), g_fine.y1(), g_fine.n(), g_fine.Nx(), g_fine.Ny());
-    cusp::coo_matrix<int, double, cusp::host_memory> A2d = projection( g2d_coarse, g2d_fine);
-
-    cusp::coo_matrix<int, double, cusp::host_memory> A( A2d.num_rows*Nz, A2d.num_cols*Nz, A2d.num_entries*Nz);
-    for( unsigned i=0; i<Nz; i++)
-        for( unsigned j=0; j<A2d.num_entries; j++)
-        {
-            A.column_indices[i*A2d.num_entries+j] = i*A2d.num_cols + A2d.column_indices[j];
-            A.row_indices[i*A2d.num_entries+j] = i*A2d.num_rows + A2d.row_indices[j];
-            A.values[i*A2d.num_entries+j] = A2d.values[j];
-        }
-    return A;
+    for( unsigned i=0; i<g.Ny(); i++)
+    for( unsigned k=0; k<g.n(); k++)
+    for( unsigned j=0; j<g.Nx(); j++)
+    for( unsigned l=0; l<g.n(); l++)
+    for( unsigned m=0; m<g.n(); m++)
+    for( unsigned o=0; o<g.n(); o++)
+        out[((i*g.n() + k)*g.Nx() + j)*g.n() + l] += forward(k,o)*forward( l, m)*in[((i*g.n() + o)*g.Nx() + j)*g.n() + m];
+    return out;
 }
-///@}
 
 }//namespace create
 
diff --git a/inc/dg/backend/projection.cuh b/inc/dg/backend/projection.cuh
index 50cd69901..95ca65f41 100644
--- a/inc/dg/backend/projection.cuh
+++ b/inc/dg/backend/projection.cuh
@@ -108,7 +108,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const Grid2d& g
  @attention Projection only works if the number of cells in the
  fine grid are multiple of the number of cells in the coarse grid
  */
-cusp::coo_matrix< int, double, cusp::host_memory> projection_( const Grid2d& g_new, const Grid2d& g_old)
+cusp::coo_matrix< int, double, cusp::host_memory> projection( const Grid2d& g_new, const Grid2d& g_old)
 {
     if( g_old.Nx() % g_new.Nx() != 0) std::cerr << "ATTENTION: you project between incompatible grids in x!!\n";
     if( g_old.Ny() % g_new.Ny() != 0) std::cerr << "ATTENTION: you project between incompatible grids in y!!\n";
@@ -167,7 +167,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const Grid3d& g
  @attention Projection only works if the number of cells in the
  fine grid are multiple of the number of cells in the coarse grid
  */
-cusp::coo_matrix< int, double, cusp::host_memory> projection_( const Grid3d& g_new, const Grid3d& g_old)
+cusp::coo_matrix< int, double, cusp::host_memory> projection( const Grid3d& g_new, const Grid3d& g_old)
 {
     if( g_old.Nx() % g_new.Nx() != 0) std::cerr << "ATTENTION: you project between incompatible grids in x!!\n";
     if( g_old.Ny() % g_new.Ny() != 0) std::cerr << "ATTENTION: you project between incompatible grids in y!!\n";
diff --git a/inc/dg/backend/projection_t.cu b/inc/dg/backend/projection_t.cu
index 878fa6fde..7c7e6b811 100644
--- a/inc/dg/backend/projection_t.cu
+++ b/inc/dg/backend/projection_t.cu
@@ -7,20 +7,19 @@
 
 double sine( double x){ return sin(x);}
 double sine( double x, double y){return sin(x)*sin(y);}
-typedef cusp::coo_matrix<int, double, cusp::host_memory> Matrix;
 
 int main()
 {
     std::cout << "TEST 1D\n";
     unsigned n_old = 4, n_new = 3, N_old = 10, N_new = 1;
-    std::cout << "Type n and N of old (fine) grid!\n";
+    std::cout << "Type n and N of old (find) grid!\n";
     std::cin >> n_old >> N_old;
     std::cout << "Type n and N of new (coarser) grid!\n";
     std::cin >> n_new >> N_new;
     dg::Grid1d go ( 0, M_PI, n_old, N_old);
     dg::Grid1d gn ( 0, M_PI, n_new, N_new);
-    Matrix proj = dg::create::projection( gn, go);
-    Matrix inte = dg::create::interpolation( gn, go);
+    cusp::coo_matrix<int, double, cusp::host_memory> proj = dg::create::projection( gn, go);
+    cusp::coo_matrix<int, double, cusp::host_memory> inte = dg::create::interpolation( gn, go);
     thrust::host_vector<double> v = dg::evaluate( sine, go);
     thrust::host_vector<double> w1do = dg::create::weights( go);
     thrust::host_vector<double> w1dn = dg::create::weights( gn);
@@ -46,8 +45,8 @@ int main()
     //old grid is larger than the new grid
     dg::Grid2d g2o (0, M_PI, 0, M_PI, n_old, N_old, N_old);
     dg::Grid2d g2n (0, M_PI, 0, M_PI, n_new, N_new, N_new);
-    Matrix proj2d = dg::create::projection_( g2n, g2o);
-    Matrix inte2d = dg::create::interpolation( g2n, g2o);
+    cusp::coo_matrix<int, double, cusp::host_memory> proj2d = dg::create::projection( g2n, g2o);
+    cusp::coo_matrix<int, double, cusp::host_memory> inte2d = dg::create::interpolation( g2n, g2o);
     const dg::HVec sinO = dg::evaluate( sine, g2o), 
                    sinN = dg::evaluate( sine, g2n);
     dg::HVec w2do = dg::create::weights( g2o);
@@ -62,7 +61,7 @@ int main()
     std::cout << "Rel Difference in L2 norm is " << sqrt(dg::blas2::dot( temp, w2dn, temp)/dg::blas2::dot( sinN, w2dn, sinN))<< std::endl;
     //std::cout << "Difference between two grid evaluations:\n";
     //std::cout << diff( sinO, sinN)<<" (should converge to zero!) \n";
-    std::cout << "Difference between projection and evaluation      ";
+    std::cout << "Difference between projection and evaluation      \n";
     dg::blas1::axpby( 1., sinN, -1., sinP);
     std::cout << dg::blas2::dot( sinP, w2dn, sinP)<<" (smaller than above)\n";
     dg::blas2::gemv( inte2d, sinO, sinP);
@@ -74,33 +73,5 @@ int main()
     << "If it works it is better than interpolation in that it conserves the integral value and has smaller L2 errors\n";
     std::cout << "Interpolation always works!\n";
 
-    std::cout << "TEST hand made projection\n";
-    const thrust::host_vector<double> xfine = dg::evaluate( sine, g2o);
-    thrust::host_vector<double> xcoarseI = dg::evaluate( sine, g2n);
-    const thrust::host_vector<double> xcoarse = dg::evaluate( sine, g2n);
-    const thrust::host_vector<double> wfine = dg::create::weights( g2o);
-    const thrust::host_vector<double> wcoarse = dg::create::weights( g2n);
-    double coarseL2 = sqrt(dg::blas2::dot( xcoarse, wcoarse, xcoarse));
-    double fineL2 =   sqrt(dg::blas2::dot( xfine, wfine, xfine));
-    std::cout << "coarse L2 norm:       "<<coarseL2<<"\n";
-    std::cout << "Fine L2 norm:         "<<fineL2<<" \n";
-
-    Matrix f2c = dg::create::projection( g2n, g2o); 
-    dg::blas2::symv( f2c, xfine, xcoarseI);
-    coarseL2 = sqrt(dg::blas2::dot( xcoarseI, wcoarse, xcoarseI));
-    std::cout << "projected    L2 norm: "<<coarseL2<<"\n";
-    std::cout << "Difference in L2      "<<fabs(fineL2-coarseL2)<<"\n";
-    //integrals
-    double coarseI = dg::blas1::dot( wcoarse, xcoarse);
-    double fineI = dg::blas1::dot( wfine, xfine);
-    std::cout << "coarse integral:      "<<coarseI<<"\n";
-    std::cout << "Fine integral:        "<<fineI<<" \n";
-    coarseI = dg::blas1::dot( wcoarse, xcoarseI);
-    std::cout << "projected    integral "<<coarseI<<"\n";
-    std::cout << "Difference Integral   "<<fabs(fineI-coarseI)/fabs(fineI)<<"\n";
-    dg::blas1::axpby( 1., xcoarseI, -1., xcoarse, xcoarseI);
-    double norm = dg::blas2::dot( xcoarseI, wcoarse, xcoarseI);
-    std::cout << "Difference evaluated to interpolated: "<<norm/coarseL2<<"\n";
-
     return 0;
 }
-- 
GitLab


From 5fd6bbe48e29a43da44a67196562e38e116158e3 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 6 Jul 2017 21:02:58 +0200
Subject: [PATCH 019/453] made transformation functions

---
 inc/dg/backend/projection.cuh  | 127 +++++++++++++++++++++++++++++++++
 inc/dg/backend/projection_t.cu |  28 ++------
 2 files changed, 133 insertions(+), 22 deletions(-)

diff --git a/inc/dg/backend/projection.cuh b/inc/dg/backend/projection.cuh
index 95ca65f41..aa09543aa 100644
--- a/inc/dg/backend/projection.cuh
+++ b/inc/dg/backend/projection.cuh
@@ -13,6 +13,40 @@ namespace dg{
 ///@addtogroup interpolation
 ///@{
 
+/**
+ * @brief Greatest common divisor
+ *
+ * @param a First number
+ * @param b Second number
+ *
+ * @return greatest common divisor
+ */
+unsigned gcd( unsigned a, unsigned b)
+{
+    unsigned r2 = std::max(a,b);
+    unsigned r1 = std::min(a,b);
+    while( r1!=0)
+    {
+        r2 = r2%r1;
+        std::swap( r1, r2);
+    }
+    return r2;
+}
+
+/**
+ * @brief Least common multiple
+ *
+ * @param a Fist number
+ * @param b Second number 
+ *
+ * @return Least common multiple
+ */
+unsigned lcm( unsigned a, unsigned b)
+{
+    unsigned g = gcd( a,b);
+    return a/g*b;
+}
+
 namespace create{
 
 /**
@@ -193,6 +227,99 @@ cusp::coo_matrix< int, double, cusp::host_memory> projection( const Grid3d& g_ne
     return A;
 }
 
+/**
+ * @brief Create a transformation matrix between two grids
+ *
+ * The transformation matrix is probably the most correct way of 
+ transforming dG vectors between two grids of differen resolution. 
+ It first finds the least common multiple grid (lcm) of the old and the new grid. Then
+ it interpolates the values to the lcm grid and finally projects them back to
+ the new grid. In total we have
+ \f[
+ \mathcal T = P Q 
+ \f] 
+ where \f$ Q\f$ is the interpolation matrix and \f$ P \f$ the projection. If either new or
+ old grid is already the lcm grid this function reduces to the interpolation/projection function. 
+ * 
+ * @param g_new The new grid 
+ * @param g_old The old grid
+ *
+ * @return transformation matrix
+ * @note The boundaries of the old grid must lie within the boundaries of the new grid
+ * @note If the grid are very incompatible the matrix-matrix multiplication can take a while
+ */
+cusp::coo_matrix< int, double, cusp::host_memory> transformation( const Grid3d& g_new, const Grid3d& g_old)
+{
+    Grid3d g_lcm(g_new.x0(), g_new.x1(), g_new.y0(), g_new.y1(), g_new.z0(), g_new.z1(), 
+                 lcm(g_new.n(), g_old.n()), lcm(g_new.Nx(), g_old.Nx()), lcm(g_new.Ny(), g_old.Ny()), 
+                 lcm(g_new.Nz(), g_old.Nz()));
+    cusp::coo_matrix< int, double, cusp::host_memory> Q = create::interpolation( g_lcm, g_old);
+    cusp::coo_matrix< int, double, cusp::host_memory> P = create::projection( g_new, g_lcm), Y;
+    cusp::multiply( P, Q, Y);
+    Y.sort_by_row_and_column();
+    return Y;
+}
+/**
+ * @brief Create a transformation matrix between two grids
+ *
+ * The transformation matrix is probably the most correct way of 
+ transforming dG vectors between two grids of differen resolution. 
+ It first finds the least common multiple grid (lcm) of the old and the new grid. Then
+ it interpolates the values to the lcm grid and finally projects them back to
+ the new grid. In total we have
+ \f[
+ \mathcal T = P Q 
+ \f] 
+ where \f$ Q\f$ is the interpolation matrix and \f$ P \f$ the projection. If either new or
+ old grid is already the lcm grid this function reduces to the interpolation/projection function. 
+ * 
+ * @param g_new The new grid 
+ * @param g_old The old grid
+ *
+ * @return transformation matrix
+ * @note The boundaries of the old grid must lie within the boundaries of the new grid
+ * @note If the grid are very incompatible the matrix-matrix multiplication can take a while
+ */
+cusp::coo_matrix< int, double, cusp::host_memory> transformation( const Grid2d& g_new, const Grid2d& g_old)
+{
+    Grid2d g_lcm(g_new.x0(), g_new.x1(), g_new.y0(), g_new.y1(), 
+                 lcm(g_new.n(), g_old.n()), lcm(g_new.Nx(), g_old.Nx()), lcm(g_new.Ny(), g_old.Ny()));
+    cusp::coo_matrix< int, double, cusp::host_memory> Q = create::interpolation( g_lcm, g_old);
+    cusp::coo_matrix< int, double, cusp::host_memory> P = create::projection( g_new, g_lcm), Y;
+    cusp::multiply( P, Q, Y);
+    Y.sort_by_row_and_column();
+    return Y;
+}
+/**
+ * @brief Create a transformation matrix between two grids
+ *
+ * The transformation matrix is probably the most correct way of 
+ transforming dG vectors between two grids of differen resolution. 
+ It first finds the least common multiple grid (lcm) of the old and the new grid. Then
+ it interpolates the values to the lcm grid and finally projects them back to
+ the new grid. In total we have
+ \f[
+ \mathcal T = P Q 
+ \f] 
+ where \f$ Q\f$ is the interpolation matrix and \f$ P \f$ the projection. If either new or
+ old grid is already the lcm grid this function reduces to the interpolation/projection function. 
+ * 
+ * @param g_new The new grid 
+ * @param g_old The old grid
+ *
+ * @return transformation matrix
+ * @note The boundaries of the old grid must lie within the boundaries of the new grid
+ * @note If the grid are very incompatible the matrix-matrix multiplication can take a while
+ */
+cusp::coo_matrix< int, double, cusp::host_memory> transformation( const Grid1d& g_new, const Grid1d& g_old)
+{
+    Grid1d g_lcm(g_new.x0(), g_new.x1(), lcm(g_new.n(), g_old.n()), lcm(g_new.N(), g_old.N()));
+    cusp::coo_matrix< int, double, cusp::host_memory> Q = create::interpolation( g_lcm, g_old);
+    cusp::coo_matrix< int, double, cusp::host_memory> P = create::projection( g_new, g_lcm), Y;
+    cusp::multiply( P, Q, Y);
+    Y.sort_by_row_and_column();
+    return Y;
+}
 ///@}
 
 }//namespace create
diff --git a/inc/dg/backend/projection_t.cu b/inc/dg/backend/projection_t.cu
index 7c7e6b811..c1120917c 100644
--- a/inc/dg/backend/projection_t.cu
+++ b/inc/dg/backend/projection_t.cu
@@ -12,13 +12,13 @@ int main()
 {
     std::cout << "TEST 1D\n";
     unsigned n_old = 4, n_new = 3, N_old = 10, N_new = 1;
-    std::cout << "Type n and N of old (find) grid!\n";
+    std::cout << "Type n and N of old (fine) grid!\n";
     std::cin >> n_old >> N_old;
     std::cout << "Type n and N of new (coarser) grid!\n";
     std::cin >> n_new >> N_new;
     dg::Grid1d go ( 0, M_PI, n_old, N_old);
     dg::Grid1d gn ( 0, M_PI, n_new, N_new);
-    cusp::coo_matrix<int, double, cusp::host_memory> proj = dg::create::projection( gn, go);
+    cusp::coo_matrix<int, double, cusp::host_memory> proj = dg::create::transformation( gn, go);
     cusp::coo_matrix<int, double, cusp::host_memory> inte = dg::create::interpolation( gn, go);
     thrust::host_vector<double> v = dg::evaluate( sine, go);
     thrust::host_vector<double> w1do = dg::create::weights( go);
@@ -36,16 +36,10 @@ int main()
     std::cout << "Difference       "<<dg::blas2::dot( oneo, w1do, v) - dg::blas2::dot( onen, w1dn, w) << "\n"<<std::endl;
 
     std::cout << "TEST 2D\n";
-    //n_old = 7, n_new = 3, N_old = 4, N_new = 3;
-    //std::cout << "Type n and N of old (find) grid!\n";
-    //std::cin >> n_old >> N_old;
-    //std::cout << "Type n and N of new (coarser) grid!\n";
-    //std::cin >> n_new >> N_new;
     
-    //old grid is larger than the new grid
     dg::Grid2d g2o (0, M_PI, 0, M_PI, n_old, N_old, N_old);
     dg::Grid2d g2n (0, M_PI, 0, M_PI, n_new, N_new, N_new);
-    cusp::coo_matrix<int, double, cusp::host_memory> proj2d = dg::create::projection( g2n, g2o);
+    cusp::coo_matrix<int, double, cusp::host_memory> proj2d = dg::create::transformation( g2n, g2o);
     cusp::coo_matrix<int, double, cusp::host_memory> inte2d = dg::create::interpolation( g2n, g2o);
     const dg::HVec sinO = dg::evaluate( sine, g2o), 
                    sinN = dg::evaluate( sine, g2n);
@@ -55,23 +49,13 @@ int main()
     dg::blas2::gemv( proj2d, sinO, sinP);
     std::cout << "Original vector     "<<sqrt(dg::blas2::dot( sinO, w2do, sinO)) << "\n";
     std::cout << "Projected vector    "<<sqrt(dg::blas2::dot( sinP, w2dn, sinP)) << "\n";
-    std::cout << "Difference in Norms "<<sqrt(dg::blas2::dot( sinO, w2do, sinO)) - sqrt(dg::blas2::dot( sinP, w2dn, sinP)) << "\n" << std::endl;
-    dg::HVec temp(sinP);
-    dg::blas1::axpby(1.,sinN,-1.,sinP,temp);
-    std::cout << "Rel Difference in L2 norm is " << sqrt(dg::blas2::dot( temp, w2dn, temp)/dg::blas2::dot( sinN, w2dn, sinN))<< std::endl;
-    //std::cout << "Difference between two grid evaluations:\n";
-    //std::cout << diff( sinO, sinN)<<" (should converge to zero!) \n";
-    std::cout << "Difference between projection and evaluation      \n";
+    std::cout << "Difference in Norms "<<sqrt(dg::blas2::dot( sinO, w2do, sinO)) - sqrt(dg::blas2::dot( sinP, w2dn, sinP)) << std::endl;
+    std::cout << "Difference between projection and evaluation      ";
     dg::blas1::axpby( 1., sinN, -1., sinP);
-    std::cout << dg::blas2::dot( sinP, w2dn, sinP)<<" (smaller than above)\n";
+    std::cout << sqrt(dg::blas2::dot( sinP, w2dn, sinP)/dg::blas2::dot(sinN, w2dn, sinN))<<"\n";
     dg::blas2::gemv( inte2d, sinO, sinP);
-    std::cout << "Original vector     "<<sqrt(dg::blas2::dot( sinO, w2do, sinO)) << "\n";
     std::cout << "Interpolated vec    "<<sqrt(dg::blas2::dot( sinP, w2dn, sinP)) << "\n";
     std::cout << "Difference in Norms "<<sqrt(dg::blas2::dot( sinO, w2do, sinO)) - sqrt(dg::blas2::dot( sinP, w2dn, sinP)) << "\n" << std::endl;
 
-    std::cout << "What you should observe is that the projection does only work if the coarse grid is a division of the fine grid!\n"
-    << "If it works it is better than interpolation in that it conserves the integral value and has smaller L2 errors\n";
-    std::cout << "Interpolation always works!\n";
-
     return 0;
 }
-- 
GitLab


From 5d4f8464a9bd83d5ba629e04d4da8667a21ef837 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 6 Jul 2017 21:16:16 +0200
Subject: [PATCH 020/453] update documentation of interpolation and projection

---
 inc/dg/backend/interpolation.cuh |  23 +-----
 inc/dg/backend/projection.cuh    | 127 +++++++------------------------
 2 files changed, 29 insertions(+), 121 deletions(-)

diff --git a/inc/dg/backend/interpolation.cuh b/inc/dg/backend/interpolation.cuh
index b4b4174bb..b759a9065 100644
--- a/inc/dg/backend/interpolation.cuh
+++ b/inc/dg/backend/interpolation.cuh
@@ -412,6 +412,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const thrust::ho
  *
  * @return Interpolation matrix
  * @note The boundaries of the old grid must lie within the boundaries of the new grid
+ * @note also check the transformation matrix, which is the more general solution
  */
 cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const Grid1d& g_new, const Grid1d& g_old)
 {
@@ -423,16 +424,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const Grid1d& g_
 
 }
 /**
- * @brief Create interpolation between two grids
- *
- * This matrix can be applied to vectors defined on the old grid to obtain
- * its values on the new grid.
- * 
- * @param g_new The new points 
- * @param g_old The old grid
- *
- * @return Interpolation matrix
- * @note The boundaries of the old grid must lie within the boundaries of the new grid
+ * @copydoc interpolation(const Grid1d&,const Grid1d&)
  */
 cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const Grid2d& g_new, const Grid2d& g_old)
 {
@@ -449,16 +441,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const Grid2d& g_
 }
 
 /**
- * @brief Create interpolation between two grids
- *
- * This matrix can be applied to vectors defined on the old grid to obtain
- * its values on the new grid.
- * 
- * @param g_new The new points 
- * @param g_old The old grid
- *
- * @return Interpolation matrix
- * @note The boundaries of the old grid must lie within the boundaries of the new grid
+ * @copydoc interpolation(const Grid1d&,const Grid1d&)
  */
 cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const Grid3d& g_new, const Grid3d& g_old)
 {
diff --git a/inc/dg/backend/projection.cuh b/inc/dg/backend/projection.cuh
index aa09543aa..d80ac1442 100644
--- a/inc/dg/backend/projection.cuh
+++ b/inc/dg/backend/projection.cuh
@@ -20,6 +20,7 @@ namespace dg{
  * @param b Second number
  *
  * @return greatest common divisor
+ * @ingroup misc
  */
 unsigned gcd( unsigned a, unsigned b)
 {
@@ -40,6 +41,7 @@ unsigned gcd( unsigned a, unsigned b)
  * @param b Second number 
  *
  * @return Least common multiple
+ * @ingroup misc
  */
 unsigned lcm( unsigned a, unsigned b)
 {
@@ -64,6 +66,24 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const Grid1d& g
     cusp::transpose( temp, A);
     return A;
 }
+/**
+ * @copydoc interpolationT
+ */
+cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const Grid2d& g_new, const Grid2d& g_old)
+{
+    cusp::coo_matrix<int, double, cusp::host_memory> temp = interpolation( g_old, g_new), A;
+    cusp::transpose( temp, A);
+    return A;
+}
+/**
+ * @copydoc interpolationT
+ */
+cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const Grid3d& g_new, const Grid3d& g_old)
+{
+    cusp::coo_matrix<int, double, cusp::host_memory> temp = interpolation( g_old, g_new), A;
+    cusp::transpose( temp, A);
+    return A;
+}
 
 /**
  * @brief Create a projection between two grids
@@ -80,6 +100,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const Grid1d& g
  *
  * @return Projection matrix
  * @note The boundaries of the old grid must lie within the boundaries of the new grid
+ * @note also check the transformation matrix, which is the more general solution
  @attention Projection only works if the number of cells in the
  fine grid are multiple of the number of cells in the coarse grid
  */
@@ -108,39 +129,9 @@ cusp::coo_matrix< int, double, cusp::host_memory> projection( const Grid1d& g_ne
     return A;
 }
 
-/**
- * @brief Create the transpose of the interpolation matrix from new to old
- *
- * @param g_new The new grid 
- * @param g_old The old grid
- *
- * @return transposed interpolation matrix
- * @note The boundaries of the old grid must lie within the boundaries of the new grid
- */
-cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const Grid2d& g_new, const Grid2d& g_old)
-{
-    cusp::coo_matrix<int, double, cusp::host_memory> temp = interpolation( g_old, g_new), A;
-    cusp::transpose( temp, A);
-    return A;
-}
 
 /**
- * @brief Create a projection between two grids
- *
- * The projection matrix is the adjoint of the interpolation matrix
- * This matrix can be applied to vectors defined on the old (fine) grid to obtain
- * its values on the new (coarse) grid. 
- * If the fine grid is a multiple of the coarse grid, the integral value
- of the projected vector will be conserved and the difference in the L2 norm 
- between old and new vector small. 
- * 
- * @param g_new The new (coarse) grid 
- * @param g_old The old (fine) grid
- *
- * @return transposed interpolation matrix
- * @note The boundaries of the old grid must lie within the boundaries of the new grid
- @attention Projection only works if the number of cells in the
- fine grid are multiple of the number of cells in the coarse grid
+ * @copydoc projection
  */
 cusp::coo_matrix< int, double, cusp::host_memory> projection( const Grid2d& g_new, const Grid2d& g_old)
 {
@@ -167,39 +158,9 @@ cusp::coo_matrix< int, double, cusp::host_memory> projection( const Grid2d& g_ne
     A.sort_by_row_and_column();
     return A;
 }
-/**
- * @brief Create the transpose of the interpolation matrix from new to old
- *
- * @param g_new The new grid 
- * @param g_old The old grid
- *
- * @return transposed interpolation matrix
- * @note The boundaries of the old grid must lie within the boundaries of the new grid
- */
-cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const Grid3d& g_new, const Grid3d& g_old)
-{
-    cusp::coo_matrix<int, double, cusp::host_memory> temp = interpolation( g_old, g_new), A;
-    cusp::transpose( temp, A);
-    return A;
-}
 
 /**
- * @brief Create a projection between two grids
- *
- * The projection matrix is the adjoint of the interpolation matrix
- * This matrix can be applied to vectors defined on the old (fine) grid to obtain
- * its values on the new (coarse) grid. 
- * If the fine grid is a multiple of the coarse grid, the integral value
- of the projected vector will be conserved and the difference in the L2 norm 
- between old and new vector small. 
- * 
- * @param g_new The new (coarse) grid 
- * @param g_old The old (fine) grid
- *
- * @return transposed interpolation matrix
- * @note The boundaries of the old grid must lie within the boundaries of the new grid
- @attention Projection only works if the number of cells in the
- fine grid are multiple of the number of cells in the coarse grid
+ * @copydoc projection
  */
 cusp::coo_matrix< int, double, cusp::host_memory> projection( const Grid3d& g_new, const Grid3d& g_old)
 {
@@ -231,7 +192,7 @@ cusp::coo_matrix< int, double, cusp::host_memory> projection( const Grid3d& g_ne
  * @brief Create a transformation matrix between two grids
  *
  * The transformation matrix is probably the most correct way of 
- transforming dG vectors between two grids of differen resolution. 
+ transforming dG vectors between any two grids of different resolution. 
  It first finds the least common multiple grid (lcm) of the old and the new grid. Then
  it interpolates the values to the lcm grid and finally projects them back to
  the new grid. In total we have
@@ -260,25 +221,7 @@ cusp::coo_matrix< int, double, cusp::host_memory> transformation( const Grid3d&
     return Y;
 }
 /**
- * @brief Create a transformation matrix between two grids
- *
- * The transformation matrix is probably the most correct way of 
- transforming dG vectors between two grids of differen resolution. 
- It first finds the least common multiple grid (lcm) of the old and the new grid. Then
- it interpolates the values to the lcm grid and finally projects them back to
- the new grid. In total we have
- \f[
- \mathcal T = P Q 
- \f] 
- where \f$ Q\f$ is the interpolation matrix and \f$ P \f$ the projection. If either new or
- old grid is already the lcm grid this function reduces to the interpolation/projection function. 
- * 
- * @param g_new The new grid 
- * @param g_old The old grid
- *
- * @return transformation matrix
- * @note The boundaries of the old grid must lie within the boundaries of the new grid
- * @note If the grid are very incompatible the matrix-matrix multiplication can take a while
+ * @copydoc transformation
  */
 cusp::coo_matrix< int, double, cusp::host_memory> transformation( const Grid2d& g_new, const Grid2d& g_old)
 {
@@ -291,25 +234,7 @@ cusp::coo_matrix< int, double, cusp::host_memory> transformation( const Grid2d&
     return Y;
 }
 /**
- * @brief Create a transformation matrix between two grids
- *
- * The transformation matrix is probably the most correct way of 
- transforming dG vectors between two grids of differen resolution. 
- It first finds the least common multiple grid (lcm) of the old and the new grid. Then
- it interpolates the values to the lcm grid and finally projects them back to
- the new grid. In total we have
- \f[
- \mathcal T = P Q 
- \f] 
- where \f$ Q\f$ is the interpolation matrix and \f$ P \f$ the projection. If either new or
- old grid is already the lcm grid this function reduces to the interpolation/projection function. 
- * 
- * @param g_new The new grid 
- * @param g_old The old grid
- *
- * @return transformation matrix
- * @note The boundaries of the old grid must lie within the boundaries of the new grid
- * @note If the grid are very incompatible the matrix-matrix multiplication can take a while
+ * @copydoc transformation
  */
 cusp::coo_matrix< int, double, cusp::host_memory> transformation( const Grid1d& g_new, const Grid1d& g_old)
 {
-- 
GitLab


From 5fccd405ee7ad1812a9ed26e00dfffa0b545c46d Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Mon, 10 Jul 2017 21:37:29 +0200
Subject: [PATCH 021/453] update dg doc with gather and scatter matrix

---
 inc/dg/dg_doc.h | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index e780d13b0..2141a622d 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -134,11 +134,11 @@
  perform a reduction operation (we sum up all elements). If we throw away all
  unused elements in w the, then w is always equal or smaller in size than v. 
 
-Think of the index map as establishing fixed connections between two vectors. 
-When you apply scatter and gather operations you send data back and forth
-between these two vectors along these connections.
- However, only if the index map is bijective, the scatter operation is actually the inverse of 
- the gather operation. 
+Since it is a vector operation the gather and scatter operation can 
+also be represented by a matrix. The gather matrix is just a 
+(permutation) matrix of 1's and 0's with exactly one "1" in each line.
+The corresponding scatter-and-reduce matrix is just the transpose of the gather matrix. The scatter matrix can have zero, one or more "1"s in each line.
+
 
  Now, consider that both vectors v and w are distributed across processes.
  That means when you send data with MPI you will probably need a communication buffers.
@@ -187,10 +187,10 @@ struct aCommunicator
      * @brief Scatters data accross processes and reduces on double indices
      *
      * The order of the received elements is according to their original array index (i.e. a[0] appears before a[1]) and their process rank of origin ( i.e. values from rank 0 appear before values from rank 1)
-        2. globally scatter the values in this buffer to a recv buffer (b) (every value in the result belongs to exactly one line/process)  
-        3. then permute and reduce the recv buffer on double indices and store result in output vector (c) 
+        1. globally scatter the values in this buffer to a recv buffer (b) (every value in the result belongs to exactly one line/process)  
+        2. then permute and reduce the recv buffer on double indices and store result in output vector (c) 
      * @tparam LocalContainer a container on a shared memory system
-     * @param gatherFrom other processes collect data from this vector (has to be of size given by recv_size())
+     * @param toScatter other processes collect data from this vector (has to be of size given by recv_size())
      * @param values contains values from other processes sent back to the origin (or send_size())
      */
     template< class LocalContainer>
@@ -207,7 +207,6 @@ struct aCommunicator
     /**
      * @brief return # of elements the calling process has to send in a scatter process (or receive in the gather process)
      *
-     * equals the size of the map given in the constructor
      * @return # of elements to send
      */
     unsigned send_size() const; 
-- 
GitLab


From 646d9ee391c53f659cd673049936bf68dd4b45ce Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Tue, 11 Jul 2017 18:37:00 +0200
Subject: [PATCH 022/453] added a virtual multiply/divide function to the grid
 class

---
 inc/dg/backend/grid.h     | 51 +++++++++++++++++++++++++++++++++++++++
 inc/dg/backend/mpi_grid.h | 50 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 101 insertions(+)

diff --git a/inc/dg/backend/grid.h b/inc/dg/backend/grid.h
index 298008801..0a3881360 100644
--- a/inc/dg/backend/grid.h
+++ b/inc/dg/backend/grid.h
@@ -289,6 +289,31 @@ struct Grid2d
      * @return 
      */
     Grid2d local_grid() const {return *this;}
+
+    /**
+    * @brief Return a copy of the grid with increased number of cells
+    *
+    * @param nx multiply # of cells in x 
+    * @param ny multiply # of cells in y
+    *
+    * @return a copy of this grid with nx*Nx and ny*Ny cells in x and y
+    */
+    virtual Grid2d multiply( unsigned nx, unsigned ny) const {
+        return Grid2d( x0_, x1_, y0_, y1_, n_, nx*Nx_, ny*Ny_, bcx_, bcy_);
+    }
+    /**
+    * @brief Return a copy of the grid with reduced number of cells
+    *
+    * @param nx divide # of cells in x 
+    * @param ny divide # of cells in y
+    *
+    * @return a copy of this grid with Nx/nx and Ny/ny cells in x and y
+    * @attention The function won't check if the number of cells are divisible without rest
+    */
+    virtual Grid2d divide( unsigned nx, unsigned ny) const {
+        return Grid2d( x0_, x1_, y0_, y1_, n_, Nx_/nx, Ny_/ny, bcx_, bcy_);
+    }
+
     /**
      * @brief discrete legendre trafo
      *
@@ -442,6 +467,32 @@ struct Grid3d
         lx_ = (x1_-x0_), ly_ = (y1_-y0_), lz_ = (z1_-z0_);
         hx_ = lx_/(double)Nx_, hy_ = ly_/(double)Ny_, hz_ = lz_/(double)Nz_;
     }
+    /**
+    * @brief Return a copy of the grid with increased number of cells
+    *
+    * @param nx multiply # of cells in x 
+    * @param ny multiply # of cells in y
+    * @param nz multiply # of cells in z 
+    *
+    * @return a copy of this grid with nx*Nx, ny*Ny and nz*Nz cells in x, y and z
+    */
+    virtual Grid3d multiply( unsigned nx, unsigned ny, unsigned nz) const {
+        return Grid3d( x0_, x1_, y0_, y1_, z0_, z1_, n_, nx*Nx_, ny*Ny_, nz*Nz_, bcx_, bcy_, bcz_);
+    }
+    /**
+    * @brief Return a copy of the grid with reduced number of cells
+    *
+    * @param nx divide # of cells in x 
+    * @param ny divide # of cells in y
+    * @param nz divide # of cells in z 
+    *
+    * @return a copy of this grid with Nx/nx, Ny/ny cells in x, y and z
+    * @attention The function won't check if the number of cells are divisible without rest
+    */
+    virtual Grid3d divide( unsigned nx, unsigned ny, unsigned nz) const {
+        return Grid3d( x0_, x1_, y0_, y1_, z0_, z1_, n_, Nx_/nx, Ny_/ny, Nz_/nz, bcx_, bcy_, bcz_);
+    }
+
     /**
      * @brief left boundary in x
      *
diff --git a/inc/dg/backend/mpi_grid.h b/inc/dg/backend/mpi_grid.h
index 9b442ced4..1baab344d 100644
--- a/inc/dg/backend/mpi_grid.h
+++ b/inc/dg/backend/mpi_grid.h
@@ -95,6 +95,30 @@ struct MPIGrid2d
             else assert( periods[1] == false);
         }
     }
+    /**
+    * @brief Return a copy of the grid with increased number of cells
+    *
+    * @param nx multiply # of cells in x 
+    * @param ny multiply # of cells in y
+    *
+    * @return a copy of this grid with nx*Nx and ny*Ny cells in x and y
+    */
+    virtual MPIGrid2d multiply( unsigned nx, unsigned ny) const {
+        return MPIGrid2d( x0_, x1_, y0_, y1_, n_, nx*Nx_, ny*Ny_, bcx_, bcy_, comm_);
+    }
+    /**
+    * @brief Return a copy of the grid with reduced number of cells
+    *
+    * @param nx divide # of cells in x 
+    * @param ny divide # of cells in y
+    *
+    * @return a copy of this grid with Nx/nx and Ny/ny cells in x and y
+    * @attention The function won't check if the number of cells are divisible without rest
+         but it does check if the number of processes is still a divisor
+    */
+    virtual MPIGrid2d divide( unsigned nx, unsigned ny) const {
+        return MPIGrid2d( x0_, x1_, y0_, y1_, n_, Nx_/nx, Ny_/ny, bcx_, bcy_, comm_);
+    }
 
     /**
      * @brief Return local x0
@@ -387,6 +411,32 @@ struct MPIGrid3d
         }
     }
 
+    /**
+    * @brief Return a copy of the grid with increased number of cells
+    *
+    * @param nx multiply # of cells in x 
+    * @param ny multiply # of cells in y
+    * @param nz multiply # of cells in z 
+    *
+    * @return a copy of this grid with nx*Nx, ny*Ny and nz*Nz cells in x, y and z
+    */
+    virtual MPIGrid3d multiply( unsigned nx, unsigned ny, unsigned nz) const {
+        return MPIGrid3d( x0_, x1_, y0_, y1_, z0_, z1_, n_, nx*Nx_, ny*Ny_, nz*Nz_, bcx_, bcy_, bcz_, comm_);
+    }
+    /**
+    * @brief Return a copy of the grid with reduced number of cells
+    *
+    * @param nx divide # of cells in x 
+    * @param ny divide # of cells in y
+    * @param nz divide # of cells in z 
+    *
+    * @return a copy of this grid with Nx/nx, Ny/ny cells in x, y and z
+    * @attention The function won't check if the number of cells are divisible without rest
+    *   but it does check if the number of processes is still a divisor
+    */
+    virtual MPIGrid3d divide( unsigned nx, unsigned ny, unsigned nz) const {
+        return MPIGrid3d( x0_, x1_, y0_, y1_, z0_, z1_, n_, Nx_/nx, Ny_/ny, Nz_/nz, bcx_, bcy_, bcz_, comm_);
+    }
     /**
      * @brief Return local x0
      *
-- 
GitLab


From 6016572c31063ad5259ea55aedae0db91f1ca4ac Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Tue, 11 Jul 2017 18:41:19 +0200
Subject: [PATCH 023/453] removed old dxmatrix

---
 inc/dg/backend/dx_matrix_blas.cuh | 56 -------------------------------
 inc/dg/backend/dxmatrix_traits.h  | 44 ------------------------
 2 files changed, 100 deletions(-)
 delete mode 100644 inc/dg/backend/dx_matrix_blas.cuh
 delete mode 100644 inc/dg/backend/dxmatrix_traits.h

diff --git a/inc/dg/backend/dx_matrix_blas.cuh b/inc/dg/backend/dx_matrix_blas.cuh
deleted file mode 100644
index 0cf578762..000000000
--- a/inc/dg/backend/dx_matrix_blas.cuh
+++ /dev/null
@@ -1,56 +0,0 @@
-#include "dx_matrix.cuh"
-
-/***********************************************************************************
- *
- * Implementation of matrix vector multiplication of thrust_vectors and dx_matrix
- *
- ***********************************************************************************/
-
-
-namespace dg
-{
-namespace blas2
-{
-namespace detail
-{
-
-template < class Matrix, class Vector1, class Vector2>
-void doSymv(Matrix& mat, Vector1& x, Vector2& y, dx_matrixTag, ThrustVectorTag, ThrustVectorTag)
-{
-    typedef thrust::host_vector<double>::iterator ElementIterator;
-    typedef thrust::host_vector<int>::iterator IndexIterator;
-        // Here comes the implementation of the matrix-vector dot product
-    dx_matrix_row_val rowval(mat, 0);
-    dx_matrix_row_idx rowidx(mat, 0);
-    double ip = 0.0;
-
-    Vector1 tmp(y);
-    for(int i = 0; i < mat.get_N() * mat.get_P(); i++)
-    {
-        rowval.update_row(mat, i);
-        rowidx.update_row(mat, i);
-
-        //thrust::permutation_iterator<ElementIterator, IndexIterator> x_it(x.begin(), rowidx.get_data().begin());
-        //thrust::permutation_iterator<ElementIterator, IndexIterator> x_it_end(x.end(), rowidx.get_data().end());
-        // 
-        //std::cout << "********************doSymv************************" << std::endl;
-        //thrust::host_vector<double>::iterator a_it = rowval.get_data().begin();
-        //for(x_it = thrust::make_permutation_iterator(x.begin(), rowidx.get_data().begin()); x_it != x_it_end; x_it++)
-        //{
-        //    std::cout << *x_it << " *  " << *a_it++ << "  +  ";
-        //}
-
-        // Try out a cool scalar product
-        ip = thrust::inner_product(rowval.get_data().begin(), rowval.get_data().end(),
-                                   thrust::make_permutation_iterator(x.begin(), rowidx.get_data().begin()),
-                                   0.0);
-        //std::cout << " = "  << ip << std::endl;
-        tmp[i] = ip;
-    }
-    y = tmp;
-}
-
-} //namespace detail
-} //namespace blas2
-} //namespace dg
-
diff --git a/inc/dg/backend/dxmatrix_traits.h b/inc/dg/backend/dxmatrix_traits.h
deleted file mode 100644
index 6ddaebcb0..000000000
--- a/inc/dg/backend/dxmatrix_traits.h
+++ /dev/null
@@ -1,44 +0,0 @@
-#ifndef _DG_DXMATRIX_TRAITS
-#define _DG_DXMATRIX_TRAITS
-
-
-#include "thrust/device_vector.h"
-#include "thrust/host_vector.h"
-#include "matrix_traits.h"
-#include "matrix_categories.h"
-
-
-//exclude from doxygen Docu
-///@cond
-
-namespace dg{
-
-    template <typename T>
-    struct MatrixTraits<thrust::host_vector<T> > {
-        typedef T value_type;
-        typedef dx_matrixTag matrix_category;
-    };
-
-    template <typename T>
-    struct MatrixTraits<thrust::device_vector<T> > {
-        typedef T value_type;
-        typedef dx_matrixTag matrix_category;
-    };
-
-    template <typename T>
-    struct MatrixTraits<const thrust::host_vector<T> > {
-        typedef T value_type;
-        typedef dx_matrixTag matrix_category;
-    };
-
-    template <typename T>
-    struct MatrixTraits<const thrust::device_vector<T> > {
-        typedef T value_type;
-        typedef dx_matrixTag matrix_category;
-    };
-
-} //namespace dg
-///@endcond
-
-#endif //_DG_DXMATRIX_TRAITS
-
-- 
GitLab


From ef5ca285cd7ca9b09179ba6a00a3469d2cb2643b Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Tue, 11 Jul 2017 18:45:58 +0200
Subject: [PATCH 024/453] provided a little test of grid multiply/divide in
 interpolation

---
 inc/dg/backend/interpolation_t.cu | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/inc/dg/backend/interpolation_t.cu b/inc/dg/backend/interpolation_t.cu
index 3fe691134..27243997e 100644
--- a/inc/dg/backend/interpolation_t.cu
+++ b/inc/dg/backend/interpolation_t.cu
@@ -21,6 +21,12 @@ int main()
 
     {
     dg::Grid2d g( -10, 10, -5, 5, n, Nx, Ny);
+    std::cout << "First test grid multiply divide functions: \n";
+    g.display( std::cout);
+    g = g.multiply(2,3);
+    g.display( std::cout);
+    g = g.divide(2,3);
+    g.display( std::cout);
     Matrix A = dg::create::backscatter( g);
     //A.sort_by_row_and_column();
 
@@ -88,6 +94,8 @@ int main()
     ////////////////////////////////////////////////////////////////////////////
     {
     dg::Grid3d g( -10, 10, -5, 5, -7, -3, n, Nx, Ny, Nz);
+    g = g.multiply( 2,2,2);
+    g = g.divide( 2,2,2);
     Matrix A = dg::create::backscatter( g);
     //A.sort_by_row_and_column();
 
-- 
GitLab


From 92f475ff1bcf5e1412259ce924348b02bed0d8a2 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Tue, 11 Jul 2017 22:15:11 +0200
Subject: [PATCH 025/453] added slab toroidal field

---
 inc/geometries/README           |  1 +
 inc/geometries/guenther.h       |  4 ++--
 inc/geometries/magnetic_field.h | 16 ++++++++--------
 inc/geometries/solovev.h        |  4 ++--
 inc/geometries/solovev_doc.h    |  8 ++++++++
 inc/geometries/taylor.h         |  3 ++-
 inc/geometries/toroidal.h       | 34 +++++++++++++++++++++++++++++++++
 7 files changed, 57 insertions(+), 13 deletions(-)
 create mode 100644 inc/geometries/toroidal.h

diff --git a/inc/geometries/README b/inc/geometries/README
index 1268614a6..55e1fa4ab 100644
--- a/inc/geometries/README
+++ b/inc/geometries/README
@@ -32,6 +32,7 @@
     taylor.h
     guenther.h
     guenther_parameters.h
+    toroidal.h
 
     testfunctors.h 
     init.h
diff --git a/inc/geometries/guenther.h b/inc/geometries/guenther.h
index 77329fcda..7cf14f824 100644
--- a/inc/geometries/guenther.h
+++ b/inc/geometries/guenther.h
@@ -1,7 +1,6 @@
 #pragma once
 
 #include <iostream>
-#include <fstream>
 #include <cmath>
 #include <vector>
 
@@ -168,7 +167,8 @@ struct IpolZ
  */
 struct MagneticField
 {
-    MagneticField( double R_0, double I_0):psip(R_0), psipR(R_0), psipZ(R_0), psipRR(R_0), psipRZ(R_0), psipZZ(R_0), laplacePsip(R_0), ipol(I_0), ipolR(), ipolZ(){}
+    MagneticField( double R_0, double I_0): R_0(R_0), psip(R_0), psipR(R_0), psipZ(R_0), psipRR(R_0), psipRZ(R_0), psipZZ(R_0), laplacePsip(R_0), ipol(I_0), ipolR(), ipolZ(){}
+    double R_0;
     Psip psip;
     PsipR psipR;
     PsipZ psipZ;
diff --git a/inc/geometries/magnetic_field.h b/inc/geometries/magnetic_field.h
index 2f03cc39b..238a92fd8 100644
--- a/inc/geometries/magnetic_field.h
+++ b/inc/geometries/magnetic_field.h
@@ -19,7 +19,7 @@ namespace geo
 template<class MagneticField>
 struct Bmodule
 {
-    Bmodule( const MagneticField& c, double R0 ):  R_0_(R0), c_(c)  { }
+    Bmodule( const MagneticField& c):  R_0_(c.R_0), c_(c)  { }
     /**
     * @brief \f[   \hat{B} \f]
     */ 
@@ -47,7 +47,7 @@ struct Bmodule
 template<class MagneticField>
 struct InvB
 {
-    InvB(  const MagneticField& c, double R0 ):  R_0_(R0), c_(c)  { }
+    InvB(  const MagneticField& c):  R_0_(c.R_0), c_(c)  { }
     /**
     * @brief \f[   \frac{1}{\hat{B}} = 
         \frac{\hat{R}}{\hat{R}_0}\frac{1}{ \sqrt{ \hat{I}^2  + \left(\frac{\partial \hat{\psi}_p }{ \partial \hat{R}}\right)^2
@@ -77,7 +77,7 @@ struct InvB
 template<class MagneticField>
 struct LnB
 {
-    LnB(  const MagneticField& c, double R0 ):  R_0_(R0), c_(c)  { }
+    LnB(  const MagneticField& c):  R_0_(c.R_0), c_(c)  { }
 /**
  * @brief \f[   \ln{(   \hat{B})} = \ln{\left[
       \frac{\hat{R}_0}{\hat{R}} \sqrt{ \hat{I}^2  + \left(\frac{\partial \hat{\psi}_p }{ \partial \hat{R}}\right)^2
@@ -107,7 +107,7 @@ struct LnB
 template<class MagneticField>
 struct BR
 {
-    BR(const MagneticField& c, double R0):  R_0_(R0), invB_(c, R0), c_(c) { }
+    BR(const MagneticField& c):  R_0_(c.R_0), invB_(c, R0), c_(c) { }
 /**
  * @brief \f[  \frac{\partial \hat{B} }{ \partial \hat{R}} = 
       -\frac{1}{\hat B \hat R}   
@@ -142,7 +142,7 @@ template<class MagneticField>
 struct BZ
 {
 
-    BZ(const MagneticField& c, double R0):  R_0_(R0), c_(c), invB_(c, R0) { }
+    BZ(const MagneticField& c ):  R_0_(c.R_0), c_(c), invB_(c, R0) { }
     /**
      * @brief \f[  \frac{\partial \hat{B} }{ \partial \hat{Z}} = 
      \frac{ \hat I \left(\frac{\partial \hat I}{\partial\hat Z}    \right)+
@@ -307,7 +307,7 @@ struct DivCurvatureKappa
 template<class MagneticField>
 struct GradLnB
 {
-    GradLnB( const MagneticField& c, double R0): R_0_(R0), c_(c), invB_(c, R0), bR_(c, R0), bZ_(c, R0) { } 
+    GradLnB( const MagneticField& c): R_0_(c.R_0), c_(c), invB_(c, R0), bR_(c, R0), bZ_(c, R0) { } 
     /**
  * @brief \f[  \hat{\nabla}_\parallel \ln{(\hat{B})} = \frac{1}{\hat{R}\hat{B}^2 } \left[ \hat{B}, \hat{\psi}_p\right]_{\hat{R}\hat{Z}} \f]
  */ 
@@ -403,7 +403,7 @@ struct FieldZ
 template<class MagneticField>
 struct FieldT
 {
-    FieldT( const MagneticField& c, double R0):  R_0_(R0), fieldR_(c, R0), fieldZ_(c, R0){}
+    FieldT( const MagneticField& c):  R_0_(c.R_0), fieldR_(c, R0), fieldZ_(c, R0){}
   /**
  * @brief \f[  B^{\theta} = 
  * B^R\partial_R\theta + B^Z\partial_Z\theta\f]
@@ -497,7 +497,7 @@ struct BHatP
 template<class MagneticField>
 struct Field
 {
-    Field( const MagneticField& c, double R0):c_(c), R_0_(R0), invB_(c, R0) { }
+    Field( const MagneticField& c):c_(c), R_0_(c.R_0), invB_(c, R0) { }
     /**
      * @brief \f[ \frac{d \hat{R} }{ d \varphi}  = \frac{\hat{R}}{\hat{I}} \frac{\partial\hat{\psi}_p}{\partial \hat{Z}}, \hspace {3 mm}
      \frac{d \hat{Z} }{ d \varphi}  =- \frac{\hat{R}}{\hat{I}} \frac{\partial \hat{\psi}_p}{\partial \hat{R}} , \hspace {3 mm}
diff --git a/inc/geometries/solovev.h b/inc/geometries/solovev.h
index 0f1e332de..3a6a61037 100644
--- a/inc/geometries/solovev.h
+++ b/inc/geometries/solovev.h
@@ -1,7 +1,6 @@
 #pragma once
 
 #include <iostream>
-#include <fstream>
 #include <cmath>
 #include <vector>
 
@@ -554,7 +553,8 @@ struct IpolZ
  */
 struct MagneticField
 {
-    MagneticField( GeomParameters gp):psip(gp), psipR(gp), psipZ(gp), psipRR(gp), psipRZ(gp), psipZZ(gp), laplacePsip(gp), ipol(gp), ipolR(gp), ipolZ(gp){}
+    MagneticField( GeomParameters gp): R_0(gp.R_0), psip(gp), psipR(gp), psipZ(gp), psipRR(gp), psipRZ(gp), psipZZ(gp), laplacePsip(gp), ipol(gp), ipolR(gp), ipolZ(gp){}
+    double R_0;
     Psip psip;
     PsipR psipR;
     PsipZ psipZ;
diff --git a/inc/geometries/solovev_doc.h b/inc/geometries/solovev_doc.h
index 738aa9b2d..17636f9b9 100644
--- a/inc/geometries/solovev_doc.h
+++ b/inc/geometries/solovev_doc.h
@@ -159,6 +159,13 @@ struct aContainer
 /**
 * @brief The tokamak magnetic geometry template model
 
+This is the representation of magnetic fields that can be modeled in the form
+\f[
+\vec B = \frac{R_0}{R} \left( I \hat e_\varphi + \nabla \psi_p \times \hat e_\varphi\right)
+\f]
+where \f$ R_0\f$ is a normalization constant, \f$ I\f$ the current 
+and \f$ \psi_p\f$ the poloidal flux function.
+
 A tokamak geometry must contain the functors named as follows, all of
 which must model aBinaryOperator.
  @attention this is not a real class it's there for documentation only
@@ -167,6 +174,7 @@ which must model aBinaryOperator.
 */
 struct aTokamakMagneticField
 {
+    double R_0; //!< \f$ R_0 \f$ 
     Psip psip; //!< \f$ \psi_p(R,Z)\f$, where R, Z are cylindrical coordinates
     PsipR psipR;//!< \f$ \partial_R \psi_p(R,Z)\f$, where R, Z are cylindrical coordinates
     PsipZ psipZ;//!< \f$ \partial_Z \psi_p(R,Z)\f$, where R, Z are cylindrical coordinates
diff --git a/inc/geometries/taylor.h b/inc/geometries/taylor.h
index d7f867d93..1a29b88ff 100644
--- a/inc/geometries/taylor.h
+++ b/inc/geometries/taylor.h
@@ -414,7 +414,8 @@ struct IpolZ
  */
 struct MagneticField
 {
-    MagneticField( solovev::GeomParameters gp):psip(gp), psipR(gp), psipZ(gp), psipRR(gp), psipRZ(gp), psipZZ(gp), laplacePsip(gp), ipol(gp), ipolR(gp), ipolZ(gp){}
+    MagneticField( solovev::GeomParameters gp): R_0(gp.R_0), psip(gp), psipR(gp), psipZ(gp), psipRR(gp), psipRZ(gp), psipZZ(gp), laplacePsip(gp), ipol(gp), ipolR(gp), ipolZ(gp){}
+    double R_0;
     Psip psip;
     PsipR psipR;
     PsipZ psipZ;
diff --git a/inc/geometries/toroidal.h b/inc/geometries/toroidal.h
new file mode 100644
index 000000000..b22564cdf
--- /dev/null
+++ b/inc/geometries/toroidal.h
@@ -0,0 +1,34 @@
+#pragma once
+
+#include "functors.h"
+#include "magnetic_field.h"
+
+
+namespace dg{
+namespace geo{
+namespace toroidal{
+
+/**
+ * @brief Models a slab toroidal field (models aTokamakMagneticField)
+ *
+ * \f$ \psi_p = 1\f$ and \f$ I = R \f$.
+ *
+ */
+struct MagneticField
+{
+    MagneticField(): R_0(1.), psip(1), psipR(0), psipZ(0), psipRR(0), psipRZ(0), psipZZ(0), laplacePsip(0), ipol(1,0), ipolR(1), ipolZ(0){}
+    double R_0;
+    dg::CONSTANT psip;
+    dg::CONSTANT psipR;
+    dg::CONSTANT psipZ;
+    dg::CONSTANT psipRR;
+    dg::CONSTANT psipRZ;
+    dg::CONSTANT psipZZ;
+    dg::CONSTANT laplacePsip;
+    dg::LinearX ipol;
+    dg::CONSTANT ipolR;
+    dg::CONSTANT ipolZ;
+};
+}//namespace toroidal
+}//namespace geo
+}//namespace dg
-- 
GitLab


From 54b4af0f9a9e234398c74450d573c0ad2a7dde40 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Tue, 11 Jul 2017 23:25:23 +0200
Subject: [PATCH 026/453] made toroidal slab with curvature

---
 inc/geometries/toroidal.h | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/inc/geometries/toroidal.h b/inc/geometries/toroidal.h
index b22564cdf..79eda28cf 100644
--- a/inc/geometries/toroidal.h
+++ b/inc/geometries/toroidal.h
@@ -11,12 +11,11 @@ namespace toroidal{
 /**
  * @brief Models a slab toroidal field (models aTokamakMagneticField)
  *
- * \f$ \psi_p = 1\f$ and \f$ I = R \f$.
- *
+ * \f$ B=\frac{R_0}{R}\f$, \f$ \psi_p = 1\f$ and \f$ I = 1\f$.
  */
 struct MagneticField
 {
-    MagneticField(): R_0(1.), psip(1), psipR(0), psipZ(0), psipRR(0), psipRZ(0), psipZZ(0), laplacePsip(0), ipol(1,0), ipolR(1), ipolZ(0){}
+    MagneticField(double R0): R_0(R0), psip(1), psipR(0), psipZ(0), psipRR(0), psipRZ(0), psipZZ(0), laplacePsip(0), ipol(1), ipolR(0), ipolZ(0){}
     double R_0;
     dg::CONSTANT psip;
     dg::CONSTANT psipR;
@@ -25,10 +24,11 @@ struct MagneticField
     dg::CONSTANT psipRZ;
     dg::CONSTANT psipZZ;
     dg::CONSTANT laplacePsip;
-    dg::LinearX ipol;
+    dg::CONSTANT ipol;
     dg::CONSTANT ipolR;
     dg::CONSTANT ipolZ;
 };
+
 }//namespace toroidal
 }//namespace geo
 }//namespace dg
-- 
GitLab


From c53e0d814afc6fba282d9d476920ba14f819ac1e Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Wed, 12 Jul 2017 15:49:41 +0200
Subject: [PATCH 027/453] updated docu and communicator names

---
 inc/dg/backend/collective_mpit.cu  |  4 +-
 inc/dg/backend/mpi_collective.h    | 47 +++++----------
 inc/dg/backend/mpi_vector.h        |  2 +-
 inc/dg/dg_doc.h                    | 94 +++++++++++-------------------
 inc/dg/geometry/mpi_fieldaligned.h | 20 +++----
 5 files changed, 62 insertions(+), 105 deletions(-)

diff --git a/inc/dg/backend/collective_mpit.cu b/inc/dg/backend/collective_mpit.cu
index 634cf8de3..46aa4a77f 100644
--- a/inc/dg/backend/collective_mpit.cu
+++ b/inc/dg/backend/collective_mpit.cu
@@ -27,7 +27,7 @@ int main( int argc, char * argv[])
     const thrust::host_vector<double> w(v);
     dg::BijectiveComm<thrust::host_vector<int>, thrust::host_vector<double> > c(m, MPI_COMM_WORLD);
     thrust::host_vector<double> receive(c.size());
-    receive = c.collect( v);
+    receive = c.global_gather( v);
     //for( unsigned i=0; i<receive.size(); i++)
     //{
     //    if( rank==0)
@@ -39,7 +39,7 @@ int main( int argc, char * argv[])
     //    if( rank==1)
     //        std::cout << receive[i]<<std::endl;
     //}
-    c.send_and_reduce( receive, v);
+    c.global_scatter_reduce( receive, v);
     bool equal = true;
     for( unsigned i=0; i<m.size(); i++)
     {
diff --git a/inc/dg/backend/mpi_collective.h b/inc/dg/backend/mpi_collective.h
index 26da83447..c9627d285 100644
--- a/inc/dg/backend/mpi_collective.h
+++ b/inc/dg/backend/mpi_collective.h
@@ -159,11 +159,8 @@ void Collective::gather_( const thrust::host_vector<double>& gatherFrom, thrust:
 //
 /**
  * @ingroup mpi_structures
- * @brief Struct that performs collective scatter and gather operations across processes
+ * @brief Struct that performs bijective collective scatter and gather operations across processes
  * on distributed vectors using mpi
- * In the bijective operations every element in a vector belongs to 
- exactly one MPI process, i.e. one value is not sent to more than 
- one process and the reduction in the global_scatter_reduce is not necessary
  *
  * @code
  int i = myrank;
@@ -190,10 +187,13 @@ struct BijectiveComm
      */
     BijectiveComm( ){ }
     /**
-     * @brief Construct from a given map 
+     * @brief Construct from a given map with respect to the data vector
      *
-     * @param pids Gives to every point of the values array the rank to which to send this data element. The rank needs to be element of the given communicator.
+     * @param pids Gives to every point of the values/data vector (not the buffer vector!) 
+     *   the rank to which to send this data element. 
+     *   The rank needs to be element of the given communicator.
      * @param comm An MPI Communicator that contains the participants of the scatter/gather
+     * @note The actual scatter/gather map is constructed from the given map 
      */
     BijectiveComm( thrust::host_vector<int> pids, MPI_Comm comm): idx_(pids)
     {
@@ -220,13 +220,14 @@ struct BijectiveComm
     }
 
     /**
-     * @brief Scatters data according to the map given in the Constructor
+     * @brief Globally gathers data into a buffer according to the map given in the Constructor
      *
-     * The order of the received elements is according to their original array index (i.e. a[0] appears before a[1]) and their process rank of origin ( i.e. values from rank 0 appear before values from rank 1)
+     * The order of the received elements is according to their original array index 
+     * (i.e. a[0] appears before a[1]) and their process rank of origin ( i.e. values from rank 0 appear before values from rank 1)
      * @param values data to send (must have the size given 
-     * by the map in the constructor, s.a. send_size())
+     * by the map in the constructor, s.a. size())
      *
-     * @return received data from other processes of size recv_size()
+     * @return received data from other processes of size size()
      * @note a scatter followed by a gather of the received values restores the original array
      */
      Vector global_gather( const Vector& values)const
@@ -242,14 +243,14 @@ struct BijectiveComm
     }
 
     /**
-     * @brief Gather data according to the map given in the constructor 
+     * @brief Scatter data according to the map given in the constructor 
      *
-     * This method is the inverse of scatter 
-     * @param gatherFrom other processes collect data from this vector (has to be of size given by recv_size())
+     * This method is the inverse of gather
+     * @param toScatter other processes collect data from this vector (has to be of size given by recv_size())
      * @param values contains values from other processes sent back to the origin (must have the size of the map given in the constructor, or send_size())
      * @note a scatter followed by a gather of the received values restores the original array
      */
-    void global_scatter_reduce( const Vector& gatherFrom, Vector& values) const
+    void global_scatter_reduce( const Vector& toScatter, Vector& values) const
     {
         Vector values_(values.size());
         //sammeln
@@ -265,23 +266,7 @@ struct BijectiveComm
      *
      * @return # of elements to receive
      */
-    unsigned recv_size() const {
-        return p_.store_size();}
-    /**
-     * @brief return # of elements the calling process has to send in a scatter process (or receive in the gather process)
-     *
-     * equals the size of the map given in the constructor
-     * @return # of elements to send
-     */
-    unsigned send_size() const {
-        return p_.values_size();}
-    /**
-    * @brief The size of the collected vector
-    *
-    * @return 
-    */
-    unsigned size() const {
-        return p_.store_size();}
+    unsigned size() const { return p_.store_size();}
     /**
     * @brief The internal communicator used 
     *
diff --git a/inc/dg/backend/mpi_vector.h b/inc/dg/backend/mpi_vector.h
index b2077c074..af367fdfa 100644
--- a/inc/dg/backend/mpi_vector.h
+++ b/inc/dg/backend/mpi_vector.h
@@ -220,7 +220,7 @@ struct NearestNeighborComm
     */
     const Vector& global_gather( const Vector& input)const;
     /**
-    * @brief Size of the output of global_gather
+    * @brief local size of the output of global_gather
     *
     * @return size
     */
diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index 2141a622d..207965d09 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -119,53 +119,39 @@
  * In order to understand the issue you must first really(!) understand what 
  gather and scatter operations are, so grab pen and paper: 
 
- Gather: imagine a vector v and a map that gives to every element in this vector v
- an index into a source vector w where the value of this element should be taken from
- i.e. \f$ v[i] = w[idx[i]] \f$ 
+ First we note that gather and scatter are most often used in the context
+ of memory buffers. The buffer needs to be filled wih values (gather) or these
+ values need to be written back into the original place (scatter).
+
+ Gather: imagine a buffer vector w and a map that gives to every element in this vector w
+ an index into a source vector v where the value of this element should be taken from
+ i.e. \f$ w[i] = v[\text{idx}[i]] \f$ 
  Note that an index in the source vector can appear several times or not at all. 
- This is why the source vector w can have any size and even be smaller than v. 
- If we throw away all unused elements in w the source vector is always equal 
- or smaller in size than v. 
+ This is why the source vector w can have any size and even be smaller than w. 
 
- Scatter: imagine a vector w and a map that gives to every element in the vector w an
+ Scatter: imagine a buffer vector w and a map that gives to every element in the buffer w an
  index in a target vector v where this element should go to, 
- i.e. \f$ w[idx[i]] = v[i] \f$. 
- Note again that a target index can appear several times. Then in our case we 
- perform a reduction operation (we sum up all elements). If we throw away all
- unused elements in w the, then w is always equal or smaller in size than v. 
+ i.e. \f$ v[\text{idx}[i]] = w[i] \f$. This is ill-defined.
+ Note again that an index in v can appear several times or never at all. 
+ Then in our case we perform a reduction operation (we sum up all elements) beginning
+ with 0 which remedies the defintion. 
 
 Since it is a vector operation the gather and scatter operation can 
-also be represented by a matrix. The gather matrix is just a 
+also be represented/defined by a matrix. The gather matrix is just a 
 (permutation) matrix of 1's and 0's with exactly one "1" in each line.
-The corresponding scatter-and-reduce matrix is just the transpose of the gather matrix. The scatter matrix can have zero, one or more "1"s in each line.
-
-
- Now, consider that both vectors v and w are distributed across processes.
- That means when you send data with MPI you will probably need a communication buffers.
-    There are three types of indices that you need to consider: 
+We uniquely define the corresponding scatter matrix as the transpose of the gather matrix. 
+The scatter matrix can have zero, one or more "1"s in each line.
+\f[ w = G v \\
+    v = S w \f]
 
-    a) the global vector index is the index of an element if there was only one vector that lay contiguously in memory. 
+The scatter matrix S is the actual inverse of G if and only if the gather map is bijective.
+In this case the buffer and the vector can swap their roles. 
 
-    b) the local vector index is the index of the local chunk a process has. 
-
-    c) the buffer index is the index into the communication buffer. 
+Finally note that when v is filled with its indices, i.e. \f$ v[i] = i \f$, then
+the gather operation will reproduce the index map in the buffer w \f$ w[i] = \text{idx}[i]\f$ .
 
  * @ingroup templates
  @attention this is not a real class it's there for documentation only
- *
- * @code
- int i = myrank;
- double values[10] = {i,i,i,i, 9,9,9,9};
- thrust::host_vector<double> hvalues( values, values+10);
- int pids[10] =      {0,1,2,3, 0,1,2,3};
- thrust::host_vector<int> hpids( pids, pids+10);
- BijectiveComm coll( hpids, MPI_COMM_WORLD);
- thrust::host_vector<double> hrecv = coll.scatter( hvalues);
- //hrecv is now {0,9,1,9,2,9,3,9} e.g. for process 0 
- thrust::host_vector<double> hrecv2( coll.send_size());
- coll.gather( hrecv, hrecv2);
- //hrecv2 now equals hvalues independent of process rank
- @endcode
  */
 struct aCommunicator
 {
@@ -173,12 +159,10 @@ struct aCommunicator
     /**
      * @brief Gather data across processes
      *
-        1. create a local send buffer and locally gather values from input vector (c) into a send buffer (order with PID, note that a given value can be sent to several processes -> that's why it's a gather)
-        2. globally scatter these values into recv buffer (b)  
-     * @param values data to send (s.a. send_size())
+     * @param values data to gather from
      * @tparam LocalContainer a container on a shared memory system
      *
-     * @return received data from other processes of size recv_size()
+     * @return the buffer vector of size size()
      */
     template< class LocalContainer>
     LocalContainer global_gather( const LocalContainer& values)const;
@@ -187,34 +171,22 @@ struct aCommunicator
      * @brief Scatters data accross processes and reduces on double indices
      *
      * The order of the received elements is according to their original array index (i.e. a[0] appears before a[1]) and their process rank of origin ( i.e. values from rank 0 appear before values from rank 1)
-        1. globally scatter the values in this buffer to a recv buffer (b) (every value in the result belongs to exactly one line/process)  
-        2. then permute and reduce the recv buffer on double indices and store result in output vector (c) 
      * @tparam LocalContainer a container on a shared memory system
-     * @param toScatter other processes collect data from this vector (has to be of size given by recv_size())
-     * @param values contains values from other processes sent back to the origin (or send_size())
+     * @param toScatter buffer vector (has to be of size given by size())
+     * @param values contains values from other processes sent back to the origin 
      */
     template< class LocalContainer>
     void global_scatter_reduce( const LocalContainer& toScatter, LocalContainer& values) const;
 
     /**
-     * @brief compute total # of elements the calling process receives in the scatter process (or sends in the gather process)
-     *
-     * (which might not equal the send size in each process)
-     *
-     * @return # of elements to receive
-     */
-    unsigned recv_size() const;
-    /**
-     * @brief return # of elements the calling process has to send in a scatter process (or receive in the gather process)
-     *
-     * @return # of elements to send
-     */
-    unsigned send_size() const; 
-    /**
-    * @brief The size of the collected vector = recv_size()
+    * @brief The size of the local buffer = local map size
     *
-    * may return 0
-    * @return 
+ Consider that both the vector v and the buffer w are distributed across processes.
+ In Feltor the vector v is distributed equally among processes and the local size
+ of v is the same for all processes. However the buffer size might be different for each process. 
+    * @note may return 0 to indicate identity between v and w and that no MPI communication is needed 
+    * @note we assume that the vector size is always the local size of a dg::MPI_Vector
+    * @return buffer size
     */
     unsigned size() const;
     /**
diff --git a/inc/dg/geometry/mpi_fieldaligned.h b/inc/dg/geometry/mpi_fieldaligned.h
index e065774bc..ba1ea9d6b 100644
--- a/inc/dg/geometry/mpi_fieldaligned.h
+++ b/inc/dg/geometry/mpi_fieldaligned.h
@@ -334,8 +334,8 @@ FieldAligned<MPIGeometry, RowDistMat<LocalMatrix, CommunicatorXY>, MPI_Vector<Lo
     CommunicatorXY cp( pids, g2d.communicator());
     commXYplus_ = cp;
     thrust::host_vector<double> pX, pY;
-    dg::blas1::transfer( cp.collect( yp[0]), pX);
-    dg::blas1::transfer( cp.collect( yp[1]), pY);
+    dg::blas1::transfer( cp.global_gather( yp[0]), pX);
+    dg::blas1::transfer( cp.global_gather( yp[1]), pY);
 
     //construt interpolation matrix
     plus = dg::create::interpolation( pX, pY, g2d.local(), globalbcz); //inner points hopefully never lie exactly on local boundary
@@ -353,8 +353,8 @@ FieldAligned<MPIGeometry, RowDistMat<LocalMatrix, CommunicatorXY>, MPI_Vector<Lo
     }
     CommunicatorXY cm( pids, g2d.communicator());
     commXYminus_ = cm;
-    dg::blas1::transfer( cm.collect( ym[0]), pX);
-    dg::blas1::transfer( cm.collect( ym[1]), pY);
+    dg::blas1::transfer( cm.global_gather( ym[0]), pX);
+    dg::blas1::transfer( cm.global_gather( ym[1]), pY);
     minus = dg::create::interpolation( pX, pY, g2d.local(), globalbcz); //inner points hopefully never lie exactly on local boundary
     cusp::transpose( minus, minusT);
     //copy to device
@@ -412,14 +412,14 @@ MPI_Vector<container> FieldAligned<G,RowDistMat<M,C>, MPI_Vector<container> >::e
             {
                 if( sizeXY != 1){
                     dg::blas2::symv( plus, tempP, tXYplus);
-                    commXYplus_.send_and_reduce( tXYplus, temp);
+                    commXYplus_.global_scatter_reduce( tXYplus, temp);
                 }
                 else
                     dg::blas2::symv( plus, tempP, temp);
                 temp.swap( tempP);
                 if( sizeXY != 1){
                     dg::blas2::symv( minus, tempM, tXYminus);
-                    commXYminus_.send_and_reduce( tXYminus, temp);
+                    commXYminus_.global_scatter_reduce( tXYminus, temp);
                 }
                 else
                     dg::blas2::symv( minus, tempM, temp);
@@ -483,7 +483,7 @@ void FieldAligned<G,RowDistMat<M,C>, MPI_Vector<container> >::einsPlus( const MP
             View tempV( tempXYplus_[i0].begin(), tempXYplus_[i0].end() );
             cusp::multiply( plus, inV, tempV);
             //exchange data in XY
-            commXYplus_.send_and_reduce( tempXYplus_[i0], temp_[i0]);
+            commXYplus_.global_scatter_reduce( tempXYplus_[i0], temp_[i0]);
         }
     }
     else //directly compute in temp_
@@ -558,7 +558,7 @@ void FieldAligned<G,RowDistMat<M,C>,MPI_Vector<container> >::einsMinus( const MP
             View tempV( tempXYminus_[i0].begin(), tempXYminus_[i0].end());
             cusp::multiply( minus, inV, tempV);
             //exchange data in XY
-            commXYminus_.send_and_reduce( tempXYminus_[i0], temp_[i0]);
+            commXYminus_.global_scatter_reduce( tempXYminus_[i0], temp_[i0]);
         }
     }
     else //directly compute in temp_
@@ -628,7 +628,7 @@ void FieldAligned<G,RowDistMat<M,C>,MPI_Vector<container> >::einsMinusT( const M
         for( int i0=0; i0<(int)g_.Nz(); i0++)
         {
             thrust::copy( in.cbegin() + i0*size2d, in.cbegin() + (i0+1)*size2d, temp_[i0].begin());
-            tempXYminus_[i0] = commXYminus_.collect( temp_[i0] );
+            tempXYminus_[i0] = commXYminus_.global_gather( temp_[i0] );
             cView inV( tempXYminus_[i0].cbegin(), tempXYminus_[i0].cend() );
             View tempV( temp_[i0].begin(), temp_[i0].end() );
             cusp::multiply( minusT, inV, tempV);
@@ -702,7 +702,7 @@ void FieldAligned<G,RowDistMat<M,C>,MPI_Vector<container> >::einsPlusT( const MP
         {
             //first exchange data in XY
             thrust::copy( in.cbegin() + i0*size2d, in.cbegin() + (i0+1)*size2d, temp_[i0].begin());
-            tempXYplus_[i0] = commXYplus_.collect( temp_[i0]);
+            tempXYplus_[i0] = commXYplus_.global_gather( temp_[i0]);
             cView inV( tempXYplus_[i0].cbegin(), tempXYplus_[i0].cend() );
             View tempV( temp_[i0].begin(), temp_[i0].end() );
             cusp::multiply( plusT, inV, tempV);
-- 
GitLab


From fbfaa4a0ab23ce11dcbcc85b32f73549515e99be Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Wed, 12 Jul 2017 17:41:45 +0200
Subject: [PATCH 028/453] update docu on mpi_collective, bijective and write
 general collective

---
 inc/dg/backend/mpi_collective.h | 74 +++++++++++++++++++++++++++------
 1 file changed, 62 insertions(+), 12 deletions(-)

diff --git a/inc/dg/backend/mpi_collective.h b/inc/dg/backend/mpi_collective.h
index c9627d285..86a9465cc 100644
--- a/inc/dg/backend/mpi_collective.h
+++ b/inc/dg/backend/mpi_collective.h
@@ -10,13 +10,19 @@
 #include <thrust/device_vector.h>
 #include "thrust_vector_blas.cuh"
 
+//TODO: Make Collective cuda-aware
 namespace dg{
 
 
 ///@cond
 
 /**
- * @brief Stores the sendTo and the recvFrom maps
+ * @brief engine class for mpi gather and scatter operations
+ * 
+ * This class takes a buffer with indices sorted according to the PID to which 
+ * to send (or gather from) and connects it to an intermediate "store"
+ * In this way gather and scatter are defined with respect to the buffer and 
+ * the store is the vector.
  */
 struct Collective
 {
@@ -25,8 +31,8 @@ struct Collective
      * @brief Construct from a map: PID -> howmanyToSend
      *
      * The size of sendTo must match the number of processes in the communicator
-     * @param sendTo howmany points to send 
-     * @param comm Communicator
+     * @param sendTo sendTo[PID] equals the number of points the calling process has to send to the given PID
+     * @param comm Communicator 
      */
     Collective( const thrust::host_vector<int>& sendTo, MPI_Comm comm) { 
         construct( sendTo, comm);}
@@ -96,8 +102,8 @@ struct Collective
     void scatter_( const thrust::host_vector<double>& values, thrust::host_vector<double>& store) const;
     void gather_( const thrust::host_vector<double>& store, thrust::host_vector<double>& values) const;
     unsigned sum;
-    thrust::host_vector<int> sendTo_,   accS_;
-    thrust::host_vector<int> recvFrom_, accR_;
+    thrust::host_vector<int> sendTo_,   accS_; //accumulated send
+    thrust::host_vector<int> recvFrom_, accR_; //accumulated recv
     MPI_Comm comm_;
 };
 
@@ -176,10 +182,9 @@ void Collective::gather_( const thrust::host_vector<double>& gatherFrom, thrust:
  //hrecv2 now equals hvalues independent of process rank
  @endcode
  @tparam Index an integer Vector
- @tparam Vector a Vector
  @note models aCommunicator
  */
-template< class Index, class Vector>
+template< class Index>
 struct BijectiveComm
 {
     /**
@@ -193,7 +198,7 @@ struct BijectiveComm
      *   the rank to which to send this data element. 
      *   The rank needs to be element of the given communicator.
      * @param comm An MPI Communicator that contains the participants of the scatter/gather
-     * @note The actual scatter/gather map is constructed from the given map 
+     * @note The actual scatter/gather map is constructed from the given map so the result behaves as if pids was the actual scatter/gather map on the buffer
      */
     BijectiveComm( thrust::host_vector<int> pids, MPI_Comm comm): idx_(pids)
     {
@@ -204,9 +209,13 @@ struct BijectiveComm
             assert( 0 <= pids[i] && pids[i] < size);
         thrust::host_vector<int> index(pids);
         thrust::sequence( index.begin(), index.end());
-        thrust::host_vector<int> one( pids.size(), 1), keys(one), number(one);
         thrust::stable_sort_by_key( pids.begin(), pids.end(), index.begin());
+        idx_=index;
+        //now we can repeat/invert the sort by a gather/scatter operation with index as map 
+        //i.e. we could sort pids by a gather 
 
+        //Now construct the collective object by getting the number of elements to send
+        thrust::host_vector<int> one( pids.size(), 1), keys(one), number(one);
         typedef thrust::host_vector<int>::iterator iterator;
         thrust::pair< iterator, iterator> new_end = 
             thrust::reduce_by_key( pids.begin(), pids.end(), one.begin(), 
@@ -216,7 +225,6 @@ struct BijectiveComm
         for( unsigned i=0; i<distance; i++)
             sendTo[keys[i]] = number[i];
         p_.construct( sendTo, comm);
-        idx_=index;
     }
 
     /**
@@ -224,14 +232,18 @@ struct BijectiveComm
      *
      * The order of the received elements is according to their original array index 
      * (i.e. a[0] appears before a[1]) and their process rank of origin ( i.e. values from rank 0 appear before values from rank 1)
+     * @tparam Vector a Vector
      * @param values data to send (must have the size given 
      * by the map in the constructor, s.a. size())
      *
      * @return received data from other processes of size size()
      * @note a scatter followed by a gather of the received values restores the original array
      */
-     Vector global_gather( const Vector& values)const
+    template<class Vector >
+    Vector global_gather( const Vector& values)const
     {
+        //actually this is a scatter but we constructed it invertedly
+        //we could maybe transpose the Collective object!?
         assert( values.size() == idx_.size());
         Vector values_(values);
         //nach PID ordnen
@@ -250,11 +262,13 @@ struct BijectiveComm
      * @param values contains values from other processes sent back to the origin (must have the size of the map given in the constructor, or send_size())
      * @note a scatter followed by a gather of the received values restores the original array
      */
+    template<class Vector >
     void global_scatter_reduce( const Vector& toScatter, Vector& values) const
     {
+        //actually this is a gather but we constructed it invertedly
         Vector values_(values.size());
         //sammeln
-        p_.gather( gatherFrom, values_);
+        p_.gather( toScatter, values_);
         //nach PID geordnete Werte wieder umsortieren
         thrust::scatter( values_.begin(), values_.end(), idx_.begin(), values.begin());
     }
@@ -278,5 +292,41 @@ struct BijectiveComm
     Collective p_;
 };
 
+/**
+ * @ingroup mpi_structures
+ * @brief Struct that performs general collective scatter and gather operations across processes
+ * on distributed vectors using mpi
+ *
+ @tparam Index an integer Vector
+ @note models aCommunicator
+ */
+template< class Index>
+class GeneralComm
+{
+    /**
+     * @brief Construct empty class
+     */
+    GeneralComm(){}
+    /**
+    * @brief 
+    *
+    * @param localGatherMap The gather map containing local vector indices
+    * @param pidGatherMap The gather map containing the pids from where to gather the local index.
+    Same size as localGatherMap.
+     *   The rank needs to be element of the given communicator.
+    * @param comm The MPI communicator participating in the scatter/gather operations
+    */
+    GeneralComm( const thrust::host_vector<int>& localGatherMap, const thrust::host_vector<int>& pidGatherMap, MPI_Comm comm): bijectiveComm(pidGatherMap, comm)
+    {
+        //the bijectiveComm behaves as if we had given the gather map for the store
+        //now gather the localGatherMap from the buffer to the store to get the final gather map 
+        Index gatherMap = bijectiveComm_.global_gather( localGatherMap);
+        gatherMap_ = gatherMap;
+    }
+    private:
+    unsigned vector_size_, buffer_size_, store_size_;
+    BijectiveComm<Index> bijectiveComm_;
+    Index gatherMap_;
+};
 
 }//namespace dg
-- 
GitLab


From 9044b9ce2b5a416451c4d9ba152d8a78e6c75d60 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Wed, 12 Jul 2017 19:13:00 +0200
Subject: [PATCH 029/453] update general Communicator but scatter reduction
 operation should be done in two steps not three

---
 inc/dg/backend/mpi_collective.h | 67 +++++++++++++++++++++++++++++----
 inc/dg/dg_doc.h                 |  2 +
 2 files changed, 61 insertions(+), 8 deletions(-)

diff --git a/inc/dg/backend/mpi_collective.h b/inc/dg/backend/mpi_collective.h
index 86a9465cc..4b863d309 100644
--- a/inc/dg/backend/mpi_collective.h
+++ b/inc/dg/backend/mpi_collective.h
@@ -209,7 +209,7 @@ struct BijectiveComm
             assert( 0 <= pids[i] && pids[i] < size);
         thrust::host_vector<int> index(pids);
         thrust::sequence( index.begin(), index.end());
-        thrust::stable_sort_by_key( pids.begin(), pids.end(), index.begin());
+        thrust::stable_sort_by_key( pids.begin(), pids.end(), index.begin());//note: this also sorts the pids
         idx_=index;
         //now we can repeat/invert the sort by a gather/scatter operation with index as map 
         //i.e. we could sort pids by a gather 
@@ -218,8 +218,8 @@ struct BijectiveComm
         thrust::host_vector<int> one( pids.size(), 1), keys(one), number(one);
         typedef thrust::host_vector<int>::iterator iterator;
         thrust::pair< iterator, iterator> new_end = 
-            thrust::reduce_by_key( pids.begin(), pids.end(), one.begin(), 
-                                                     keys.begin(), number.begin() ); 
+            thrust::reduce_by_key( pids.begin(), pids.end(), //sorted!
+                one.begin(), keys.begin(), number.begin() ); 
         unsigned distance = thrust::distance( keys.begin(), new_end.first);
         thrust::host_vector<int> sendTo( size, 0 );
         for( unsigned i=0; i<distance; i++)
@@ -294,9 +294,10 @@ struct BijectiveComm
 
 /**
  * @ingroup mpi_structures
- * @brief Struct that performs general collective scatter and gather operations across processes
- * on distributed vectors using mpi
+ * @brief Struct that performs general collective scatter and gather operations across processes on distributed vectors using mpi
  *
+ * This Communicator can perform the most general global gather and
+ scatter operations 
  @tparam Index an integer Vector
  @note models aCommunicator
  */
@@ -308,7 +309,7 @@ class GeneralComm
      */
     GeneralComm(){}
     /**
-    * @brief 
+    * @brief Construct from local indices and PIDs
     *
     * @param localGatherMap The gather map containing local vector indices
     * @param pidGatherMap The gather map containing the pids from where to gather the local index.
@@ -318,15 +319,65 @@ class GeneralComm
     */
     GeneralComm( const thrust::host_vector<int>& localGatherMap, const thrust::host_vector<int>& pidGatherMap, MPI_Comm comm): bijectiveComm(pidGatherMap, comm)
     {
+        buffer_size_ = localGatherMap.size();
+        assert( buffer_size_ == pidGatherMap.size());
         //the bijectiveComm behaves as if we had given the gather map for the store
         //now gather the localGatherMap from the buffer to the store to get the final gather map 
         Index gatherMap = bijectiveComm_.global_gather( localGatherMap);
-        gatherMap_ = gatherMap;
+        dg::blas1::transfer(gatherMap, gatherMap_);
+        store_size_ = gatherMap_.size();
+
+        //now prepare a reduction map and a scatter map
+        thrust::host_vector<int> sortMap(gatherMap);
+        thrust::sequence( sortMap.begin(), sortMap.end());
+        thrust::stable_sort_by_key( gatherMap.begin(), gatherMap.end(), sortMap.begin());//note: this also sorts the gatherMap
+        dg::blas1::transfer( sortMap, sortMap_);
+        dg::blas1::transfer( gatherMap, sortedGatherMap_);
+        //now we can repeat/invert the sort by a gather/scatter operation with sortMap as map 
+
+        thrust::host_vector<int> one( gatherMap.size(), 1), keys(one), number(one);
+        typedef thrust::host_vector<int>::iterator iterator;
+        thrust::pair< iterator, iterator> new_end = 
+            thrust::reduce_by_key( gatherMap.begin(), gatherMap.end(), //sorted!
+                one.begin(), keys.begin(), number.begin() ); 
+        unsigned distance = thrust::distance( keys.begin(), new_end.first);
+        vector_size_ = distance;
+        thrust::host_vector<int> scatterMap( size, 0 );
+        thrust::copy( keys.begin(), keys.begin()+distance, scatterMap.begin());
+
+            
     }
+    template<class Vector >
+    Vector global_gather( const Vector& values)const
+    {
+        assert( values.size() == vector_size_);
+        //gather values to store
+        Vector store_(store_size_);
+        thrust::gather( gatherMap_.begin(), gatherMap_.end(), values.begin(), store_.begin());
+        //now gather from store into buffer
+        Vector buffer( buffer_size_);
+        bijectiveComm_.global_scatter_reduce( store_, buffer);
+        return buffer;
+    }
+    template<class Vector>
+    void global_scatter_reduce( const Vector& toScatter, Vector& values)
+    {
+        //first gather values into store
+        Vector store_ = bijectiveComm_.global_gather( toScatter);
+        //now perform a local sort, reduce and scatter operation
+        Vector sortedStore(store_size_), values_(vector_size_);
+        thrust::gather( sortMap_.begin(), sortMap_.end(), store_.begin(), sortedStore.begin());
+        Index keys( vector_size_);
+        thrust::reduce_by_key( sortedGatherMap_.begin(), sortedGatherMap_.end(), sortedStore.begin(), keys.begin(), values_.begin());
+        thrust::scatter( values_.begin(), values_.end(), scatterMap_.begin(), values.begin()  );
+    }
+    unsigned size() const {return buffer_size_;}
     private:
     unsigned vector_size_, buffer_size_, store_size_;
     BijectiveComm<Index> bijectiveComm_;
-    Index gatherMap_;
+    Index gatherMap_; 
+    Index sortMap_, sortedGatherMap_, scatterMap_;
+
 };
 
 }//namespace dg
diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index 207965d09..afba8559d 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -139,6 +139,8 @@
 Since it is a vector operation the gather and scatter operation can 
 also be represented/defined by a matrix. The gather matrix is just a 
 (permutation) matrix of 1's and 0's with exactly one "1" in each line.
+In a "coo" formatted sparse matrix format the values array would consist only of "1"s, 
+row array is just the index and column array is the gather map.
 We uniquely define the corresponding scatter matrix as the transpose of the gather matrix. 
 The scatter matrix can have zero, one or more "1"s in each line.
 \f[ w = G v \\
-- 
GitLab


From f74635cb00486d647145d61b2adf8fca6685a69e Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Wed, 12 Jul 2017 23:15:27 +0200
Subject: [PATCH 030/453] made Collective cuda-aware and made GeneralComm a
 SurjectiveComm

---
 inc/dg/backend/collective_mpit.cu |   2 +-
 inc/dg/backend/mpi_collective.h   | 151 +++++++++++++-----------------
 2 files changed, 67 insertions(+), 86 deletions(-)

diff --git a/inc/dg/backend/collective_mpit.cu b/inc/dg/backend/collective_mpit.cu
index 46aa4a77f..18f3a01cf 100644
--- a/inc/dg/backend/collective_mpit.cu
+++ b/inc/dg/backend/collective_mpit.cu
@@ -25,7 +25,7 @@ int main( int argc, char * argv[])
         v[i] = v[i] + (double)(i + 17%(rank+1));
     }
     const thrust::host_vector<double> w(v);
-    dg::BijectiveComm<thrust::host_vector<int>, thrust::host_vector<double> > c(m, MPI_COMM_WORLD);
+    dg::BijectiveComm<thrust::host_vector<int> > c(m, MPI_COMM_WORLD);
     thrust::host_vector<double> receive(c.size());
     receive = c.global_gather( v);
     //for( unsigned i=0; i<receive.size(); i++)
diff --git a/inc/dg/backend/mpi_collective.h b/inc/dg/backend/mpi_collective.h
index 4b863d309..247614ffa 100644
--- a/inc/dg/backend/mpi_collective.h
+++ b/inc/dg/backend/mpi_collective.h
@@ -9,6 +9,7 @@
 #include <thrust/host_vector.h>
 #include <thrust/device_vector.h>
 #include "thrust_vector_blas.cuh"
+#include "dg/blas1.h"
 
 //TODO: Make Collective cuda-aware
 namespace dg{
@@ -24,6 +25,7 @@ namespace dg{
  * In this way gather and scatter are defined with respect to the buffer and 
  * the store is the vector.
  */
+template<class Index>
 struct Collective
 {
     Collective(){}
@@ -39,20 +41,22 @@ struct Collective
 
     void construct( const thrust::host_vector<int>& map, MPI_Comm comm){
         //sollte schnell sein
-        sendTo_=map, recvFrom_=sendTo_, comm_=comm;
-        accS_ = sendTo_, accR_ = recvFrom_;
+        thrust::host_vector<int> sendTo=map, recvFrom=sendTo; 
+        comm_=comm;
+        thrust::host_vector<int> accS = sendTo, accR = recvFrom;
         int rank, size; 
         MPI_Comm_rank( comm_, &rank);
         MPI_Comm_size( comm_, &size);
-        assert( sendTo_.size() == (unsigned)size);
-        thrust::host_vector<unsigned> global_( size*size);
-        MPI_Allgather( sendTo_.data(),  size, MPI_UNSIGNED,
-                       global_.data(), size, MPI_UNSIGNED,
+        assert( sendTo.size() == (unsigned)size);
+        thrust::host_vector<unsigned> global( size*size);
+        MPI_Allgather( sendTo.data(), size, MPI_UNSIGNED,
+                       global.data(), size, MPI_UNSIGNED,
                        comm_);
         for( unsigned i=0; i<(unsigned)size; i++)
-            recvFrom_[i] = global_[i*size+rank]; 
-        thrust::exclusive_scan( sendTo_.begin(),   sendTo_.end(),   accS_.begin());
-        thrust::exclusive_scan( recvFrom_.begin(), recvFrom_.end(), accR_.begin());
+            recvFrom[i] = global[i*size+rank]; 
+        thrust::exclusive_scan( sendTo.begin(),   sendTo.end(),   accS.begin());
+        thrust::exclusive_scan( recvFrom.begin(), recvFrom.end(), accR.begin());
+        sendTo_=sendTo, recvFrom_=recvFrom, accS_=accS, accR_=accR;
     }
     /**
      * @brief Number of processes in the communicator
@@ -64,23 +68,6 @@ struct Collective
     unsigned size() const {return sendTo_.size();}
     MPI_Comm comm() const {return comm_;}
 
-    /**
-     * @brief Number of elements to send to process pid 
-     *
-     * @param pid Process ID
-     *
-     * @return Number
-     */
-    unsigned sendTo( unsigned pid) const {return sendTo_[pid];}
-
-    /**
-     * @brief Number of elements received from process pid
-     *
-     * @param pid Process ID
-     *
-     * @return Number
-     */
-    unsigned recvFrom( unsigned pid) const {return recvFrom_[pid];}
 
     /**
      * @brief swaps the send and receive maps 
@@ -90,7 +77,6 @@ struct Collective
     void transpose(){ sendTo_.swap( recvFrom_);}
     void invert(){ sendTo_.swap( recvFrom_);}
 
-    thrust::host_vector<double> scatter( const thrust::host_vector<double>& values)const;
     template<class Device>
     void scatter( const Device& values, Device& store) const;
     template<class Device>
@@ -99,66 +85,63 @@ struct Collective
     unsigned values_size() const{ return thrust::reduce( sendTo_.begin(), sendTo_.end() );}
     MPI_Comm communicator() const{return comm_;}
     private:
-    void scatter_( const thrust::host_vector<double>& values, thrust::host_vector<double>& store) const;
-    void gather_( const thrust::host_vector<double>& store, thrust::host_vector<double>& values) const;
+    /**
+     * @brief Number of elements to send to process pid 
+     *
+     * @param pid Process ID
+     *
+     * @return Number
+     */
+    unsigned sendTo( unsigned pid) const {return sendTo_[pid];}
+
+    /**
+     * @brief Number of elements received from process pid
+     *
+     * @param pid Process ID
+     *
+     * @return Number
+     */
+    unsigned recvFrom( unsigned pid) const {return recvFrom_[pid];}
     unsigned sum;
-    thrust::host_vector<int> sendTo_,   accS_; //accumulated send
-    thrust::host_vector<int> recvFrom_, accR_; //accumulated recv
+    Index sendTo_,   accS_; //accumulated send
+    Index recvFrom_, accR_; //accumulated recv
     MPI_Comm comm_;
 };
 
+template< class Index>
 template<class Device>
-void Collective::scatter( const Device& values, Device& store) const
-{
-    //transfer to host, then scatter and transfer result to device
-    thrust::host_vector<double> hvalues, hstore(store.size());
-    dg::blas1::detail::doTransfer( values, hvalues, typename VectorTraits<Device>::vector_category(), ThrustVectorTag()) ;
-    scatter_( hvalues, hstore);
-    dg::blas1::detail::doTransfer( hstore, store, ThrustVectorTag(), typename VectorTraits<Device>::vector_category()) ;
-    thrust::copy( hstore.begin(), hstore.end(), store.begin());
-}
-
-void Collective::scatter_( const thrust::host_vector<double>& values, thrust::host_vector<double>& store) const
+void Collective<Index>::scatter( const Device& values, Device& store) const
 {
     assert( store.size() == store_size() );
-    MPI_Alltoallv( const_cast<double*>(values.data()), 
-                   const_cast<int*>(sendTo_.data()), 
-                   const_cast<int*>(accS_.data()), MPI_DOUBLE,
-                   store.data(), 
-                   const_cast<int*>(recvFrom_.data()), 
-                   const_cast<int*>(accR_.data()), MPI_DOUBLE, comm_);
-                   //the const_cast shouldn't be necessary any more in MPI-3 standard
-}
-
-thrust::host_vector<double> Collective::scatter( const thrust::host_vector<double>& values) const 
-{
-    thrust::host_vector<double> received( store_size() );
-    scatter_( values, received);
-    return received;
+#if THRUST_DEVICE_SYSTEM==THRUST_DEVICE_SYSTEM_CUDA
+    cudaDeviceSynchronize(); //needs to be called 
+#endif //THRUST_DEVICE_SYSTEM
+    MPI_Alltoallv( 
+            thrust::raw_pointer_cast( values.data()), 
+            thrust::raw_pointer_cast( sendTo_.data()), 
+            thrust::raw_pointer_cast( accS_.data()), MPI_DOUBLE, 
+            thrust::raw_pointer_cast( store.data()),
+            thrust::raw_pointer_cast( recvFrom_.data()),
+            thrust::raw_pointer_cast( accR_.data()), MPI_DOUBLE, comm_);
 }
 
+template< class Index>
 template<class Device>
-void Collective::gather( const Device& gatherFrom, Device& values) const 
-{
-    //transfer to host, then gather and transfer result to device
-    thrust::host_vector<double> hvalues(values.size()), hgatherFrom;
-    dg::blas1::detail::doTransfer( (gatherFrom), hgatherFrom, typename VectorTraits<Device>::vector_category(), ThrustVectorTag()) ;
-    gather_( hgatherFrom, hvalues);
-    dg::blas1::detail::doTransfer( hvalues, values, ThrustVectorTag(), typename VectorTraits<Device>::vector_category()) ;
-}
-
-void Collective::gather_( const thrust::host_vector<double>& gatherFrom, thrust::host_vector<double>& values) const 
+void Collective<Index>::gather( const Device& gatherFrom, Device& values) const 
 {
     //std::cout << gatherFrom.size()<<" "<<store_size()<<std::endl;
     assert( gatherFrom.size() == store_size() );
     values.resize( values_size() );
+#if THRUST_DEVICE_SYSTEM==THRUST_DEVICE_SYSTEM_CUDA
+    cudaDeviceSynchronize(); //needs to be called 
+#endif //THRUST_DEVICE_SYSTEM
     MPI_Alltoallv( 
-            const_cast<double*>(gatherFrom.data()), 
-            const_cast<int*>(recvFrom_.data()),
-            const_cast<int*>(accR_.data()), MPI_DOUBLE, 
-            values.data(), 
-            const_cast<int*>(sendTo_.data()), 
-            const_cast<int*>(accS_.data()), MPI_DOUBLE, comm_);
+            thrust::raw_pointer_cast( gatherFrom.data()), 
+            thrust::raw_pointer_cast( recvFrom_.data()),
+            thrust::raw_pointer_cast( accR_.data()), MPI_DOUBLE, 
+            thrust::raw_pointer_cast( values.data()), 
+            thrust::raw_pointer_cast( sendTo_.data()), 
+            thrust::raw_pointer_cast( accS_.data()), MPI_DOUBLE, comm_);
 }
 //BijectiveComm ist der Spezialfall, dass jedes Element nur ein einziges Mal gebraucht wird. 
 ///@endcond
@@ -289,25 +272,27 @@ struct BijectiveComm
     MPI_Comm communicator() const {return p_.communicator();}
     private:
     Index idx_;
-    Collective p_;
+    Collective<Index> p_;
 };
 
 /**
  * @ingroup mpi_structures
  * @brief Struct that performs general collective scatter and gather operations across processes on distributed vectors using mpi
  *
- * This Communicator can perform the most general global gather and
- scatter operations 
+ * This Communicator can perform general global gather and
+ scatter operations. We only assume that the gather/scatter map
+ is surjective, i.e. all elements in a vector get gathered. This
+ is important only in the global_scatter_reduce function.
  @tparam Index an integer Vector
  @note models aCommunicator
  */
 template< class Index>
-class GeneralComm
+class SurjectiveComm
 {
     /**
      * @brief Construct empty class
      */
-    GeneralComm(){}
+    SurjectiveComm(){}
     /**
     * @brief Construct from local indices and PIDs
     *
@@ -316,8 +301,9 @@ class GeneralComm
     Same size as localGatherMap.
      *   The rank needs to be element of the given communicator.
     * @param comm The MPI communicator participating in the scatter/gather operations
+    * @note we assume that the gather map is surjective
     */
-    GeneralComm( const thrust::host_vector<int>& localGatherMap, const thrust::host_vector<int>& pidGatherMap, MPI_Comm comm): bijectiveComm(pidGatherMap, comm)
+    SurjectiveComm( const thrust::host_vector<int>& localGatherMap, const thrust::host_vector<int>& pidGatherMap, MPI_Comm comm): bijectiveComm_(pidGatherMap, comm)
     {
         buffer_size_ = localGatherMap.size();
         assert( buffer_size_ == pidGatherMap.size());
@@ -342,10 +328,6 @@ class GeneralComm
                 one.begin(), keys.begin(), number.begin() ); 
         unsigned distance = thrust::distance( keys.begin(), new_end.first);
         vector_size_ = distance;
-        thrust::host_vector<int> scatterMap( size, 0 );
-        thrust::copy( keys.begin(), keys.begin()+distance, scatterMap.begin());
-
-            
     }
     template<class Vector >
     Vector global_gather( const Vector& values)const
@@ -365,11 +347,10 @@ class GeneralComm
         //first gather values into store
         Vector store_ = bijectiveComm_.global_gather( toScatter);
         //now perform a local sort, reduce and scatter operation
-        Vector sortedStore(store_size_), values_(vector_size_);
+        Vector sortedStore(store_size_);
         thrust::gather( sortMap_.begin(), sortMap_.end(), store_.begin(), sortedStore.begin());
         Index keys( vector_size_);
-        thrust::reduce_by_key( sortedGatherMap_.begin(), sortedGatherMap_.end(), sortedStore.begin(), keys.begin(), values_.begin());
-        thrust::scatter( values_.begin(), values_.end(), scatterMap_.begin(), values.begin()  );
+        thrust::reduce_by_key( sortedGatherMap_.begin(), sortedGatherMap_.end(), sortedStore.begin(), keys.begin(), values.begin());
     }
     unsigned size() const {return buffer_size_;}
     private:
-- 
GitLab


From 72225c0a7b5c7afbbaf353f70be30099fbd7d31f Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Wed, 12 Jul 2017 23:53:34 +0200
Subject: [PATCH 031/453] found two TODOs in mpi_collective

---
 inc/dg/backend/collective_mpit.cu | 26 +++++++++++++++-----------
 inc/dg/backend/mpi_collective.h   | 11 ++++++++---
 2 files changed, 23 insertions(+), 14 deletions(-)

diff --git a/inc/dg/backend/collective_mpit.cu b/inc/dg/backend/collective_mpit.cu
index 18f3a01cf..b5370aee8 100644
--- a/inc/dg/backend/collective_mpit.cu
+++ b/inc/dg/backend/collective_mpit.cu
@@ -16,7 +16,7 @@ int main( int argc, char * argv[])
     if(rank==0)std::cout <<"Size =  " <<size <<std::endl;
     if(size!=4 && rank == 0){std::cerr <<"You run with "<<size<<" processes. Run with 4 processes!\n"; MPI_Finalize(); return -1;}
 
-    if(rank==0)std::cout << "Test if scatter followed by gather leaves the input vector intact\n";
+    if(rank==0)std::cout << "Test BijectiveComm: scatter followed by gather leaves the input vector intact\n";
     thrust::host_vector<double> v( Nx*Ny, 3*rank);
     thrust::host_vector<double> m( Nx*Ny);
     for( unsigned i=0; i<m.size(); i++)
@@ -28,17 +28,7 @@ int main( int argc, char * argv[])
     dg::BijectiveComm<thrust::host_vector<int> > c(m, MPI_COMM_WORLD);
     thrust::host_vector<double> receive(c.size());
     receive = c.global_gather( v);
-    //for( unsigned i=0; i<receive.size(); i++)
-    //{
-    //    if( rank==0)
-    //        std::cout << receive[i]<<std::endl;
-    //}
     MPI_Barrier( MPI_COMM_WORLD);
-    //for( unsigned i=0; i<receive.size(); i++)
-    //{
-    //    if( rank==1)
-    //        std::cout << receive[i]<<std::endl;
-    //}
     c.global_scatter_reduce( receive, v);
     bool equal = true;
     for( unsigned i=0; i<m.size(); i++)
@@ -56,6 +46,20 @@ int main( int argc, char * argv[])
         else
             std::cerr << "TEST FAILED\n";
     }
+    if(rank==0)std::cout << "Test SurjectiveComm:\n";
+    Nx = 3, Ny = 3; 
+    thrust::host_vector<double> vec( Nx*Ny, rank);
+    thrust::host_vector<int> idx( (Nx+1)*(Ny+1)), pids( (Nx+1)*(Ny+1));
+    for( unsigned i=0; i<(Nx+1)*(Ny+1); i++)
+    {
+        idx[i] = i%(Nx*Ny);
+        pids[i] = rank;
+        if( i>=Nx*Ny) pids[i] = (rank+1)%size;
+    }
+    dg::SurjectiveComm<thrust::host_vector<int> > s( idx, pids, MPI_COMM_WORLD);
+    //receive = s.global_gather( vec);
+    //for( unsigned i=0; i<(Nx+1)*(Ny+1); i++)
+    //    if(rank==0) std::cout << i << std::endl;
 
     MPI_Finalize();
 
diff --git a/inc/dg/backend/mpi_collective.h b/inc/dg/backend/mpi_collective.h
index 247614ffa..c66f78db1 100644
--- a/inc/dg/backend/mpi_collective.h
+++ b/inc/dg/backend/mpi_collective.h
@@ -11,7 +11,11 @@
 #include "thrust_vector_blas.cuh"
 #include "dg/blas1.h"
 
-//TODO: Make Collective cuda-aware
+//TODO: use Buffer class from MPI_Vector, since Device cannot(!) be 
+//arbitrary type (has to be on device if Index is on device e.g.)
+
+//TODO: The MPI_Alltoallv function only send DOUBLEs so no int vector can be used 
+//
 namespace dg{
 
 
@@ -287,7 +291,7 @@ struct BijectiveComm
  @note models aCommunicator
  */
 template< class Index>
-class SurjectiveComm
+struct SurjectiveComm
 {
     /**
      * @brief Construct empty class
@@ -309,7 +313,8 @@ class SurjectiveComm
         assert( buffer_size_ == pidGatherMap.size());
         //the bijectiveComm behaves as if we had given the gather map for the store
         //now gather the localGatherMap from the buffer to the store to get the final gather map 
-        Index gatherMap = bijectiveComm_.global_gather( localGatherMap);
+        thrust::device_vector<double> localGatherMap_d = localGatherMap;
+        Index gatherMap = bijectiveComm_.global_gather( localGatherMap_d);
         dg::blas1::transfer(gatherMap, gatherMap_);
         store_size_ = gatherMap_.size();
 
-- 
GitLab


From fcc5caca9eb599ca7005732abca1894a3cf64e3d Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 13 Jul 2017 12:06:01 +0200
Subject: [PATCH 032/453] made first test for surjective Comm

---
 inc/dg/backend/collective_mpit.cu | 37 ++++++++-----
 inc/dg/backend/mpi_collective.h   | 87 +++++++++++--------------------
 inc/dg/dg_doc.h                   |  1 +
 3 files changed, 56 insertions(+), 69 deletions(-)

diff --git a/inc/dg/backend/collective_mpit.cu b/inc/dg/backend/collective_mpit.cu
index b5370aee8..fc84f60a1 100644
--- a/inc/dg/backend/collective_mpit.cu
+++ b/inc/dg/backend/collective_mpit.cu
@@ -25,7 +25,7 @@ int main( int argc, char * argv[])
         v[i] = v[i] + (double)(i + 17%(rank+1));
     }
     const thrust::host_vector<double> w(v);
-    dg::BijectiveComm<thrust::host_vector<int> > c(m, MPI_COMM_WORLD);
+    dg::BijectiveComm<thrust::host_vector<int>, thrust::host_vector<double> > c(m, MPI_COMM_WORLD);
     thrust::host_vector<double> receive(c.size());
     receive = c.global_gather( v);
     MPI_Barrier( MPI_COMM_WORLD);
@@ -33,22 +33,19 @@ int main( int argc, char * argv[])
     bool equal = true;
     for( unsigned i=0; i<m.size(); i++)
     {
-        if( v[i] != w[i])
-        {
-            equal = false;
-        }
+        if( v[i] != w[i]) { equal = false; }
         //if( rank==0) std::cout << i << " "<<v[i]<<" "<<w[i]<<"\n";
     }
-    if( rank==0)
     {
         if( equal) 
-            std::cout <<"TEST PASSED\n";
+            std::cout <<"Rank "<<rank<<" TEST PASSED"<<std::endl;
         else
-            std::cerr << "TEST FAILED\n";
+            std::cerr <<"Rank "<<rank<<" TEST FAILED"<<std::endl;
     }
+    MPI_Barrier(MPI_COMM_WORLD);
     if(rank==0)std::cout << "Test SurjectiveComm:\n";
     Nx = 3, Ny = 3; 
-    thrust::host_vector<double> vec( Nx*Ny, rank);
+    thrust::host_vector<double> vec( Nx*Ny, rank), result( Nx*Ny);
     thrust::host_vector<int> idx( (Nx+1)*(Ny+1)), pids( (Nx+1)*(Ny+1));
     for( unsigned i=0; i<(Nx+1)*(Ny+1); i++)
     {
@@ -56,10 +53,26 @@ int main( int argc, char * argv[])
         pids[i] = rank;
         if( i>=Nx*Ny) pids[i] = (rank+1)%size;
     }
-    dg::SurjectiveComm<thrust::host_vector<int> > s( idx, pids, MPI_COMM_WORLD);
-    //receive = s.global_gather( vec);
+    dg::SurjectiveComm<thrust::host_vector<int>, thrust::host_vector<double> > s( idx, pids, MPI_COMM_WORLD);
+    receive = s.global_gather( vec);
     //for( unsigned i=0; i<(Nx+1)*(Ny+1); i++)
-    //    if(rank==0) std::cout << i << std::endl;
+    //    if(rank==0) std::cout << i<<"\t "<< receive[i] << std::endl;
+    s.global_scatter_reduce( receive, vec);
+    equal=true;
+    for( unsigned i=0; i<(Nx)*(Ny); i++)
+    {
+        //if(rank==1) std::cout << i<<"\t "<< vec[i] << std::endl;
+        result[i] = rank; 
+        if( i < (Nx+1)*(Ny+1) - Nx*Ny) result[i] += (rank)%size;
+        if( vec[i] != result[i]) equal = false;
+    }
+    {
+        if( equal) 
+            std::cout <<"Rank "<<rank<<" TEST PASSED"<<std::endl;
+        else
+            std::cerr <<"Rank "<<rank<<" TEST FAILED"<<std::endl;
+    }
+
 
     MPI_Finalize();
 
diff --git a/inc/dg/backend/mpi_collective.h b/inc/dg/backend/mpi_collective.h
index c66f78db1..32c118996 100644
--- a/inc/dg/backend/mpi_collective.h
+++ b/inc/dg/backend/mpi_collective.h
@@ -10,6 +10,7 @@
 #include <thrust/device_vector.h>
 #include "thrust_vector_blas.cuh"
 #include "dg/blas1.h"
+#include "mpi_vector.h"
 
 //TODO: use Buffer class from MPI_Vector, since Device cannot(!) be 
 //arbitrary type (has to be on device if Index is on device e.g.)
@@ -28,8 +29,9 @@ namespace dg{
  * to send (or gather from) and connects it to an intermediate "store"
  * In this way gather and scatter are defined with respect to the buffer and 
  * the store is the vector.
+ @note the data type of the Vector class has to be double
  */
-template<class Index>
+template<class Index, class Vector>
 struct Collective
 {
     Collective(){}
@@ -72,49 +74,24 @@ struct Collective
     unsigned size() const {return sendTo_.size();}
     MPI_Comm comm() const {return comm_;}
 
-
-    /**
-     * @brief swaps the send and receive maps 
-     *
-     * Now the pattern works backwards
-     */
     void transpose(){ sendTo_.swap( recvFrom_);}
     void invert(){ sendTo_.swap( recvFrom_);}
 
-    template<class Device>
-    void scatter( const Device& values, Device& store) const;
-    template<class Device>
-    void gather( const Device& store, Device& values) const;
+    void scatter( const Vector& values, Vector& store) const;
+    void gather( const Vector& store, Vector& values) const;
     unsigned store_size() const{ return thrust::reduce( recvFrom_.begin(), recvFrom_.end() );}
     unsigned values_size() const{ return thrust::reduce( sendTo_.begin(), sendTo_.end() );}
     MPI_Comm communicator() const{return comm_;}
     private:
-    /**
-     * @brief Number of elements to send to process pid 
-     *
-     * @param pid Process ID
-     *
-     * @return Number
-     */
     unsigned sendTo( unsigned pid) const {return sendTo_[pid];}
-
-    /**
-     * @brief Number of elements received from process pid
-     *
-     * @param pid Process ID
-     *
-     * @return Number
-     */
     unsigned recvFrom( unsigned pid) const {return recvFrom_[pid];}
-    unsigned sum;
     Index sendTo_,   accS_; //accumulated send
     Index recvFrom_, accR_; //accumulated recv
     MPI_Comm comm_;
 };
 
-template< class Index>
-template<class Device>
-void Collective<Index>::scatter( const Device& values, Device& store) const
+template< class Index, class Device>
+void Collective<Index, Device>::scatter( const Device& values, Device& store) const
 {
     assert( store.size() == store_size() );
 #if THRUST_DEVICE_SYSTEM==THRUST_DEVICE_SYSTEM_CUDA
@@ -129,9 +106,8 @@ void Collective<Index>::scatter( const Device& values, Device& store) const
             thrust::raw_pointer_cast( accR_.data()), MPI_DOUBLE, comm_);
 }
 
-template< class Index>
-template<class Device>
-void Collective<Index>::gather( const Device& gatherFrom, Device& values) const 
+template< class Index, class Device>
+void Collective<Index, Device>::gather( const Device& gatherFrom, Device& values) const 
 {
     //std::cout << gatherFrom.size()<<" "<<store_size()<<std::endl;
     assert( gatherFrom.size() == store_size() );
@@ -171,7 +147,7 @@ void Collective<Index>::gather( const Device& gatherFrom, Device& values) const
  @tparam Index an integer Vector
  @note models aCommunicator
  */
-template< class Index>
+template< class Index, class Vector>
 struct BijectiveComm
 {
     /**
@@ -212,6 +188,7 @@ struct BijectiveComm
         for( unsigned i=0; i<distance; i++)
             sendTo[keys[i]] = number[i];
         p_.construct( sendTo, comm);
+        values_.data()->resize( idx_.size());
     }
 
     /**
@@ -226,18 +203,16 @@ struct BijectiveComm
      * @return received data from other processes of size size()
      * @note a scatter followed by a gather of the received values restores the original array
      */
-    template<class Vector >
     Vector global_gather( const Vector& values)const
     {
         //actually this is a scatter but we constructed it invertedly
         //we could maybe transpose the Collective object!?
         assert( values.size() == idx_.size());
-        Vector values_(values);
         //nach PID ordnen
-        thrust::gather( idx_.begin(), idx_.end(), values.begin(), values_.begin());
+        thrust::gather( idx_.begin(), idx_.end(), values.begin(), values_.data()->begin());
         //senden
         Vector store( p_.store_size());
-        p_.scatter( values_, store);
+        p_.scatter( *values_.data(), store);
         return store;
     }
 
@@ -249,15 +224,12 @@ struct BijectiveComm
      * @param values contains values from other processes sent back to the origin (must have the size of the map given in the constructor, or send_size())
      * @note a scatter followed by a gather of the received values restores the original array
      */
-    template<class Vector >
     void global_scatter_reduce( const Vector& toScatter, Vector& values) const
     {
         //actually this is a gather but we constructed it invertedly
-        Vector values_(values.size());
-        //sammeln
-        p_.gather( toScatter, values_);
+        p_.gather( toScatter, *values_.data());
         //nach PID geordnete Werte wieder umsortieren
-        thrust::scatter( values_.begin(), values_.end(), idx_.begin(), values.begin());
+        thrust::scatter( values_.data()->begin(), values_.data()->end(), idx_.begin(), values.begin());
     }
 
     /**
@@ -275,8 +247,9 @@ struct BijectiveComm
     */
     MPI_Comm communicator() const {return p_.communicator();}
     private:
+    Buffer<Vector> values_;
     Index idx_;
-    Collective<Index> p_;
+    Collective<Index, Vector> p_;
 };
 
 /**
@@ -290,7 +263,7 @@ struct BijectiveComm
  @tparam Index an integer Vector
  @note models aCommunicator
  */
-template< class Index>
+template< class Index, class Vector>
 struct SurjectiveComm
 {
     /**
@@ -313,7 +286,8 @@ struct SurjectiveComm
         assert( buffer_size_ == pidGatherMap.size());
         //the bijectiveComm behaves as if we had given the gather map for the store
         //now gather the localGatherMap from the buffer to the store to get the final gather map 
-        thrust::device_vector<double> localGatherMap_d = localGatherMap;
+        Vector localGatherMap_d;
+        dg::blas1::transfer( localGatherMap, localGatherMap_d);
         Index gatherMap = bijectiveComm_.global_gather( localGatherMap_d);
         dg::blas1::transfer(gatherMap, gatherMap_);
         store_size_ = gatherMap_.size();
@@ -333,36 +307,35 @@ struct SurjectiveComm
                 one.begin(), keys.begin(), number.begin() ); 
         unsigned distance = thrust::distance( keys.begin(), new_end.first);
         vector_size_ = distance;
+        store_.data()->resize( store_size_);
+        keys_.data()->resize( vector_size_);
     }
-    template<class Vector >
     Vector global_gather( const Vector& values)const
     {
         assert( values.size() == vector_size_);
         //gather values to store
-        Vector store_(store_size_);
-        thrust::gather( gatherMap_.begin(), gatherMap_.end(), values.begin(), store_.begin());
+        thrust::gather( gatherMap_.begin(), gatherMap_.end(), values.begin(), store_.data()->begin());
         //now gather from store into buffer
         Vector buffer( buffer_size_);
-        bijectiveComm_.global_scatter_reduce( store_, buffer);
+        bijectiveComm_.global_scatter_reduce( *store_.data(), buffer);
         return buffer;
     }
-    template<class Vector>
     void global_scatter_reduce( const Vector& toScatter, Vector& values)
     {
         //first gather values into store
-        Vector store_ = bijectiveComm_.global_gather( toScatter);
+        Vector store_t = bijectiveComm_.global_gather( toScatter);
         //now perform a local sort, reduce and scatter operation
-        Vector sortedStore(store_size_);
-        thrust::gather( sortMap_.begin(), sortMap_.end(), store_.begin(), sortedStore.begin());
-        Index keys( vector_size_);
-        thrust::reduce_by_key( sortedGatherMap_.begin(), sortedGatherMap_.end(), sortedStore.begin(), keys.begin(), values.begin());
+        thrust::gather( sortMap_.begin(), sortMap_.end(), store_t.begin(), store_.data()->begin());
+        thrust::reduce_by_key( sortedGatherMap_.begin(), sortedGatherMap_.end(), store_.data()->begin(), keys_.data()->begin(), values.begin());
     }
     unsigned size() const {return buffer_size_;}
     private:
     unsigned vector_size_, buffer_size_, store_size_;
-    BijectiveComm<Index> bijectiveComm_;
+    BijectiveComm<Index, Vector> bijectiveComm_;
     Index gatherMap_; 
     Index sortMap_, sortedGatherMap_, scatterMap_;
+    Buffer<Index> keys_;
+    Buffer<Vector> store_;
 
 };
 
diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index afba8559d..fe3337c25 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -168,6 +168,7 @@ struct aCommunicator
      */
     template< class LocalContainer>
     LocalContainer global_gather( const LocalContainer& values)const;
+    //actually the return type in NNC is const LocalContainer& 
 
     /**
      * @brief Scatters data accross processes and reduces on double indices
-- 
GitLab


From 82405f932933395d01fe5fad8715c7a4fcd86577 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 13 Jul 2017 12:10:45 +0200
Subject: [PATCH 033/453] final docu bug

---
 inc/dg/backend/mpi_collective.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/inc/dg/backend/mpi_collective.h b/inc/dg/backend/mpi_collective.h
index 32c118996..6abbc9f49 100644
--- a/inc/dg/backend/mpi_collective.h
+++ b/inc/dg/backend/mpi_collective.h
@@ -145,6 +145,7 @@ void Collective<Index, Device>::gather( const Device& gatherFrom, Device& values
  //hrecv2 now equals hvalues independent of process rank
  @endcode
  @tparam Index an integer Vector
+ @tparam Vector a Vector (the data type of Vector must be double)
  @note models aCommunicator
  */
 template< class Index, class Vector>
@@ -261,6 +262,7 @@ struct BijectiveComm
  is surjective, i.e. all elements in a vector get gathered. This
  is important only in the global_scatter_reduce function.
  @tparam Index an integer Vector
+ @tparam Vector a Vector (the data type of Vector must be double)
  @note models aCommunicator
  */
 template< class Index, class Vector>
-- 
GitLab


From 71ec925ea5fa282c4cfd86480098cf5a468b1eca Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 13 Jul 2017 17:01:21 +0200
Subject: [PATCH 034/453] moved template parameters in DS

---
 inc/dg/ds.h                     | 76 ++++++++++++++-------------------
 inc/geometries/magnetic_field.h | 61 +++++++++++++++++---------
 inc/geometries/toroidal.h       |  1 +
 3 files changed, 73 insertions(+), 65 deletions(-)

diff --git a/inc/dg/ds.h b/inc/dg/ds.h
index 5fe837609..dc0f05cb5 100644
--- a/inc/dg/ds.h
+++ b/inc/dg/ds.h
@@ -21,17 +21,16 @@ namespace dg{
 *
 * This class discretizes the operators \f$ \nabla_\parallel = 
 \mathbf{b}\cdot \nabla = b_R\partial_R + b_Z\partial_Z + b_\phi\partial_\phi \f$, \f$\nabla_\parallel^\dagger\f$ and \f$\Delta_\parallel=\nabla_\parallel^\dagger\cdot\nabla_\parallel\f$ in
-cylindrical coordinates
+arbitrary coordinates
 * @ingroup fieldaligned
-* @tparam FA Engine class for interpolation, provides the necessary interpolation operations
+* @tparam Geometry The grid geometry
+* @tparam IMatrix The type of the interpolation matrix
 * @tparam Matrix The matrix class of the jump matrix
 * @tparam container The container-class on which the interpolation matrix operates on (does not need to be dg::HVec)
 */
-template< class FA, class Matrix, class container >
+template< class Geometry, class IMatrix, class Matrix, class container >
 struct DS
 {
-    typedef FA FieldAligned;//!< typedef for easier construction of corresponding fieldaligned object
-
     /**
     * @brief Construct from a field and a grid
     *
@@ -213,7 +212,7 @@ struct DS
     private:
     FA f_;
     Matrix jumpX, jumpY;
-    typename FA::InterpolationMatrix f2c, c2f, f2cT, c2fT;
+    IMatrix f2c, c2f, f2cT, c2fT;
     container tempP, temp0, tempM;
     container tempPc, tempMc, temp0c;
     container f, dsf;
@@ -231,9 +230,9 @@ struct DS
 ///@cond
 ////////////////////////////////////DEFINITIONS////////////////////////////////////////
 
-template<class FA, class M, class container>
-template <class Field, class Geometry>
-DS<FA, M,container>::DS(const FA& field, Geometry gridc, Field inverseB, dg::norm no, dg::direction dir, bool jumpX):
+template<class G, class I, class M, class container>
+template <class MagneticField>
+DS<G, I, M,container>::DS(const FA& field, Geometry gridc, Field inverseB, dg::norm no, dg::direction dir, bool jumpX):
         f_(field),
         jumpX( dg::create::jumpX( gridc)),
         jumpY( dg::create::jumpY( gridc)),
@@ -257,8 +256,8 @@ DS<FA, M,container>::DS(const FA& field, Geometry gridc, Field inverseB, dg::nor
 
 }
 
-template<class F, class M, class container>
-inline void DS<F,M,container>::operator()( const container& f, container& dsf) { 
+template<class G, class I, class M, class container>
+inline void DS<G,I,M,container>::operator()( const container& f, container& dsf) { 
     if( dir_ == dg::centered)
         return centered( f, dsf);
     else if( dir_ == dg::forward)
@@ -268,8 +267,8 @@ inline void DS<F,M,container>::operator()( const container& f, container& dsf) {
 }
 
 
-template<class F, class M, class container>
-void DS<F,M,container>::centered( const container& fc, container& dsfc)
+template<class G, class I, class M, class container>
+void DS<G,I,M,container>::centered( const container& fc, container& dsfc)
 {
     //direct discretisation
     assert( &fc != &dsfc);
@@ -295,8 +294,8 @@ void DS<F,M,container>::centered( const container& fc, container& dsfc)
 //     dg::blas1::pointwiseDot( inv3d, dsfc, dsfc); 
 }
 
-template<class F, class M, class container>
-void DS<F,M,container>::centeredT( const container& fc, container& dsfc)
+template<class G, class I, class M, class container>
+void DS<G,I,M,container>::centeredT( const container& fc, container& dsfc)
 {               
     //adjoint discretisation
     assert( &fc != &dsfc);    
@@ -337,8 +336,8 @@ void DS<F,M,container>::centeredT( const container& fc, container& dsfc)
 //         dg::blas1::pointwiseDivide( dsf,w2d,  dsf);  
 }
 
-template<class F, class M, class container>
-void DS<F,M,container>::centeredTD( const container& f, container& dsf)
+template<class G, class I, class M, class container>
+void DS<G,I,M,container>::centeredTD( const container& f, container& dsf)
 {       
 //     Direct discretisation
     assert( &f != &dsf);    
@@ -350,8 +349,8 @@ void DS<F,M,container>::centeredTD( const container& f, container& dsf)
     dg::blas1::pointwiseDivide( dsf, invB, dsf);
 }
 
-template<class F, class M, class container>
-void DS<F,M,container>::forward( const container& fc, container& dsfc)
+template<class G, class I, class M, class container>
+void DS<G,I,M,container>::forward( const container& fc, container& dsfc)
 {
     //direct
     assert( &fc != &dsfc);
@@ -371,8 +370,8 @@ void DS<F,M,container>::forward( const container& fc, container& dsfc)
     cusp::multiply( f2c, dsf, dsfc);
 }
 
-template<class F, class M, class container>
-void DS<F,M,container>::forwardT( const container& fc, container& dsfc)
+template<class G, class I, class M, class container>
+void DS<G,I,M,container>::forwardT( const container& fc, container& dsfc)
 {    
     //adjoint discretisation
     assert( &fc != &dsfc);
@@ -386,8 +385,8 @@ void DS<F,M,container>::forwardT( const container& fc, container& dsfc)
     dg::blas1::pointwiseDot( inv3d, dsfc, dsfc); 
 }
 
-template<class F, class M, class container>
-void DS<F,M,container>::forwardTD( const container& fc, container& dsfc)
+template<class G, class I, class M, class container>
+void DS<G,I,M,container>::forwardTD( const container& fc, container& dsfc)
 {
     //direct discretisation
     assert( &fc != &dsfc);
@@ -403,8 +402,8 @@ void DS<F,M,container>::forwardTD( const container& fc, container& dsfc)
 
 
 }
-template<class F, class M, class container>
-void DS<F,M,container>::backward( const container& fc, container& dsfc)
+template<class G, class I, class M, class container>
+void DS<G,I,M,container>::backward( const container& fc, container& dsfc)
 {
     //direct
     assert( &fc != &dsfc);
@@ -424,8 +423,8 @@ void DS<F,M,container>::backward( const container& fc, container& dsfc)
 //     dg::blas1::pointwiseDot( dsf, invB, dsf);
     cusp::multiply( f2c, dsf, dsfc);
 }
-template<class F, class M, class container>
-void DS<F,M,container>::backwardT( const container& fc, container& dsfc)
+template<class G, class I, class M, class container>
+void DS<G,I,M,container>::backwardT( const container& fc, container& dsfc)
 {    
     //adjoint discretisation
     assert( &fc != &dsfc);
@@ -440,8 +439,8 @@ void DS<F,M,container>::backwardT( const container& fc, container& dsfc)
     dg::blas1::pointwiseDot( inv3d, dsfc, dsfc); 
 }
 
-template<class F, class M, class container>
-void DS<F,M,container>::backwardTD( const container& fc, container& dsfc)
+template<class G, class I, class M, class container>
+void DS<G,I,M,container>::backwardTD( const container& fc, container& dsfc)
 {
     //direct
     assert( &fc != &dsfc);
@@ -455,8 +454,8 @@ void DS<F,M,container>::backwardTD( const container& fc, container& dsfc)
     cusp::multiply( f2c, dsf, dsfc);
 }
 
-template< class F, class M, class container >
-void DS<F,M,container>::symv( const container& f, container& dsTdsf)
+template<class G, class I, class M, class container>
+void DS<G,I,M,container>::symv( const container& f, container& dsTdsf)
 {
     if(dir_ == dg::centered)
     {
@@ -508,8 +507,8 @@ void DS<F,M,container>::symv( const container& f, container& dsTdsf)
 
 
 //enables the use of the dg::blas2::symv function 
-template< class F, class M, class V>
-struct MatrixTraits< DS<F,M, V> >
+template< class G, class I, class M, class V>
+struct MatrixTraits< DS<G,I,M, V> >
 {
     typedef double value_type;
     typedef SelfMadeMatrixTag matrix_category;
@@ -517,16 +516,5 @@ struct MatrixTraits< DS<F,M, V> >
 
 ///@endcond
 
-///@addtogroup typedefs
-///@{
-typedef dg::DS<dg::FieldAligned<dg::CylindricalGrid3d<dg::DVec>, dg::IDMatrix, dg::DVec>, dg::DMatrix, dg::DVec> DDS;//!< device DS type
-typedef dg::DS<dg::FieldAligned<dg::CylindricalGrid3d<dg::HVec>, dg::IHMatrix, dg::HVec>, dg::HMatrix, dg::HVec> HDS; //!< host DS type
-#ifdef MPI_VERSION
-typedef dg::DS< dg::MPI_FieldAligned<dg::CylindricalMPIGrid3d<dg::MDVec>, dg::IDMatrix, dg::BijectiveComm< dg::iDVec, dg::DVec >, dg::DVec>, dg::MDMatrix, dg::MDVec > MDDS; //!< MPI device DS type
-typedef dg::DS< dg::MPI_FieldAligned<dg::CylindricalMPIGrid3d<dg::MHVec>, dg::IHMatrix, dg::BijectiveComm< dg::iHVec, dg::HVec >, dg::HVec>, dg::MHMatrix, dg::MHVec > MHDS; //!< MPI host DS type
-#endif //MPI_VERSION
-///@}
-
-
 }//namespace dg
 
diff --git a/inc/geometries/magnetic_field.h b/inc/geometries/magnetic_field.h
index 238a92fd8..45925d665 100644
--- a/inc/geometries/magnetic_field.h
+++ b/inc/geometries/magnetic_field.h
@@ -510,26 +510,6 @@ struct Field
         yp[0] =  y[0]*c_.psipZ(y[0],y[1])/ipol;              //dR/dphi =  R/I Psip_Z
         yp[1] = -y[0]*c_.psipR(y[0],y[1])/ipol ;             //dZ/dphi = -R/I Psip_R
 
-    }
-    /**
-     * @brief \f[   \frac{1}{\hat{B}} = 
-      \frac{\hat{R}}{\hat{R}_0}\frac{1}{ \sqrt{ \hat{I}^2  + \left(\frac{\partial \hat{\psi}_p }{ \partial \hat{R}}\right)^2
-      + \left(\frac{\partial \hat{\psi}_p }{ \partial \hat{Z}}\right)^2}}  \f]
-     */ 
-    double operator()( double R, double Z) const
-    {
-        //modified
-//          return invB_(R,Z)* invB_(R,Z)*ipol_(R,Z)*gp_.R_0/R;
-        return invB_(R,Z);
-    }
-    /**
-     * @brief == operator()(R,Z)
-     */ 
-    double operator()( double R, double Z, double phi) const
-    {
-        return invB_(R,Z,phi);
-
-//         return invB_(R,Z,phi)*invB_(R,Z,phi)*ipol_(R,Z,phi)*gp_.R_0/R;
     }
     double error( const dg::HVec& x0, const dg::HVec& x1)
     {
@@ -552,7 +532,46 @@ struct Field
     MagneticField c_;
     double R_0_;
     InvB<MagneticField>   invB_;
-   
+};
+
+
+template<class container>
+struct DSField
+{
+    DSField( const MagneticField& c)
+
+    private:
+    thrust::host_vector<double> dzetadphi_, detadphi_, dsdphi_;
+
+};
+struct Interpolate
+{
+    Interpolate( const thrust::host_vector<double>& fZeta, 
+                 const thrust::host_vector<double>& fEta, 
+                 const dg::Grid2d& g2d ): 
+        iter0_( dg::create::forward_transform( fZeta, g2d) ), 
+        iter1_( dg::create::forward_transform(  fEta, g2d) ), 
+        g_(g2d), zeta1_(g2d.x1()), eta1_(g2d.y1()){}
+    void operator()(const thrust::host_vector<double>& zeta, thrust::host_vector<double>& fZeta)
+    {
+        fZeta[0] = interpolate( fmod( zeta[0]+zeta1_, zeta1_), fmod( zeta[1]+eta1_, eta1_), iter0_, g_);
+        fZeta[1] = interpolate( fmod( zeta[0]+zeta1_, zeta1_), fmod( zeta[1]+eta1_, eta1_), iter1_, g_);
+        //fZeta[0] = interpolate(  zeta[0], zeta[1], iter0_, g_);
+        //fZeta[1] = interpolate(  zeta[0], zeta[1], iter1_, g_);
+    }
+    void operator()(const std::vector<thrust::host_vector<double> >& zeta, std::vector< thrust::host_vector<double> >& fZeta)
+    {
+        for( unsigned i=0; i<zeta[0].size(); i++)
+        {
+            fZeta[0][i] = interpolate( fmod( zeta[0][i]+zeta1_, zeta1_), fmod( zeta[1][i]+eta1_, eta1_), iter0_, g_);
+            fZeta[1][i] = interpolate( fmod( zeta[0][i]+zeta1_, zeta1_), fmod( zeta[1][i]+eta1_, eta1_), iter1_, g_);
+        }
+    }
+    private:
+    thrust::host_vector<double> iter0_;
+    thrust::host_vector<double> iter1_;
+    dg::Grid2d g_;
+    double zeta1_, eta1_;
 };
 
 ///**
diff --git a/inc/geometries/toroidal.h b/inc/geometries/toroidal.h
index 79eda28cf..8d4bf2088 100644
--- a/inc/geometries/toroidal.h
+++ b/inc/geometries/toroidal.h
@@ -12,6 +12,7 @@ namespace toroidal{
  * @brief Models a slab toroidal field (models aTokamakMagneticField)
  *
  * \f$ B=\frac{R_0}{R}\f$, \f$ \psi_p = 1\f$ and \f$ I = 1\f$.
+ @note The solovev field can also be made to model a todoidal slab field
  */
 struct MagneticField
 {
-- 
GitLab


From 812ace4b7c43606809018e142a1ec41a60551a4b Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 13 Jul 2017 17:27:31 +0200
Subject: [PATCH 035/453] use copydoc on geometry generators and begin DSField

---
 inc/geometries/flux.h              | 24 ++----------------------
 inc/geometries/magnetic_field.h    |  5 +++--
 inc/geometries/simple_orthogonal.h | 13 +------------
 3 files changed, 6 insertions(+), 36 deletions(-)

diff --git a/inc/geometries/flux.h b/inc/geometries/flux.h
index 0231b016c..7ad228f90 100644
--- a/inc/geometries/flux.h
+++ b/inc/geometries/flux.h
@@ -195,17 +195,7 @@ struct FluxGenerator
      */
     double height() const{return 2.*M_PI;}
     /**
-     * @brief Generate the points and the elements of the Jacobian
-     *
-     * Call the width() and height() function before calling this function!
-     * @param zeta1d one-dimensional list of points inside the zeta-domain (0<zeta<width())
-     * @param eta1d one-dimensional list of points inside the eta-domain (0<eta<height())
-     * @param x  = x(zeta,eta)
-     * @param y  = y(zeta,eta)
-     * @param zetaX = zeta_x(zeta,eta)
-     * @param zetaY = zeta_y(zeta,eta)
-     * @param etaX = eta_x(zeta,eta)
-     * @param etaY = eta_y(zeta,eta)
+     * @copydoc aGenerator::operator()(const thrust::host_vector<double>&,const thrust::host_vector<double>&,thrust::host_vector<double>&,thrust::host_vector<double>&,thrust::host_vector<double>&,thrust::host_vector<double>&,thrust::host_vector<double>&,thrust::host_vector<double>&)
      * @note All the resulting vectors are write-only and get properly resized
      */
     void operator()( 
@@ -322,17 +312,7 @@ struct RibeiroFluxGenerator
      */
     double height() const{return 2.*M_PI;}
     /**
-     * @brief Generate the points and the elements of the Jacobian
-     *
-     * Call the width() and height() function before calling this function!
-     * @param zeta1d one-dimensional list of points inside the zeta-domain (0<zeta<width())
-     * @param eta1d one-dimensional list of points inside the eta-domain (0<eta<height())
-     * @param x  = x(zeta,eta)
-     * @param y  = y(zeta,eta)
-     * @param zetaX = zeta_x(zeta,eta)
-     * @param zetaY = zeta_y(zeta,eta)
-     * @param etaX = eta_x(zeta,eta)
-     * @param etaY = eta_y(zeta,eta)
+     * @copydoc aGenerator::operator()(const thrust::host_vector<double>&,const thrust::host_vector<double>&,thrust::host_vector<double>&,thrust::host_vector<double>&,thrust::host_vector<double>&,thrust::host_vector<double>&,thrust::host_vector<double>&,thrust::host_vector<double>&)
      * @note All the resulting vectors are write-only and get properly resized
      */
     void operator()( 
diff --git a/inc/geometries/magnetic_field.h b/inc/geometries/magnetic_field.h
index 45925d665..9f23cad2d 100644
--- a/inc/geometries/magnetic_field.h
+++ b/inc/geometries/magnetic_field.h
@@ -535,15 +535,16 @@ struct Field
 };
 
 
-template<class container>
+template<class Field>
+
 struct DSField
 {
     DSField( const MagneticField& c)
 
     private:
-    thrust::host_vector<double> dzetadphi_, detadphi_, dsdphi_;
 
 };
+//interpolate the two components of a vector field
 struct Interpolate
 {
     Interpolate( const thrust::host_vector<double>& fZeta, 
diff --git a/inc/geometries/simple_orthogonal.h b/inc/geometries/simple_orthogonal.h
index 9d0d512cc..6c8b49da4 100644
--- a/inc/geometries/simple_orthogonal.h
+++ b/inc/geometries/simple_orthogonal.h
@@ -344,19 +344,8 @@ struct SimpleOrthogonal
      */
     bool isConformal()  const{return false;}
     /**
-     * @brief Generate the points and the elements of the Jacobian
-     *
-     * Call the width() and height() function before calling this function!
-     * @param zeta1d one-dimensional list of points inside the zeta-domain (0<zeta<width())
-     * @param eta1d one-dimensional list of points inside the eta-domain (0<eta<height())
-     * @param x  \f$= x(\zeta,\eta)\f$
-     * @param y  \f$= y(\zeta,\eta)\f$
-     * @param zetaX \f$= \zeta_x(\zeta,\eta)\f$
-     * @param zetaY \f$= \zeta_y(\zeta,\eta)\f$
-     * @param etaX \f$= \eta_x(\zeta,\eta)\f$
-     * @param etaY \f$= \eta_y(\zeta,\eta)\f$
+     * @copydoc aGenerator::operator()(const thrust::host_vector<double>&,const thrust::host_vector<double>&,thrust::host_vector<double>&,thrust::host_vector<double>&,thrust::host_vector<double>&,thrust::host_vector<double>&,thrust::host_vector<double>&,thrust::host_vector<double>&)
      * @note All the resulting vectors are write-only and get properly resized
-     * @note The \f$ \zeta\f$ direction is continuous in memory
      */
     void operator()( 
          const thrust::host_vector<double>& zeta1d, 
-- 
GitLab


From 682fe4366530a0e8500fee59b7b4b0c66488394a Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 13 Jul 2017 17:44:11 +0200
Subject: [PATCH 036/453] update documentation on shift_topologic and contains
 in grid classes

---
 inc/dg/backend/grid.h | 58 ++++++++++++++++++++-----------------------
 1 file changed, 27 insertions(+), 31 deletions(-)

diff --git a/inc/dg/backend/grid.h b/inc/dg/backend/grid.h
index 0a3881360..5e8cbce9b 100644
--- a/inc/dg/backend/grid.h
+++ b/inc/dg/backend/grid.h
@@ -115,16 +115,14 @@ struct Grid1d
     }
 
     /**
-     * @brief Shifts a point coordinate due to topology
+     * @brief Shifts a point coordinate if periodic
      *
-     * If you want to construct a point by adding delta X to a given point
-     * x0 then the resulting coordinate x1 might be incorrect due to topologic reasons (periodic boundaries). This function corrects this coordinate
-     * @param x0 starting point (must lie inside of the grid)
+     * This function shifts a point coordinate to its value between x0() and x1() if bcx() returns dg::PER
+     * @param x0 arbitrary point (irrelevant for the function, it's there to be consistent with GridX1d)
      * @param x1 end point (inout)
      */
     void shift_topologic( double x0, double& x1)const
     {
-        assert( contains(x0));
         double deltaX;
         if( x1 > x0_) deltaX = x1 -x0_;
         else deltaX = x1_ - x1;
@@ -139,7 +137,7 @@ struct Grid1d
      * @note Doesn't check periodicity!!
      * @param x point to check
      *
-     * @return true if x is between x0 and x1, false else
+     * @return true if x0()<x<x1(), false else
      */
     bool contains( double x)const
     {
@@ -351,18 +349,17 @@ struct Grid2d
             <<"    "<<bc2str(bcy_)<<"\n";
     }
     /**
-     * @brief Shifts a point coordinate due to topology
+     * @brief Shifts point coordinates if periodic
      *
-     * If you want to construct a point by adding (delta X, delta Y) to a given point
-     * (x0, y0) then the resulting coordinate (x1, y1) might be incorrect due to topologic reasons (periodic boundaries). This function corrects this coordinate
-     * @param x0 starting x-point (must lie inside of the grid)
-     * @param y0 starting y-point (must lie inside of the grid)
-     * @param x1 end x-point (inout)
-     * @param y1 end y-point (inout)
+     * This function shifts point coordinates to its values inside
+     the domain if the respective boundary condition is periodic
+     * @param x0 arbitrary coordinate (irrelevant for the function, it's there to be consistent with GridX2d)
+     * @param y0 arbitrary coordinate (irrelevant for the function, it's there to be consistent with GridX2d)
+     * @param x1 x-coordinate to shift (inout)
+     * @param y1 y-coordinate to shift (inout)
      */
     void shift_topologic( double x0, double y0, double& x1, double& y1)const
     {
-        assert( contains(x0, y0));
         double deltaX;
         if( x1 > x0_) deltaX = (x1 -x0_);
         else deltaX = x1_ - x1;
@@ -381,10 +378,10 @@ struct Grid2d
      * @brief Check if the grid contains a point
      *
      * @note doesn't check periodicity!!
-     * @param x x-point to check
-     * @param y y-point to check
+     * @param x x-coordinate to check
+     * @param y y-coordinate to check
      *
-     * @return true if point is inside, false else
+     * @return true if x0()<x<x1() and y0()<y<y1(), false else
      */
     bool contains( double x, double y)const
     {
@@ -656,20 +653,19 @@ struct Grid3d
     }
 
     /**
-     * @brief Shifts a point coordinate due to topology
+     * @brief Shifts point coordinates if periodic
      *
-     * If you want to construct a point by adding (delta X, delta Y, delta Z) to a given point
-     * (x0, y0, z0) then the resulting coordinate (x1, y1, z1) might be incorrect due to topologic reasons (periodic boundaries). This function corrects this coordinate
-     * @param x0 starting x-point (must lie inside of the grid)
-     * @param y0 starting y-point (must lie inside of the grid)
-     * @param z0 starting y-point (must lie inside of the grid)
-     * @param x1 end x-point (inout)
-     * @param y1 end y-point (inout)
-     * @param z1 end z-point (inout)
+     * This function shifts point coordinates to its values inside
+     the domain if the respective boundary condition is periodic
+     * @param x0 arbitrary x-coordinate (irrelevant for the function, it's there to be consistent with GridX3d)
+     * @param y0 arbitrary y-coordinate (irrelevant for the function, it's there to be consistent with GridX3d)
+     * @param z0 arbitrary z-coordinate (irrelevant for the function, it's there to be consistent with GridX3d)
+     * @param x1 x-coordinate to shift (inout)
+     * @param y1 y-coordinate to shift (inout)
+     * @param z1 z-coordinate to shift (inout)
      */
     void shift_topologic( double x0, double y0, double z0, double& x1, double& y1, double& z1)const
     {
-        assert( contains(x0, y0, z0));
         double deltaX;
         if( x1 > x0_) deltaX = (x1 -x0_);
         else deltaX = x1_ - x1;
@@ -694,11 +690,11 @@ struct Grid3d
      * @brief Check if the grid contains a point
      *
      * @note doesn't check periodicity!!
-     * @param x x-point to check
-     * @param y y-point to check
-     * @param z z-point to check
+     * @param x x-coordinate to check
+     * @param y y-coordinate to check
+     * @param z z-coordinate to check
      *
-     * @return true if x is between x0 and x1, false else
+     * @return true if x0()<x<x1() and y0()<y<y1() and z0()<z<z1() , false else
      */
     bool contains( double x, double y, double z)const
     {
-- 
GitLab


From 4d453d19000563bc3bc9348459d7c632651a99aa Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 13 Jul 2017 18:04:23 +0200
Subject: [PATCH 037/453] further removed R0 from magnetic field quantities

---
 inc/dg/backend/interpolation.cuh |   4 +-
 inc/geometries/magnetic_field.h  | 118 +++++++++----------------------
 2 files changed, 36 insertions(+), 86 deletions(-)

diff --git a/inc/dg/backend/interpolation.cuh b/inc/dg/backend/interpolation.cuh
index b759a9065..389bc934e 100644
--- a/inc/dg/backend/interpolation.cuh
+++ b/inc/dg/backend/interpolation.cuh
@@ -467,6 +467,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const Grid3d& g_
  * @param in input
  * @param g grid
  *
+ * @ingroup misc
  * @return the vector in LSPACE
  */
 thrust::host_vector<double> forward_transform( const thrust::host_vector<double>& in, const Grid2d& g)
@@ -490,11 +491,12 @@ thrust::host_vector<double> forward_transform( const thrust::host_vector<double>
  *
  * @param x X-coordinate of interpolation point
  * @param y Y-coordinate of interpolation point
- * @param v The vector to interpolate in LSPACE
+ * @param v The vector to interpolate in LSPACE, s.a. dg::forward_transform( )
  * @param g The Grid on which to operate
  *
  * @ingroup interpolation
  * @return interpolated point
+ * @note g.contains(x,y) must return true
  */
 double interpolate( double x, double y,  const thrust::host_vector<double>& v, const Grid2d& g )
 {
diff --git a/inc/geometries/magnetic_field.h b/inc/geometries/magnetic_field.h
index 9f23cad2d..12abe69da 100644
--- a/inc/geometries/magnetic_field.h
+++ b/inc/geometries/magnetic_field.h
@@ -174,7 +174,7 @@ struct BZ
 template<class MagneticField>
 struct CurvatureNablaBR
 {
-    CurvatureNablaBR(const MagneticField& c, double R0 ): invB_(c, R0), bZ_(c, R0) { }
+    CurvatureNablaBR(const MagneticField& c): invB_(c), bZ_(c) { }
     /**
      * @brief \f[ \mathcal{\hat{K}}^{\hat{R}}_{\nabla B} =-\frac{1}{ \hat{B}^2}  \frac{\partial \hat{B}}{\partial \hat{Z}}  \f]
      */ 
@@ -202,7 +202,7 @@ struct CurvatureNablaBR
 template<class MagneticField>
 struct CurvatureNablaBZ
 {
-    CurvatureNablaBZ( const MagneticField& c, double R0): invB_(c, R0), bR_(c, R0) { }
+    CurvatureNablaBZ( const MagneticField& c): invB_(c), bR_(c) { }
  /**
  * @brief \f[  \mathcal{\hat{K}}^{\hat{Z}}_{\nabla B} =\frac{1}{ \hat{B}^2}   \frac{\partial \hat{B}}{\partial \hat{R}} \f]
  */    
@@ -251,8 +251,8 @@ struct CurvatureKappaR
 template<class MagneticField>
 struct CurvatureKappaZ
 {
-    CurvatureKappaZ( const MagneticField c, double R0):
-        invB_(c, R0) { }
+    CurvatureKappaZ( const MagneticField c):
+        invB_(c) { }
  /**
  * @brief \f[  \mathcal{\hat{K}}^{\hat{Z}}_{\vec{\kappa}} = - \frac{1}{\hat{R} \hat{B}} \f]
  */    
@@ -278,9 +278,9 @@ struct CurvatureKappaZ
 template<class MagneticField>
 struct DivCurvatureKappa
 {
-    DivCurvatureKappa( const MagneticField& c, double R0):
-        invB_(c, R0),
-        bZ_(c, R0){ }
+    DivCurvatureKappa( const MagneticField& c):
+        invB_(c),
+        bZ_(c){ }
  /**
  * @brief \f[  \vec{\hat{\nabla}}\cdot \mathcal{\hat{K}}_{\vec{\kappa}}  = \frac{1}{\hat{R}  \hat{B}^2 } \partial_{\hat{Z}} \hat{B}\f]
  */    
@@ -307,7 +307,7 @@ struct DivCurvatureKappa
 template<class MagneticField>
 struct GradLnB
 {
-    GradLnB( const MagneticField& c): R_0_(c.R_0), c_(c), invB_(c, R0), bR_(c, R0), bZ_(c, R0) { } 
+    GradLnB( const MagneticField& c): R_0_(c.R_0), c_(c), invB_(c), bR_(c), bZ_(c) { } 
     /**
  * @brief \f[  \hat{\nabla}_\parallel \ln{(\hat{B})} = \frac{1}{\hat{R}\hat{B}^2 } \left[ \hat{B}, \hat{\psi}_p\right]_{\hat{R}\hat{Z}} \f]
  */ 
@@ -335,7 +335,7 @@ struct GradLnB
 template<class MagneticField>
 struct FieldP
 {
-    FieldP( const MagneticField& c, double R0): R_0(R0), c_(c){}
+    FieldP( const MagneticField& c): R_0(c.R_0), c_(c){}
     double operator()( double R, double Z, double phi) const
     {
         return R_0*c_.ipol(R,Z)/R/R;
@@ -353,7 +353,7 @@ struct FieldP
 template<class MagneticField>
 struct FieldR
 {
-    FieldR( const MagneticField& c, double R0): R_0(R0), c_(c){}
+    FieldR( const MagneticField& c): R_0(c.R_0), c_(c){}
     double operator()( double R, double Z) const
     {
         return  R_0/R*c_.psipZ(R,Z);
@@ -378,7 +378,7 @@ struct FieldR
 template<class MagneticField>
 struct FieldZ
 {
-    FieldZ( const MagneticField& c, double R0): R_0(R0), c_(c){}
+    FieldZ( const MagneticField& c): R_0(c.R_0), c_(c){}
     double operator()( double R, double Z) const
     {
         return  -R_0/R*c_.psipR(R,Z);
@@ -403,7 +403,7 @@ struct FieldZ
 template<class MagneticField>
 struct FieldT
 {
-    FieldT( const MagneticField& c):  R_0_(c.R_0), fieldR_(c, R0), fieldZ_(c, R0){}
+    FieldT( const MagneticField& c):  R_0_(c.R_0), fieldR_(c), fieldZ_(c){}
   /**
  * @brief \f[  B^{\theta} = 
  * B^R\partial_R\theta + B^Z\partial_Z\theta\f]
@@ -435,7 +435,7 @@ struct FieldT
 template<class MagneticField>
 struct BHatR
 {
-    BHatR( const MagneticField& c, double R0): c_(c), R_0(R0), invB_(c, R0){ }
+    BHatR( const MagneticField& c): c_(c), R_0(c.R_0), invB_(c){ }
     double operator()( double R, double Z, double phi) const
     {
         return  invB_(R,Z)*R_0/R*c_.psipZ(R,Z);
@@ -454,7 +454,7 @@ struct BHatR
 template<class MagneticField>
 struct BHatZ
 {
-    BHatZ( const MagneticField& c, double R0): c_(c), R_0(R0), invB_(c, R0){ }
+    BHatZ( const MagneticField& c): c_(c), R_0(c.R_0), invB_(c){ }
 
     double operator()( double R, double Z, double phi) const
     {
@@ -474,7 +474,7 @@ struct BHatZ
 template<class MagneticField>
 struct BHatP
 {
-    BHatP( const MagneticField& c, double R0): c_(c), R_0(R0), invB_(c, R0){ }
+    BHatP( const MagneticField& c): c_(c), R_0(c.R_0), invB_(c){ }
     double operator()( double R, double Z, double phi) const
     {
         return invB_(R,Z)*R_0*c_.ipol(R,Z)/R/R;
@@ -497,7 +497,7 @@ struct BHatP
 template<class MagneticField>
 struct Field
 {
-    Field( const MagneticField& c):c_(c), R_0_(c.R_0), invB_(c, R0) { }
+    Field( const MagneticField& c):c_(c), invB_(c), R_0_(c.R_0) { }
     /**
      * @brief \f[ \frac{d \hat{R} }{ d \varphi}  = \frac{\hat{R}}{\hat{I}} \frac{\partial\hat{\psi}_p}{\partial \hat{Z}}, \hspace {3 mm}
      \frac{d \hat{Z} }{ d \varphi}  =- \frac{\hat{R}}{\hat{I}} \frac{\partial \hat{\psi}_p}{\partial \hat{R}} , \hspace {3 mm}
@@ -530,8 +530,8 @@ struct Field
     
     private:
     MagneticField c_;
+    InvB invB_;
     double R_0_;
-    InvB<MagneticField>   invB_;
 };
 
 
@@ -539,9 +539,24 @@ template<class Field>
 
 struct DSField
 {
-    DSField( const MagneticField& c)
+    DSField( const MagneticField& c, const Geometry& g)
+    {
+        InvB invB(c);
+        FieldR fieldR(c);
+
+    }
 
+    void operator()(const thrust::host_vector<double>& y, thrust::host_vector<double>& yp)
+    {
+        g_.shift_topologic( y[0], y[1]
+        //shift points onto domain
+        //if contains is false return 0
+        //else interpolate
+        interpolate( zeta, eta, 
+    }
     private:
+    thrust::host_vector<double> dzetadphi_, detadphi_, dsdphi_;
+    Geometry g_;
 
 };
 //interpolate the two components of a vector field
@@ -575,73 +590,6 @@ struct Interpolate
     double zeta1_, eta1_;
 };
 
-///**
-// * @brief Integrates the equations for a field line and 1/B
-// */ 
-//template<class MagneticField>
-//struct OrthogonalField
-//{
-//    OrthogonalField( const MagneticField& c, double R0, dg::solovev::GeomParameters gp, const dg::Grid2d& gXY, const thrust::host_vector<double>& f2):
-//        c_(c), R_0_(R0), invB_(c, R0), gXY_(gXY), 
-//        g_(dg::create::forward_transform(f2, gXY)) 
-//    { }
-//
-//    /**
-//     * @brief \f[ \frac{d \hat{R} }{ d \varphi}  = \frac{\hat{R}}{\hat{I}} \frac{\partial\hat{\psi}_p}{\partial \hat{Z}}, \hspace {3 mm}
-//     \frac{d \hat{Z} }{ d \varphi}  =- \frac{\hat{R}}{\hat{I}} \frac{\partial \hat{\psi}_p}{\partial \hat{R}} , \hspace {3 mm}
-//     \frac{d \hat{l} }{ d \varphi}  =\frac{\hat{R}^2 \hat{B}}{\hat{I}  \hat{R}_0}  \f]
-//     */ 
-//    void operator()( const dg::HVec& y, dg::HVec& yp)
-//    {
-//        //x,y,s,R,Z
-//        double psipR = c_.psipR(y[3],y[4]), psipZ = c_.psipZ(y[3],y[4]), ipol = c_.ipol( y[3],y[4]);
-//        double xs = y[0],ys=y[1];
-//        gXY_.shift_topologic( y[0], M_PI, xs,ys);
-//        double g = dg::interpolate( xs,  ys, g_, gXY_);
-//        yp[0] = 0;
-//        yp[1] = y[3]*g*(psipR*psipR+psipZ*psipZ)/ipol;
-//        //yp[1] = g/ipol;
-//        yp[2] =  y[3]*y[3]/invB_(y[3],y[4])/ipol/R_0_; //ds/dphi =  R^2 B/I/R_0_hat
-//        yp[3] =  y[3]*psipZ/ipol;              //dR/dphi =  R/I Psip_Z
-//        yp[4] = -y[3]*psipR/ipol;             //dZ/dphi = -R/I Psip_R
-//
-//    }
-//    /**
-//     * @brief \f[   \frac{1}{\hat{B}} = 
-//      \frac{\hat{R}}{\hat{R}_0}\frac{1}{ \sqrt{ \hat{I}^2  + \left(\frac{\partial \hat{\psi}_p }{ \partial \hat{R}}\right)^2
-//      + \left(\frac{\partial \hat{\psi}_p }{ \partial \hat{Z}}\right)^2}}  \f]
-//     */ 
-//    double operator()( double R, double Z) const { return invB_(R,Z); }
-//    /**
-//     * @brief == operator()(R,Z)
-//     */ 
-//    double operator()( double R, double Z, double phi) const { return invB_(R,Z,phi); }
-//    double error( const dg::HVec& x0, const dg::HVec& x1)
-//    {
-//        //compute error in x,y,s
-//        return sqrt( (x0[0]-x1[0])*(x0[0]-x1[0]) +(x0[1]-x1[1])*(x0[1]-x1[1])+(x0[2]-x1[2])*(x0[2]-x1[2]));
-//    }
-//    bool monitor( const dg::HVec& end){ 
-//        if ( std::isnan(end[1]) || std::isnan(end[2]) || std::isnan(end[3])||std::isnan( end[4]) ) 
-//        {
-//            return false;
-//        }
-//        if( (end[3] < 1e-5) || end[3]*end[3] > 1e10 ||end[1]*end[1] > 1e10 ||end[2]*end[2] > 1e10 ||(end[4]*end[4] > 1e10) )
-//        {
-//            return false;
-//        }
-//        return true;
-//    }
-//    
-//    private:
-//    MagneticField c_;
-//    double R_0_;
-//    dg::magnetic::InvB<MagneticField>   invB_;
-//    const dg::Grid2d gXY_;
-//    thrust::host_vector<double> g_;
-//};
-
-
 
 } //namespace geo
 } //namespace dg
-- 
GitLab


From bf5f8a1b5c43cef426efec7560cfe34d97137a82 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 13 Jul 2017 18:32:57 +0200
Subject: [PATCH 038/453] created a new DSField in magnetic_field

---
 inc/geometries/magnetic_field.h | 87 +++++++++++++++++----------------
 1 file changed, 45 insertions(+), 42 deletions(-)

diff --git a/inc/geometries/magnetic_field.h b/inc/geometries/magnetic_field.h
index 12abe69da..993f0b852 100644
--- a/inc/geometries/magnetic_field.h
+++ b/inc/geometries/magnetic_field.h
@@ -490,7 +490,7 @@ struct BHatP
 ///@} 
 
 /**
- * @brief Integrates the equations for a field line and 1/B
+ * @brief Integrates the equations for a field line 
  * @tparam MagneticField models aTokamakMagneticField
  * @ingroup misc
  */ 
@@ -535,59 +535,62 @@ struct Field
 };
 
 
-template<class Field>
-
+template< class GeometryPerp>
 struct DSField
 {
-    DSField( const MagneticField& c, const Geometry& g)
+    template<class MagneticField>
+    DSField( const MagneticField& c, const GeometryPerp& g)
     {
-        InvB invB(c);
-        FieldR fieldR(c);
-
-    }
-
-    void operator()(const thrust::host_vector<double>& y, thrust::host_vector<double>& yp)
+        InvB<MagneticField> invB(c);
+        FieldR<MagneticField> fieldR(c);
+        FieldZ<MagneticField> fieldZ(c);
+        thrust::host_vector<double> b_zeta, b_eta;
+        dg::geo::pushForwardPerp( fieldR, fieldZ, b_zeta, b_eta, g);
+        FieldP<MagneticField> fieldP(c);
+        thrust::host_vector<double> b_phi = dg::pullback( fieldP, g);
+        Bmodule<MagneticField> bmod( c);
+        thrust::host_vector<double> b_mod = dg::pullback( bmod, g);
+        dg::blas1::pointwiseDivide( b_zeta, b_phi, b_zeta);
+        dg::blas1::pointwiseDivide( b_eta,  b_phi, b_eta);
+        dg::blas1::pointwiseDivide( b_mod,  b_phi, b_mod);
+        dzetadphi_ = dg::forward_transform( b_zeta, g );
+        detadphi_  = dg::forward_transform( b_eta, g );
+        dsdphi_    = dg::forward_transform( b_mod, g );
+    }
+
+    void operator()(thrust::host_vector<double> y, thrust::host_vector<double>& yp)
     {
-        g_.shift_topologic( y[0], y[1]
-        //shift points onto domain
-        //if contains is false return 0
-        //else interpolate
-        interpolate( zeta, eta, 
+        g_.shift_topologic( y[0], y[1], y[0], y[1]); //shift points onto domain
+        if( !g_.contains( y[0], y[1])) yp[0] = yp[1]= yp[2] = 0;
+        else
+        {
+            //else interpolate
+            yp[0] = interpolate( y[0], y[1], dzetadphi_, g_);
+            yp[1] = interpolate( y[0], y[1], detadphi_, g_);
+            yp[2] = interpolate( y[0], y[1], dsphi_, g_);
+        }
     }
-    private:
-    thrust::host_vector<double> dzetadphi_, detadphi_, dsdphi_;
-    Geometry g_;
 
-};
-//interpolate the two components of a vector field
-struct Interpolate
-{
-    Interpolate( const thrust::host_vector<double>& fZeta, 
-                 const thrust::host_vector<double>& fEta, 
-                 const dg::Grid2d& g2d ): 
-        iter0_( dg::create::forward_transform( fZeta, g2d) ), 
-        iter1_( dg::create::forward_transform(  fEta, g2d) ), 
-        g_(g2d), zeta1_(g2d.x1()), eta1_(g2d.y1()){}
-    void operator()(const thrust::host_vector<double>& zeta, thrust::host_vector<double>& fZeta)
+    double error( const dg::HVec& x0, const dg::HVec& x1)
     {
-        fZeta[0] = interpolate( fmod( zeta[0]+zeta1_, zeta1_), fmod( zeta[1]+eta1_, eta1_), iter0_, g_);
-        fZeta[1] = interpolate( fmod( zeta[0]+zeta1_, zeta1_), fmod( zeta[1]+eta1_, eta1_), iter1_, g_);
-        //fZeta[0] = interpolate(  zeta[0], zeta[1], iter0_, g_);
-        //fZeta[1] = interpolate(  zeta[0], zeta[1], iter1_, g_);
+        return sqrt( (x0[0]-x1[0])*(x0[0]-x1[0]) +(x0[1]-x1[1])*(x0[1]-x1[1])+(x0[2]-x1[2])*(x0[2]-x1[2]));
     }
-    void operator()(const std::vector<thrust::host_vector<double> >& zeta, std::vector< thrust::host_vector<double> >& fZeta)
-    {
-        for( unsigned i=0; i<zeta[0].size(); i++)
+    bool monitor( const dg::HVec& end){ 
+        if ( std::isnan(end[0]) || std::isnan(end[1]) || std::isnan(end[2]) ) 
+        {
+            return false;
+        }
+        //if new integrated point outside domain
+        if ((1e-5 > end[0]  ) || (1e10 < end[0])  ||(-1e10  > end[1]  ) || (1e10 < end[1])||(-1e10 > end[2]  ) || (1e10 < end[2])  )
         {
-            fZeta[0][i] = interpolate( fmod( zeta[0][i]+zeta1_, zeta1_), fmod( zeta[1][i]+eta1_, eta1_), iter0_, g_);
-            fZeta[1][i] = interpolate( fmod( zeta[0][i]+zeta1_, zeta1_), fmod( zeta[1][i]+eta1_, eta1_), iter1_, g_);
+            return false;
         }
+        return true;
     }
     private:
-    thrust::host_vector<double> iter0_;
-    thrust::host_vector<double> iter1_;
-    dg::Grid2d g_;
-    double zeta1_, eta1_;
+    thrust::host_vector<double> dzetadphi_, detadphi_, dsdphi_;
+    GeometryPerp g_;
+
 };
 
 
-- 
GitLab


From 912ba1ad680b64574a4504b306ba2b1ebed6db20 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 13 Jul 2017 20:28:11 +0200
Subject: [PATCH 039/453] removed commented field in ribeiro.h

---
 inc/geometries/ribeiro.h | 81 ----------------------------------------
 1 file changed, 81 deletions(-)

diff --git a/inc/geometries/ribeiro.h b/inc/geometries/ribeiro.h
index 287de3bc7..69a5da064 100644
--- a/inc/geometries/ribeiro.h
+++ b/inc/geometries/ribeiro.h
@@ -308,86 +308,5 @@ struct Ribeiro
     int mode_; //0 = ribeiro, 1 = equalarc
 };
 
-///**
-// * @brief Integrates the equations for a field line and 1/B
-// */ 
-//struct Field
-//{
-//    Field( dg::geo::detail::GeomParameters gp,const thrust::host_vector<double>& x, const thrust::host_vector<double>& f_x):
-//        gp_(gp),
-//        psipR_(gp), psipZ_(gp),
-//        ipol_(gp), invB_(gp), last_idx(0), x_(x), fx_(f_x)
-//    { }
-//
-//    /**
-//     * @brief \f[ \frac{d \hat{R} }{ d \varphi}  = \frac{\hat{R}}{\hat{I}} \frac{\partial\hat{\psi}_p}{\partial \hat{Z}}, \hspace {3 mm}
-//     \frac{d \hat{Z} }{ d \varphi}  =- \frac{\hat{R}}{\hat{I}} \frac{\partial \hat{\psi}_p}{\partial \hat{R}} , \hspace {3 mm}
-//     \frac{d \hat{l} }{ d \varphi}  =\frac{\hat{R}^2 \hat{B}}{\hat{I}  \hat{R}_0}  \f]
-//     */ 
-//    void operator()( const dg::HVec& y, dg::HVec& yp)
-//    {
-//        //x,y,s,R,Z
-//        double psipR = psipR_(y[3],y[4]), psipZ = psipZ_(y[3],y[4]), ipol = ipol_( y[3],y[4]);
-//        double fx = find_fx( y[0]);
-//        yp[0] = 0;
-//        yp[1] = fx*y[3]*(0.0+1.00*(psipR*psipR+psipZ*psipZ))/ipol;
-//        yp[2] =  y[3]*y[3]/invB_(y[3],y[4])/ipol/gp_.R_0; //ds/dphi =  R^2 B/I/R_0_hat
-//        yp[3] =  y[3]*psipZ/ipol;              //dR/dphi =  R/I Psip_Z
-//        yp[4] = -y[3]*psipR/ipol;             //dZ/dphi = -R/I Psip_R
-//
-//    }
-//    /**
-//     * @brief \f[   \frac{1}{\hat{B}} = 
-//      \frac{\hat{R}}{\hat{R}_0}\frac{1}{ \sqrt{ \hat{I}^2  + \left(\frac{\partial \hat{\psi}_p }{ \partial \hat{R}}\right)^2
-//      + \left(\frac{\partial \hat{\psi}_p }{ \partial \hat{Z}}\right)^2}}  \f]
-//     */ 
-//    double operator()( double R, double Z) const { return invB_(R,Z); }
-//    /**
-//     * @brief == operator()(R,Z)
-//     */ 
-//    double operator()( double R, double Z, double phi) const { return invB_(R,Z,phi); }
-//    double error( const dg::HVec& x0, const dg::HVec& x1)
-//    {
-//        //compute error in x,y,s
-//        return sqrt( (x0[0]-x1[0])*(x0[0]-x1[0]) +(x0[1]-x1[1])*(x0[1]-x1[1])+(x0[2]-x1[2])*(x0[2]-x1[2]));
-//    }
-//    bool monitor( const dg::HVec& end){ 
-//        if ( std::isnan(end[1]) || std::isnan(end[2]) || std::isnan(end[3])||std::isnan( end[4]) ) 
-//        {
-//            return false;
-//        }
-//        if( (end[3] < 1e-5) || end[3]*end[3] > 1e10 ||end[1]*end[1] > 1e10 ||end[2]*end[2] > 1e10 ||(end[4]*end[4] > 1e10) )
-//        {
-//            return false;
-//        }
-//        return true;
-//    }
-//    
-//    private:
-//    double find_fx(double x) 
-//    {
-//        if( fabs(x-x_[last_idx]) < 1e-12)
-//            return fx_[last_idx];
-//        for( unsigned i=0; i<x_.size(); i++)
-//            if( fabs(x-x_[i]) < 1e-12)
-//            {
-//                last_idx = (int)i;
-//                return fx_[i];
-//            }
-//        std::cerr << "x not found!!\n";
-//        return 0;
-//    }
-//    
-//    dg::geo::GeomParameters gp_;
-//    dg::geo::PsipR  psipR_;
-//    dg::geo::PsipZ  psipZ_;
-//    dg::geo::Ipol   ipol_;
-//    dg::geo::InvB   invB_;
-//    int last_idx;
-//    thrust::host_vector<double> x_;
-//    thrust::host_vector<double> fx_;
-//   
-//};
-//
 } //namespace geo
 } //namespace dg
-- 
GitLab


From 0159d080f6ee9c25998653adf4e66d8a5d0fcf93 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 13 Jul 2017 21:31:41 +0200
Subject: [PATCH 040/453] introduced GeneralComm, other copy operators in
 collective

---
 inc/dg/backend/collective_mpit.cu |   4 +-
 inc/dg/backend/mpi_collective.h   | 197 +++++++++++++++++++++---------
 inc/dg/dg_doc.h                   |   1 -
 3 files changed, 139 insertions(+), 63 deletions(-)

diff --git a/inc/dg/backend/collective_mpit.cu b/inc/dg/backend/collective_mpit.cu
index fc84f60a1..d2e7a4728 100644
--- a/inc/dg/backend/collective_mpit.cu
+++ b/inc/dg/backend/collective_mpit.cu
@@ -43,7 +43,7 @@ int main( int argc, char * argv[])
             std::cerr <<"Rank "<<rank<<" TEST FAILED"<<std::endl;
     }
     MPI_Barrier(MPI_COMM_WORLD);
-    if(rank==0)std::cout << "Test SurjectiveComm:\n";
+    if(rank==0)std::cout << "Test SurjectiveComm and GeneralComm:\n";
     Nx = 3, Ny = 3; 
     thrust::host_vector<double> vec( Nx*Ny, rank), result( Nx*Ny);
     thrust::host_vector<int> idx( (Nx+1)*(Ny+1)), pids( (Nx+1)*(Ny+1));
@@ -53,7 +53,7 @@ int main( int argc, char * argv[])
         pids[i] = rank;
         if( i>=Nx*Ny) pids[i] = (rank+1)%size;
     }
-    dg::SurjectiveComm<thrust::host_vector<int>, thrust::host_vector<double> > s( idx, pids, MPI_COMM_WORLD);
+    dg::GeneralComm<thrust::host_vector<int>, thrust::host_vector<double> > s( idx, pids, MPI_COMM_WORLD);
     receive = s.global_gather( vec);
     //for( unsigned i=0; i<(Nx+1)*(Ny+1); i++)
     //    if(rank==0) std::cout << i<<"\t "<< receive[i] << std::endl;
diff --git a/inc/dg/backend/mpi_collective.h b/inc/dg/backend/mpi_collective.h
index 6abbc9f49..fefa46a38 100644
--- a/inc/dg/backend/mpi_collective.h
+++ b/inc/dg/backend/mpi_collective.h
@@ -12,11 +12,6 @@
 #include "dg/blas1.h"
 #include "mpi_vector.h"
 
-//TODO: use Buffer class from MPI_Vector, since Device cannot(!) be 
-//arbitrary type (has to be on device if Index is on device e.g.)
-
-//TODO: The MPI_Alltoallv function only send DOUBLEs so no int vector can be used 
-//
 namespace dg{
 
 
@@ -164,32 +159,15 @@ struct BijectiveComm
      * @param comm An MPI Communicator that contains the participants of the scatter/gather
      * @note The actual scatter/gather map is constructed from the given map so the result behaves as if pids was the actual scatter/gather map on the buffer
      */
-    BijectiveComm( thrust::host_vector<int> pids, MPI_Comm comm): idx_(pids)
+    BijectiveComm( const thrust::host_vector<int>& pids, MPI_Comm comm)
     {
-        int rank, size; 
-        MPI_Comm_size( comm, &size);
-        MPI_Comm_rank( comm, &rank);
-        for( unsigned i=0; i<pids.size(); i++)
-            assert( 0 <= pids[i] && pids[i] < size);
-        thrust::host_vector<int> index(pids);
-        thrust::sequence( index.begin(), index.end());
-        thrust::stable_sort_by_key( pids.begin(), pids.end(), index.begin());//note: this also sorts the pids
-        idx_=index;
-        //now we can repeat/invert the sort by a gather/scatter operation with index as map 
-        //i.e. we could sort pids by a gather 
+        construct( pids, comm);
+    }
 
-        //Now construct the collective object by getting the number of elements to send
-        thrust::host_vector<int> one( pids.size(), 1), keys(one), number(one);
-        typedef thrust::host_vector<int>::iterator iterator;
-        thrust::pair< iterator, iterator> new_end = 
-            thrust::reduce_by_key( pids.begin(), pids.end(), //sorted!
-                one.begin(), keys.begin(), number.begin() ); 
-        unsigned distance = thrust::distance( keys.begin(), new_end.first);
-        thrust::host_vector<int> sendTo( size, 0 );
-        for( unsigned i=0; i<distance; i++)
-            sendTo[keys[i]] = number[i];
-        p_.construct( sendTo, comm);
-        values_.data()->resize( idx_.size());
+    template<class OtherIndex, class OtherVector>
+    BijectiveComm( const BijectiveComm<OtherIndex, OtherVector>& src)
+    {
+        construct( src.get_pids(), src.communicator());
     }
 
     /**
@@ -197,7 +175,6 @@ struct BijectiveComm
      *
      * The order of the received elements is according to their original array index 
      * (i.e. a[0] appears before a[1]) and their process rank of origin ( i.e. values from rank 0 appear before values from rank 1)
-     * @tparam Vector a Vector
      * @param values data to send (must have the size given 
      * by the map in the constructor, s.a. size())
      *
@@ -247,15 +224,52 @@ struct BijectiveComm
     * @return MPI Communicator
     */
     MPI_Comm communicator() const {return p_.communicator();}
+    /**
+    * @brief These are the pids that was given in the Constructor
+    *
+    * @return the vector given in the constructor
+    */
+    const thrust::host_vector<int> get_pids()const{return pids_;}
     private:
+    void construct( thrust::host_vector<int> pids, MPI_Comm comm)
+    {
+        pids_ = pids;
+        idx_.resize( pids.size());
+        dg::blas1::transfer( pids, idx_);
+        int rank, size; 
+        MPI_Comm_size( comm, &size);
+        MPI_Comm_rank( comm, &rank);
+        for( unsigned i=0; i<pids.size(); i++)
+            assert( 0 <= pids[i] && pids[i] < size);
+        thrust::host_vector<int> index(pids);
+        thrust::sequence( index.begin(), index.end());
+        thrust::stable_sort_by_key( pids.begin(), pids.end(), index.begin());//note: this also sorts the pids
+        idx_=index;
+        //now we can repeat/invert the sort by a gather/scatter operation with index as map 
+        //i.e. we could sort pids by a gather 
+
+        //Now construct the collective object by getting the number of elements to send
+        thrust::host_vector<int> one( pids.size(), 1), keys(one), number(one);
+        typedef thrust::host_vector<int>::iterator iterator;
+        thrust::pair< iterator, iterator> new_end = 
+            thrust::reduce_by_key( pids.begin(), pids.end(), //sorted!
+                one.begin(), keys.begin(), number.begin() ); 
+        unsigned distance = thrust::distance( keys.begin(), new_end.first);
+        thrust::host_vector<int> sendTo( size, 0 );
+        for( unsigned i=0; i<distance; i++)
+            sendTo[keys[i]] = number[i];
+        p_.construct( sendTo, comm);
+        values_.data()->resize( idx_.size());
+    }
     Buffer<Vector> values_;
     Index idx_;
     Collective<Index, Vector> p_;
+    thrust::host_vector<int> pids_;
 };
 
 /**
  * @ingroup mpi_structures
- * @brief Struct that performs general collective scatter and gather operations across processes on distributed vectors using mpi
+ * @brief Struct that performs surjective collective scatter and gather operations across processes on distributed vectors using mpi
  *
  * This Communicator can perform general global gather and
  scatter operations. We only assume that the gather/scatter map
@@ -284,6 +298,40 @@ struct SurjectiveComm
     */
     SurjectiveComm( const thrust::host_vector<int>& localGatherMap, const thrust::host_vector<int>& pidGatherMap, MPI_Comm comm): bijectiveComm_(pidGatherMap, comm)
     {
+        construct( localGatherMap, pidGatherMap, comm);
+    }
+
+    template<class OtherIndex, class OtherVector>
+    SurjectiveComm( const SurjectiveComm<OtherIndex, OtherVector>& src): bijectiveComm_(src.getPidGatherMap(), src.communicator())
+    {
+        construct( src.getLocalGatherMap(), src.getPidGatherMap(), src.communicator());
+    }
+    Vector global_gather( const Vector& values)const
+    {
+        //gather values to store
+        thrust::gather( gatherMap_.begin(), gatherMap_.end(), values.begin(), store_.data()->begin());
+        //now gather from store into buffer
+        Vector buffer( buffer_size_);
+        bijectiveComm_.global_scatter_reduce( *store_.data(), buffer);
+        return buffer;
+    }
+    void global_scatter_reduce( const Vector& toScatter, Vector& values)
+    {
+        //first gather values into store
+        Vector store_t = bijectiveComm_.global_gather( toScatter);
+        //now perform a local sort, reduce and scatter operation
+        thrust::gather( sortMap_.begin(), sortMap_.end(), store_t.begin(), store_.data()->begin());
+        thrust::reduce_by_key( sortedGatherMap_.begin(), sortedGatherMap_.end(), store_.data()->begin(), keys_.data()->begin(), values.begin());
+    }
+    unsigned size() const {return buffer_size_;}
+    const thrust::host_vector<int> getLocalGatherMap() const {return localGatherMap_;}
+    const thrust::host_vector<int> getPidGatherMap() const {return pidGatherMap_;}
+    MPI_Comm communicator()const{return bijectiveComm_.communicator();}
+    const Index& getSortedGatherMap() const {return sortedGatherMap_;}
+    private:
+    void construct( thrust::host_vector<int> localGatherMap, thrust::host_vector<int> pidGatherMap, MPI_Comm comm)
+    {
+        localGatherMap_ = localGatherMap, pidGatherMap_ = pidGatherMap;
         buffer_size_ = localGatherMap.size();
         assert( buffer_size_ == pidGatherMap.size());
         //the bijectiveComm behaves as if we had given the gather map for the store
@@ -293,6 +341,8 @@ struct SurjectiveComm
         Index gatherMap = bijectiveComm_.global_gather( localGatherMap_d);
         dg::blas1::transfer(gatherMap, gatherMap_);
         store_size_ = gatherMap_.size();
+        store_.data()->resize( store_size_);
+        keys_.data()->resize( store_size_);
 
         //now prepare a reduction map and a scatter map
         thrust::host_vector<int> sortMap(gatherMap);
@@ -301,44 +351,71 @@ struct SurjectiveComm
         dg::blas1::transfer( sortMap, sortMap_);
         dg::blas1::transfer( gatherMap, sortedGatherMap_);
         //now we can repeat/invert the sort by a gather/scatter operation with sortMap as map 
+    }
+    unsigned buffer_size_, store_size_;
+    BijectiveComm<Index, Vector> bijectiveComm_;
+    Index gatherMap_; 
+    Index sortMap_, sortedGatherMap_;
+    Buffer<Index> keys_;
+    Buffer<Vector> store_;
+    thrust::host_vector<int> localGatherMap_, pidGatherMap_;
+};
 
-        thrust::host_vector<int> one( gatherMap.size(), 1), keys(one), number(one);
-        typedef thrust::host_vector<int>::iterator iterator;
-        thrust::pair< iterator, iterator> new_end = 
-            thrust::reduce_by_key( gatherMap.begin(), gatherMap.end(), //sorted!
-                one.begin(), keys.begin(), number.begin() ); 
-        unsigned distance = thrust::distance( keys.begin(), new_end.first);
-        vector_size_ = distance;
-        store_.data()->resize( store_size_);
-        keys_.data()->resize( vector_size_);
+/**
+ * @ingroup mpi_structures
+ * @brief Struct that performs general collective scatter and gather operations across processes on distributed vectors using mpi
+ *
+ * This Communicator can perform general global gather and
+ scatter operations. 
+ @tparam Index an integer Vector
+ @tparam Vector a Vector (the data type of Vector must be double)
+ @note models aCommunicator
+ */
+template< class Index, class Vector>
+struct GeneralComm
+{
+    GeneralComm(){}
+    GeneralComm( const thrust::host_vector<int>& localGatherMap, const thrust::host_vector<int>& pidGatherMap, MPI_Comm comm): surjectiveComm_(localGatherMap, pidGatherMap, comm)
+    {
+        construct( localGatherMap, pidGatherMap, comm);
+    }
+    template<class OtherIndex, class OtherVector>
+    GeneralComm( const GeneralComm<OtherIndex, OtherVector>& src): surjectiveComm_(src.getLocalGatherMap(), src.getPidGatherMap(), src.communicator())
+    {
+        construct( src.getLocalGatherMap(), src.getPidGatherMap(), src.communicator());
     }
     Vector global_gather( const Vector& values)const
     {
-        assert( values.size() == vector_size_);
-        //gather values to store
-        thrust::gather( gatherMap_.begin(), gatherMap_.end(), values.begin(), store_.data()->begin());
-        //now gather from store into buffer
-        Vector buffer( buffer_size_);
-        bijectiveComm_.global_scatter_reduce( *store_.data(), buffer);
-        return buffer;
+        return surjectiveComm_.global_gather( values);
     }
     void global_scatter_reduce( const Vector& toScatter, Vector& values)
     {
-        //first gather values into store
-        Vector store_t = bijectiveComm_.global_gather( toScatter);
-        //now perform a local sort, reduce and scatter operation
-        thrust::gather( sortMap_.begin(), sortMap_.end(), store_t.begin(), store_.data()->begin());
-        thrust::reduce_by_key( sortedGatherMap_.begin(), sortedGatherMap_.end(), store_.data()->begin(), keys_.data()->begin(), values.begin());
+        surjectiveComm_.global_scatter_reduce( toScatter, *store_.data());
+        thrust::scatter( store_.data()->begin(), store_.data()->end(), scatterMap_.begin(), values.begin());
     }
-    unsigned size() const {return buffer_size_;}
+
+    unsigned size() const{return surjectiveComm_.size();}
+    const thrust::host_vector<int> getLocalGatherMap() const {return surjectiveComm_.getLocalGatherMap();}
+    const thrust::host_vector<int> getPidGatherMap() const {return surjectiveComm_.getPidGatherMap();}
+    MPI_Comm communicator()const{return surjectiveComm_.communicator();}
     private:
-    unsigned vector_size_, buffer_size_, store_size_;
-    BijectiveComm<Index, Vector> bijectiveComm_;
-    Index gatherMap_; 
-    Index sortMap_, sortedGatherMap_, scatterMap_;
-    Buffer<Index> keys_;
+    void construct( const thrust::host_vector<int>& localGatherMap, const thrust::host_vector<int>& pidGatherMap, MPI_Comm comm)
+    {
+        const Index& sortedGatherMap_ = surjectiveComm_.getSortedGatherMap();
+        thrust::host_vector<int> gatherMap;
+        dg::blas1::transfer( sortedGatherMap_, gatherMap);
+        thrust::host_vector<int> one( gatherMap.size(), 1), keys(one), number(one);
+        typedef thrust::host_vector<int>::iterator iterator;
+        thrust::pair< iterator, iterator> new_end = 
+            thrust::reduce_by_key( gatherMap.begin(), gatherMap.end(), //sorted!
+                one.begin(), keys.begin(), number.begin() ); 
+        unsigned distance = thrust::distance( keys.begin(), new_end.first);
+        store_.data()->resize( distance);
+        scatterMap_.resize(distance);
+        thrust::copy( keys.begin(), keys.begin() + distance, scatterMap_.begin());
+    }
+    SurjectiveComm<Index, Vector> surjectiveComm_;
     Buffer<Vector> store_;
-
+    Index scatterMap_;
 };
-
 }//namespace dg
diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index fe3337c25..28b6d8162 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -173,7 +173,6 @@ struct aCommunicator
     /**
      * @brief Scatters data accross processes and reduces on double indices
      *
-     * The order of the received elements is according to their original array index (i.e. a[0] appears before a[1]) and their process rank of origin ( i.e. values from rank 0 appear before values from rank 1)
      * @tparam LocalContainer a container on a shared memory system
      * @param toScatter buffer vector (has to be of size given by size())
      * @param values contains values from other processes sent back to the origin 
-- 
GitLab


From d8a42b19cdacc1062d5d0e10e05aa05de92e1dc8 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 13 Jul 2017 23:02:53 +0200
Subject: [PATCH 041/453] added local2global and global2local index maps in MPI
 grids

---
 inc/dg/backend/evaluation_mpit.cu | 18 ++++++
 inc/dg/backend/mpi_collective.h   |  1 +
 inc/dg/backend/mpi_grid.h         | 99 +++++++++++++++++++++++++++++--
 3 files changed, 114 insertions(+), 4 deletions(-)

diff --git a/inc/dg/backend/evaluation_mpit.cu b/inc/dg/backend/evaluation_mpit.cu
index 804caf658..67b4f58b9 100644
--- a/inc/dg/backend/evaluation_mpit.cu
+++ b/inc/dg/backend/evaluation_mpit.cu
@@ -67,6 +67,24 @@ int main(int argc, char** argv)
     double solution3 = solution2*solution;
     if(rank==0)std::cout << "Correct square norm is    "<<solution3<<std::endl;
     if(rank==0)std::cout << "Relative 3d error is      "<<(norm3d-solution3)/solution3<<"\n";
+
+    if(rank==0)
+    {
+        int globalIdx, localIdx, PID, result;
+        std::cout << "Type in global vector index: \n";
+        std::cin >> globalIdx;
+        if( g2d.global2localIdx( globalIdx, localIdx, PID) == MPI_SUCCESS)
+            std::cout <<"2d Local Index "<<localIdx<<" with rank "<<PID<<"\n";
+        g2d.local2globalIdx( localIdx, PID, result);
+        if( globalIdx !=  result)
+            std::cerr <<"Inversion failed "<<result<<"\n";
+        if( g3d.global2localIdx( globalIdx, localIdx, PID) == MPI_SUCCESS)
+            std::cout <<"3d Local Index "<<localIdx<<" with rank "<<PID<<"\n";
+        g3d.local2globalIdx( localIdx, PID, result);
+        if( globalIdx != result)
+            std::cerr <<"Inversion failed "<<result<<"\n";
+    }
+
     MPI_Finalize();
     return 0;
 } 
diff --git a/inc/dg/backend/mpi_collective.h b/inc/dg/backend/mpi_collective.h
index fefa46a38..80d08364a 100644
--- a/inc/dg/backend/mpi_collective.h
+++ b/inc/dg/backend/mpi_collective.h
@@ -306,6 +306,7 @@ struct SurjectiveComm
     {
         construct( src.getLocalGatherMap(), src.getPidGatherMap(), src.communicator());
     }
+
     Vector global_gather( const Vector& values)const
     {
         //gather values to store
diff --git a/inc/dg/backend/mpi_grid.h b/inc/dg/backend/mpi_grid.h
index 1baab344d..3a919d25b 100644
--- a/inc/dg/backend/mpi_grid.h
+++ b/inc/dg/backend/mpi_grid.h
@@ -104,7 +104,7 @@ struct MPIGrid2d
     * @return a copy of this grid with nx*Nx and ny*Ny cells in x and y
     */
     virtual MPIGrid2d multiply( unsigned nx, unsigned ny) const {
-        return MPIGrid2d( x0_, x1_, y0_, y1_, n_, nx*Nx_, ny*Ny_, bcx_, bcy_, comm_);
+        return MPIGrid2d( g.x0(), g.x1(), g.y0(), g.y1(), g.n(), nx*g.Nx(), ny*g.Ny(), g.bcx(), g.bcy(), comm);
     }
     /**
     * @brief Return a copy of the grid with reduced number of cells
@@ -117,7 +117,7 @@ struct MPIGrid2d
          but it does check if the number of processes is still a divisor
     */
     virtual MPIGrid2d divide( unsigned nx, unsigned ny) const {
-        return MPIGrid2d( x0_, x1_, y0_, y1_, n_, Nx_/nx, Ny_/ny, bcx_, bcy_, comm_);
+        return MPIGrid2d( g.x0(), g.x1(), g.y0(), g.y1(), g.n(), g.Nx()/nx, g.Ny()/ny, g.bcx(), g.bcy(), comm);
     }
 
     /**
@@ -299,6 +299,57 @@ struct MPIGrid2d
      * @return pid of a process, or -1 if non of the grids matches
      */
     int pidOf( double x, double y) const;
+
+    /**
+    * @brief Map a local index plus the PID to a global vector index
+    *
+    * @param localIdx a local vector index
+    * @param PID a PID in the communicator
+    * @param globalIdx the corresponding global vector Index (contains result on output)
+    * @return MPI_SUCESS if successful, -1 if localIdx or PID is not part of the grid
+    */
+    int local2globalIdx( int localIdx, int PID, int& globalIdx)
+    {
+        if( localIdx < 0 || localIdx >= (int)size()) return -1;
+        int coords[2];
+        if( MPI_Cart_coords( comm, PID, 2, coords) != MPI_SUCCESS)
+            return -1;
+        int lIdx0 = localIdx %(n()*Nx());
+        int lIdx1 = localIdx /(n()*Nx());
+        int gIdx0 = coords[0]*n()*Nx()+lIdx0;
+        int gIdx1 = coords[1]*n()*Ny()+lIdx1;
+        globalIdx = gIdx1*g.n()*g.Nx() + gIdx0;
+        return MPI_SUCCESS;
+    }
+    /**
+    * @brief Map a global vector index to a local vector Index and the corresponding PID
+    *
+    * @param globalIdx a global vector Index
+    * @param localIdx contains local vector index on output
+    * @param PID contains corresponding PID in the communicator on output
+    * @return MPI_SUCESS if successful, -1 if globalIdx is not part of the grid
+    */
+    int global2localIdx( int globalIdx, int& localIdx, int& PID)
+    {
+        if( globalIdx < 0 || globalIdx >= (int)g.size()) return -1;
+        int coords[2];
+        int gIdx0 = globalIdx%(g.n()*g.Nx());
+        int gIdx1 = globalIdx/(g.n()*g.Nx());
+        coords[0] = gIdx0/(n()*Nx());
+        coords[1] = gIdx1/(n()*Ny());
+        int lIdx0 = gIdx0%(n()*Nx());
+        int lIdx1 = gIdx1%(n()*Ny());
+        localIdx = lIdx1*n()*Nx() + lIdx0;
+        std::cout<< gIdx0<<" "<<gIdx1<<" "<<coords[0]<<" "<<coords[1]<<" "<<lIdx0<<" "<<lIdx1<<" "<<localIdx<<std::endl;
+        if( MPI_Cart_rank( comm, coords, &PID) == MPI_SUCCESS ) 
+            return MPI_SUCCESS;
+        else
+        {
+            std::cout<<"Failed "<<PID<<"\n";
+            return -1;
+        }
+    }
+
     protected:
     void init_X_boundaries( double global_x0, double global_x1)
     {
@@ -421,7 +472,7 @@ struct MPIGrid3d
     * @return a copy of this grid with nx*Nx, ny*Ny and nz*Nz cells in x, y and z
     */
     virtual MPIGrid3d multiply( unsigned nx, unsigned ny, unsigned nz) const {
-        return MPIGrid3d( x0_, x1_, y0_, y1_, z0_, z1_, n_, nx*Nx_, ny*Ny_, nz*Nz_, bcx_, bcy_, bcz_, comm_);
+        return MPIGrid3d( g.x0(), g.x1(), g.y0(), g.y1(), g.z0(), g.z1(), g.n(), nx*g.Nx(), ny*g.Ny(), nz*g.Nz(), g.bcx(), g.bcy(), g.bcz(), comm);
     }
     /**
     * @brief Return a copy of the grid with reduced number of cells
@@ -435,7 +486,7 @@ struct MPIGrid3d
     *   but it does check if the number of processes is still a divisor
     */
     virtual MPIGrid3d divide( unsigned nx, unsigned ny, unsigned nz) const {
-        return MPIGrid3d( x0_, x1_, y0_, y1_, z0_, z1_, n_, Nx_/nx, Ny_/ny, Nz_/nz, bcx_, bcy_, bcz_, comm_);
+        return MPIGrid3d( g.x0(), g.x1(), g.y0(), g.y1(), g.z0(), g.z1(), g.n(), g.Nx()/nx, g.Ny()/ny, g.Nz()/nz, g.bcx(), g.bcy(), g.bcz(), comm);
     }
     /**
      * @brief Return local x0
@@ -651,6 +702,46 @@ struct MPIGrid3d
      * @return pid of a process, or -1 if non of the grids matches
      */
     int pidOf( double x, double y, double z) const;
+    /**
+    * @copydoc MPIGrid2d::local2globalIdx(int,int,int&)
+    */
+    int local2globalIdx( int localIdx, int PID, int& globalIdx)
+    {
+        if( localIdx < 0 || localIdx >= (int)size()) return -1;
+        int coords[3];
+        if( MPI_Cart_coords( comm, PID, 3, coords) != MPI_SUCCESS)
+            return -1;
+        int lIdx0 = localIdx %(n()*Nx());
+        int lIdx1 = (localIdx /(n()*Nx())) % (n()*Ny());
+        int lIdx2 = localIdx / (n()*n()*Nx()*Ny());
+        int gIdx0 = coords[0]*n()*Nx()+lIdx0;
+        int gIdx1 = coords[1]*n()*Ny()+lIdx1;
+        int gIdx2 = coords[2]*Nz()  + lIdx2;
+        globalIdx = (gIdx2*g.n()*g.Ny() + gIdx1)*g.n()*g.Nx() + gIdx0;
+        return MPI_SUCCESS;
+    }
+    /**
+    * @copydoc MPIGrid2d::global2localIdx(int,int&,int&)
+    */
+    int global2localIdx( int globalIdx, int& localIdx, int& PID)
+    {
+        if( globalIdx < 0 || globalIdx >= (int)g.size()) return -1;
+        int coords[3];
+        int gIdx0 = globalIdx%(g.n()*g.Nx());
+        int gIdx1 = (globalIdx/(g.n()*g.Nx())) % (g.n()*g.Ny());
+        int gIdx2 = globalIdx/(g.n()*g.n()*g.Nx()*g.Ny());
+        coords[0] = gIdx0/(n()*Nx());
+        coords[1] = gIdx1/(n()*Ny());
+        coords[2] = gIdx2/Nz();
+        int lIdx0 = gIdx0%(n()*Nx());
+        int lIdx1 = gIdx1%(n()*Ny());
+        int lIdx2 = gIdx2%Nz();
+        localIdx = (lIdx2*n()*Ny() + lIdx1)*n()*Nx() + lIdx0;
+        if( MPI_Cart_rank( comm, coords, &PID) == MPI_SUCCESS ) 
+            return MPI_SUCCESS;
+        else
+            return -1;
+    }
     protected:
     void init_X_boundaries( double global_x0, double global_x1)
     {
-- 
GitLab


From 2563cc960289c3929267301aad1d8b19dbee360e Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 13 Jul 2017 23:22:28 +0200
Subject: [PATCH 042/453] added global gather constructors for SurjectiveComm
 and GeneralComm

---
 inc/dg/backend/evaluation_mpit.cu |  4 ++--
 inc/dg/backend/mpi_collective.h   | 36 +++++++++++++++++++++++++------
 inc/dg/backend/mpi_grid.h         | 32 +++++++++++++--------------
 3 files changed, 48 insertions(+), 24 deletions(-)

diff --git a/inc/dg/backend/evaluation_mpit.cu b/inc/dg/backend/evaluation_mpit.cu
index 67b4f58b9..9bbde95a9 100644
--- a/inc/dg/backend/evaluation_mpit.cu
+++ b/inc/dg/backend/evaluation_mpit.cu
@@ -73,12 +73,12 @@ int main(int argc, char** argv)
         int globalIdx, localIdx, PID, result;
         std::cout << "Type in global vector index: \n";
         std::cin >> globalIdx;
-        if( g2d.global2localIdx( globalIdx, localIdx, PID) == MPI_SUCCESS)
+        if( g2d.global2localIdx( globalIdx, localIdx, PID) )
             std::cout <<"2d Local Index "<<localIdx<<" with rank "<<PID<<"\n";
         g2d.local2globalIdx( localIdx, PID, result);
         if( globalIdx !=  result)
             std::cerr <<"Inversion failed "<<result<<"\n";
-        if( g3d.global2localIdx( globalIdx, localIdx, PID) == MPI_SUCCESS)
+        if( g3d.global2localIdx( globalIdx, localIdx, PID) )
             std::cout <<"3d Local Index "<<localIdx<<" with rank "<<PID<<"\n";
         g3d.local2globalIdx( localIdx, PID, result);
         if( globalIdx != result)
diff --git a/inc/dg/backend/mpi_collective.h b/inc/dg/backend/mpi_collective.h
index 80d08364a..14c796901 100644
--- a/inc/dg/backend/mpi_collective.h
+++ b/inc/dg/backend/mpi_collective.h
@@ -296,13 +296,25 @@ struct SurjectiveComm
     * @param comm The MPI communicator participating in the scatter/gather operations
     * @note we assume that the gather map is surjective
     */
-    SurjectiveComm( const thrust::host_vector<int>& localGatherMap, const thrust::host_vector<int>& pidGatherMap, MPI_Comm comm): bijectiveComm_(pidGatherMap, comm)
+    SurjectiveComm( const thrust::host_vector<int>& localGatherMap, const thrust::host_vector<int>& pidGatherMap, MPI_Comm comm)
     {
         construct( localGatherMap, pidGatherMap, comm);
     }
 
+    template<class MPIGeometry>
+    SurjectiveComm( const thrust::host_vector<int>& globalGatherMap, const MPIGeometry& g)
+    {
+        thrust::host_vector<int> local(globalGatherMap.size()), pids(globalGatherMap.size());
+        bool success = true;
+        for(unsigned i=0; i<local.size(); i++)
+            if( !g.global2localIdx(globalGatherMap[i], pids[i], local[i]) ) success = false;
+
+        assert( success);
+        construct( local, pids, g.communicator());
+    }
+
     template<class OtherIndex, class OtherVector>
-    SurjectiveComm( const SurjectiveComm<OtherIndex, OtherVector>& src): bijectiveComm_(src.getPidGatherMap(), src.communicator())
+    SurjectiveComm( const SurjectiveComm<OtherIndex, OtherVector>& src)
     {
         construct( src.getLocalGatherMap(), src.getPidGatherMap(), src.communicator());
     }
@@ -332,6 +344,7 @@ struct SurjectiveComm
     private:
     void construct( thrust::host_vector<int> localGatherMap, thrust::host_vector<int> pidGatherMap, MPI_Comm comm)
     {
+        bijectiveComm_ = BijectiveComm<Index, Vector>( pidGatherMap, comm);
         localGatherMap_ = localGatherMap, pidGatherMap_ = pidGatherMap;
         buffer_size_ = localGatherMap.size();
         assert( buffer_size_ == pidGatherMap.size());
@@ -376,15 +389,24 @@ template< class Index, class Vector>
 struct GeneralComm
 {
     GeneralComm(){}
-    GeneralComm( const thrust::host_vector<int>& localGatherMap, const thrust::host_vector<int>& pidGatherMap, MPI_Comm comm): surjectiveComm_(localGatherMap, pidGatherMap, comm)
-    {
+    GeneralComm( const thrust::host_vector<int>& localGatherMap, const thrust::host_vector<int>& pidGatherMap, MPI_Comm comm) {
         construct( localGatherMap, pidGatherMap, comm);
     }
     template<class OtherIndex, class OtherVector>
-    GeneralComm( const GeneralComm<OtherIndex, OtherVector>& src): surjectiveComm_(src.getLocalGatherMap(), src.getPidGatherMap(), src.communicator())
-    {
+    GeneralComm( const GeneralComm<OtherIndex, OtherVector>& src){
         construct( src.getLocalGatherMap(), src.getPidGatherMap(), src.communicator());
     }
+    template<class MPIGeometry>
+    GeneralComm( const thrust::host_vector<int>& globalGatherMap, MPIGeometry& g)
+    {
+        thrust::host_vector<int> local(globalGatherMap.size()), pids(globalGatherMap.size());
+        bool success = true;
+        for(unsigned i=0; i<local.size(); i++)
+            if( !g.global2localIdx(globalGatherMap[i], pids[i], local[i]) ) success = false;
+        assert( success);
+        construct( local, pids, g.communicator());
+    }
+
     Vector global_gather( const Vector& values)const
     {
         return surjectiveComm_.global_gather( values);
@@ -402,6 +424,8 @@ struct GeneralComm
     private:
     void construct( const thrust::host_vector<int>& localGatherMap, const thrust::host_vector<int>& pidGatherMap, MPI_Comm comm)
     {
+        surjectiveComm_ = SurjectiveComm<Index,Vector>(localGatherMap, pidGatherMap, comm);
+
         const Index& sortedGatherMap_ = surjectiveComm_.getSortedGatherMap();
         thrust::host_vector<int> gatherMap;
         dg::blas1::transfer( sortedGatherMap_, gatherMap);
diff --git a/inc/dg/backend/mpi_grid.h b/inc/dg/backend/mpi_grid.h
index 3a919d25b..c7530cc44 100644
--- a/inc/dg/backend/mpi_grid.h
+++ b/inc/dg/backend/mpi_grid.h
@@ -306,20 +306,20 @@ struct MPIGrid2d
     * @param localIdx a local vector index
     * @param PID a PID in the communicator
     * @param globalIdx the corresponding global vector Index (contains result on output)
-    * @return MPI_SUCESS if successful, -1 if localIdx or PID is not part of the grid
+    * @return true if successful, false if localIdx or PID is not part of the grid
     */
-    int local2globalIdx( int localIdx, int PID, int& globalIdx)
+    bool local2globalIdx( int localIdx, int PID, int& globalIdx)
     {
         if( localIdx < 0 || localIdx >= (int)size()) return -1;
         int coords[2];
         if( MPI_Cart_coords( comm, PID, 2, coords) != MPI_SUCCESS)
-            return -1;
+            return false;
         int lIdx0 = localIdx %(n()*Nx());
         int lIdx1 = localIdx /(n()*Nx());
         int gIdx0 = coords[0]*n()*Nx()+lIdx0;
         int gIdx1 = coords[1]*n()*Ny()+lIdx1;
         globalIdx = gIdx1*g.n()*g.Nx() + gIdx0;
-        return MPI_SUCCESS;
+        return true;
     }
     /**
     * @brief Map a global vector index to a local vector Index and the corresponding PID
@@ -327,9 +327,9 @@ struct MPIGrid2d
     * @param globalIdx a global vector Index
     * @param localIdx contains local vector index on output
     * @param PID contains corresponding PID in the communicator on output
-    * @return MPI_SUCESS if successful, -1 if globalIdx is not part of the grid
+    * @return true if successful, false if globalIdx is not part of the grid
     */
-    int global2localIdx( int globalIdx, int& localIdx, int& PID)
+    bool global2localIdx( int globalIdx, int& localIdx, int& PID)
     {
         if( globalIdx < 0 || globalIdx >= (int)g.size()) return -1;
         int coords[2];
@@ -342,11 +342,11 @@ struct MPIGrid2d
         localIdx = lIdx1*n()*Nx() + lIdx0;
         std::cout<< gIdx0<<" "<<gIdx1<<" "<<coords[0]<<" "<<coords[1]<<" "<<lIdx0<<" "<<lIdx1<<" "<<localIdx<<std::endl;
         if( MPI_Cart_rank( comm, coords, &PID) == MPI_SUCCESS ) 
-            return MPI_SUCCESS;
+            return true;
         else
         {
             std::cout<<"Failed "<<PID<<"\n";
-            return -1;
+            return false;
         }
     }
 
@@ -705,12 +705,12 @@ struct MPIGrid3d
     /**
     * @copydoc MPIGrid2d::local2globalIdx(int,int,int&)
     */
-    int local2globalIdx( int localIdx, int PID, int& globalIdx)
+    bool local2globalIdx( int localIdx, int PID, int& globalIdx)
     {
-        if( localIdx < 0 || localIdx >= (int)size()) return -1;
+        if( localIdx < 0 || localIdx >= (int)size()) return false;
         int coords[3];
         if( MPI_Cart_coords( comm, PID, 3, coords) != MPI_SUCCESS)
-            return -1;
+            return false;
         int lIdx0 = localIdx %(n()*Nx());
         int lIdx1 = (localIdx /(n()*Nx())) % (n()*Ny());
         int lIdx2 = localIdx / (n()*n()*Nx()*Ny());
@@ -718,14 +718,14 @@ struct MPIGrid3d
         int gIdx1 = coords[1]*n()*Ny()+lIdx1;
         int gIdx2 = coords[2]*Nz()  + lIdx2;
         globalIdx = (gIdx2*g.n()*g.Ny() + gIdx1)*g.n()*g.Nx() + gIdx0;
-        return MPI_SUCCESS;
+        return true;
     }
     /**
     * @copydoc MPIGrid2d::global2localIdx(int,int&,int&)
     */
-    int global2localIdx( int globalIdx, int& localIdx, int& PID)
+    bool global2localIdx( int globalIdx, int& localIdx, int& PID)
     {
-        if( globalIdx < 0 || globalIdx >= (int)g.size()) return -1;
+        if( globalIdx < 0 || globalIdx >= (int)g.size()) return false;
         int coords[3];
         int gIdx0 = globalIdx%(g.n()*g.Nx());
         int gIdx1 = (globalIdx/(g.n()*g.Nx())) % (g.n()*g.Ny());
@@ -738,9 +738,9 @@ struct MPIGrid3d
         int lIdx2 = gIdx2%Nz();
         localIdx = (lIdx2*n()*Ny() + lIdx1)*n()*Nx() + lIdx0;
         if( MPI_Cart_rank( comm, coords, &PID) == MPI_SUCCESS ) 
-            return MPI_SUCCESS;
+            return true;
         else
-            return -1;
+            return false;
     }
     protected:
     void init_X_boundaries( double global_x0, double global_x1)
-- 
GitLab


From 9499ad80da25b3075ba0c1d48bef9988cf3d3040 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sun, 16 Jul 2017 16:50:37 +0200
Subject: [PATCH 043/453] changed matrix types in fieldaligned

---
 inc/dg/geometry/fieldaligned.h | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/inc/dg/geometry/fieldaligned.h b/inc/dg/geometry/fieldaligned.h
index e0492ba30..f9655bc17 100644
--- a/inc/dg/geometry/fieldaligned.h
+++ b/inc/dg/geometry/fieldaligned.h
@@ -191,11 +191,11 @@ cylindrical coordinates
 * @tparam Matrix The matrix class of the interpolation matrix
 * @tparam container The container-class on which the interpolation matrix operates on (does not need to be dg::HVec)
 */
-template< class Geometry, class Matrix, class container >
+template< class Geometry, class IMatrix, class container >
 struct FieldAligned
 {
 
-    typedef Matrix InterpolationMatrix;
+    typedef IMatrix InterpolationMatrix;
 
     /**
     * @brief Construct from a field and a grid
@@ -218,8 +218,7 @@ struct FieldAligned
         If there is no limiter the boundary condition is periodic.
     */
     template <class Field, class Limiter>
-    FieldAligned(Field field, Geometry grid, double eps = 1e-4, Limiter limit = DefaultLimiter(), dg::bc globalbcz = dg::DIR, double deltaPhi = -1);
-
+    FieldAligned(Field field, Geometry grid, unsigned multiplyX, unsigned multiplyY, double eps = 1e-4, Limiter limit = DefaultLimiter(), dg::bc globalbcz = dg::DIR, double deltaPhi = -1);
 
     /**
     * @brief Set boundary conditions in the limiter region
@@ -355,7 +354,7 @@ struct FieldAligned
     private:
     typedef cusp::array1d_view< typename container::iterator> View;
     typedef cusp::array1d_view< typename container::const_iterator> cView;
-    Matrix plus, minus, plusT, minusT; //interpolation matrices
+    IMatrix plus, minus, plusT, minusT; //interpolation matrices
     container hz_, hp_,hm_, ghostM, ghostP;
     Geometry g_;
     dg::bc bcz_;
-- 
GitLab


From 79554a76d16ad601b68d098541b49eed01514166 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sun, 16 Jul 2017 19:20:52 +0200
Subject: [PATCH 044/453] updated toefl.tex

---
 src/toefl/toefl.tex | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/toefl/toefl.tex b/src/toefl/toefl.tex
index 0b0ceaf64..03de26962 100644
--- a/src/toefl/toefl.tex
+++ b/src/toefl/toefl.tex
@@ -266,9 +266,13 @@ mass                     & Dataset & 1 (energy\_time) & mass integral   \\
 \bottomrule
 \end{longtable}
 \section{Diagnostics toeflRdiag.cu}
+There only is a shared memory version available
 \begin{verbatim}
+cd path/to/feltor/diag
+make toeflRdiag 
 path/to/feltor/diag/toeflRdiag input.nc output.nc
 \end{verbatim}
+
 Input file format: netcdf-4/hdf5
 %
 %Name | Type | Dimensionality | Description
-- 
GitLab


From 5ca16f9bf2f665cd7eeaeb3c1aecce803c946b1d Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Wed, 19 Jul 2017 12:05:35 +0200
Subject: [PATCH 045/453] update documentation for globalbcz in interpolation

---
 inc/dg/backend/interpolation.cuh | 29 ++++++++++++++++-------------
 1 file changed, 16 insertions(+), 13 deletions(-)

diff --git a/inc/dg/backend/interpolation.cuh b/inc/dg/backend/interpolation.cuh
index 389bc934e..88780f7d4 100644
--- a/inc/dg/backend/interpolation.cuh
+++ b/inc/dg/backend/interpolation.cuh
@@ -119,11 +119,13 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const thrust::ho
  * @param x X-coordinates of interpolation points
  * @param y Y-coordinates of interpolation points
  * @param g The Grid on which to operate
- * @param globalbcz NEU for common interpolation. DIR for zeros at Box
+ * @param boundary determines what to do when a point
+ lies exactly on the boundary:  DIR generates zeroes in the interpolation matrix, NEU and other boundaries interpolate the inner side polynomial.
+ * @attention all points (x,y) must lie within or on the boundaries of g.
  *
  * @return interpolation matrix
  */
-cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const thrust::host_vector<double>& x, const thrust::host_vector<double>& y, const Grid2d& g , dg::bc globalbcz = dg::NEU)
+cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const thrust::host_vector<double>& x, const thrust::host_vector<double>& y, const Grid2d& g , dg::bc boundary = dg::NEU)
 {
     assert( x.size() == y.size());
     std::vector<double> gauss_nodes = g.dlt().abscissas(); 
@@ -187,7 +189,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const thrust::ho
             for(unsigned k=0; k<pyF.size(); k++)
                 for( unsigned l=0; l<pxF.size(); l++)
                     pxy[k*px.size()+l]= pyF[k]*pxF[l];
-            if (globalbcz == dg::DIR)
+            if (boundary == dg::DIR)
             {
                 if ( x[i]==g.x0() || x[i]==g.x1()  || y[i]==g.y0()  || y[i]==g.y1())
     //             if ( fabs(x[i]-g.x0())<1e-10 || fabs(x[i]-g.x1())<1e-10  || fabs(y[i]-g.y0())<1e-10  || fabs(y[i]-g.y1())<1e-10)
@@ -245,9 +247,9 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const thrust::ho
     cusp::coo_matrix<int, double, cusp::host_memory> A( x.size(), g.size(), values.size());
     A.row_indices = row_indices; A.column_indices = column_indices; A.values = values;
 
-    if (globalbcz == DIR_NEU ) std::cerr << "DIR_NEU NOT IMPLEMENTED "<<std::endl;
-    if (globalbcz == NEU_DIR ) std::cerr << "NEU_DIR NOT IMPLEMENTED "<<std::endl;
-    if (globalbcz == dg::PER ) std::cerr << "PER NOT IMPLEMENTED "<<std::endl;
+    if (boundary == DIR_NEU ) std::cerr << "DIR_NEU NOT IMPLEMENTED "<<std::endl;
+    if (boundary == NEU_DIR ) std::cerr << "NEU_DIR NOT IMPLEMENTED "<<std::endl;
+    if (boundary == dg::PER ) std::cerr << "PER NOT IMPLEMENTED "<<std::endl;
     
     return A;
 }
@@ -262,12 +264,13 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const thrust::ho
  * @param y Y-coordinates of interpolation points
  * @param z Z-coordinates of interpolation points
  * @param g The Grid on which to operate
- * @param globalbcz determines what to do if values lie exactly on the boundary
+ * @param boundary determines what to do when a point
+ lies exactly on the boundary:  DIR generates zeroes in the interpolation matrix, NEU and other boundaries interpolate the inner side polynomial.
  *
  * @return interpolation matrix
- * @note The values of x, y and z must lie within the boundaries of g
+ * @attention all points (x, y, z) must lie within or on the boundaries of g
  */
-cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const thrust::host_vector<double>& x, const thrust::host_vector<double>& y, const thrust::host_vector<double>& z, const Grid3d& g, dg::bc globalbcz= dg::NEU)
+cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const thrust::host_vector<double>& x, const thrust::host_vector<double>& y, const thrust::host_vector<double>& z, const Grid3d& g, dg::bc boundary= dg::NEU)
 {
     assert( x.size() == y.size());
     assert( y.size() == z.size());
@@ -338,7 +341,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const thrust::ho
             for(unsigned k=0; k<pyF.size(); k++)
                 for( unsigned l=0; l<pxF.size(); l++)
                     pxyz[k*g.n()+l]= 1.*pyF[k]*pxF[l];
-            if (globalbcz == dg::DIR)
+            if (boundary == dg::DIR)
             {
                 if ( x[i]==g.x0() || x[i]==g.x1()  ||y[i]==g.y0()  || y[i]==g.y1())
                 {
@@ -395,9 +398,9 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const thrust::ho
     cusp::coo_matrix<int, double, cusp::host_memory> A( x.size(), g.size(), values.size());
     A.row_indices = row_indices; A.column_indices = column_indices; A.values = values;
 
-    if (globalbcz == DIR_NEU ) std::cerr << "DIR_NEU NOT IMPLEMENTED "<<std::endl;
-    if (globalbcz == NEU_DIR ) std::cerr << "NEU_DIR NOT IMPLEMENTED "<<std::endl;
-    if (globalbcz == dg::PER ) std::cerr << "PER NOT IMPLEMENTED "<<std::endl;
+    if (boundary == DIR_NEU ) std::cerr << "DIR_NEU NOT IMPLEMENTED "<<std::endl;
+    if (boundary == NEU_DIR ) std::cerr << "NEU_DIR NOT IMPLEMENTED "<<std::endl;
+    if (boundary == dg::PER ) std::cerr << "PER NOT IMPLEMENTED "<<std::endl;
     
     return A;
 }
-- 
GitLab


From 971862a5d97805212598f92c1a5d713819df1911 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Wed, 19 Jul 2017 17:21:40 +0200
Subject: [PATCH 046/453] removed interpolation in ds and made enum for
 fieldaligned

---
 inc/dg/ds.h                    | 269 +++++++++------------------------
 inc/dg/geometry/fieldaligned.h | 184 +++++++---------------
 2 files changed, 125 insertions(+), 328 deletions(-)

diff --git a/inc/dg/ds.h b/inc/dg/ds.h
index dc0f05cb5..6a3166ca3 100644
--- a/inc/dg/ds.h
+++ b/inc/dg/ds.h
@@ -14,6 +14,7 @@
  * This file includes the appropriate headers for parallel derivatives
  */
 
+//TODO: use buffers to make symv const
 namespace dg{
 
 /**
@@ -45,7 +46,7 @@ struct DS
     DS(const FA& field, Geometry, InvB invB, dg::norm no=dg::normed, dg::direction dir = dg::centered, bool jumpX = true);
 
     /**
-    * @brief Apply the derivative on a 3d vector
+    * @brief Apply the forward derivative on a 3d vector
     *
     * forward derivative \f$ \frac{1}{h_z^+}(f_{i+1} - f_{i})\f$
     * @param f The vector to derive
@@ -53,7 +54,7 @@ struct DS
     */
     void forward( const container& f, container& dsf);
     /**
-    * @brief Apply the derivative on a 3d vector
+    * @brief Apply the backward derivative on a 3d vector
     *
     * backward derivative \f$ \frac{1}{2h_z^-}(f_{i} - f_{i-1})\f$
     * @param f The vector to derive
@@ -61,7 +62,7 @@ struct DS
     */
     void backward( const container& f, container& dsf);
     /**
-    * @brief Apply the derivative on a 3d vector
+    * @brief Apply the centered derivative on a 3d vector
     *
     * centered derivative \f$ \frac{1}{2h_z}(f_{i+1} - f_{i-1})\f$
     * @param f The vector to derive
@@ -72,27 +73,24 @@ struct DS
     /**
     * @brief Apply the negative adjoint derivative on a 3d vector
     *
-    * forward derivative \f$ \frac{1}{h_z^+}(f_{i+1} - f_{i})\f$
     * @param f The vector to derive
     * @param dsf contains result on output (write only)
     */
-    void forwardT( const container& f, container& dsf);
+    void forwardAdj( const container& f, container& dsf);
     /**
     * @brief Apply the negative adjoint derivative on a 3d vector
     *
-    * backward derivative \f$ \frac{1}{2h_z^-}(f_{i} - f_{i-1})\f$
     * @param f The vector to derive
     * @param dsf contains result on output (write only)
     */
-    void backwardT( const container& f, container& dsf);
+    void backwardAdj( const container& f, container& dsf);
     /**
     * @brief Apply the negative adjoint derivative on a 3d vector
     *
-    * centered derivative \f$ \frac{1}{2h_z}(f_{i+1} - f_{i-1})\f$
     * @param f The vector to derive
     * @param dsf contains result on output (write only)
     */
-    void centeredTD( const container& f, container& dsf);
+    void centeredAdjDir( const container& f, container& dsf);
     /**
     * @brief Apply the negative adjoint derivative on a 3d vector with the direct method
     *
@@ -100,49 +98,39 @@ struct DS
     * @param f The vector to derive
     * @param dsf contains result on output (write only)
     */
-    void forwardTD( const container& f, container& dsf);
+    void forwardAdjDir( const container& f, container& dsf);
     /**
     * @brief Apply the negative adjoint derivative on a 3d vector with the direct method
     *
-    * backward derivative \f$ \frac{1}{2h_z^-}(f_{i} - f_{i-1})\f$
     * @param f The vector to derive
     * @param dsf contains result on output (write only)
     */
-    void backwardTD( const container& f, container& dsf);
+    void backwardAdjDir( const container& f, container& dsf);
     /**
     * @brief Apply the negative adjoint derivative on a 3d vector with the direct method
     *
-    * centered derivative \f$ \frac{1}{2h_z}(f_{i+1} - f_{i-1})\f$
     * @param f The vector to derive
     * @param dsf contains result on output (write only)
     */
-    void centeredT( const container& f, container& dsf);
+    void centeredAdj( const container& f, container& dsf);
 
     /**
-    * @brief is the negative transposed of centered
+    * @brief compute parallel derivative
     *
-    * redirects to centered
+    * dependent on dir redirects to either forward(), backward() or centered()
     * @param f The vector to derive
     * @param dsf contains result on output (write only)
     */
     void operator()( const container& f, container& dsf);
 
 
-    /**
-     * @brief Compute the second derivative using finite differences
-     *
-     * discretizes \f$ \nabla_\parallel\cdot \nabla_\parallel\f$
-     * @param f input function
-     * @param dssf output (write-only)
-     */
-    void dss( const container& f, container& dssf);
-
     /**
      * @brief Discretizes the parallel Laplacian as a symmetric matrix
      *
-     * forward followed by forwardT and adding jump terms
+     * if direction is centered then centered followed by centeredAdj and adding jump terms
      * @param f The vector to derive
      * @param dsTdsf contains result on output (write only)
+     * @note if apply_jumpX is false then no jumpy terms will be added in the x-direction
      */
     void symv( const container& f, container& dsTdsf);
 
@@ -210,20 +198,16 @@ struct DS
     */
     const FA& fieldaligned() const{return f_;}
     private:
-    FA f_;
+    FieldAligned<G,I,container> f_;
     Matrix jumpX, jumpY;
-    IMatrix f2c, c2f, f2cT, c2fT;
     container tempP, temp0, tempM;
-    container tempPc, tempMc, temp0c;
     container f, dsf;
     container vol3d, inv3d;
-    container vol3df, inv3df;
     container invB;
     //container R_;
     dg::norm no_;
     dg::direction dir_;
     bool apply_jumpX_;
-
     container volume_;
 };
 
@@ -232,28 +216,18 @@ struct DS
 
 template<class G, class I, class M, class container>
 template <class MagneticField>
-DS<G, I, M,container>::DS(const FA& field, Geometry gridc, Field inverseB, dg::norm no, dg::direction dir, bool jumpX):
+DS<G, I, M,container>::DS(MagneticField mag, Geometry grid, dg::norm no, dg::direction dir, bool jumpX):
         f_(field),
-        jumpX( dg::create::jumpX( gridc)),
-        jumpY( dg::create::jumpY( gridc)),
+        jumpX( dg::create::jumpX( grid)),
+        jumpY( dg::create::jumpY( grid)),
         tempP( dg::evaluate( dg::zero, field.grid())), temp0( tempP), tempM( tempP), 
-        tempPc( dg::evaluate( dg::zero, gridc)), temp0c( tempPc), tempMc( tempPc), 
         f(tempP), dsf(tempP),
-        vol3d( dg::create::volume( gridc)), inv3d( dg::create::inv_volume( gridc)),
-        vol3df( dg::create::volume( field.grid())), inv3df( dg::create::inv_volume( field.grid())),
+        vol3d( dg::create::volume( grid)), inv3d( dg::create::inv_volume( grid)),
         invB(dg::pullback(inverseB,field.grid())), //R_(dg::evaluate(dg::coo1,grid)), 
         no_(no), dir_(dir), apply_jumpX_(jumpX)
 {
-
-    f2c = dg::create::projection( gridc, field.grid());
-    c2f = dg::create::interpolation( field.grid(), gridc);
-    cusp::transpose( f2c, f2cT);
-    cusp::transpose( c2f, c2fT);     
-
-    volume_ = dg::evaluate( dg::one, gridc);
-    dg::geo::multiplyVolume( volume_, gridc);
-
-
+    volume_ = dg::evaluate( dg::one, grid);
+    dg::geo::multiplyVolume( volume_, grid);
 }
 
 template<class G, class I, class M, class container>
@@ -268,190 +242,108 @@ inline void DS<G,I,M,container>::operator()( const container& f, container& dsf)
 
 
 template<class G, class I, class M, class container>
-void DS<G,I,M,container>::centered( const container& fc, container& dsfc)
+void DS<G,I,M,container>::centered( const container& f, container& dsf)
 {
     //direct discretisation
-    assert( &fc != &dsfc);
-    cusp::multiply( c2f, fc, f);
-    f_.einsPlus( f, tempP);
-    f_.einsMinus( f, tempM);
+    assert( &f != &dsf);
+    f_(einsPlus, f, tempP);
+    f_(einsMinus, f, tempM);
     dg::blas1::axpby( 1., tempP, -1., tempM);
     dg::blas1::pointwiseDivide( tempM, f_.hz(), dsf);
-      cusp::multiply( f2c, dsf, dsfc);  
-
-    
-    ////adjoint discretisation
-//     assert( &fc != &dsfc);
-//     dg::blas1::pointwiseDot( vol3d, fc, dsfc);
-//     cusp::multiply( f2cT, dsfc, dsf);
-//     dg::blas1::pointwiseDivide( dsf, f_.hz(), dsf);
-//     dg::blas1::pointwiseDivide( dsf, invB, dsf);
-//     f_.einsPlusT( dsf, tempP);
-//     f_.einsMinusT( dsf, tempM);
-//     dg::blas1::axpby( 1., tempM, -1., tempP);
-//     dg::blas1::pointwiseDot( tempP, invB, tempP);
-//     cusp::multiply( c2fT, tempP, dsfc);
-//     dg::blas1::pointwiseDot( inv3d, dsfc, dsfc); 
 }
 
 template<class G, class I, class M, class container>
-void DS<G,I,M,container>::centeredT( const container& fc, container& dsfc)
+void DS<G,I,M,container>::centeredAdj( const container& f, container& dsf)
 {               
     //adjoint discretisation
-    assert( &fc != &dsfc);    
-    dg::blas1::pointwiseDot( vol3d, fc, dsfc);
-    cusp::multiply( f2cT, dsfc, dsf);
-
+    assert( &f != &dsf);    
+    dg::blas1::pointwiseDot( vol3d, f, dsf);
     dg::blas1::pointwiseDivide( dsf, f_.hz(), dsf);
-    f_.einsPlusT( dsf, tempP);
-    f_.einsMinusT( dsf, tempM);
-    dg::blas1::axpby( 1., tempM, -1., tempP);        
-
-    cusp::multiply( c2fT, tempP, dsfc);
-    dg::blas1::pointwiseDot( inv3d, dsfc, dsfc); 
-    
-    //non adjoint direct 
-//     assert( &fc != &dsfc);     
-//     cusp::multiply( c2f, fc, f);
-//     dg::blas1::pointwiseDot( f, invB, dsf);
-//     f_.einsPlus( dsf, tempP);
-//     f_.einsMinus( dsf, tempM);
-//     dg::blas1::axpby( 1., tempP, -1., tempM);
-//     dg::blas1::pointwiseDivide( tempM, f_.hz(), dsf);        
-//     dg::blas1::pointwiseDivide( dsf, invB, dsf);
-//     cusp::multiply( f2c, dsf, dsfc);  
-//       dg::blas1::pointwiseDot( inv3d, tempP,tempP); //make it symmetric
-        //stegmeir weights
-//         dg::blas1::pointwiseDot( f_.hz()h, f, dsf);
-//         dg::blas1::pointwiseDot( invB, dsf, dsf);
-//         dg::blas1::pointwiseDot( w2d, dsf, dsf);
-//         dg::blas1::pointwiseDivide( dsf, f_.hz(), dsf);
-//         einsPlusT( dsf, tempP);
-//         einsMinusT( dsf, tempM);
-//         dg::blas1::axpby( 1., tempM, -1., tempP);        
-//         dg::blas1::pointwiseDot( inv3d, tempP, dsf);
-//         dg::blas1::scal(dsf,0.5);
-//         dg::blas1::pointwiseDivide( tempP,f_.hz()h,  dsf);
-//         dg::blas1::pointwiseDivide(  dsf,invB, dsf);
-//         dg::blas1::pointwiseDivide( dsf,w2d,  dsf);  
+    f_(einsPlusT, dsf, tempP);
+    f_(einsMinusT, dsf, tempM);
+    dg::blas1::axpby( 1., tempM, -1., tempP, dsf);        
+    dg::blas1::pointwiseDot( inv3d, dsf, dsf); 
 }
 
 template<class G, class I, class M, class container>
-void DS<G,I,M,container>::centeredTD( const container& f, container& dsf)
+void DS<G,I,M,container>::centeredAdjDir( const container& f, container& dsf)
 {       
 //     Direct discretisation
     assert( &f != &dsf);    
     dg::blas1::pointwiseDot( f, invB, dsf);
-    f_.einsPlus( dsf, tempP);
-    f_.einsMinus( dsf, tempM);
+    f_(einsPlus, dsf, tempP);
+    f_(einsMinus, dsf, tempM);
     dg::blas1::axpby( 1., tempP, -1., tempM);
     dg::blas1::pointwiseDivide( tempM, f_.hz(), dsf);        
     dg::blas1::pointwiseDivide( dsf, invB, dsf);
 }
 
 template<class G, class I, class M, class container>
-void DS<G,I,M,container>::forward( const container& fc, container& dsfc)
+void DS<G,I,M,container>::forward( const container& f, container& dsf)
 {
     //direct
-    assert( &fc != &dsfc);
-    cusp::multiply( c2f, fc, f);
-    f_.einsPlus( f, tempP);
+    assert( &f != &dsf);
+    f_(einsPlus, f, tempP);
     dg::blas1::axpby( 1., tempP, -1., f, tempP);
     dg::blas1::pointwiseDivide( tempP, f_.hp(), dsf);
-    //adjoint discretisation
-//     assert( &f != &dsf);    
-//     dg::blas1::pointwiseDot( vol3d, f, dsf);
-//     dg::blas1::pointwiseDivide( dsf, f_.hm(), dsf);
-//     dg::blas1::pointwiseDivide( dsf, invB, dsf);
-//     einsMinusT( dsf, tempP);
-//     dg::blas1::axpby( 1., tempP,-1.,dsf,dsf);
-//     dg::blas1::pointwiseDot( inv3d, dsf, dsf);
-//     dg::blas1::pointwiseDot( dsf, invB, dsf);
-    cusp::multiply( f2c, dsf, dsfc);
 }
 
 template<class G, class I, class M, class container>
-void DS<G,I,M,container>::forwardT( const container& fc, container& dsfc)
+void DS<G,I,M,container>::forwardAdj( const container& f, container& dsf)
 {    
     //adjoint discretisation
-    assert( &fc != &dsfc);
-    dg::blas1::pointwiseDot( vol3d, fc, dsfc);
-    cusp::multiply( f2cT, dsfc, dsf);
+    assert( &f != &dsf);
+    dg::blas1::pointwiseDot( vol3d, f, dsf);
     dg::blas1::pointwiseDivide( dsf, f_.hp(), dsf);
-    f_.einsPlusT( dsf, tempP);
+    f_(einsPlusT, dsf, tempP);
     dg::blas1::axpby( -1., tempP, 1., dsf, dsf);
-
-    cusp::multiply( c2fT, dsf, dsfc);
-    dg::blas1::pointwiseDot( inv3d, dsfc, dsfc); 
+    dg::blas1::pointwiseDot( inv3d, dsf, dsf); 
 }
 
 template<class G, class I, class M, class container>
-void DS<G,I,M,container>::forwardTD( const container& fc, container& dsfc)
+void DS<G,I,M,container>::forwardAdjDir( const container& f, container& dsf)
 {
     //direct discretisation
-    assert( &fc != &dsfc);
-    cusp::multiply( c2f, fc, f);
-
+    assert( &f != &dsf);
     dg::blas1::pointwiseDot( f, invB, dsf);
-    f_.einsMinus( dsf, tempP);
+    f_(einsMinus, dsf, tempP);
     dg::blas1::axpby( -1., tempP, 1., dsf, dsf);
     dg::blas1::pointwiseDivide( dsf, f_.hm(), dsf);        
     dg::blas1::pointwiseDivide( dsf, invB, dsf);
-
-    cusp::multiply( f2c, dsf, dsfc);
-
-
 }
+
 template<class G, class I, class M, class container>
-void DS<G,I,M,container>::backward( const container& fc, container& dsfc)
+void DS<G,I,M,container>::backward( const container& f, container& dsf)
 {
     //direct
-    assert( &fc != &dsfc);
-    cusp::multiply( c2f, fc, f);
-    f_.einsMinus( f, tempM);
+    assert( &f != &dsf);
+    f_(einsMinus, f, tempM);
     dg::blas1::axpby( 1., tempM, -1., f, tempM);
     dg::blas1::pointwiseDivide( tempM, f_.hm(), dsf);
-    
-    //adjoint discretisation
-//     assert( &f != &dsf);    
-//     dg::blas1::pointwiseDot( vol3d, f, dsf);
-//     dg::blas1::pointwiseDivide( dsf, f_.hp(), dsf);
-//     dg::blas1::pointwiseDivide( dsf, invB, dsf);
-//     einsPlusT( dsf, tempM);
-//     dg::blas1::axpby( 1., tempM, -1.,dsf,dsf);
-//     dg::blas1::pointwiseDot( inv3d,dsf, dsf);
-//     dg::blas1::pointwiseDot( dsf, invB, dsf);
-    cusp::multiply( f2c, dsf, dsfc);
 }
+
 template<class G, class I, class M, class container>
-void DS<G,I,M,container>::backwardT( const container& fc, container& dsfc)
+void DS<G,I,M,container>::backwardAdj( const container& f, container& dsf)
 {    
     //adjoint discretisation
-    assert( &fc != &dsfc);
-    dg::blas1::pointwiseDot( vol3d, fc, dsfc);
-    cusp::multiply( f2cT, dsfc, dsf);
-
+    assert( &f != &dsf);
+    dg::blas1::pointwiseDot( vol3d, f, dsf);
     dg::blas1::pointwiseDivide( dsf, f_.hm(), dsf);
-    f_.einsMinusT( dsf, tempM);
+    f_(einsMinusT, dsf, tempM);
     dg::blas1::axpby( -1., tempM, 1., dsf, dsf);
-
-    cusp::multiply( c2fT, dsf, dsfc);
-    dg::blas1::pointwiseDot( inv3d, dsfc, dsfc); 
+    dg::blas1::pointwiseDot( inv3d, dsf, dsf); 
 }
 
 template<class G, class I, class M, class container>
-void DS<G,I,M,container>::backwardTD( const container& fc, container& dsfc)
+void DS<G,I,M,container>::backwardAdjDir( const container& f, container& dsf)
 {
     //direct
-    assert( &fc != &dsfc);
-    cusp::multiply( c2f, fc, f);
+    assert( &f != &dsf);
     dg::blas1::pointwiseDot( f, invB, dsf);
-    f_.einsPlus( dsf, tempM);
+    f_(einsPlus, dsf, tempM);
     dg::blas1::axpby( -1., tempM, 1., dsf, dsf);
     dg::blas1::pointwiseDivide( dsf, f_.hp(), dsf);        
     dg::blas1::pointwiseDivide( dsf, invB, dsf);
-
-    cusp::multiply( f2c, dsf, dsfc);
 }
 
 template<class G, class I, class M, class container>
@@ -459,53 +351,34 @@ void DS<G,I,M,container>::symv( const container& f, container& dsTdsf)
 {
     if(dir_ == dg::centered)
     {
-        centered( f, tempPc);
-        centeredT( tempPc, dsTdsf);
+        centered( f, tempP);
+        centeredAdj( tempP, dsTdsf);
     }
     else 
     {
-        forward( f, tempPc);
-        forwardT( tempPc, dsTdsf);
-        backward( f, tempMc);
-        backwardT( tempMc, temp0c);
-        dg::blas1::axpby(0.5,temp0c,0.5,dsTdsf,dsTdsf);
+        forward( f, tempP);
+        forwardAdj( tempP, dsTdsf);
+        backward( f, tempM);
+        backwardAdj( tempM, temp0);
+        dg::blas1::axpby(0.5,temp0,0.5,dsTdsf,dsTdsf);
     }
 //     add jump term 
 
     if(apply_jumpX_)
     {
-        dg::blas2::symv( jumpX, f, temp0c);
-        dg::blas1::pointwiseDivide( temp0c, volume_, temp0c);
+        dg::blas2::symv( jumpX, f, temp0);
+        dg::blas1::pointwiseDivide( temp0, volume_, temp0);
         dg::blas1::axpby( -1., temp0c, 1., dsTdsf, dsTdsf);
     }
-    dg::blas2::symv( jumpY, f, temp0c);
-    //dg::geo::divideVolume( temp0c, gridc);
-    dg::blas1::pointwiseDivide( temp0c, volume_, temp0c);
-    //dg::blas1::pointwiseDivide( temp0, R_, temp0);
-    dg::blas1::axpby( -1., temp0c, 1., dsTdsf, dsTdsf);
+    dg::blas2::symv( jumpY, f, temp0);
+    dg::blas1::pointwiseDivide( temp0, volume_, temp0);
+    dg::blas1::axpby( -1., temp0, 1., dsTdsf, dsTdsf);
     if( no_ == not_normed)
     {
         dg::blas1::pointwiseDot( vol3d, dsTdsf, dsTdsf); //make it symmetric
     }
 }
 
-//template< class F, class M, class container >
-//void DS<F,M,container>::dss( const container& f, container& dssf)
-//{
-//    assert( &f != &dssf);
-//    f_.einsPlus(  f, tempP);
-//    f_.einsMinus( f, tempM);
-//    dg::blas1::pointwiseDivide( tempP, f_.hp(), tempP);
-//    dg::blas1::pointwiseDivide( tempP, f_.hz(), tempP);
-//    dg::blas1::pointwiseDivide( f, f_.hp(), temp0);
-//    dg::blas1::pointwiseDivide( temp0, f_.hm(), temp0);
-//    dg::blas1::pointwiseDivide( tempM, f_.hm(), tempM);
-//    dg::blas1::pointwiseDivide( tempM, f_.hz(), tempM);
-//    dg::blas1::axpby(  2., tempP, +2., tempM); //fp+fm
-//    dg::blas1::axpby( -2., temp0, +1., tempM, dssf); 
-//}
-
-
 //enables the use of the dg::blas2::symv function 
 template< class G, class I, class M, class V>
 struct MatrixTraits< DS<G,I,M, V> >
diff --git a/inc/dg/geometry/fieldaligned.h b/inc/dg/geometry/fieldaligned.h
index f9655bc17..22c3c78a6 100644
--- a/inc/dg/geometry/fieldaligned.h
+++ b/inc/dg/geometry/fieldaligned.h
@@ -14,6 +14,14 @@
 
 namespace dg{
 
+enum whichMatrix
+{
+    einsPlus = 0,  
+    einsPlusT = 1,  
+    einsMinus = 2,  
+    einsMinusT = 3,  
+};
+
 /**
  * @brief With the Default field ds becomes a dz
  */
@@ -150,6 +158,7 @@ void boxintegrator( Field& field, const Grid& grid,
 #endif //DG_DEBUG
         if( globalbcz == dg::DIR)
         {
+            //idea: maybe we should take the "wrong" long Delta s instead of the short one to avoid deteriorated CFL condition
             BoxIntegrator<Field, Grid> boxy( field, grid, eps);
             boxy.set_coords( coords0); //nimm alte koordinaten
             if( phi1 > 0)
@@ -159,7 +168,7 @@ void boxintegrator( Field& field, const Grid& grid,
                 phi1 = (dPhiMin+dPhiMax)/2.;
                 dg::integrateRK4( field, coords0, coords1, dPhiMax, eps); //integriere bis über 0 stelle raus damit unten Wert neu gesetzt wird
             }
-            else
+            else // phi1 < 0 
             {
                 double dPhiMin = phi1, dPhiMax = 0;
                 dg::bisection1d( boxy, dPhiMin, dPhiMax,eps);
@@ -232,7 +241,7 @@ struct FieldAligned
     void set_boundaries( dg::bc bcz, double left, double right)
     {
         bcz_ = bcz;
-        const dg::Grid2d g2d( g_.x0(), g_.x1(), g_.y0(), g_.y1(), g_.n(), g_.Nx(), g_.Ny());
+        const dg::Grid1d g2d( 0, 1, 1, perp_size_);
         left_ = dg::evaluate( dg::CONSTANT(left), g2d);
         right_ = dg::evaluate( dg::CONSTANT(right),g2d);
     }
@@ -298,6 +307,7 @@ struct FieldAligned
     template< class BinaryOp, class UnaryOp>
     container evaluate( BinaryOp f, UnaryOp g, unsigned p0, unsigned rounds) const;
 
+    void operator()(enum whichMatrix which, const container& in, container& out);
     /**
     * @brief Applies the interpolation to the next planes 
     *
@@ -312,20 +322,6 @@ struct FieldAligned
     * @param out output may not equal intpu
     */
     void einsMinus( const container& in, container& out);
-    /**
-    * @brief Applies the transposed interpolation to the previous plane 
-    *
-    * @param in input 
-    * @param out output may not equal intpu
-    */
-    void einsPlusT( const container& in, container& out);
-    /**
-    * @brief Applies the transposed interpolation to the next plane 
-    *
-    * @param in input 
-    * @param out output may not equal intpu
-    */
-    void einsMinusT( const container& in, container& out);
 
     /**
     * @brief hz is the distance between the plus and minus planes
@@ -345,18 +341,12 @@ struct FieldAligned
     * @return three-dimensional vector
     */
     const container& hm()const {return hm_;}
-    /**
-    * @brief Access the underlying grid
-    *
-    * @return the grid
-    */
-    const Geometry& grid() const{return g_;}
     private:
     typedef cusp::array1d_view< typename container::iterator> View;
     typedef cusp::array1d_view< typename container::const_iterator> cView;
     IMatrix plus, minus, plusT, minusT; //interpolation matrices
     container hz_, hp_,hm_, ghostM, ghostP;
-    Geometry g_;
+    unsigned Nz_, perp_size_;
     dg::bc bcz_;
     container left_, right_;
     container limiter_;
@@ -369,15 +359,15 @@ template<class Geometry, class M, class container>
 template <class Field, class Limiter>
 FieldAligned<Geometry, M,container>::FieldAligned(Field field, Geometry grid, double eps, Limiter limit, dg::bc globalbcz, double deltaPhi):
         hz_( dg::evaluate( dg::zero, grid)), hp_( hz_), hm_( hz_), 
-        g_(grid), bcz_(grid.bcz())
+        Nz_(grid.Nz()), bcz_(grid.bcz())
 {
     //Resize vector to 2D grid size
 
     typename Geometry::perpendicular_grid g2d = grid.perp_grid( );
-    unsigned size = g2d.size();
+    perp_size_ = g2d.size();
     limiter_ = dg::evaluate( limit, g2d);
     right_ = left_ = dg::evaluate( zero, g2d);
-    ghostM.resize( size); ghostP.resize( size);
+    ghostM.resize( perp_size_); ghostP.resize( perp_size_);
     //Set starting points
     std::vector<thrust::host_vector<double> > y( 5, dg::evaluate( dg::cooX2d, g2d)); // x
     y[1] = dg::evaluate( dg::cooY2d, g2d); //y
@@ -391,7 +381,7 @@ FieldAligned<Geometry, M,container>::FieldAligned(Field field, Geometry grid, do
 #ifdef _OPENMP
 #pragma omp parallel for shared(field)
 #endif //_OPENMP
-    for( unsigned i=0; i<size; i++)
+    for( unsigned i=0; i<perp_size_; i++)
     {
         thrust::host_vector<double> coords(5), coordsP(5), coordsM(5);
         coords[0] = y[0][i], coords[1] = y[1][i], coords[2] = y[2][i], coords[3] = y[3][i], coords[4] = y[4][i]; //x,y,s,R,Z
@@ -411,8 +401,8 @@ FieldAligned<Geometry, M,container>::FieldAligned(Field field, Geometry grid, do
 //     copy into h vectors
     for( unsigned i=0; i<grid.Nz(); i++)
     {
-        thrust::copy( yp[2].begin(), yp[2].end(), hp_.begin() + i*g2d.size());
-        thrust::copy( ym[2].begin(), ym[2].end(), hm_.begin() + i*g2d.size());
+        thrust::copy( yp[2].begin(), yp[2].end(), hp_.begin() + i*perp_size_);
+        thrust::copy( ym[2].begin(), ym[2].end(), hm_.begin() + i*perp_size_);
     }
     dg::blas1::scal( hm_, -1.);
     dg::blas1::axpby(  1., hp_, +1., hm_, hz_);    //
@@ -422,9 +412,8 @@ FieldAligned<Geometry, M,container>::FieldAligned(Field field, Geometry grid, do
 template<class G, class M, class container>
 void FieldAligned<G,M,container>::set_boundaries( dg::bc bcz, const container& global, double scal_left, double scal_right)
 {
-    unsigned size = g_.n()*g_.n()*g_.Nx()*g_.Ny();
-    cView left( global.cbegin(), global.cbegin() + size);
-    cView right( global.cbegin()+(g_.Nz()-1)*size, global.cbegin() + g_.Nz()*size);
+    cView left( global.cbegin(), global.cbegin() + perp_size_);
+    cView right( global.cbegin()+(Nz_-1)*perp_size_, global.cbegin() + Nz_*perp_size_);
     View leftView( left_.begin(), left_.end());
     View rightView( right_.begin(), right_.end());
     cusp::copy( left, leftView);
@@ -442,14 +431,15 @@ container FieldAligned<G,M,container>::evaluate( BinaryOp binary, unsigned p0) c
 }
 
 template<class G, class M, class container>
-template< class BinaryOp, class UnaryOp>
-container FieldAligned<G,M,container>::evaluate( BinaryOp binary, UnaryOp unary, unsigned p0, unsigned rounds) const
+template< class BinaryOp, class UnaryOp, class Grid>
+container FieldAligned<G,M,container>::evaluate( BinaryOp binary, UnaryOp unary, unsigned p0, unsigned rounds, Grid g_) const
 {
     //idea: simply apply I+/I- enough times on the init2d vector to get the result in each plane
     //unary function is always such that the p0 plane is at x=0
     assert( g_.Nz() > 1);
     assert( p0 < g_.Nz());
     const typename G::perpendicular_grid g2d = g_.perp_grid();
+    assert( g2d.size() == perp_size_ && Nz_== g_.Nz());
     container init2d = dg::pullback( binary, g2d), temp(init2d), tempP(init2d), tempM(init2d);
     container vec3d = dg::evaluate( dg::zero, g_);
     std::vector<container>  plus2d( g_.Nz(), (container)dg::evaluate(dg::zero, g2d) ), minus2d( plus2d), result( plus2d);
@@ -457,11 +447,11 @@ container FieldAligned<G,M,container>::evaluate( BinaryOp binary, UnaryOp unary,
     if( turns ==0) turns++;
     //first apply Interpolation many times, scale and store results
     for( unsigned r=0; r<turns; r++)
-        for( unsigned i0=0; i0<g_.Nz(); i0++)
+        for( unsigned i0=0; i0<Nz_; i0++)
         {
             dg::blas1::copy( init2d, tempP);
             dg::blas1::copy( init2d, tempM);
-            unsigned rep = i0 + r*g_.Nz();
+            unsigned rep = i0 + r*Nz_;
             for(unsigned k=0; k<rep; k++)
             {
                 dg::blas2::symv( plus, tempP, temp);
@@ -477,123 +467,58 @@ container FieldAligned<G,M,container>::evaluate( BinaryOp binary, UnaryOp unary,
     //now we have the plus and the minus filaments
     if( rounds == 0) //there is a limiter
     {
-        for( unsigned i0=0; i0<g_.Nz(); i0++)
+        for( unsigned i0=0; i0<Nz_; i0++)
         {
             int idx = (int)i0 - (int)p0;
             if(idx>=0)
                 result[i0] = plus2d[idx];
             else
                 result[i0] = minus2d[abs(idx)];
-            thrust::copy( result[i0].begin(), result[i0].end(), vec3d.begin() + i0*g2d.size());
+            thrust::copy( result[i0].begin(), result[i0].end(), vec3d.begin() + i0*perp_size_);
         }
     }
     else //sum up plus2d and minus2d
     {
-        for( unsigned i0=0; i0<g_.Nz(); i0++)
+        for( unsigned i0=0; i0<Nz_; i0++)
         {
-            unsigned revi0 = (g_.Nz() - i0)%g_.Nz(); //reverted index
+            unsigned revi0 = (Nz_ - i0)%Nz_; //reverted index
             dg::blas1::axpby( 1., plus2d[i0], 0., result[i0]);
             dg::blas1::axpby( 1., minus2d[revi0], 1., result[i0]);
         }
         dg::blas1::axpby( -1., init2d, 1., result[0]);
-        for(unsigned i0=0; i0<g_.Nz(); i0++)
+        for(unsigned i0=0; i0<Nz_; i0++)
         {
-            int idx = ((int)i0 -(int)p0 + g_.Nz())%g_.Nz(); //shift index
-            thrust::copy( result[idx].begin(), result[idx].end(), vec3d.begin() + i0*g2d.size());
+            int idx = ((int)i0 -(int)p0 + Nz_)%Nz_; //shift index
+            thrust::copy( result[idx].begin(), result[idx].end(), vec3d.begin() + i0*perp_size_);
         }
     }
     return vec3d;
 }
 
 
-template< class G, class M, class container>
-void FieldAligned<G,M, container>::einsPlus( const container& f, container& fpe)
+void FieldAligned< >::operator()(enum whichMatrix which, const container& f, container& fe)
 {
-    unsigned size = g_.n()*g_.n()*g_.Nx()*g_.Ny();
-    View ghostPV( ghostP.begin(), ghostP.end());
-    View ghostMV( ghostM.begin(), ghostM.end());
-    cView rightV( right_.begin(), right_.end());
-    for( unsigned i0=0; i0<g_.Nz(); i0++)
-    {
-        unsigned ip = (i0==g_.Nz()-1) ? 0:i0+1;
-
-        cView fp( f.cbegin() + ip*size, f.cbegin() + (ip+1)*size);
-        cView f0( f.cbegin() + i0*size, f.cbegin() + (i0+1)*size);
-        View fP( fpe.begin() + i0*size, fpe.begin() + (i0+1)*size);
-        cusp::multiply( plus, fp, fP);
-        //make ghostcells i.e. modify fpe in the limiter region
-        if( i0==g_.Nz()-1 && bcz_ != dg::PER)
-        {
-            if( bcz_ == dg::DIR || bcz_ == dg::NEU_DIR)
-            {
-                cusp::blas::axpby( rightV, f0, ghostPV, 2., -1.);
-            }
-            if( bcz_ == dg::NEU || bcz_ == dg::DIR_NEU)
-            {
-                thrust::transform( right_.begin(), right_.end(),  hp_.begin(), ghostM.begin(), thrust::multiplies<double>());
-                cusp::blas::axpby( ghostMV, f0, ghostPV, 1., 1.);
-            }
-            //interlay ghostcells with periodic cells: L*g + (1-L)*fpe
-            cusp::blas::axpby( ghostPV, fP, ghostPV, 1., -1.);
-            dg::blas1::pointwiseDot( limiter_, ghostP, ghostP);
-            cusp::blas::axpby(  ghostPV, fP, fP, 1.,1.);
-        }
-    }
+    if(which == einsPlus || which == einsMinusT) einsPlus( which, f, fe);
+    if(which == einsMinus || which == einsPlusT) einsMinus( which, f, fe);
 }
 
-template< class G,class M, class container>
-void FieldAligned<G,M, container>::einsMinus( const container& f, container& fme)
-{
-    //note that thrust functions don't work on views
-    unsigned size = g_.n()*g_.n()*g_.Nx()*g_.Ny();
-    View ghostPV( ghostP.begin(), ghostP.end());
-    View ghostMV( ghostM.begin(), ghostM.end());
-    cView leftV( left_.begin(), left_.end());
-    for( unsigned i0=0; i0<g_.Nz(); i0++)
-    {
-        unsigned im = (i0==0) ? g_.Nz()-1:i0-1;
-        cView fm( f.cbegin() + im*size, f.cbegin() + (im+1)*size);
-        cView f0( f.cbegin() + i0*size, f.cbegin() + (i0+1)*size);
-        View fM( fme.begin() + i0*size, fme.begin() + (i0+1)*size);
-        cusp::multiply( minus, fm, fM );
-        //make ghostcells
-        if( i0==0 && bcz_ != dg::PER)
-        {
-            if( bcz_ == dg::DIR || bcz_ == dg::DIR_NEU)
-            {
-                cusp::blas::axpby( leftV,  f0, ghostMV, 2., -1.);
-            }
-            if( bcz_ == dg::NEU || bcz_ == dg::NEU_DIR)
-            {
-                thrust::transform( left_.begin(), left_.end(),  hm_.begin(), ghostP.begin(), thrust::multiplies<double>());
-                cusp::blas::axpby( ghostPV, f0, ghostMV, -1., 1.);
-            }
-            //interlay ghostcells with periodic cells: L*g + (1-L)*fme
-            cusp::blas::axpby( ghostMV, fM, ghostMV, 1., -1.);
-            dg::blas1::pointwiseDot( limiter_, ghostM, ghostM);
-            cusp::blas::axpby( ghostMV, fM, fM, 1., 1.);
-
-        }
-    }
-}
-
-template< class G,class M, class container>
-void FieldAligned<G,M, container>::einsMinusT( const container& f, container& fpe)
+template< class G, class M, class container>
+void FieldAligned<G,M, container>::einsPlus( enum whichMatrix which, const container& f, container& fpe)
 {
-    unsigned size = g_.n()*g_.n()*g_.Nx()*g_.Ny();
     View ghostPV( ghostP.begin(), ghostP.end());
     View ghostMV( ghostM.begin(), ghostM.end());
     cView rightV( right_.begin(), right_.end());
-    for( unsigned i0=0; i0<g_.Nz(); i0++)
+    for( unsigned i0=0; i0<Nz_; i0++)
     {
-        unsigned ip = (i0==g_.Nz()-1) ? 0:i0+1;
+        unsigned ip = (i0==Nz_-1) ? 0:i0+1;
 
-        cView fp( f.cbegin() + ip*size, f.cbegin() + (ip+1)*size);
-        cView f0( f.cbegin() + i0*size, f.cbegin() + (i0+1)*size);
-        View fP( fpe.begin() + i0*size, fpe.begin() + (i0+1)*size);
-        cusp::multiply( minusT, fp, fP );
+        cView fp( f.cbegin() + ip*perp_size_, f.cbegin() + (ip+1)*perp_size_);
+        cView f0( f.cbegin() + i0*perp_size_, f.cbegin() + (i0+1)*perp_size_);
+        View fP( fpe.begin() + i0*perp_size_, fpe.begin() + (i0+1)*perp_size_);
+        if(which == einsPlus) cusp::multiply( plus, fp, fP);
+        else if(which == einsMinusT) cusp::multiply( minusT, fp, fP );
         //make ghostcells i.e. modify fpe in the limiter region
-        if( i0==g_.Nz()-1 && bcz_ != dg::PER)
+        if( i0==Nz_-1 && bcz_ != dg::PER)
         {
             if( bcz_ == dg::DIR || bcz_ == dg::NEU_DIR)
             {
@@ -609,25 +534,24 @@ void FieldAligned<G,M, container>::einsMinusT( const container& f, container& fp
             dg::blas1::pointwiseDot( limiter_, ghostP, ghostP);
             cusp::blas::axpby(  ghostPV, fP, fP, 1.,1.);
         }
-
     }
 }
 
 template< class G,class M, class container>
-void FieldAligned<G,M, container>::einsPlusT( const container& f, container& fme)
+void FieldAligned<G,M, container>::einsMinus( enum whichMatrix which, const container& f, container& fme)
 {
     //note that thrust functions don't work on views
-    unsigned size = g_.n()*g_.n()*g_.Nx()*g_.Ny();
     View ghostPV( ghostP.begin(), ghostP.end());
     View ghostMV( ghostM.begin(), ghostM.end());
     cView leftV( left_.begin(), left_.end());
-    for( unsigned i0=0; i0<g_.Nz(); i0++)
+    for( unsigned i0=0; i0<Nz_; i0++)
     {
-        unsigned im = (i0==0) ? g_.Nz()-1:i0-1;
-        cView fm( f.cbegin() + im*size, f.cbegin() + (im+1)*size);
-        cView f0( f.cbegin() + i0*size, f.cbegin() + (i0+1)*size);
-        View fM( fme.begin() + i0*size, fme.begin() + (i0+1)*size);
-        cusp::multiply( plusT, fm, fM );
+        unsigned im = (i0==0) ? Nz_-1:i0-1;
+        cView fm( f.cbegin() + im*perp_size_, f.cbegin() + (im+1)*perp_size_);
+        cView f0( f.cbegin() + i0*perp_size_, f.cbegin() + (i0+1)*perp_size_);
+        View fM( fme.begin() + i0*perp_size_, fme.begin() + (i0+1)*perp_size_);
+        if(which == einsPlusT) cusp::multiply( plusT, fm, fM );
+        else if (which == einsMinus) cusp::multiply( minus, fm, fM );
         //make ghostcells
         if( i0==0 && bcz_ != dg::PER)
         {
-- 
GitLab


From 8d4e442d5f813e2126f04c70e1be4360b1e73881 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Wed, 19 Jul 2017 18:23:14 +0200
Subject: [PATCH 047/453] work on new fieldaligned

---
 inc/dg/ds.h                    |  7 ++-
 inc/dg/geometry/fieldaligned.h | 95 ++++++++++++++++++++--------------
 2 files changed, 60 insertions(+), 42 deletions(-)

diff --git a/inc/dg/ds.h b/inc/dg/ds.h
index 6a3166ca3..ee3d9152c 100644
--- a/inc/dg/ds.h
+++ b/inc/dg/ds.h
@@ -196,9 +196,9 @@ struct DS
     *
     * @return acces to fieldaligned object
     */
-    const FA& fieldaligned() const{return f_;}
+    const FieldAligned<IMatrix, container>& fieldaligned() const{return f_;}
     private:
-    FieldAligned<G,I,container> f_;
+    FieldAligned<IMatrix,container> f_;
     Matrix jumpX, jumpY;
     container tempP, temp0, tempM;
     container f, dsf;
@@ -217,13 +217,12 @@ struct DS
 template<class G, class I, class M, class container>
 template <class MagneticField>
 DS<G, I, M,container>::DS(MagneticField mag, Geometry grid, dg::norm no, dg::direction dir, bool jumpX):
-        f_(field),
         jumpX( dg::create::jumpX( grid)),
         jumpY( dg::create::jumpY( grid)),
         tempP( dg::evaluate( dg::zero, field.grid())), temp0( tempP), tempM( tempP), 
         f(tempP), dsf(tempP),
         vol3d( dg::create::volume( grid)), inv3d( dg::create::inv_volume( grid)),
-        invB(dg::pullback(inverseB,field.grid())), //R_(dg::evaluate(dg::coo1,grid)), 
+        invB(dg::pullback(dg::geo::InvB<MagneticField>(mag),grid)), 
         no_(no), dir_(dir), apply_jumpX_(jumpX)
 {
     volume_ = dg::evaluate( dg::one, grid);
diff --git a/inc/dg/geometry/fieldaligned.h b/inc/dg/geometry/fieldaligned.h
index 22c3c78a6..431da8806 100644
--- a/inc/dg/geometry/fieldaligned.h
+++ b/inc/dg/geometry/fieldaligned.h
@@ -182,7 +182,7 @@ void boxintegrator( Field& field, const Grid& grid,
         }
         else if (globalbcz == dg::NEU )
         {
-             coords1[0] = coords0[0]; coords1[1] = coords0[1];  
+             coords1[0] = coords0[0]; coords1[1] = coords0[1];  //this is clearly wrong -> makes ds a d\varphi
         }
         else if (globalbcz == DIR_NEU )std::cerr << "DIR_NEU NOT IMPLEMENTED "<<std::endl;
         else if (globalbcz == NEU_DIR )std::cerr << "NEU_DIR NOT IMPLEMENTED "<<std::endl;
@@ -200,11 +200,12 @@ cylindrical coordinates
 * @tparam Matrix The matrix class of the interpolation matrix
 * @tparam container The container-class on which the interpolation matrix operates on (does not need to be dg::HVec)
 */
-template< class Geometry, class IMatrix, class container >
+template<class IMatrix, class container >
 struct FieldAligned
 {
 
     typedef IMatrix InterpolationMatrix;
+    FieldAligned(){}
 
     /**
     * @brief Construct from a field and a grid
@@ -226,7 +227,7 @@ struct FieldAligned
         by the bcz variable from the grid and can be changed by the set_boundaries function. 
         If there is no limiter the boundary condition is periodic.
     */
-    template <class Field, class Limiter>
+    template <class Field, class Geometry, class Limiter>
     FieldAligned(Field field, Geometry grid, unsigned multiplyX, unsigned multiplyY, double eps = 1e-4, Limiter limit = DefaultLimiter(), dg::bc globalbcz = dg::DIR, double deltaPhi = -1);
 
     /**
@@ -355,36 +356,38 @@ struct FieldAligned
 ///@cond 
 ////////////////////////////////////DEFINITIONS////////////////////////////////////////
 
-template<class Geometry, class M, class container>
-template <class Field, class Limiter>
-FieldAligned<Geometry, M,container>::FieldAligned(Field field, Geometry grid, double eps, Limiter limit, dg::bc globalbcz, double deltaPhi):
+template<class I, class container>
+template <class Field, class Geometry, class Limiter>
+FieldAligned<I, container>::FieldAligned(Field field, Geometry grid, unsigned mx, unsigned my, double eps, Limiter limit, dg::bc globalbcz, double deltaPhi):
         hz_( dg::evaluate( dg::zero, grid)), hp_( hz_), hm_( hz_), 
         Nz_(grid.Nz()), bcz_(grid.bcz())
 {
     //Resize vector to 2D grid size
 
-    typename Geometry::perpendicular_grid g2d = grid.perp_grid( );
-    perp_size_ = g2d.size();
-    limiter_ = dg::evaluate( limit, g2d);
-    right_ = left_ = dg::evaluate( zero, g2d);
+    typename Geometry::perpendicular_grid g2dCoarse = grid.perp_grid( );
+    perp_size_ = g2dCoarse.size();
+    limiter_ = dg::evaluate( limit, g2dCoarse);
+    right_ = left_ = dg::evaluate( zero, g2dCoarse);
     ghostM.resize( perp_size_); ghostP.resize( perp_size_);
     //Set starting points
-    std::vector<thrust::host_vector<double> > y( 5, dg::evaluate( dg::cooX2d, g2d)); // x
-    y[1] = dg::evaluate( dg::cooY2d, g2d); //y
-    y[2] = dg::evaluate( dg::zero, g2d);
-    y[3] = dg::pullback( dg::cooX2d, g2d); //R
-    y[4] = dg::pullback( dg::cooY2d, g2d); //Z
+    typename Geometry::perpendicular_grid g2dFine = g2dCoarse.multiply( mX, mY);
+    
+    std::vector<thrust::host_vector<double> > y( 3, dg::evaluate( dg::cooX2d, g2dFine)); // x
+    y[1] = dg::evaluate( dg::cooY2d, g2dFine); //y
+    y[2] = dg::evaluate( dg::zero, g2dFine);
     //integrate field lines for all points
-    std::vector<thrust::host_vector<double> > yp( 3, dg::evaluate(dg::zero, g2d)), ym(yp); 
+    std::vector<thrust::host_vector<double> > yp( 3, dg::evaluate(dg::zero, g2dFine)), ym(yp); 
     if( deltaPhi <=0) deltaPhi = grid.hz();
     else assert( grid.Nz() == 1 || grid.hz()==deltaPhi);
+    dg::Timer t;
+    t.tic();
 #ifdef _OPENMP
 #pragma omp parallel for shared(field)
 #endif //_OPENMP
-    for( unsigned i=0; i<perp_size_; i++)
+    for( unsigned i=0; i<g2dF.size(); i++)
     {
-        thrust::host_vector<double> coords(5), coordsP(5), coordsM(5);
-        coords[0] = y[0][i], coords[1] = y[1][i], coords[2] = y[2][i], coords[3] = y[3][i], coords[4] = y[4][i]; //x,y,s,R,Z
+        thrust::host_vector<double> coords(3), coordsP(3), coordsM(3);
+        coords[0] = y[0][i], coords[1] = y[1][i], coords[2] = y[2][i]; //x,y,s
         double phi1 = deltaPhi;
         boxintegrator( field, g2d, coords, coordsP, phi1, eps, globalbcz);
         phi1 =  - deltaPhi;
@@ -392,25 +395,40 @@ FieldAligned<Geometry, M,container>::FieldAligned(Field field, Geometry grid, do
         yp[0][i] = coordsP[0], yp[1][i] = coordsP[1], yp[2][i] = coordsP[2];
         ym[0][i] = coordsM[0], ym[1][i] = coordsM[1], ym[2][i] = coordsM[2];
     }
-    //fange Periodische RB ab
-    plus  = dg::create::interpolation( yp[0], yp[1], g2d, globalbcz);
-    minus = dg::create::interpolation( ym[0], ym[1], g2d, globalbcz);
-// //     Transposed matrices work only for csr_matrix due to bad matrix form for ell_matrix and MPI_Matrix lacks of transpose function!!!
+    t.toc(); 
+    std::cout << "Fieldline integration took "<<t.diff()<<"s\n";
+    t.tic();
+    IMatrix plusFine  = dg::create::interpolation( yp[0], yp[1], g2dCoarse, globalbcz);
+    IMatrix minusFine = dg::create::interpolation( ym[0], ym[1], g2dCoarse, globalbcz);
+    IMatrix interpolation = dg::create::interpolation( g2dFine, g2dCoarse);
+    IMatrix projection = dg::create::projection( g2dCoarse, g2dFine);
+    t.toc();
+    std::cout <<"Creation of interpolation/projection took "<<t.diff()<<"s\n";
+    t.tic();
+    cusp::multiply( projection, plusFine, plus);
+    cusp::multiply( projection, minusFine, minus);
+    t.toc();
+    std::cout<< "Multiplication of PI took: "<<t.diff()<<"s\n";
+    //Transposed matrices work only for csr_matrix due to bad matrix form for ell_matrix!!!
     cusp::transpose( plus, plusT);
     cusp::transpose( minus, minusT);     
 //     copy into h vectors
+    
+    thrust::host_vector<double> hp( perp_size_), hm(hp);
+    dg::blas2::symv( projection, yp[2], hp);
+    dg::blas2::symv( projection, ym[2], hm);
     for( unsigned i=0; i<grid.Nz(); i++)
     {
-        thrust::copy( yp[2].begin(), yp[2].end(), hp_.begin() + i*perp_size_);
-        thrust::copy( ym[2].begin(), ym[2].end(), hm_.begin() + i*perp_size_);
+        thrust::copy( hp.begin(), hp.end(), hp_.begin() + i*perp_size_);
+        thrust::copy( hm.begin(), hm.end(), hm_.begin() + i*perp_size_);
     }
     dg::blas1::scal( hm_, -1.);
     dg::blas1::axpby(  1., hp_, +1., hm_, hz_);    //
  
 }
 
-template<class G, class M, class container>
-void FieldAligned<G,M,container>::set_boundaries( dg::bc bcz, const container& global, double scal_left, double scal_right)
+template<class I, class container>
+void FieldAligned<I,container>::set_boundaries( dg::bc bcz, const container& global, double scal_left, double scal_right)
 {
     cView left( global.cbegin(), global.cbegin() + perp_size_);
     cView right( global.cbegin()+(Nz_-1)*perp_size_, global.cbegin() + Nz_*perp_size_);
@@ -423,16 +441,16 @@ void FieldAligned<G,M,container>::set_boundaries( dg::bc bcz, const container& g
     bcz_ = bcz;
 }
 
-template< class G, class M, class container>
-template< class BinaryOp>
-container FieldAligned<G,M,container>::evaluate( BinaryOp binary, unsigned p0) const
+template< class I, class container>
+template< class BinaryOp, class Grid>
+container FieldAligned<I,container>::evaluate( BinaryOp binary, unsigned p0, Grid grid) const
 {
-    return evaluate( binary, dg::CONSTANT(1), p0, 0);
+    return evaluate( binary, dg::CONSTANT(1), p0, 0, grid);
 }
 
-template<class G, class M, class container>
+template<class I, class container>
 template< class BinaryOp, class UnaryOp, class Grid>
-container FieldAligned<G,M,container>::evaluate( BinaryOp binary, UnaryOp unary, unsigned p0, unsigned rounds, Grid g_) const
+container FieldAligned<I,container>::evaluate( BinaryOp binary, UnaryOp unary, unsigned p0, unsigned rounds, Grid g_) const
 {
     //idea: simply apply I+/I- enough times on the init2d vector to get the result in each plane
     //unary function is always such that the p0 plane is at x=0
@@ -496,14 +514,15 @@ container FieldAligned<G,M,container>::evaluate( BinaryOp binary, UnaryOp unary,
 }
 
 
-void FieldAligned< >::operator()(enum whichMatrix which, const container& f, container& fe)
+template< class I, class container>
+void FieldAligned<I, container >::operator()(enum whichMatrix which, const container& f, container& fe)
 {
     if(which == einsPlus || which == einsMinusT) einsPlus( which, f, fe);
     if(which == einsMinus || which == einsPlusT) einsMinus( which, f, fe);
 }
 
-template< class G, class M, class container>
-void FieldAligned<G,M, container>::einsPlus( enum whichMatrix which, const container& f, container& fpe)
+template< class I, class container>
+void FieldAligned<I, container>::einsPlus( enum whichMatrix which, const container& f, container& fpe)
 {
     View ghostPV( ghostP.begin(), ghostP.end());
     View ghostMV( ghostM.begin(), ghostM.end());
@@ -537,8 +556,8 @@ void FieldAligned<G,M, container>::einsPlus( enum whichMatrix which, const conta
     }
 }
 
-template< class G,class M, class container>
-void FieldAligned<G,M, container>::einsMinus( enum whichMatrix which, const container& f, container& fme)
+template< class I, class container>
+void FieldAligned<I, container>::einsMinus( enum whichMatrix which, const container& f, container& fme)
 {
     //note that thrust functions don't work on views
     View ghostPV( ghostP.begin(), ghostP.end());
-- 
GitLab


From 2ab5dd538969e077dbdf9b5d739fe4189c445f42 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Wed, 19 Jul 2017 18:35:59 +0200
Subject: [PATCH 048/453] made pushForward use host_vector internally and
 construct container directly

---
 inc/dg/elliptic.h                 |  2 +-
 inc/dg/geometry.h                 | 14 +++---
 inc/dg/geometry/geometry_traits.h | 72 +++++++++++++++++--------------
 src/asela/asela.cuh               |  9 +---
 src/feltor/feltor.cuh             |  9 +---
 5 files changed, 49 insertions(+), 57 deletions(-)

diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index 331228bfa..98a22bb56 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -651,7 +651,7 @@ struct TensorElliptic
     template<class ChiRR, class ChiRZ, class ChiZZ>
     void set( ChiRR chiRR, ChiRZ chiRZ, ChiZZ chiZZ)
     {
-        typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector chiXX, chiXY, chiYY;
+        Vector chiXX, chiXY, chiYY;
         dg::geo::pushForwardPerp( chiRR, chiRZ, chiZZ, chiXX, chiXY, chiYY, g_);
         set( chiXX, chiXY, chiYY);
     }
diff --git a/inc/dg/geometry.h b/inc/dg/geometry.h
index 69d1595a2..8879eff02 100644
--- a/inc/dg/geometry.h
+++ b/inc/dg/geometry.h
@@ -154,21 +154,19 @@ void dividePerpVolume( container& inout, const Geometry& g)
  * @param vy y-component of vector (gets properly resized)
  * @param g The geometry object
  */
-template<class Functor1, class Functor2, class Geometry> 
+template<class Functor1, class Functor2, class container, class Geometry> 
 void pushForwardPerp( Functor1 vR, Functor2 vZ, 
-        typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector& vx, 
-        typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector& vy,
+        container& vx, container& vy,
         const Geometry& g)
 {
     dg::geo::detail::doPushForwardPerp( vR, vZ, vx, vy, g, typename GeometryTraits<Geometry>::metric_category() ); 
 }
 
 ///@cond
-template<class Geometry> 
+template<class container, class Geometry> 
 void pushForwardPerp(
         double(fR)(double,double,double), double(fZ)(double, double, double), 
-        typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector& out1, 
-        typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector& out2,
+        container& out1, container& out2,
         const Geometry& g)
 {
     pushForwardPerp<double(double, double, double), double(double, double, double), Geometry>( fR, fZ, out1, out2, g); 
@@ -195,9 +193,7 @@ void pushForwardPerp(
  */
 template<class FunctorRR, class FunctorRZ, class FunctorZZ, class Geometry> 
 void pushForwardPerp( FunctorRR chiRR, FunctorRZ chiRZ, FunctorZZ chiZZ,
-        typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector& chixx, 
-        typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector& chixy,
-        typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector& chiyy, 
+        container& chixx, container& chixy, container& chiyy,
         const Geometry& g)
 {
     dg::geo::detail::doPushForwardPerp( chiRR, chiRZ, chiZZ, chixx, chixy, chiyy, g, typename GeometryTraits<Geometry>::metric_category() ); 
diff --git a/inc/dg/geometry/geometry_traits.h b/inc/dg/geometry/geometry_traits.h
index 88a734a83..8f6cb9127 100644
--- a/inc/dg/geometry/geometry_traits.h
+++ b/inc/dg/geometry/geometry_traits.h
@@ -134,56 +134,59 @@ void doVolRaisePerpIndex( container& in1, container& in2, container& out1, conta
 };
 
 
-template<class TernaryOp1, class TernaryOp2, class Geometry> 
+template<class TernaryOp1, class TernaryOp2, class container, class Geometry> 
 void doPushForwardPerp( TernaryOp1 f1, TernaryOp2 f2, 
-        typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector& out1, 
-        typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector& out2,
+        container& vx, container& vy,
         const Geometry& g, OrthonormalCylindricalTag)
 {
+    typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector& out1, out2; 
     out1 = evaluate( f1, g);
     out2 = evaluate( f2, g);
+    dg::blas1::transfer( out1, vx);
+    dg::blas1::transfer( out2, vy);
 }
 
-template<class TernaryOp1, class TernaryOp2, class Geometry> 
+template<class TernaryOp1, class TernaryOp2, class container, class Geometry> 
 void doPushForwardPerp( TernaryOp1 f1, TernaryOp2 f2, 
-        typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector& out1, 
-        typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector& out2,
+        container& vx, container& vy,
         const Geometry& g, CurvilinearCylindricalTag)
 {
-    typedef typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector container;
-    out1 = pullback( f1, g);
-    out2 = pullback( f2, g);
-    container temp1( out1), temp2( out2);
+    typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector& out1, out2, temp1, temp2; 
+    temp1 = out1 = pullback( f1, g);
+    temp2 = out2 = pullback( f2, g);
     dg::blas1::pointwiseDot( g.xr(), temp1, out1);
     dg::blas1::pointwiseDot( 1., g.xz(), temp2, 1., out1);
     dg::blas1::pointwiseDot( g.yr(), temp1, out2);
     dg::blas1::pointwiseDot( 1., g.yz(), temp2, 1., out2);
+    dg::blas1::transfer( out1, vx);
+    dg::blas1::transfer( out2, vy);
 }
 
-template<class FunctorRR, class FunctorRZ, class FunctorZZ, class Geometry> 
+template<class FunctorRR, class FunctorRZ, class FunctorZZ, class container, class Geometry> 
 void doPushForwardPerp( FunctorRR chiRR, FunctorRZ chiRZ, FunctorZZ chiZZ,
-        typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector& chixx, 
-        typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector& chixy,
-        typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector& chiyy, 
+        container& chixx, container& chixy, container& chiyy,
         const Geometry& g, OrthonormalCylindricalTag)
 {
-    chixx = evaluate( chiRR, g);
-    chixy = evaluate( chiRZ, g);
-    chiyy = evaluate( chiZZ, g);
+    typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector& chixx_, chixy_, chiyy_; 
+    chixx_ = evaluate( chiRR, g);
+    chixy_ = evaluate( chiRZ, g);
+    chiyy_ = evaluate( chiZZ, g);
+    dg::blas1::transfer( chixx_, chixx);
+    dg::blas1::transfer( chixy_, chixy);
+    dg::blas1::transfer( chiyy_, chiyy);
 }
 
-template<class FunctorRR, class FunctorRZ, class FunctorZZ, class Geometry> 
+template<class FunctorRR, class FunctorRZ, class FunctorZZ, class container, class Geometry> 
 void doPushForwardPerp( FunctorRR chiRR_, FunctorRZ chiRZ_, FunctorZZ chiZZ_,
-        typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector& chixx, 
-        typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector& chixy,
-        typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector& chiyy, 
+        container& chixx, container& chixy, container& chiyy,
         const Geometry& g, CurvilinearCylindricalTag)
 {
     //compute the rhs
     typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector chiRR, chiRZ, chiZZ;
-    chixx = chiRR = pullback( chiRR_, g);
-    chixy = chiRZ = pullback( chiRZ_, g);
-    chiyy = chiZZ = pullback( chiZZ_, g);
+    typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector& chixx_, chixy_, chiyy_; 
+    chixx_ = chiRR = pullback( chiRR_, g);
+    chixy_ = chiRZ = pullback( chiRZ_, g);
+    chiyy_ = chiZZ = pullback( chiZZ_, g);
     //compute the transformation matrix
     typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector t00(chixx), t01(t00), t02(t00), t10(t00), t11(t00), t12(t00), t20(t00), t21(t00), t22(t00);
     dg::blas1::pointwiseDot( g.xr(), g.xr(), t00);
@@ -201,15 +204,18 @@ void doPushForwardPerp( FunctorRR chiRR_, FunctorRZ chiRZ_, FunctorZZ chiZZ_,
     dg::blas1::scal( t21, 2.);
     dg::blas1::pointwiseDot( g.yz(), g.yz(), t22);
     //now multiply
-    dg::blas1::pointwiseDot(     t00, chiRR, chixx);
-    dg::blas1::pointwiseDot( 1., t01, chiRZ, 1., chixx);
-    dg::blas1::pointwiseDot( 1., t02, chiZZ, 1., chixx);
-    dg::blas1::pointwiseDot(     t10, chiRR, chixy);
-    dg::blas1::pointwiseDot( 1., t11, chiRZ, 1., chixy);
-    dg::blas1::pointwiseDot( 1., t12, chiZZ, 1., chixy);
-    dg::blas1::pointwiseDot(     t20, chiRR, chiyy);
-    dg::blas1::pointwiseDot( 1., t21, chiRZ, 1., chiyy);
-    dg::blas1::pointwiseDot( 1., t22, chiZZ, 1., chiyy);
+    dg::blas1::pointwiseDot(     t00, chiRR, chixx_);
+    dg::blas1::pointwiseDot( 1., t01, chiRZ, 1., chixx_);
+    dg::blas1::pointwiseDot( 1., t02, chiZZ, 1., chixx_);
+    dg::blas1::pointwiseDot(     t10, chiRR, chixy_);
+    dg::blas1::pointwiseDot( 1., t11, chiRZ, 1., chixy_);
+    dg::blas1::pointwiseDot( 1., t12, chiZZ, 1., chixy_);
+    dg::blas1::pointwiseDot(     t20, chiRR, chiyy_);
+    dg::blas1::pointwiseDot( 1., t21, chiRZ, 1., chiyy_);
+    dg::blas1::pointwiseDot( 1., t22, chiZZ, 1., chiyy_);
+    dg::blas1::transfer( chixx_, chixx);
+    dg::blas1::transfer( chixy_, chixy);
+    dg::blas1::transfer( chiyy_, chiyy);
 
 }
 
diff --git a/src/asela/asela.cuh b/src/asela/asela.cuh
index 430b05467..b5e9224c7 100644
--- a/src/asela/asela.cuh
+++ b/src/asela/asela.cuh
@@ -348,16 +348,11 @@ Asela<Grid, DS, Matrix, container>::Asela( const Grid& g, Parameters p, dg::geo:
     dg::blas1::transfer(  dg::pullback(dg::geo::TanhSource<Psip>(mf.psip, gp.psipmin, gp.alpha),      g), source);
     dg::blas1::transfer(  dg::pullback(dg::geo::GaussianDamping<Psip>(mf.psip, gp.psipmax, gp.alpha), g), damping);
     ////////////////////////////transform curvature components////////
-    typename dg::HostVec< typename dg::GeometryTraits<Grid>::memory_category>::host_vector tempX, tempY;
-    dg::geo::pushForwardPerp(dg::geo::CurvatureNablaBR<MagneticField>(mf, gp.R_0), dg::geo::CurvatureNablaBZ<MagneticField>(mf, gp.R_0), tempX, tempY, g);
-    dg::blas1::transfer(  tempX, curvX);
-    dg::blas1::transfer(  tempY, curvY);
+    dg::geo::pushForwardPerp(dg::geo::CurvatureNablaBR<MagneticField>(mf, gp.R_0), dg::geo::CurvatureNablaBZ<MagneticField>(mf, gp.R_0), curvX, curvY, g);
     dg::blas1::transfer(  dg::pullback(dg::geo::DivCurvatureKappa<MagneticField>(mf, gp.R_0), g), divCurvKappa);
     if (p.curvmode==1) 
     {
-        dg::geo::pushForwardPerp(dg::geo::CurvatureKappaR(), dg::geo::CurvatureKappaZ<MagneticField>(mf, gp.R_0), tempX, tempY, g);
-        dg::blas1::transfer(  tempX, curvKappaX);
-        dg::blas1::transfer(  tempY, curvKappaY);
+        dg::geo::pushForwardPerp(dg::geo::CurvatureKappaR(), dg::geo::CurvatureKappaZ<MagneticField>(mf, gp.R_0), curvKappaX, curvKappaY, g);
         dg::blas1::axpby( 1.,curvX,1.,curvKappaX,curvX);
         dg::blas1::axpby( 1.,curvY,1.,curvKappaY,curvY);
     }
diff --git a/src/feltor/feltor.cuh b/src/feltor/feltor.cuh
index fa3c675d4..4482bb33e 100644
--- a/src/feltor/feltor.cuh
+++ b/src/feltor/feltor.cuh
@@ -343,16 +343,11 @@ Feltor<Grid, DS, Matrix, container>::Feltor( const Grid& g, eule::Parameters p,
     dg::blas1::transfer(  dg::pullback(dg::geo::TanhSource<Psip>(mf.psip, gp.psipmin, gp.alpha),      g), source);
     dg::blas1::transfer(  dg::pullback(dg::geo::GaussianDamping<Psip>(mf.psip, gp.psipmax, gp.alpha), g), damping);
     ////////////////////////////transform curvature components////////
-    typename dg::HostVec< typename dg::GeometryTraits<Grid>::memory_category>::host_vector tempX, tempY;
-    dg::geo::pushForwardPerp(dg::geo::CurvatureNablaBR<MagneticField>(mf, gp.R_0), dg::geo::CurvatureNablaBZ<MagneticField>(mf, gp.R_0), tempX, tempY, g);
-    dg::blas1::transfer(  tempX, curvX);
-    dg::blas1::transfer(  tempY, curvY);
+    dg::geo::pushForwardPerp(dg::geo::CurvatureNablaBR<MagneticField>(mf, gp.R_0), dg::geo::CurvatureNablaBZ<MagneticField>(mf, gp.R_0), curvX, curvY, g);
     dg::blas1::transfer(  dg::pullback(dg::geo::DivCurvatureKappa<MagneticField>(mf, gp.R_0), g), divCurvKappa);
     if (p.curvmode==1) 
     {
-        dg::geo::pushForwardPerp(dg::geo::CurvatureKappaR(), dg::geo::CurvatureKappaZ<MagneticField>(mf, gp.R_0), tempX, tempY, g);
-        dg::blas1::transfer(  tempX, curvKappaX);
-        dg::blas1::transfer(  tempY, curvKappaY);
+        dg::geo::pushForwardPerp(dg::geo::CurvatureKappaR(), dg::geo::CurvatureKappaZ<MagneticField>(mf, gp.R_0), curvKappaX, curvKappaY, g);
         dg::blas1::axpby( 1.,curvX,1.,curvKappaX,curvX);
         dg::blas1::axpby( 1.,curvY,1.,curvKappaY,curvY);
     }
-- 
GitLab


From d3bfa6e5425106be4b2a880df62eed0a1f3d4fa8 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Wed, 19 Jul 2017 19:31:28 +0200
Subject: [PATCH 049/453] made MPI global function virtual and implemented them
 in geometries

---
 inc/dg/backend/mpi_grid.h        | 26 +++++++++-------------
 inc/dg/ds.h                      |  9 ++++++++
 inc/geometries/mpi_conformal.h   | 38 ++++++++++++++++++--------------
 inc/geometries/mpi_curvilinear.h | 38 ++++++++++++++++++--------------
 inc/geometries/mpi_orthogonal.h  | 38 ++++++++++++++++++--------------
 5 files changed, 82 insertions(+), 67 deletions(-)

diff --git a/inc/dg/backend/mpi_grid.h b/inc/dg/backend/mpi_grid.h
index 1baab344d..5358847ec 100644
--- a/inc/dg/backend/mpi_grid.h
+++ b/inc/dg/backend/mpi_grid.h
@@ -275,21 +275,22 @@ struct MPIGrid2d
     }
 
     /**
-     * @brief Return a grid local for the calling process
+     * @brief Return a non-MPI grid local for the calling process
      *
      * The local grid returns the same values for x0(), x1(), ..., Nx(), Ny(), ... as the grid
-     * class itself
+     * class itself 
      * @return Grid object
+     * @note the boundary conditions in the local grid are not well defined since there might not actually be any boundaries
      */
     Grid2d local() const {return Grid2d(x0(), x1(), y0(), y1(), n(), Nx(), Ny(), bcx(), bcy());}
 
     /**
-     * @brief Return a grid global for the calling process
+     * @brief Return the global non-MPI grid 
      *
-     * The global grid contains the global boundaries
-     * @return Grid object
+     * The global grid contains the global boundaries and cell numbers. This is the grid that we would have to use in a non-MPI implementation.
+     * @return non-MPI Grid object
      */
-    Grid2d global() const {return g;}
+    virtual const Grid2d& global() const {return g;}
     /**
      * @brief Returns the pid of the process that holds the local grid surrounding the given point
      *
@@ -627,20 +628,13 @@ struct MPIGrid3d
 
     }
     /**
-     * @brief Return a grid local for the calling process
-     *
-     * The local grid returns the same values for x0(), x1(), ..., Nx(), Ny(), ... as the grid
-     * class itself
-     * @return Grid object
+     *@copydoc MPIGrid2d::local()const
      */
     Grid3d local() const {return Grid3d(x0(), x1(), y0(), y1(), z0(), z1(), n(), Nx(), Ny(), Nz(), bcx(), bcy(), bcz());}
     /**
-     * @brief Return a grid global for the calling process
-     *
-     * The global grid contains the global boundaries
-     * @return Grid object
+     *@copydoc MPIGrid2d::global()const
      */
-    Grid3d global() const {return g;}
+    virtual const Grid3d& global() const {return g;}
     /**
      * @brief Returns the pid of the process that holds the local grid surrounding the given point
      *
diff --git a/inc/dg/ds.h b/inc/dg/ds.h
index ee3d9152c..e364ed829 100644
--- a/inc/dg/ds.h
+++ b/inc/dg/ds.h
@@ -8,6 +8,7 @@
 #include "backend/mpi_derivatives.h"
 #include "geometry/mpi_fieldaligned.h"
 #endif //MPI_VERSION
+#include "../geometries/magnetic_field.h"
 
 /*!@file 
  *
@@ -225,6 +226,14 @@ DS<G, I, M,container>::DS(MagneticField mag, Geometry grid, dg::norm no, dg::dir
         invB(dg::pullback(dg::geo::InvB<MagneticField>(mag),grid)), 
         no_(no), dir_(dir), apply_jumpX_(jumpX)
 {
+    //if geometry = cylindrical  (use traits)
+    //construct cylinder field Field
+    Field<Magneticfield> field(mag);
+    //else 
+    //construct interpolate field DSField
+    DSField<typename G::perpendicular_grid> field( mag, grid.perp_grid());
+    //if flux-aligned
+    //use only 1d refinement
     volume_ = dg::evaluate( dg::one, grid);
     dg::geo::multiplyVolume( volume_, grid);
 }
diff --git a/inc/geometries/mpi_conformal.h b/inc/geometries/mpi_conformal.h
index 7d171ddc8..4d56d87f7 100644
--- a/inc/geometries/mpi_conformal.h
+++ b/inc/geometries/mpi_conformal.h
@@ -32,10 +32,9 @@ struct ConformalMPIGrid3d : public dg::MPIGrid3d
     ConformalMPIGrid3d( const Generator& generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx, MPI_Comm comm): 
         dg::MPIGrid3d( 0, 1, 0., 2*M_PI, 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, dg::PER, dg::PER, comm),
         r_(dg::evaluate( dg::one, *this)), z_(r_), xr_(r_), xz_(r_), yr_(r_), yz_(r_),
-        g_xx_(r_), g_xy_(g_xx_), g_yy_(g_xx_), g_pp_(g_xx_), vol_(g_xx_), vol2d_(g_xx_)
+        g_xx_(r_), g_xy_(g_xx_), g_yy_(g_xx_), g_pp_(g_xx_), vol_(g_xx_), vol2d_(g_xx_),
+        g( generator, n,Nx, Ny, local().Nz(), bcx)
     {
-        dg::ConformalGrid3d<LocalContainer> g( generator, n,Nx, Ny, local().Nz(), bcx);
-
         //divide and conquer
         int dims[3], periods[3], coords[3];
         MPI_Cart_get( comm, 3, dims, periods, coords);
@@ -79,9 +78,11 @@ struct ConformalMPIGrid3d : public dg::MPIGrid3d
     const dg::MPI_Vector<LocalContainer>& g_pp()const{return g_pp_;}
     const dg::MPI_Vector<LocalContainer>& vol()const{return vol_;}
     const dg::MPI_Vector<LocalContainer>& perpVol()const{return vol2d_;}
+    const dg::ConformalGrid3d<LocalContainer>& global() const {return g;}
     private:
     dg::MPI_Vector<thrust::host_vector<double> > r_, z_, xr_, xz_, yr_, yz_; //3d vector
     dg::MPI_Vector<LocalContainer> g_xx_, g_xy_, g_yy_, g_pp_, vol_, vol2d_;
+    dg::ConformalGrid3d<LocalContainer> g;
 };
 
 /**
@@ -96,13 +97,13 @@ struct ConformalMPIGrid2d : public dg::MPIGrid2d
     ConformalMPIGrid2d( const Generator& generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx, MPI_Comm comm2d): 
         dg::MPIGrid2d( 0, 1, 0., 2*M_PI, n, Nx, Ny, bcx, dg::PER, comm2d),
         r_(dg::evaluate( dg::one, *this)), z_(r_), xr_(r_), xz_(r_), yr_(r_), yz_(r_), 
-        g_xx_(r_), g_xy_(g_xx_), g_yy_(g_xx_), vol2d_(g_xx_)
+        g_xx_(r_), g_xy_(g_xx_), g_yy_(g_xx_), vol2d_(g_xx_),
+        g_( generator, n,Nx, Ny, bcx)
     {
-        dg::ConformalGrid2d<LocalContainer> g( generator, n,Nx, Ny, bcx);
         //divide and conquer
         int dims[2], periods[2], coords[2];
         MPI_Cart_get( communicator(), 2, dims, periods, coords);
-        init_X_boundaries( g.x0(), g.x1());
+        init_X_boundaries( g_.x0(), g_.x1());
             //for( unsigned py=0; py<dims[1]; py++)
                 for( unsigned i=0; i<this->n()*this->Ny(); i++)
                     //for( unsigned px=0; px<dims[0]; px++)
@@ -110,22 +111,23 @@ struct ConformalMPIGrid2d : public dg::MPIGrid2d
                         {
                             unsigned idx1 = i*this->n()*this->Nx() + j;
                             unsigned idx2 = ((coords[1]*this->n()*this->Ny()+i)*dims[0] + coords[0])*this->n()*this->Nx() + j;
-                            r_.data()[idx1] = g.r()[idx2];
-                            z_.data()[idx1] = g.z()[idx2];
-                            xr_.data()[idx1] = g.xr()[idx2];
-                            xz_.data()[idx1] = g.xz()[idx2];
-                            yr_.data()[idx1] = g.yr()[idx2];
-                            yz_.data()[idx1] = g.yz()[idx2];
-                            g_xx_.data()[idx1] = g.g_xx()[idx2];
-                            g_xy_.data()[idx1] = g.g_xy()[idx2];
-                            g_yy_.data()[idx1] = g.g_yy()[idx2];
-                            vol2d_.data()[idx1] = g.perpVol()[idx2];
+                            r_.data()[idx1] = g_.r()[idx2];
+                            z_.data()[idx1] = g_.z()[idx2];
+                            xr_.data()[idx1] = g_.xr()[idx2];
+                            xz_.data()[idx1] = g_.xz()[idx2];
+                            yr_.data()[idx1] = g_.yr()[idx2];
+                            yz_.data()[idx1] = g_.yz()[idx2];
+                            g_xx_.data()[idx1] = g_.g_xx()[idx2];
+                            g_xy_.data()[idx1] = g_.g_xy()[idx2];
+                            g_yy_.data()[idx1] = g_.g_yy()[idx2];
+                            vol2d_.data()[idx1] = g_.perpVol()[idx2];
                         }
     }
     ConformalMPIGrid2d( const ConformalMPIGrid3d<LocalContainer>& g):
         dg::MPIGrid2d( g.global().x0(), g.global().x1(), g.global().y0(), g.global().y1(), g.global().n(), g.global().Nx(), g.global().Ny(), g.global().bcx(), g.global().bcy(), get_reduced_comm( g.communicator() )),
         r_(dg::evaluate( dg::one, *this)), z_(r_), xr_(r_), xz_(r_), yr_(r_), yz_(r_), 
-        g_xx_(r_), g_xy_(g_xx_), g_yy_(g_xx_), vol2d_(g_xx_)
+        g_xx_(r_), g_xy_(g_xx_), g_yy_(g_xx_), vol2d_(g_xx_),
+        g_(g.global())
     {
         unsigned s = this->size();
         for( unsigned i=0; i<s; i++)
@@ -155,6 +157,7 @@ struct ConformalMPIGrid2d : public dg::MPIGrid2d
     const dg::MPI_Vector<LocalContainer>& g_xy()const{return g_xy_;}
     const dg::MPI_Vector<LocalContainer>& vol()const{return vol2d_;}
     const dg::MPI_Vector<LocalContainer>& perpVol()const{return vol2d_;}
+    const dg::ConformalGrid2d<LocalContainer>& global()const{return g_;}
     private:
     MPI_Comm get_reduced_comm( MPI_Comm src)
     {
@@ -166,6 +169,7 @@ struct ConformalMPIGrid2d : public dg::MPIGrid2d
 
     dg::MPI_Vector<thrust::host_vector<double> > r_, z_, xr_, xz_, yr_, yz_; //2d vector
     dg::MPI_Vector<LocalContainer> g_xx_, g_xy_, g_yy_, vol2d_;
+    dg::ConformalGrid2d<LocalContainer> g_;
 };
 ///@}
 }//namespace dg
diff --git a/inc/geometries/mpi_curvilinear.h b/inc/geometries/mpi_curvilinear.h
index 7e509f5c2..827c43653 100644
--- a/inc/geometries/mpi_curvilinear.h
+++ b/inc/geometries/mpi_curvilinear.h
@@ -33,10 +33,9 @@ struct CurvilinearMPIGrid3d : public dg::MPIGrid3d
     CurvilinearMPIGrid3d( const Generator& generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx, MPI_Comm comm): 
         dg::MPIGrid3d( 0, generator.width(), 0., generator.height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, dg::PER, dg::PER, comm),
         r_(dg::evaluate( dg::one, *this)), z_(r_), xr_(r_), xz_(r_), yr_(r_), yz_(r_),
-        g_xx_(r_), g_xy_(g_xx_), g_yy_(g_xx_), g_pp_(g_xx_), vol_(g_xx_), vol2d_(g_xx_)
+        g_xx_(r_), g_xy_(g_xx_), g_yy_(g_xx_), g_pp_(g_xx_), vol_(g_xx_), vol2d_(g_xx_),
+        g( generator, n,Nx, Ny, local().Nz(), bcx)
     {
-        dg::CurvilinearGrid3d<LocalContainer> g( generator, n,Nx, Ny, local().Nz(), bcx);
-
         //divide and conquer
         int dims[3], periods[3], coords[3];
         MPI_Cart_get( comm, 3, dims, periods, coords);
@@ -80,9 +79,11 @@ struct CurvilinearMPIGrid3d : public dg::MPIGrid3d
     const dg::MPI_Vector<LocalContainer>& g_pp()const{return g_pp_;}
     const dg::MPI_Vector<LocalContainer>& vol()const{return vol_;}
     const dg::MPI_Vector<LocalContainer>& perpVol()const{return vol2d_;}
+    const dg::CurvilinearGrid3d<LocalContainer>& global() const {return g;}
     private:
     dg::MPI_Vector<thrust::host_vector<double> > r_, z_, xr_, xz_, yr_, yz_; //3d vector
     dg::MPI_Vector<LocalContainer> g_xx_, g_xy_, g_yy_, g_pp_, vol_, vol2d_;
+    dg::CurvilinearGrid3d<LocalContainer> g;
 };
 
 /**
@@ -97,13 +98,13 @@ struct CurvilinearMPIGrid2d : public dg::MPIGrid2d
     CurvilinearMPIGrid2d( const Generator& generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx, MPI_Comm comm2d): 
         dg::MPIGrid2d( 0, generator.width(), 0., generator.height(), n, Nx, Ny, bcx, dg::PER, comm2d),
         r_(dg::evaluate( dg::one, *this)), z_(r_), xr_(r_), xz_(r_), yr_(r_), yz_(r_), 
-        g_xx_(r_), g_xy_(g_xx_), g_yy_(g_xx_), vol2d_(g_xx_)
+        g_xx_(r_), g_xy_(g_xx_), g_yy_(g_xx_), vol2d_(g_xx_),
+        g_( generator, n,Nx, Ny, bcx)
     {
-        dg::CurvilinearGrid2d<LocalContainer> g( generator, n,Nx, Ny, bcx);
         //divide and conquer
         int dims[2], periods[2], coords[2];
         MPI_Cart_get( communicator(), 2, dims, periods, coords);
-        init_X_boundaries( g.x0(), g.x1());
+        init_X_boundaries( g_.x0(), g_.x1());
         //for( unsigned py=0; py<dims[1]; py++)
             for( unsigned i=0; i<this->n()*this->Ny(); i++)
                 //for( unsigned px=0; px<dims[0]; px++)
@@ -111,22 +112,23 @@ struct CurvilinearMPIGrid2d : public dg::MPIGrid2d
                     {
                         unsigned idx1 = i*this->n()*this->Nx() + j;
                         unsigned idx2 = ((coords[1]*this->n()*this->Ny()+i)*dims[0] + coords[0])*this->n()*this->Nx() + j;
-                        r_.data()[idx1] = g.r()[idx2];
-                        z_.data()[idx1] = g.z()[idx2];
-                        xr_.data()[idx1] = g.xr()[idx2];
-                        xz_.data()[idx1] = g.xz()[idx2];
-                        yr_.data()[idx1] = g.yr()[idx2];
-                        yz_.data()[idx1] = g.yz()[idx2];
-                        g_xx_.data()[idx1] = g.g_xx()[idx2];
-                        g_xy_.data()[idx1] = g.g_xy()[idx2];
-                        g_yy_.data()[idx1] = g.g_yy()[idx2];
-                        vol2d_.data()[idx1] = g.perpVol()[idx2];
+                        r_.data()[idx1] = g_.r()[idx2];
+                        z_.data()[idx1] = g_.z()[idx2];
+                        xr_.data()[idx1] = g_.xr()[idx2];
+                        xz_.data()[idx1] = g_.xz()[idx2];
+                        yr_.data()[idx1] = g_.yr()[idx2];
+                        yz_.data()[idx1] = g_.yz()[idx2];
+                        g_xx_.data()[idx1] = g_.g_xx()[idx2];
+                        g_xy_.data()[idx1] = g_.g_xy()[idx2];
+                        g_yy_.data()[idx1] = g_.g_yy()[idx2];
+                        vol2d_.data()[idx1] = g_.perpVol()[idx2];
                     }
     }
     CurvilinearMPIGrid2d( const CurvilinearMPIGrid3d<LocalContainer>& g):
         dg::MPIGrid2d( g.global().x0(), g.global().x1(), g.global().y0(), g.global().y1(), g.global().n(), g.global().Nx(), g.global().Ny(), g.global().bcx(), g.global().bcy(), get_reduced_comm( g.communicator() )),
         r_(dg::evaluate( dg::one, *this)), z_(r_), xr_(r_), xz_(r_), yr_(r_), yz_(r_), 
-        g_xx_(r_), g_xy_(g_xx_), g_yy_(g_xx_), vol2d_(g_xx_)
+        g_xx_(r_), g_xy_(g_xx_), g_yy_(g_xx_), vol2d_(g_xx_),
+        g_( g.global())
     {
         unsigned s = this->size();
         for( unsigned i=0; i<s; i++)
@@ -156,6 +158,7 @@ struct CurvilinearMPIGrid2d : public dg::MPIGrid2d
     const dg::MPI_Vector<LocalContainer>& g_xy()const{return g_xy_;}
     const dg::MPI_Vector<LocalContainer>& vol()const{return vol2d_;}
     const dg::MPI_Vector<LocalContainer>& perpVol()const{return vol2d_;}
+    const dg::CurvilinearGrid2d<LocalContainer>& global() const {return g_;}
     private:
     MPI_Comm get_reduced_comm( MPI_Comm src)
     {
@@ -167,6 +170,7 @@ struct CurvilinearMPIGrid2d : public dg::MPIGrid2d
 
     dg::MPI_Vector<thrust::host_vector<double> > r_, z_, xr_, xz_, yr_, yz_; //2d vector
     dg::MPI_Vector<LocalContainer> g_xx_, g_xy_, g_yy_, vol2d_;
+    dg::CurvilinearGrid2d<LocalContainer> g_;
 };
 ///@}
 }//namespace dg
diff --git a/inc/geometries/mpi_orthogonal.h b/inc/geometries/mpi_orthogonal.h
index a461e3253..fed2151d3 100644
--- a/inc/geometries/mpi_orthogonal.h
+++ b/inc/geometries/mpi_orthogonal.h
@@ -32,10 +32,9 @@ struct OrthogonalMPIGrid3d : public dg::MPIGrid3d
     OrthogonalMPIGrid3d( const Generator& generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx, MPI_Comm comm): 
         dg::MPIGrid3d( 0, generator.width(), 0., generator.height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, dg::PER, dg::PER, comm),
         r_(dg::evaluate( dg::one, *this)), z_(r_), xr_(r_), xz_(r_), yr_(r_), yz_(r_),
-        g_xx_(r_), g_xy_(g_xx_), g_yy_(g_xx_), g_pp_(g_xx_), vol_(g_xx_), vol2d_(g_xx_)
+        g_xx_(r_), g_xy_(g_xx_), g_yy_(g_xx_), g_pp_(g_xx_), vol_(g_xx_), vol2d_(g_xx_),
+        g( generator, n, Nx, Ny, local().Nz(), bcx)
     {
-        dg::OrthogonalGrid3d<LocalContainer> g( generator, n,Nx, Ny, local().Nz(), bcx);
-
         //divide and conquer
         int dims[3], periods[3], coords[3];
         MPI_Cart_get( comm, 3, dims, periods, coords);
@@ -79,9 +78,11 @@ struct OrthogonalMPIGrid3d : public dg::MPIGrid3d
     const dg::MPI_Vector<LocalContainer>& g_pp()const{return g_pp_;}
     const dg::MPI_Vector<LocalContainer>& vol()const{return vol_;}
     const dg::MPI_Vector<LocalContainer>& perpVol()const{return vol2d_;}
+    const dg::OrthogonalGrid3d<LocalContainer>& global() const{return g;}
     private:
     dg::MPI_Vector<thrust::host_vector<double> > r_, z_, xr_, xz_, yr_, yz_; //3d vector
     dg::MPI_Vector<LocalContainer> g_xx_, g_xy_, g_yy_, g_pp_, vol_, vol2d_;
+    dg::OrthogonalGrid3d<LocalContainer> g;
 };
 
 /**
@@ -96,13 +97,13 @@ struct OrthogonalMPIGrid2d : public dg::MPIGrid2d
     OrthogonalMPIGrid2d( const Generator& generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx, MPI_Comm comm2d): 
         dg::MPIGrid2d( 0, generator.width(), 0., generator.height(), n, Nx, Ny, bcx, dg::PER, comm2d),
         r_(dg::evaluate( dg::one, *this)), z_(r_), xr_(r_), xz_(r_), yr_(r_), yz_(r_), 
-        g_xx_(r_), g_xy_(g_xx_), g_yy_(g_xx_), vol2d_(g_xx_)
+        g_xx_(r_), g_xy_(g_xx_), g_yy_(g_xx_), vol2d_(g_xx_),
+        g_( generator, n, Nx, Ny, bcx)
     {
-        dg::OrthogonalGrid2d<LocalContainer> g( generator, n,Nx, Ny, bcx);
         //divide and conquer
         int dims[2], periods[2], coords[2];
         MPI_Cart_get( communicator(), 2, dims, periods, coords);
-        init_X_boundaries( g.x0(), g.x1());
+        init_X_boundaries( g_.x0(), g_.x1());
             //for( unsigned py=0; py<dims[1]; py++)
                 for( unsigned i=0; i<this->n()*this->Ny(); i++)
                     //for( unsigned px=0; px<dims[0]; px++)
@@ -110,22 +111,23 @@ struct OrthogonalMPIGrid2d : public dg::MPIGrid2d
                         {
                             unsigned idx1 = i*this->n()*this->Nx() + j;
                             unsigned idx2 = ((coords[1]*this->n()*this->Ny()+i)*dims[0] + coords[0])*this->n()*this->Nx() + j;
-                            r_.data()[idx1] = g.r()[idx2];
-                            z_.data()[idx1] = g.z()[idx2];
-                            xr_.data()[idx1] = g.xr()[idx2];
-                            xz_.data()[idx1] = g.xz()[idx2];
-                            yr_.data()[idx1] = g.yr()[idx2];
-                            yz_.data()[idx1] = g.yz()[idx2];
-                            g_xx_.data()[idx1] = g.g_xx()[idx2];
-                            g_xy_.data()[idx1] = g.g_xy()[idx2];
-                            g_yy_.data()[idx1] = g.g_yy()[idx2];
-                            vol2d_.data()[idx1] = g.perpVol()[idx2];
+                            r_.data()[idx1] = g_.r()[idx2];
+                            z_.data()[idx1] = g_.z()[idx2];
+                            xr_.data()[idx1] = g_.xr()[idx2];
+                            xz_.data()[idx1] = g_.xz()[idx2];
+                            yr_.data()[idx1] = g_.yr()[idx2];
+                            yz_.data()[idx1] = g_.yz()[idx2];
+                            g_xx_.data()[idx1] = g_.g_xx()[idx2];
+                            g_xy_.data()[idx1] = g_.g_xy()[idx2];
+                            g_yy_.data()[idx1] = g_.g_yy()[idx2];
+                            vol2d_.data()[idx1] = g_.perpVol()[idx2];
                         }
     }
     OrthogonalMPIGrid2d( const OrthogonalMPIGrid3d<LocalContainer>& g):
         dg::MPIGrid2d( g.global().x0(), g.global().x1(), g.global().y0(), g.global().y1(), g.global().n(), g.global().Nx(), g.global().Ny(), g.global().bcx(), g.global().bcy(), get_reduced_comm( g.communicator() )),
         r_(dg::evaluate( dg::one, *this)), z_(r_), xr_(r_), xz_(r_), yr_(r_), yz_(r_), 
-        g_xx_(r_), g_xy_(g_xx_), g_yy_(g_xx_), vol2d_(g_xx_)
+        g_xx_(r_), g_xy_(g_xx_), g_yy_(g_xx_), vol2d_(g_xx_),
+        g_( g.global())
     {
         unsigned s = this->size();
         for( unsigned i=0; i<s; i++)
@@ -155,6 +157,7 @@ struct OrthogonalMPIGrid2d : public dg::MPIGrid2d
     const dg::MPI_Vector<LocalContainer>& g_xy()const{return g_xy_;}
     const dg::MPI_Vector<LocalContainer>& vol()const{return vol2d_;}
     const dg::MPI_Vector<LocalContainer>& perpVol()const{return vol2d_;}
+    const dg::OrthogonalGrid2d<LocalContainer>& global() const {return g;}
     private:
     MPI_Comm get_reduced_comm( MPI_Comm src)
     {
@@ -166,6 +169,7 @@ struct OrthogonalMPIGrid2d : public dg::MPIGrid2d
 
     dg::MPI_Vector<thrust::host_vector<double> > r_, z_, xr_, xz_, yr_, yz_; //2d vector
     dg::MPI_Vector<LocalContainer> g_xx_, g_xy_, g_yy_, vol2d_;
+    dg::OrthogonalGrid2d<LocalContainer> g_;
 };
 ///@}
 }//namespace dg
-- 
GitLab


From a80efcf9812279026726d5b6067c000f8689a691 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Wed, 19 Jul 2017 22:56:03 +0200
Subject: [PATCH 050/453] introduced set function in grid and mpi grid

---
 inc/dg/backend/grid.h     |  57 +++------
 inc/dg/backend/mpi_grid.h | 249 +++++++++++---------------------------
 2 files changed, 89 insertions(+), 217 deletions(-)

diff --git a/inc/dg/backend/grid.h b/inc/dg/backend/grid.h
index 5e8cbce9b..068f64b95 100644
--- a/inc/dg/backend/grid.h
+++ b/inc/dg/backend/grid.h
@@ -289,27 +289,16 @@ struct Grid2d
     Grid2d local_grid() const {return *this;}
 
     /**
-    * @brief Return a copy of the grid with increased number of cells
+    * @brief Set the number of polynomials and cells
     *
-    * @param nx multiply # of cells in x 
-    * @param ny multiply # of cells in y
-    *
-    * @return a copy of this grid with nx*Nx and ny*Ny cells in x and y
+    * @param new_n new number of %Gaussian nodes
+    * @param new_Nx new number of cells in x 
+    * @param new_Ny new number of cells in y
     */
-    virtual Grid2d multiply( unsigned nx, unsigned ny) const {
-        return Grid2d( x0_, x1_, y0_, y1_, n_, nx*Nx_, ny*Ny_, bcx_, bcy_);
-    }
-    /**
-    * @brief Return a copy of the grid with reduced number of cells
-    *
-    * @param nx divide # of cells in x 
-    * @param ny divide # of cells in y
-    *
-    * @return a copy of this grid with Nx/nx and Ny/ny cells in x and y
-    * @attention The function won't check if the number of cells are divisible without rest
-    */
-    virtual Grid2d divide( unsigned nx, unsigned ny) const {
-        return Grid2d( x0_, x1_, y0_, y1_, n_, Nx_/nx, Ny_/ny, bcx_, bcy_);
+    virtual void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny) {
+        n_ = new_n, Nx_ = new_Nx, Ny_ = new_Ny;
+        hx_ = lx_/(double)Nx_, hy_ = ly_/(double)Ny_;
+        dlt_ = DLT<double>( new_n);
     }
 
     /**
@@ -465,29 +454,17 @@ struct Grid3d
         hx_ = lx_/(double)Nx_, hy_ = ly_/(double)Ny_, hz_ = lz_/(double)Nz_;
     }
     /**
-    * @brief Return a copy of the grid with increased number of cells
-    *
-    * @param nx multiply # of cells in x 
-    * @param ny multiply # of cells in y
-    * @param nz multiply # of cells in z 
-    *
-    * @return a copy of this grid with nx*Nx, ny*Ny and nz*Nz cells in x, y and z
-    */
-    virtual Grid3d multiply( unsigned nx, unsigned ny, unsigned nz) const {
-        return Grid3d( x0_, x1_, y0_, y1_, z0_, z1_, n_, nx*Nx_, ny*Ny_, nz*Nz_, bcx_, bcy_, bcz_);
-    }
-    /**
-    * @brief Return a copy of the grid with reduced number of cells
-    *
-    * @param nx divide # of cells in x 
-    * @param ny divide # of cells in y
-    * @param nz divide # of cells in z 
+    * @brief Set the number of polynomials and cells
     *
-    * @return a copy of this grid with Nx/nx, Ny/ny cells in x, y and z
-    * @attention The function won't check if the number of cells are divisible without rest
+    * @param new_n new number of %Gaussian nodes
+    * @param new_Nx new number of cells in x 
+    * @param new_Ny new number of cells in y
+    * @param new_Nz new number of cells in z
     */
-    virtual Grid3d divide( unsigned nx, unsigned ny, unsigned nz) const {
-        return Grid3d( x0_, x1_, y0_, y1_, z0_, z1_, n_, Nx_/nx, Ny_/ny, Nz_/nz, bcx_, bcy_, bcz_);
+    virtual void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz) {
+        n_ = new_n, Nx_ = new_Nx, Ny_ = new_Ny, Nz_ = new_Nz;
+        hx_ = lx_/(double)Nx_, hy_ = ly_/(double)Ny_, hz_ =lz_/(double)Nz_;
+        dlt_ = DLT<double>( new_n);
     }
 
     /**
diff --git a/inc/dg/backend/mpi_grid.h b/inc/dg/backend/mpi_grid.h
index 5358847ec..ca7a4dc8a 100644
--- a/inc/dg/backend/mpi_grid.h
+++ b/inc/dg/backend/mpi_grid.h
@@ -19,105 +19,42 @@ namespace dg
  *
  * Represents the local grid coordinates and the process topology. 
  * It just divides the given (global) box into nonoverlapping (local) subboxes that are attributed to each process
+ * @note a single cell is never divided across processes.
  * @attention
  * The boundaries in the constructors are global boundaries, the boundaries returned by the access functions are local boundaries, this is because the grid represents the information given to one process
  *
- * @note Note that a single cell is never divided across processes.
  */
 struct MPIGrid2d
 {
     typedef MPITag memory_category;
     typedef TwoDimensionalTag dimensionality;
     /**
-     * @brief Construct mpi grid
-     *
-     * @param x0
-     * @param x1
-     * @param y0
-     * @param y1
-     * @param n
-     * @param Nx
-     * @param Ny
-     * @param comm
+     * @copydoc dg::Grid2d::Grid2d()
+     * @param comm a two-dimensional Cartesian communicator
+     * @note the paramateres given in the constructor are global parameters 
      */
     MPIGrid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, MPI_Comm comm):
         g( x0, x1, y0, y1, n, Nx, Ny), comm( comm)
     {
-        int rank, dims[2], periods[2], coords[2];
-        MPI_Cart_get( comm, 2, dims, periods, coords);
-        MPI_Comm_rank( comm, &rank);
-        if( rank == 0)
-        {
-            if(Nx%dims[0]!=0)
-                std::cerr << "Nx "<<Nx<<" npx "<<dims[0]<<std::endl;
-            assert( Nx%dims[0] == 0);
-            if(Ny%dims[1]!=0)
-                std::cerr << "Ny "<<Ny<<" npy "<<dims[1]<<std::endl;
-            assert( Ny%dims[1] == 0);
-            if( g.bcx() == dg::PER) assert( periods[0] == true);
-            else assert( periods[0] == false);
-            if( g.bcy() == dg::PER) assert( periods[1] == true);
-            else assert( periods[1] == false);
-        }
+        check_division( Nx, Ny, g.bcx(), g.bcy());
     }
 
     /**
-     * @brief Construct mpi grid
-     *
-     * @param x0
-     * @param x1
-     * @param y0
-     * @param y1
-     * @param n
-     * @param Nx
-     * @param Ny
-     * @param bcx
-     * @param bcy
-     * @param comm
+     * @copydoc dg::Grid2d::Grid2d()
+     * @param comm a two-dimensional Cartesian communicator
+     * @note the paramateres given in the constructor are global parameters 
      */
     MPIGrid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx, bc bcy, MPI_Comm comm):
         g( x0, x1, y0, y1, n, Nx, Ny, bcx, bcy), comm( comm)
     {
-        int rank, dims[2], periods[2], coords[2];
-        MPI_Cart_get( comm, 2, dims, periods, coords);
-        MPI_Comm_rank( comm, &rank);
-        if( rank == 0)
-        {
-            if(Nx%dims[0]!=0)
-                std::cerr << "Nx "<<Nx<<" npx "<<dims[0]<<std::endl;
-            assert( Nx%dims[0] == 0);
-            if(Ny%dims[1]!=0)
-                std::cerr << "Ny "<<Ny<<" npy "<<dims[1]<<std::endl;
-            assert( Ny%dims[1] == 0);
-            if( bcx == dg::PER) assert( periods[0] == true);
-            else assert( periods[0] == false);
-            if( bcy == dg::PER) assert( periods[1] == true);
-            else assert( periods[1] == false);
-        }
-    }
-    /**
-    * @brief Return a copy of the grid with increased number of cells
-    *
-    * @param nx multiply # of cells in x 
-    * @param ny multiply # of cells in y
-    *
-    * @return a copy of this grid with nx*Nx and ny*Ny cells in x and y
-    */
-    virtual MPIGrid2d multiply( unsigned nx, unsigned ny) const {
-        return MPIGrid2d( x0_, x1_, y0_, y1_, n_, nx*Nx_, ny*Ny_, bcx_, bcy_, comm_);
+        check_division( Nx, Ny, bcx, bcy);
     }
-    /**
-    * @brief Return a copy of the grid with reduced number of cells
-    *
-    * @param nx divide # of cells in x 
-    * @param ny divide # of cells in y
-    *
-    * @return a copy of this grid with Nx/nx and Ny/ny cells in x and y
-    * @attention The function won't check if the number of cells are divisible without rest
-         but it does check if the number of processes is still a divisor
-    */
-    virtual MPIGrid2d divide( unsigned nx, unsigned ny) const {
-        return MPIGrid2d( x0_, x1_, y0_, y1_, n_, Nx_/nx, Ny_/ny, bcx_, bcy_, comm_);
+
+    ///@copydoc Grid2d::set(unsigned,unsigned,unsigned)
+    ///@note these are global parameters
+    virtual void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny) {
+        g.set(new_n,new_Nx,new_Ny);
+        check_division( new_Nx, new_Ny, g.bcx(), g.bcy());
     }
 
     /**
@@ -306,6 +243,25 @@ struct MPIGrid2d
         g.init_X_boundaries(global_x0, global_x1);
     }
     private:
+    void check_division( unsigned Nx, unsigned Ny, bc bcx, bc bcy)
+    {
+        int rank, dims[2], periods[2], coords[2];
+        MPI_Cart_get( comm, 2, dims, periods, coords);
+        MPI_Comm_rank( comm, &rank);
+        if( rank == 0)
+        {
+            if(Nx%dims[0]!=0)
+                std::cerr << "Nx "<<Nx<<" npx "<<dims[0]<<std::endl;
+            assert( Nx%dims[0] == 0);
+            if(Ny%dims[1]!=0)
+                std::cerr << "Ny "<<Ny<<" npy "<<dims[1]<<std::endl;
+            assert( Ny%dims[1] == 0);
+            if( bcx == dg::PER) assert( periods[0] == true);
+            else assert( periods[0] == false);
+            if( bcy == dg::PER) assert( periods[1] == true);
+            else assert( periods[1] == false);
+        }
+    }
     Grid2d g; //global grid
     MPI_Comm comm; //just an integer...
 
@@ -326,118 +282,33 @@ struct MPIGrid3d
     typedef MPITag memory_category;
     typedef ThreeDimensionalTag dimensionality;
     /**
-     * @brief Construct a 3D grid
-     *
-     * @param x0 left boundary in x
-     * @param x1 right boundary in x 
-     * @param y0 lower boundary in y
-     * @param y1 upper boundary in y 
-     * @param z0 lower boundary in z
-     * @param z1 upper boundary in z 
-     * @param n  # of polynomial coefficients per (x-,y-) dimension
-     * @param Nx # of points in x 
-     * @param Ny # of points in y
-     * @param Nz # of points in z
-     * @param comm mpi communicator
-     * @attention # of polynomial coefficients in z direction is always 1
+     * @copydoc Grid3d::Grid3d()
+     * @param comm a three-dimensional Cartesian communicator
+     * @note the paramateres given in the constructor are global parameters 
      */
     MPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, MPI_Comm comm):
         g( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz), comm( comm)
     {
-        int rank, dims[3], periods[3], coords[3];
-        MPI_Cart_get( comm, 3, dims, periods, coords);
-        MPI_Comm_rank( comm, &rank);
-        if( rank == 0)
-        {
-            if(!(Nx%dims[0]==0))
-                std::cerr << "Nx "<<Nx<<" npx "<<dims[0]<<std::endl;
-            assert( Nx%dims[0] == 0);
-            if( !(Ny%dims[1]==0))
-                std::cerr << "Ny "<<Ny<<" npy "<<dims[1]<<std::endl;
-            assert( Ny%dims[1] == 0);
-            if( !(Nz%dims[2]==0))
-                std::cerr << "Nz "<<Nz<<" npz "<<dims[2]<<std::endl;
-            assert( Nz%dims[2] == 0);
-            if( g.bcx() == dg::PER) assert( periods[0] == true);
-            else assert( periods[0] == false);
-            if( g.bcy() == dg::PER) assert( periods[1] == true);
-            else assert( periods[1] == false);
-            if( g.bcz() == dg::PER) assert( periods[2] == true);
-            else assert( periods[2] == false);
-        }
+        check_division( Nx, Ny, Nz, bcx(), bcy(), bcz());
     }
 
     /**
-     * @brief Construct a 3D grid
-     *
-     * @param x0 left boundary in x
-     * @param x1 right boundary in x 
-     * @param y0 lower boundary in y
-     * @param y1 upper boundary in y 
-     * @param z0 lower boundary in z
-     * @param z1 upper boundary in z 
-     * @param n  # of polynomial coefficients per (x-,y-) dimension
-     * @param Nx # of points in x 
-     * @param Ny # of points in y
-     * @param Nz # of points in z
-     * @param bcx boundary condition in x
-     * @param bcy boundary condition in y
-     * @param bcz boundary condition in z
-     * @param comm mpi communicator
-     * @attention # of polynomial coefficients in z direction is always 1
+     * @copydoc Grid3d::Grid3d()
+     * @param comm a three-dimensional Cartesian communicator
+     * @note the paramateres given in the constructor are global parameters 
      */
     MPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz, MPI_Comm comm):
         g( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, bcx, bcy, bcz), comm( comm)
     {
-        int rank, dims[3], periods[3], coords[3];
-        MPI_Cart_get( comm, 3, dims, periods, coords);
-        MPI_Comm_rank( comm, &rank);
-        if( rank == 0)
-        {
-            if(!(Nx%dims[0]==0))
-                std::cerr << "Nx "<<Nx<<" npx "<<dims[0]<<std::endl;
-            assert( Nx%dims[0] == 0);
-            if( !(Ny%dims[1]==0))
-                std::cerr << "Ny "<<Ny<<" npy "<<dims[1]<<std::endl;
-            assert( Ny%dims[1] == 0);
-            if( !(Nz%dims[2]==0))
-                std::cerr << "Nz "<<Nz<<" npz "<<dims[2]<<std::endl;
-            assert( Nz%dims[2] == 0);
-            if( bcx == dg::PER) assert( periods[0] == true);
-            else assert( periods[0] == false);
-            if( bcy == dg::PER) assert( periods[1] == true);
-            else assert( periods[1] == false);
-            if( bcz == dg::PER) assert( periods[2] == true);
-            else assert( periods[2] == false);
-        }
-    }
-
-    /**
-    * @brief Return a copy of the grid with increased number of cells
-    *
-    * @param nx multiply # of cells in x 
-    * @param ny multiply # of cells in y
-    * @param nz multiply # of cells in z 
-    *
-    * @return a copy of this grid with nx*Nx, ny*Ny and nz*Nz cells in x, y and z
-    */
-    virtual MPIGrid3d multiply( unsigned nx, unsigned ny, unsigned nz) const {
-        return MPIGrid3d( x0_, x1_, y0_, y1_, z0_, z1_, n_, nx*Nx_, ny*Ny_, nz*Nz_, bcx_, bcy_, bcz_, comm_);
+        check_division( Nx, Ny, Nz, bcx, bcy, bcz);
     }
-    /**
-    * @brief Return a copy of the grid with reduced number of cells
-    *
-    * @param nx divide # of cells in x 
-    * @param ny divide # of cells in y
-    * @param nz divide # of cells in z 
-    *
-    * @return a copy of this grid with Nx/nx, Ny/ny cells in x, y and z
-    * @attention The function won't check if the number of cells are divisible without rest
-    *   but it does check if the number of processes is still a divisor
-    */
-    virtual MPIGrid3d divide( unsigned nx, unsigned ny, unsigned nz) const {
-        return MPIGrid3d( x0_, x1_, y0_, y1_, z0_, z1_, n_, Nx_/nx, Ny_/ny, Nz_/nz, bcx_, bcy_, bcz_, comm_);
+    ///@copydoc Grid3d::set(unsigned,unsigned,unsigned,unsigned)
+    ///@note these are global parameters
+    virtual void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz) {
+        g.set(new_n,new_Nx,new_Ny,new_Nz);
+        check_division( new_Nx,new_Ny,new_Nz,g.bcx(),g.bcy(),g.bcz());
     }
+
     /**
      * @brief Return local x0
      *
@@ -651,6 +522,30 @@ struct MPIGrid3d
         g.init_X_boundaries(global_x0, global_x1);
     }
     private:
+    void check_division( unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz)
+    {
+        int rank, dims[3], periods[3], coords[3];
+        MPI_Cart_get( comm, 3, dims, periods, coords);
+        MPI_Comm_rank( comm, &rank);
+        if( rank == 0)
+        {
+            if(!(Nx%dims[0]==0))
+                std::cerr << "Nx "<<Nx<<" npx "<<dims[0]<<std::endl;
+            assert( Nx%dims[0] == 0);
+            if( !(Ny%dims[1]==0))
+                std::cerr << "Ny "<<Ny<<" npy "<<dims[1]<<std::endl;
+            assert( Ny%dims[1] == 0);
+            if( !(Nz%dims[2]==0))
+                std::cerr << "Nz "<<Nz<<" npz "<<dims[2]<<std::endl;
+            assert( Nz%dims[2] == 0);
+            if( bcx == dg::PER) assert( periods[0] == true);
+            else assert( periods[0] == false);
+            if( bcy == dg::PER) assert( periods[1] == true);
+            else assert( periods[1] == false);
+            if( bcz == dg::PER) assert( periods[2] == true);
+            else assert( periods[2] == false);
+        }
+    }
     Grid3d g; //global grid
     MPI_Comm comm; //just an integer...
 };
-- 
GitLab


From af61b22bd6ce592bcd31c059dc335153eabee6ab Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Wed, 19 Jul 2017 23:06:37 +0200
Subject: [PATCH 051/453] commented on design flaw in geometry

---
 inc/dg/geometry.h | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/inc/dg/geometry.h b/inc/dg/geometry.h
index 8879eff02..ffc70c54a 100644
--- a/inc/dg/geometry.h
+++ b/inc/dg/geometry.h
@@ -23,6 +23,21 @@
  * geometry functions
  */
 
+/*
+Comment: there is a little design flaw in basing the geometry routines also on template programming. 
+For example it is a flaw that we have to recompile when we want to use an Orthogonal instead
+of a Curvilinear grid. Also, the performance gain from using template should be minimal.
+At the same time we cannot get rid of the distinction between Grids and MPIGrids. 
+There is also nothing wrong with recompiling there 
+as this is always accompanied with changing the hardware the program runs on. 
+Now, the question is how severe this flaw actually affects execution in 
+real life, i.e. do users want to change between grids during a parameter scan?
+It might not be that severe especially with the generator mechanism to 
+construct grids. 
+If ever a problem: A possible solution could be to make the geometry functions virtual 
+in the Grid and the MPI_Grid each (we still need both). Then objects would have to
+keep pointers to the grid-base-class (Grid* or MPIGrid*) and work through these.
+*/
 namespace dg{
 
 /*! @brief Geometry routines 
-- 
GitLab


From abafc64ebf978d6d010b847d746aad9fa069a956 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Wed, 19 Jul 2017 23:36:57 +0200
Subject: [PATCH 052/453] shortened docu in geometry by copydocs

---
 inc/dg/backend/interpolation_t.cu | 10 +++---
 inc/dg/geometry/cartesian.h       | 43 ++------------------------
 inc/dg/geometry/cartesianX.h      | 51 ++-----------------------------
 inc/dg/geometry/cylindrical.h     | 36 +++++++++-------------
 inc/dg/geometry/mpi_grids.h       | 35 +++++++++++++++++++++
 inc/dg/geometry/refined_grid.h    | 51 +++----------------------------
 inc/dg/geometry/refined_gridX.h   | 28 ++---------------
 7 files changed, 67 insertions(+), 187 deletions(-)

diff --git a/inc/dg/backend/interpolation_t.cu b/inc/dg/backend/interpolation_t.cu
index 27243997e..b0b6c5053 100644
--- a/inc/dg/backend/interpolation_t.cu
+++ b/inc/dg/backend/interpolation_t.cu
@@ -21,11 +21,11 @@ int main()
 
     {
     dg::Grid2d g( -10, 10, -5, 5, n, Nx, Ny);
-    std::cout << "First test grid multiply divide functions: \n";
+    std::cout << "First test grid set functions: \n";
     g.display( std::cout);
-    g = g.multiply(2,3);
+    g.set(2,2,3);
     g.display( std::cout);
-    g = g.divide(2,3);
+    g.set(n, Nx, Ny);
     g.display( std::cout);
     Matrix A = dg::create::backscatter( g);
     //A.sort_by_row_and_column();
@@ -94,8 +94,8 @@ int main()
     ////////////////////////////////////////////////////////////////////////////
     {
     dg::Grid3d g( -10, 10, -5, 5, -7, -3, n, Nx, Ny, Nz);
-    g = g.multiply( 2,2,2);
-    g = g.divide( 2,2,2);
+    g.set( 2,2,2,3);
+    g.set( n, Nx,Ny,Nz);
     Matrix A = dg::create::backscatter( g);
     //A.sort_by_row_and_column();
 
diff --git a/inc/dg/geometry/cartesian.h b/inc/dg/geometry/cartesian.h
index b26696bdb..2b44e8096 100644
--- a/inc/dg/geometry/cartesian.h
+++ b/inc/dg/geometry/cartesian.h
@@ -14,15 +14,7 @@ namespace dg
 struct CartesianGrid1d: public dg::Grid1d
 {
     typedef OrthonormalTag metric_category; 
-    /**
-     * @brief Constructor is the same as 1d Grid
-     *
-     @param x0 left boundary
-     @param x1 right boundary
-     @param n # of polynomial coefficients
-     @param N # of cells
-     @param bcx boundary conditions
-     */
+    ///@copydoc Grid1d::Grid1d()
     CartesianGrid1d( double x0, double x1, unsigned n, unsigned N, bc bcx = PER): dg::Grid1d(x0,x1,n,N,bcx){}
     /**
      * @brief Construct from existing topology
@@ -38,19 +30,7 @@ struct CartesianGrid1d: public dg::Grid1d
 struct CartesianGrid2d: public dg::Grid2d
 {
     typedef OrthonormalTag metric_category; 
-    /**
-     * @brief Construct a 2D grid
-     *
-     * @param x0 left boundary in x
-     * @param x1 right boundary in x 
-     * @param y0 lower boundary in y
-     * @param y1 upper boundary in y 
-     * @param n  # of polynomial coefficients per dimension
-     * @param Nx # of points in x 
-     * @param Ny # of points in y
-     * @param bcx boundary condition in x
-     * @param bcy boundary condition in y
-     */
+    ///@copydoc Grid2d::Grid2d()
     CartesianGrid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx = PER, bc bcy = PER): dg::Grid2d(x0,x1,y0,y1,n,Nx,Ny,bcx,bcy){}
     /**
      * @brief Construct from existing topology
@@ -66,24 +46,7 @@ struct CartesianGrid2d: public dg::Grid2d
 struct CartesianGrid3d: public dg::Grid3d
 {
     typedef OrthonormalTag metric_category; 
-    /**
-     * @brief Construct a 3D grid
-     *
-     * @param x0 left boundary in x
-     * @param x1 right boundary in x 
-     * @param y0 lower boundary in y
-     * @param y1 upper boundary in y 
-     * @param z0 lower boundary in z
-     * @param z1 upper boundary in z 
-     * @param n  # of polynomial coefficients per (x-,y-) dimension
-     * @param Nx # of points in x 
-     * @param Ny # of points in y
-     * @param Nz # of points in z
-     * @param bcx boundary condition in x
-     * @param bcy boundary condition in y
-     * @param bcz boundary condition in z
-     * @attention # of polynomial coefficients in z direction is always 1
-     */
+    ///@copydoc Grid3d::Grid3d()
     CartesianGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = PER, bc bcy = PER, bc bcz = PER): dg::Grid3d(x0,x1,y0,y1,z0,z1,n,Nx,Ny,Nz,bcx,bcy,bcz){}
     /**
      * @brief Construct from existing topology
diff --git a/inc/dg/geometry/cartesianX.h b/inc/dg/geometry/cartesianX.h
index 98a85f1fe..de1b6001d 100644
--- a/inc/dg/geometry/cartesianX.h
+++ b/inc/dg/geometry/cartesianX.h
@@ -15,21 +15,11 @@ namespace dg
 struct CartesianGridX1d: public dg::GridX1d
 {
     typedef OrthonormalTag metric_category; 
-    /**
-     * @brief 1D grid
-     * 
-     @param x0 left boundary
-     @param x1 right boundary
-     @param f factor 0<f<0.5 divides the domain
-     @param n # of polynomial coefficients
-     @param N # of cells
-     @param bcx boundary conditions
-     */
+    ///@copydoc GridX1d::GridX1d()
     CartesianGridX1d( double x0, double x1, double f, unsigned n, unsigned N, bc bcx = NEU):
         dg::GridX1d(x0,x1,f,n,N,bcx){}
     /**
      * @brief Construct from existing topology
-     *
      * @param grid existing grid class
      */
     CartesianGridX1d( const dg::GridX1d& grid):dg::GridX1d(grid){}
@@ -41,25 +31,10 @@ struct CartesianGridX1d: public dg::GridX1d
 struct CartesianGridX2d: public dg::GridX2d
 {
     typedef OrthonormalTag metric_category; 
-    /**
-     * @brief Construct a 2D grid
-     *
-     * @param x0 left boundary in x
-     * @param x1 right boundary in x 
-     * @param y0 lower boundary in y
-     * @param y1 upper boundary in y 
-     * @param fx factor for x-direction (fx*Nx must be a natural number)
-     * @param fy factor for y-direction (fy*Ny must be a natural number)
-     * @param n  # of polynomial coefficients per dimension
-     * @param Nx # of points in x 
-     * @param Ny # of points in y
-     * @param bcx boundary condition in x
-     * @param bcy boundary condition in y
-     */
+    ///@copydoc GridX2d::GridX2d()
     CartesianGridX2d( double x0, double x1, double y0, double y1, double fx, double fy, unsigned n, unsigned Nx, unsigned Ny, bc bcx = PER, bc bcy = NEU):dg::GridX2d(x0,x1,y0,y1,fx,fy,n,Nx,Ny,bcx,bcy){}
     /**
      * @brief Construct from existing topology
-     *
      * @param grid existing grid class
      */
     CartesianGridX2d( const dg::GridX2d& grid):dg::GridX2d(grid){}
@@ -71,30 +46,10 @@ struct CartesianGridX2d: public dg::GridX2d
 struct CartesianGridX3d: public dg::GridX3d
 {
     typedef OrthonormalTag metric_category; 
-    /**
-     * @brief Construct a 3D grid
-     *
-     * @param x0 left boundary in x
-     * @param x1 right boundary in x 
-     * @param y0 lower boundary in y
-     * @param y1 upper boundary in y 
-     * @param z0 lower boundary in z
-     * @param z1 upper boundary in z 
-     * @param fx factor for x-direction
-     * @param fy factor for y-direction
-     * @param n  # of polynomial coefficients per (x-,y-) dimension
-     * @param Nx # of points in x 
-     * @param Ny # of points in y
-     * @param Nz # of points in z
-     * @param bcx boundary condition in x
-     * @param bcy boundary condition in y
-     * @param bcz boundary condition in z
-     * @attention # of polynomial coefficients in z direction is always 1
-     */
+    ///@copydoc GridX3d::GridX3d()
     CartesianGridX3d( double x0, double x1, double y0, double y1, double z0, double z1, double fx, double fy, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = PER, bc bcy = NEU, bc bcz = PER): dg::GridX3d(x0,x1,y0,y1,z0,z1,fx,fy,n,Nx,Ny,Nz,bcx,bcy,bcz){}
     /**
      * @brief Construct from existing topology
-     *
      * @param grid existing grid class
      */
     CartesianGridX3d( const dg::GridX3d& grid):dg::GridX3d(grid){}
diff --git a/inc/dg/geometry/cylindrical.h b/inc/dg/geometry/cylindrical.h
index 1cb28d588..c834c6d88 100644
--- a/inc/dg/geometry/cylindrical.h
+++ b/inc/dg/geometry/cylindrical.h
@@ -20,24 +20,7 @@ struct CylindricalGrid3d : public dg::Grid3d
 {
     typedef OrthonormalCylindricalTag metric_category; 
     typedef dg::CartesianGrid2d perpendicular_grid;
-    /**
-     * @brief Construct a 3D grid
-     *
-     * @param x0 left boundary in x
-     * @param x1 right boundary in x 
-     * @param y0 lower boundary in y
-     * @param y1 upper boundary in y 
-     * @param z0 lower boundary in z
-     * @param z1 upper boundary in z 
-     * @param n  # of polynomial coefficients per (x-,y-) dimension
-     * @param Nx # of points in x 
-     * @param Ny # of points in y
-     * @param Nz # of points in z
-     * @param bcx boundary condition in x
-     * @param bcy boundary condition in y
-     * @param bcz boundary condition in z
-     * @attention # of polynomial coefficients in z direction is always 1
-     */
+    ///@copydoc Grid3d()
     CylindricalGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = PER, bc bcy = PER, bc bcz = PER): 
         dg::Grid3d(x0,x1,y0,y1,z0,z1,n,Nx,Ny,Nz,bcx,bcy,bcz),
         R_(dg::evaluate( dg::cooX3d, *this)){}
@@ -50,14 +33,23 @@ struct CylindricalGrid3d : public dg::Grid3d
     CylindricalGrid3d( const dg::Grid3d& grid):
         dg::Grid3d(grid),
         R_(dg::evaluate( dg::cooX3d, *this)){}
+
     /**
-     * @brief The volume element
+    * @brief Return the grid of the R-Z planes
+    *
+    * @return a Cartesian 2d grid of the R-Z plane
+    */
+    perpendicular_grid perp_grid() const { return dg::CartesianGrid2d( x0(), x1(), y0(), y1(), n(), Nx(), Ny(), bcx(), bcy());}
+    /**
+     * @brief The volume element R
      *
-     * @return the volume element
+     * @return the volume element R
      */
-
-    perpendicular_grid perp_grid() const { return dg::CartesianGrid2d( x0(), x1(), y0(), y1(), n(), Nx(), Ny(), bcx(), bcy());}
     const container& vol()const {return R_;}
+    void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz){
+        this->set(new_n,new_Ny,new_Nz);
+        R_=dg::evaluate(dg::cooX3d, *this);
+    }
     private:
     container R_;
 };
diff --git a/inc/dg/geometry/mpi_grids.h b/inc/dg/geometry/mpi_grids.h
index c22877bfa..cd5cad564 100644
--- a/inc/dg/geometry/mpi_grids.h
+++ b/inc/dg/geometry/mpi_grids.h
@@ -18,8 +18,18 @@ struct CartesianMPIGrid2d : public dg::MPIGrid2d
 {
     typedef OrthonormalTag metric_category; 
 
+    /**
+     * @copydoc Grid2d::Grid2d()
+     * @param comm a two-dimensional Cartesian communicator
+     * @note the paramateres given in the constructor are global parameters 
+     */
     CartesianMPIGrid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, MPI_Comm comm): dg::MPIGrid2d( x0, x1, y0, y1, n, Nx, Ny, comm){}
 
+    /**
+     * @copydoc Grid2d::Grid2d()
+     * @param comm a two-dimensional Cartesian communicator
+     * @note the paramateres given in the constructor are global parameters 
+     */
     CartesianMPIGrid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx, bc bcy, MPI_Comm comm):dg::MPIGrid2d( x0, x1, y0, y1, n, Nx, Ny,bcx, bcy, comm){}
     CartesianMPIGrid2d( const dg::MPIGrid2d& grid ):MPIGrid2d( grid){}
 };
@@ -31,8 +41,18 @@ struct CartesianMPIGrid3d : public dg::MPIGrid3d
 {
     typedef OrthonormalTag metric_category; 
 
+    /**
+     * @copydoc Grid3d::Grid3d()
+     * @param comm a three-dimensional Cartesian communicator
+     * @note the paramateres given in the constructor are global parameters 
+     */
     CartesianMPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, MPI_Comm comm): dg::MPIGrid3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, comm){}
 
+    /**
+     * @copydoc Grid3d::Grid3d()
+     * @param comm a three-dimensional Cartesian communicator
+     * @note the paramateres given in the constructor are global parameters 
+     */
     CartesianMPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz, MPI_Comm comm):dg::MPIGrid3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, bcx, bcy, bcz, comm){}
     CartesianMPIGrid3d( const dg::MPIGrid3d& grid ): dg::MPIGrid3d( grid){}
 };
@@ -51,10 +71,20 @@ struct CylindricalMPIGrid3d : public MPIGrid3d
     typedef OrthonormalCylindricalTag metric_category; 
     typedef dg::CartesianMPIGrid2d perpendicular_grid;
 
+    /**
+     * @copydoc Grid3d::Grid3d()
+     * @param comm a three-dimensional Cartesian communicator
+     * @note the paramateres given in the constructor are global parameters 
+     */
     CylindricalMPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, MPI_Comm comm): 
         dg::MPIGrid3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, comm), 
         R_( dg::evaluate( dg::cooX3d, *this)) { }
 
+    /**
+     * @copydoc Grid3d::Grid3d()
+     * @param comm a three-dimensional Cartesian communicator
+     * @note the paramateres given in the constructor are global parameters 
+     */
     CylindricalMPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz, MPI_Comm comm):
         dg::MPIGrid3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, bcx, bcy, bcz, comm),
         R_( dg::evaluate( dg::cooX3d, *this))
@@ -66,12 +96,17 @@ struct CylindricalMPIGrid3d : public MPIGrid3d
     {}
 
     const container& vol() const { return R_;}
+    ///@copydoc CylindricalGrid3d::perp_grid()
     perpendicular_grid perp_grid() const { 
         MPI_Comm planeComm;
         int remain_dims[] = {true,true,false}; //true true false
         MPI_Cart_sub( communicator(), remain_dims, &planeComm);
         return dg::CartesianMPIGrid2d( global().x0(), global().x1(), global().y0(), global().y1(), global().n(), global().Nx(), global().Ny(), global().bcx(), global().bcy(), planeComm);
     }
+    void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz){
+        this->set(new_n,new_Ny,new_Nz);
+        R_=dg::evaluate(dg::cooX3d, *this);
+    }
     private:
     container R_;
 };
diff --git a/inc/dg/geometry/refined_grid.h b/inc/dg/geometry/refined_grid.h
index 84ad12f9c..c132f766f 100644
--- a/inc/dg/geometry/refined_grid.h
+++ b/inc/dg/geometry/refined_grid.h
@@ -268,15 +268,7 @@ struct RefinedGrid2d : public dg::Grid2d
      * @param add_y Add number of cells to the existing one
      * @param howmanyX Add number of cells to the existing one
      * @param howmanyY Add number of cells to the existing one
-     * @param x0
-     * @param x1
-     * @param y0
-     * @param y1
-     * @param n
-     * @param Nx
-     * @param Ny
-     * @param bcx
-     * @param bcy
+     * @copydetails Grid2d::Grid2d()
      */
     RefinedGrid2d( unsigned node_x, unsigned node_y, unsigned add_x, unsigned add_y, 
             unsigned howmanyX, unsigned howmanyY,
@@ -308,16 +300,8 @@ struct RefinedGrid2d : public dg::Grid2d
      *
      * @param multiple_x refine all cells in x 
      * @param multiple_y refine all cells in y
-     * @param x0
-     * @param x1
-     * @param y0
-     * @param y1
-     * @param n
-     * @param n_old
-     * @param Nx
-     * @param Ny
-     * @param bcx
-     * @param bcy
+     * @param n_old the polynomials in the old grid
+     * @copydetails Grid2d::Grid2d()
      */
     RefinedGrid2d( unsigned multiple_x, unsigned multiple_y,
             double x0, double x1, double y0, double y1, unsigned n,
@@ -418,19 +402,7 @@ struct RefinedGrid3d : public dg::Grid3d
      * @param add_y Add number of cells to the existing one
      * @param howmanyX howmany cells should be refined in x
      * @param howmanyY howmany cells should be refined in y
-     * @param x0
-     * @param x1
-     * @param y0
-     * @param y1
-     * @param z0
-     * @param z1
-     * @param n
-     * @param Nx
-     * @param Ny
-     * @param Nz
-     * @param bcx
-     * @param bcy
-     * @param bcz
+     * @copydetails Grid3d::Grid3d()
      */
     RefinedGrid3d( unsigned node_x, unsigned node_y, unsigned add_x, unsigned add_y, 
             unsigned howmanyX, unsigned howmanyY,
@@ -463,20 +435,7 @@ struct RefinedGrid3d : public dg::Grid3d
      *
      * @param multiple_x Multiply all cells in x - direction
      * @param multiple_y Multiply all cells in y - direction
-     * @param x0
-     * @param x1
-     * @param y0
-     * @param y1
-     * @param z0
-     * @param z1
-     * @param n
-     * @param n_old
-     * @param Nx
-     * @param Ny
-     * @param Nz
-     * @param bcx
-     * @param bcy
-     * @param bcz
+     * @copydetails Grid3d::Grid3d()
      */
     RefinedGrid3d( unsigned multiple_x, unsigned multiple_y,
             double x0, double x1, double y0, double y1, double z0, double z1, 
diff --git a/inc/dg/geometry/refined_gridX.h b/inc/dg/geometry/refined_gridX.h
index 664861094..f5c58c792 100644
--- a/inc/dg/geometry/refined_gridX.h
+++ b/inc/dg/geometry/refined_gridX.h
@@ -100,18 +100,8 @@ struct RefinedGridX2d : public dg::GridX2d
      * @param add_y Add number of cells to the existing one
      * @param howmanyX Add number of cells to the existing one
      * @param howmanyY Add number of cells to the existing one
-     * @param x0
-     * @param x1
-     * @param y0
-     * @param y1
-     * @param fx
-     * @param fy
      * @param n_ref The new number of polynomial coefficients
-     * @param n
-     * @param Nx
-     * @param Ny
-     * @param bcx
-     * @param bcy
+     * @copydetails GridX2d::GridX2d()
      */
     RefinedGridX2d( unsigned add_x, unsigned add_y, unsigned howmanyX, unsigned howmanyY, 
             double x0, double x1, double y0, double y1, 
@@ -256,22 +246,8 @@ struct RefinedGridX3d : public dg::GridX3d
      * @param add_y Add number of cells to the existing one
      * @param howmanyX howmany cells are refined in x
      * @param howmanyY howmany cells are refined in x
-     * @param x0
-     * @param x1
-     * @param y0
-     * @param y1
-     * @param z0
-     * @param z1
-     * @param fx the ratio of cells outside to inside the separatrix
-     * @param fy
      * @param n_ref number of polynomial coefficients in the refined grid
-     * @param n
-     * @param Nx
-     * @param Ny
-     * @param Nz
-     * @param bcx
-     * @param bcy
-     * @param bcz
+     * @copydetails GridX3d::GridX3d()
      */
     RefinedGridX3d( unsigned add_x, unsigned add_y, unsigned howmanyX,  unsigned howmanyY,
             double x0, double x1, double y0, double y1, double z0, double z1,
-- 
GitLab


From 54307360da9756ed5454c8a34ad024209ecd7bf2 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 20 Jul 2017 00:05:15 +0200
Subject: [PATCH 053/453] begin to make Generators virtual

---
 inc/dg/geometry/cylindrical.h      |  2 +-
 inc/dg/geometry/mpi_grids.h        |  2 +-
 inc/geometries/generator.h         | 46 ++++++++++++++++++++++++++++++
 inc/geometries/orthogonal.h        | 36 ++++++++++++++---------
 inc/geometries/simple_orthogonal.h |  4 +--
 5 files changed, 73 insertions(+), 17 deletions(-)
 create mode 100644 inc/geometries/generator.h

diff --git a/inc/dg/geometry/cylindrical.h b/inc/dg/geometry/cylindrical.h
index c834c6d88..5d8c1a156 100644
--- a/inc/dg/geometry/cylindrical.h
+++ b/inc/dg/geometry/cylindrical.h
@@ -47,7 +47,7 @@ struct CylindricalGrid3d : public dg::Grid3d
      */
     const container& vol()const {return R_;}
     void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz){
-        this->set(new_n,new_Ny,new_Nz);
+        dg::Grid3d::set(new_n,new_Ny,new_Nz);
         R_=dg::evaluate(dg::cooX3d, *this);
     }
     private:
diff --git a/inc/dg/geometry/mpi_grids.h b/inc/dg/geometry/mpi_grids.h
index cd5cad564..e94ee46a8 100644
--- a/inc/dg/geometry/mpi_grids.h
+++ b/inc/dg/geometry/mpi_grids.h
@@ -104,7 +104,7 @@ struct CylindricalMPIGrid3d : public MPIGrid3d
         return dg::CartesianMPIGrid2d( global().x0(), global().x1(), global().y0(), global().y1(), global().n(), global().Nx(), global().Ny(), global().bcx(), global().bcy(), planeComm);
     }
     void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz){
-        this->set(new_n,new_Ny,new_Nz);
+        MPIGrid3d::set(new_n,new_Ny,new_Nz);
         R_=dg::evaluate(dg::cooX3d, *this);
     }
     private:
diff --git a/inc/geometries/generator.h b/inc/geometries/generator.h
new file mode 100644
index 000000000..9e0d7332b
--- /dev/null
+++ b/inc/geometries/generator.h
@@ -0,0 +1,46 @@
+#pragma once
+
+namespace dg
+{
+namespace geo
+{
+/**
+* @brief The abstract generator base class 
+
+A generator is there to construct coordinate transformations from physical coordinates
+\f$ x,y\f$ to the computational domain \f$\zeta, \eta\f$, which
+is a product space. 
+ @ingroup generators
+*/
+struct aGenerator
+{
+    virtual double width() =0 const; //!<length in \f$ \zeta\f$ 
+    virtual double height()=0 const; //!<length in \f$ \eta\f$
+    virtual bool isOrthogonal()=0 const; //!< true if coordinate system is orthogonal
+    virtual bool isConformal()=0 const; //!< true if coordinate system is conformal
+    /**
+    * @brief Generate grid points and elements of the Jacobian 
+    *
+    * @param zeta1d (input) a list of \f$ N_\zeta\f$ points \f$ 0<\zeta_i<\f$width() 
+    * @param eta1d (input) a list of \f$ N_\eta\f$ points \f$ 0<\eta_j<\f$height() 
+    * @param x (output) the list of \f$ N_\eta N_\zeta\f$ coordinates \f$ x(\zeta_i, \eta_j)\f$ 
+    * @param y (output) the list of \f$ N_\eta N_\zeta\f$ coordinates \f$ y(\zeta_i, \eta_j)\f$ 
+    * @param zetaX (output) the list of \f$ N_\eta N_\zeta\f$ elements \f$ \partial\zeta/\partial x (\zeta_i, \eta_j)\f$ 
+    * @param zetaY (output) the list of \f$ N_\eta N_\zeta\f$ elements \f$ \partial\zeta/\partial y (\zeta_i, \eta_j)\f$ 
+    * @param etaX (output) the list of \f$ N_\eta N_\zeta\f$ elements \f$ \partial\eta/\partial x (\zeta_i, \eta_j)\f$ 
+    * @param etaY (output) the list of \f$ N_\eta N_\zeta\f$ elements \f$ \partial\eta/\partial y (\zeta_i, \eta_j)\f$ 
+    @note the \f$ \zeta\f$ coordinate is contiguous in memory
+    */
+    virtual void operator()( 
+         const thrust::host_vector<double>& zeta1d, 
+         const thrust::host_vector<double>& eta1d, 
+         thrust::host_vector<double>& x, 
+         thrust::host_vector<double>& y, 
+         thrust::host_vector<double>& zetaX, 
+         thrust::host_vector<double>& zetaY, 
+         thrust::host_vector<double>& etaX, 
+         thrust::host_vector<double>& etaY)=0;
+};
+
+}//namespace geo
+}//namespace dg
diff --git a/inc/geometries/orthogonal.h b/inc/geometries/orthogonal.h
index ad5e040eb..37fcf1eee 100644
--- a/inc/geometries/orthogonal.h
+++ b/inc/geometries/orthogonal.h
@@ -3,6 +3,7 @@
 #include "dg/backend/grid.h"
 #include "dg/blas1.h"
 #include "dg/geometry/geometry_traits.h"
+#include "dg/generator.h"
 
 namespace dg
 {
@@ -26,20 +27,29 @@ struct OrthogonalGrid3d : public dg::Grid3d
 
     /*!@brief Constructor
     
-     * @tparam Generator models aGenerator
-     * @param generator must generate an orthogonal grid
+     * @param generator the pointer must not be deleted as long as you intend to call the set() function
      * @param n 
      * @param Nx
      @param Ny
      @param Nz 
      @param bcx
      */
-    template< class Generator>
-    OrthogonalGrid3d( const Generator& generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx=dg::DIR):
-        dg::Grid3d( 0, generator.width(), 0., generator.height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, dg::PER, dg::PER)
+    OrthogonalGrid3d( aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx=dg::DIR):
+        dg::Grid3d( 0, generator->width(), 0., generator->height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, dg::PER, dg::PER)
     { 
-        assert( generator.isOrthogonal());
-        construct( generator, n, Nx, Ny);
+        assert( generator->isOrthogonal());
+        generator_ = generator;
+        construct( n, Nx, Ny);
+    }
+    /**
+    * @brief Reconstruct the grid coordinates
+    *
+    * @copydetails Grid3d::set()
+    * @attention the generator must still live when this function is called
+    */
+    void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny){
+        dg::Grid3d::set( new_n, new_Nx, new_Ny);
+        construct( new_n, new_Nx, new_Ny);
     }
 
     perpendicular_grid perp_grid() const { return OrthogonalGrid2d<container>(*this);}
@@ -56,15 +66,14 @@ struct OrthogonalGrid3d : public dg::Grid3d
     const container& vol()const{return vol_;}
     const container& perpVol()const{return vol2d_;}
     private:
-    template< class Generator>
-    void construct( Generator generator, unsigned n, unsigned Nx, unsigned Ny)
+    void construct( unsigned n, unsigned Nx, unsigned Ny)
     {
-        dg::Grid1d gY1d( 0, generator.height(), n, Ny, dg::PER);
-        dg::Grid1d gX1d( 0., generator.width(), n, Nx);
+        dg::Grid1d gY1d( 0, generator_->height(), n, Ny, dg::PER);
+        dg::Grid1d gX1d( 0., generator_->width(), n, Nx);
         thrust::host_vector<double> x_vec = dg::evaluate( dg::cooX1d, gX1d);
         thrust::host_vector<double> y_vec = dg::evaluate( dg::cooX1d, gY1d);
-        generator( x_vec, y_vec, r_, z_, xr_, xz_, yr_, yz_);
-        init_X_boundaries( 0., generator.width());
+        *generator_( x_vec, y_vec, r_, z_, xr_, xz_, yr_, yz_);
+        init_X_boundaries( 0., generator->width());
         lift3d( ); //lift to 3D grid
         construct_metric();
     }
@@ -106,6 +115,7 @@ struct OrthogonalGrid3d : public dg::Grid3d
     }
     thrust::host_vector<double> r_, z_, xr_, xz_, yr_, yz_;
     container g_xx_, g_xy_, g_yy_, g_pp_, vol_, vol2d_;
+    aGenerator* generator_;
 };
 
 /**
diff --git a/inc/geometries/simple_orthogonal.h b/inc/geometries/simple_orthogonal.h
index 6c8b49da4..28f4ff15f 100644
--- a/inc/geometries/simple_orthogonal.h
+++ b/inc/geometries/simple_orthogonal.h
@@ -284,7 +284,7 @@ void construct_rz( Nemov nemov,
  * @tparam Psi All the template parameters must model a Binary-operator i.e. the bracket operator() must be callable with two arguments and return a double. 
  */
 template< class Psi, class PsiX, class PsiY, class LaplacePsi>
-struct SimpleOrthogonal
+struct SimpleOrthogonal : public aGenerator
 {
     /**
      * @brief Construct a simple orthogonal grid 
@@ -344,7 +344,7 @@ struct SimpleOrthogonal
      */
     bool isConformal()  const{return false;}
     /**
-     * @copydoc aGenerator::operator()(const thrust::host_vector<double>&,const thrust::host_vector<double>&,thrust::host_vector<double>&,thrust::host_vector<double>&,thrust::host_vector<double>&,thrust::host_vector<double>&,thrust::host_vector<double>&,thrust::host_vector<double>&)
+     * @copydoc aGenerator::operator()()
      * @note All the resulting vectors are write-only and get properly resized
      */
     void operator()( 
-- 
GitLab


From 3e340362c946dc18fb1788e733c60034e94f4417 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 20 Jul 2017 11:50:07 +0200
Subject: [PATCH 054/453] added a resize function

---
 inc/dg/backend/grid.h     | 16 ++++++++++++++++
 inc/dg/backend/mpi_grid.h | 28 ++++++++++++++++++++++++----
 2 files changed, 40 insertions(+), 4 deletions(-)

diff --git a/inc/dg/backend/grid.h b/inc/dg/backend/grid.h
index 068f64b95..0fdbd49bc 100644
--- a/inc/dg/backend/grid.h
+++ b/inc/dg/backend/grid.h
@@ -288,6 +288,16 @@ struct Grid2d
      */
     Grid2d local_grid() const {return *this;}
 
+    /**
+    * @brief Resize the number of cells relative to the old ones
+    *
+    * With this function you can resize the grid ignorantly of its current size
+    * @param fx new number of cells is fx*Nx()
+    * @param fy new number of cells is fy*Ny()
+    */
+    void resize( double fx, double fy){
+        set(n_, floor(fx*(double)Nx_+0.5), floor(fy*(double)Ny_+0.5));
+    }
     /**
     * @brief Set the number of polynomials and cells
     *
@@ -299,6 +309,7 @@ struct Grid2d
         n_ = new_n, Nx_ = new_Nx, Ny_ = new_Ny;
         hx_ = lx_/(double)Nx_, hy_ = ly_/(double)Ny_;
         dlt_ = DLT<double>( new_n);
+        assert( n_> 0 && Nx_ > 0  && Ny_ > 0);
     }
 
     /**
@@ -453,6 +464,10 @@ struct Grid3d
         lx_ = (x1_-x0_), ly_ = (y1_-y0_), lz_ = (z1_-z0_);
         hx_ = lx_/(double)Nx_, hy_ = ly_/(double)Ny_, hz_ = lz_/(double)Nz_;
     }
+    ///@copydoc Grid2d::resize()
+    void resize( double fx, double fy){
+        set(n_, floor(fx*(double)Nx_+0.5), floor(fy*(double)Ny_+0.5), Nz());
+    }
     /**
     * @brief Set the number of polynomials and cells
     *
@@ -465,6 +480,7 @@ struct Grid3d
         n_ = new_n, Nx_ = new_Nx, Ny_ = new_Ny, Nz_ = new_Nz;
         hx_ = lx_/(double)Nx_, hy_ = ly_/(double)Ny_, hz_ =lz_/(double)Nz_;
         dlt_ = DLT<double>( new_n);
+        assert( n_>0); assert( Nx_ > 0  && Ny_ > 0); assert( Nz_ > 0);
     }
 
     /**
diff --git a/inc/dg/backend/mpi_grid.h b/inc/dg/backend/mpi_grid.h
index ca7a4dc8a..fc092c7ec 100644
--- a/inc/dg/backend/mpi_grid.h
+++ b/inc/dg/backend/mpi_grid.h
@@ -50,8 +50,21 @@ struct MPIGrid2d
         check_division( Nx, Ny, bcx, bcy);
     }
 
-    ///@copydoc Grid2d::set(unsigned,unsigned,unsigned)
-    ///@note these are global parameters
+    /**
+    * @brief Resize the number of cells relative to the old ones
+    *
+    * With this function you can resize the grid ignorantly of its current size
+    * @param fx new global number of cells is fx*global().Nx()
+    * @param fy new global number of cells is fy*global().Ny()
+    */
+    void resize( double fx, double fy){
+        set(n_, floor(fx*(double)global.Nx()+0.5), floor(fy*(double)global.Ny()+0.5));
+    }
+    /**
+    * @copydoc Grid2d::set(unsigned,unsigned,unsigned)
+    * @attention these are global parameters, i.e. set( g.n(), 2*g.Nx(), 2*g.Ny()) is NOT(!) what you want
+    *           use the resize function instead, or set( g.n(), 2*g.global().Nx(), 2*g.global().Ny())
+    */
     virtual void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny) {
         g.set(new_n,new_Nx,new_Ny);
         check_division( new_Nx, new_Ny, g.bcx(), g.bcy());
@@ -302,8 +315,15 @@ struct MPIGrid3d
     {
         check_division( Nx, Ny, Nz, bcx, bcy, bcz);
     }
-    ///@copydoc Grid3d::set(unsigned,unsigned,unsigned,unsigned)
-    ///@note these are global parameters
+    ///@copydoc MPIGrid2d::resize()
+    void resize( double fx, double fy){
+        set(n_, floor(fx*(double)global.Nx()+0.5), floor(fy*(double)global.Ny()+0.5), global.Nz());
+    }
+    /**
+     * @copydoc Grid3d::set(unsigned,unsigned,unsigned,unsigned)
+     * @attention these are global parameters, i.e. set( g.n(), 2*g.Nx(), 2*g.Ny(), 2*g.Nz()) is NOT(!) what you want
+     *           use the resize function instead, or set( g.n(), 2*g.global().Nx(), 2*g.global().Ny(), 2*g.global().Nz())
+     */
     virtual void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz) {
         g.set(new_n,new_Nx,new_Ny,new_Nz);
         check_division( new_Nx,new_Ny,new_Nz,g.bcx(),g.bcy(),g.bcz());
-- 
GitLab


From 76ee5c55f25602645f681009f84c9353d2078e0d Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 20 Jul 2017 12:59:42 +0200
Subject: [PATCH 055/453] cusp mpi_vector symv, global function in grid,
 MPI_Vector in MPIGrids

---
 inc/dg/backend/grid.h            | 10 ++++++++++
 inc/dg/backend/mpi_matrix_blas.h | 15 ++++++++++++++-
 inc/dg/geometry/cylindrical.h    |  2 --
 inc/dg/geometry/mpi_grids.h      |  9 +++++----
 inc/geometries/mpi_orthogonal.h  |  6 ++++--
 inc/geometries/orthogonal.h      |  2 ++
 6 files changed, 35 insertions(+), 9 deletions(-)

diff --git a/inc/dg/backend/grid.h b/inc/dg/backend/grid.h
index 0fdbd49bc..622697744 100644
--- a/inc/dg/backend/grid.h
+++ b/inc/dg/backend/grid.h
@@ -388,6 +388,11 @@ struct Grid2d
         if( (x>=x0_ && x <= x1_) && (y>=y0_ && y <= y1_)) return true; 
         return false;
     }
+    /**
+    * @brief This function corresponds to MPIGrid2d::global()
+    * @return *this
+    */
+    virtual const Grid2d& global()const{return *this;}
   protected:
     virtual void init_X_boundaries( double x0, double x1)
     {
@@ -695,6 +700,11 @@ struct Grid3d
             return true; 
         return false;
     }
+    /**
+    * @brief This function corresponds to MPIGrid3d::global()
+    * @return *this
+    */
+    virtual const Grid3d& global()const{return *this;}
   protected:
     virtual void init_X_boundaries( double x0, double x1)
     {
diff --git a/inc/dg/backend/mpi_matrix_blas.h b/inc/dg/backend/mpi_matrix_blas.h
index 32c2091fb..1250cd18e 100644
--- a/inc/dg/backend/mpi_matrix_blas.h
+++ b/inc/dg/backend/mpi_matrix_blas.h
@@ -13,7 +13,7 @@ template< class Matrix1, class Matrix2>
 inline void doTransfer( const Matrix1& m1, Matrix2& m2, MPIMatrixTag, MPIMatrixTag)
 {
     Matrix2 m(m1);
-    m2 = m1;
+    m2 = m;
 }
 
 template< class Matrix, class Vector1, class Vector2>
@@ -27,6 +27,19 @@ inline void doGemv( Matrix& m, Vector1&x, Vector2& y, MPIMatrixTag, MPIVectorTag
 {
     doSymv( m, x, y, MPIMatrixTag(), MPIVectorTag(), MPIVectorTag());
 }
+template< class Matrix, class Vector>
+inline void doSymv( Matrix& m, Vector& x, Vector& y, CuspMatrixTag, MPIVectorTag, MPIVectorTag )
+{
+    typedef typename Vector::container_type container;
+    doSymv(m,x.data(),y.data(),CuspMatrixTag(),typename VectorTraits<container>::vector_category(),
+                                             typename VectorTraits<container>::vector_category());
+}
+
+template< class Matrix, class Vector>
+inline void doGemv( Matrix& m, Vector&x, Vector& y, CuspMatrixTag, MPIVectorTag, MPIVectorTag  )
+{
+    doSymv( m, x, y, CuspMatrixTag(), MPIVectorTag(), MPIVectorTag());
+}
 
 } //namespace detail
 } //namespace blas2
diff --git a/inc/dg/geometry/cylindrical.h b/inc/dg/geometry/cylindrical.h
index 5d8c1a156..4e8735b7f 100644
--- a/inc/dg/geometry/cylindrical.h
+++ b/inc/dg/geometry/cylindrical.h
@@ -36,13 +36,11 @@ struct CylindricalGrid3d : public dg::Grid3d
 
     /**
     * @brief Return the grid of the R-Z planes
-    *
     * @return a Cartesian 2d grid of the R-Z plane
     */
     perpendicular_grid perp_grid() const { return dg::CartesianGrid2d( x0(), x1(), y0(), y1(), n(), Nx(), Ny(), bcx(), bcy());}
     /**
      * @brief The volume element R
-     *
      * @return the volume element R
      */
     const container& vol()const {return R_;}
diff --git a/inc/dg/geometry/mpi_grids.h b/inc/dg/geometry/mpi_grids.h
index e94ee46a8..da9503143 100644
--- a/inc/dg/geometry/mpi_grids.h
+++ b/inc/dg/geometry/mpi_grids.h
@@ -63,11 +63,12 @@ struct CartesianMPIGrid3d : public dg::MPIGrid3d
  * @brief MPI version of Cylindrical grid
  *
  * @ingroup basicgrids
- * @tparam container The MPI Vector container
+ * @tparam MPIContainer The MPI Vector container
  */
-template<class container>
+template<class MPIContainer>
 struct CylindricalMPIGrid3d : public MPIGrid3d
 {
+    typedef typename MPIContainer::container_type LocalContainer; //!< the local container type
     typedef OrthonormalCylindricalTag metric_category; 
     typedef dg::CartesianMPIGrid2d perpendicular_grid;
 
@@ -95,7 +96,7 @@ struct CylindricalMPIGrid3d : public MPIGrid3d
         R_( dg::evaluate( dg::cooX3d, *this))
     {}
 
-    const container& vol() const { return R_;}
+    const MPIContainer& vol() const { return R_;}
     ///@copydoc CylindricalGrid3d::perp_grid()
     perpendicular_grid perp_grid() const { 
         MPI_Comm planeComm;
@@ -108,7 +109,7 @@ struct CylindricalMPIGrid3d : public MPIGrid3d
         R_=dg::evaluate(dg::cooX3d, *this);
     }
     private:
-    container R_;
+    MPIContainer R_;
 };
 ///@}
 
diff --git a/inc/geometries/mpi_orthogonal.h b/inc/geometries/mpi_orthogonal.h
index fed2151d3..44ec29af1 100644
--- a/inc/geometries/mpi_orthogonal.h
+++ b/inc/geometries/mpi_orthogonal.h
@@ -22,11 +22,12 @@ struct OrthogonalMPIGrid2d;
 /**
  * @tparam LocalContainer Vector class that holds metric coefficients
  */
-template<class LocalContainer>
+template<class MPIContainer>
 struct OrthogonalMPIGrid3d : public dg::MPIGrid3d
 {
     typedef dg::OrthogonalTag metric_category; //!< metric tag
     typedef dg::OrthogonalMPIGrid2d<LocalContainer> perpendicular_grid; //!< the two-dimensional grid
+    typedef typename MPIContainer::container_type LocalContainer; //!< the local container type
 
     template< class Generator>
     OrthogonalMPIGrid3d( const Generator& generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx, MPI_Comm comm): 
@@ -88,10 +89,11 @@ struct OrthogonalMPIGrid3d : public dg::MPIGrid3d
 /**
  * @tparam LocalContainer Vector class that holds metric coefficients
  */
-template<class LocalContainer>
+template<class MPIContainer>
 struct OrthogonalMPIGrid2d : public dg::MPIGrid2d
 {
     typedef dg::OrthogonalTag metric_category; 
+    typedef typename MPIContainer::container_type LocalContainer; //!< the local container type
 
     template< class Generator>
     OrthogonalMPIGrid2d( const Generator& generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx, MPI_Comm comm2d): 
diff --git a/inc/geometries/orthogonal.h b/inc/geometries/orthogonal.h
index 37fcf1eee..ae1b4f49c 100644
--- a/inc/geometries/orthogonal.h
+++ b/inc/geometries/orthogonal.h
@@ -65,6 +65,7 @@ struct OrthogonalGrid3d : public dg::Grid3d
     const container& g_pp()const{return g_pp_;}
     const container& vol()const{return vol_;}
     const container& perpVol()const{return vol2d_;}
+    const OrthogonalGrid3d& global() const{return *this;}
     private:
     void construct( unsigned n, unsigned Nx, unsigned Ny)
     {
@@ -171,6 +172,7 @@ struct OrthogonalGrid2d : public dg::Grid2d
     const container& g_xy()const{return g_xy_;}
     const container& vol()const{return vol2d_;}
     const container& perpVol()const{return vol2d_;}
+    const OrthogonalGrid2d& global() const{return *this;}
     private:
     thrust::host_vector<double> r_, z_, xr_, xz_, yr_, yz_;
     container g_xx_, g_xy_, g_yy_, vol2d_;
-- 
GitLab


From 2346e5bf046f4b768f5508447a5007edb9943b23 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 20 Jul 2017 17:30:58 +0200
Subject: [PATCH 056/453] made generators polymorphic and mpigrids use
 MPIContainer

---
 inc/dg/backend/grid.h              |  10 --
 inc/geometries/README              |   1 +
 inc/geometries/conformal.h         |  54 +++++++----
 inc/geometries/curvilinear.h       |  49 +++++++---
 inc/geometries/flux.h              |  15 +--
 inc/geometries/generator.h         |   3 +-
 inc/geometries/hector.h            |   3 +-
 inc/geometries/mpi_conformal.h     | 127 +++++++++++++++-----------
 inc/geometries/mpi_curvilinear.h   | 142 +++++++++++++++++------------
 inc/geometries/mpi_orthogonal.h    | 140 ++++++++++++++++------------
 inc/geometries/orthogonal.h        |  30 ++++--
 inc/geometries/ribeiro.h           |  18 +---
 inc/geometries/simple_orthogonal.h |   6 +-
 inc/geometries/solovev_doc.h       |  40 --------
 14 files changed, 346 insertions(+), 292 deletions(-)

diff --git a/inc/dg/backend/grid.h b/inc/dg/backend/grid.h
index 622697744..0fdbd49bc 100644
--- a/inc/dg/backend/grid.h
+++ b/inc/dg/backend/grid.h
@@ -388,11 +388,6 @@ struct Grid2d
         if( (x>=x0_ && x <= x1_) && (y>=y0_ && y <= y1_)) return true; 
         return false;
     }
-    /**
-    * @brief This function corresponds to MPIGrid2d::global()
-    * @return *this
-    */
-    virtual const Grid2d& global()const{return *this;}
   protected:
     virtual void init_X_boundaries( double x0, double x1)
     {
@@ -700,11 +695,6 @@ struct Grid3d
             return true; 
         return false;
     }
-    /**
-    * @brief This function corresponds to MPIGrid3d::global()
-    * @return *this
-    */
-    virtual const Grid3d& global()const{return *this;}
   protected:
     virtual void init_X_boundaries( double x0, double x1)
     {
diff --git a/inc/geometries/README b/inc/geometries/README
index 55e1fa4ab..dac07a2fd 100644
--- a/inc/geometries/README
+++ b/inc/geometries/README
@@ -15,6 +15,7 @@
     refined_orthogonalX.h
 
 #generators
+    generator.h
     simple_orthogonal.h
     separatrix_orthogonal.h
     ribeiro.h
diff --git a/inc/geometries/conformal.h b/inc/geometries/conformal.h
index 2ad983a8c..682c745a1 100644
--- a/inc/geometries/conformal.h
+++ b/inc/geometries/conformal.h
@@ -3,6 +3,7 @@
 #include "dg/backend/grid.h"
 #include "dg/blas1.h"
 #include "dg/geometry/geometry_traits.h"
+#include "generator.h"
 
 namespace dg
 {
@@ -28,7 +29,6 @@ struct ConformalGrid3d : public dg::Grid3d
     /**
      * @brief 
      *
-     * @tparam Generator models aGenerator
      * @param generator must generate a conformal grid
      * @param n
      * @param Nx
@@ -36,12 +36,24 @@ struct ConformalGrid3d : public dg::Grid3d
      * @param Nz
      * @param bcx
      */
-    template< class Generator>
-    ConformalGrid3d( const Generator& generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx=dg::DIR) :
-        dg::Grid3d( 0, generator.width(), 0., generator.height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, dg::PER, dg::PER)
+    ConformalGrid3d( aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx=dg::DIR) :
+        dg::Grid3d( 0, generator->width(), 0., generator->height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, dg::PER, dg::PER)
     {
-        construct( generator, n,Nx, Ny, Nz, bcx);
+        assert( generator.isConformal());
+        generator_=generator;
+        construct(n,Nx, Ny);
     }
+    /**
+    * @brief Reconstruct the grid coordinates
+    *
+    * @copydetails dg::Grid3d::set()
+    * @attention the generator must still live when this function is called
+    */
+    void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz){
+        dg::Grid3d::set( new_n, new_Nx, new_Ny, new_Nz);
+        construct( new_n, new_Nx, new_Ny);
+    }
+
     perpendicular_grid perp_grid() const { return ConformalGrid2d<container>(*this);}
     const thrust::host_vector<double>& r()const{return r_;}
     const thrust::host_vector<double>& z()const{return z_;}
@@ -54,17 +66,16 @@ struct ConformalGrid3d : public dg::Grid3d
     const container& g_pp()const{return g_pp_;}
     const container& vol()const{return vol_;}
     const container& perpVol()const{return vol2d_;}
+    aGenerator* const generator() const{return generator_;}
     private:
-    template< class Generator>
-    void construct( const Generator& hector, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx) 
+    void construct( unsigned n, unsigned Nx, unsigned Ny) 
     {
-        assert( hector.isConformal());
-        dg::Grid1d gu( 0., hector.width(), n, Nx);
-        dg::Grid1d gv( 0., hector.height(), n, Ny);
+        dg::Grid1d gu( 0., generator_->width(), n, Nx);
+        dg::Grid1d gv( 0., generator_->height(), n, Ny);
         const thrust::host_vector<double> u1d = dg::evaluate( dg::cooX1d, gu);
         const thrust::host_vector<double> v1d = dg::evaluate( dg::cooX1d, gv);
-        hector( u1d, v1d, r_, z_, xr_, xz_, yr_, yz_);
-        init_X_boundaries( 0., hector.width());
+        *generator_( u1d, v1d, r_, z_, xr_, xz_, yr_, yz_);
+        init_X_boundaries( 0., generator_->width());
         lift3d( ); //lift to 3D grid
         construct_metric();
     }
@@ -106,6 +117,7 @@ struct ConformalGrid3d : public dg::Grid3d
     
     thrust::host_vector<double> r_, z_, xr_, xz_, yr_, yz_; //3d vector
     container gradU2_, g_pp_, vol_, vol2d_;
+    aGenerator* generator_;
 
 };
 
@@ -119,17 +131,16 @@ struct ConformalGrid2d : public dg::Grid2d
     /**
      * @brief 
      *
-     * @tparam Generator models aGenerator
      * @param generator must generate a conformal grid
      * @param n
      * @param Nx
      * @param Ny
      * @param bcx
      */
-    template< class Generator>
-    ConformalGrid2d( const Generator& generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx=dg::DIR):
-        dg::Grid2d( 0, generator.width(), 0., generator.height(), n,Nx,Ny, bcx, dg::PER)
+    ConformalGrid2d( aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx=dg::DIR):
+        dg::Grid2d( 0, generator->width(), 0., generator->height(), n,Nx,Ny, bcx, dg::PER)
     {
+        generator_=generator;
         ConformalGrid3d<container> g( generator, n,Nx,Ny,1,bcx);
         init_X_boundaries( g.x0(), g.x1());
         r_=g.r(), z_=g.z(), xr_=g.xr(), xz_=g.xz(), yr_=g.yr(), yz_=g.yz();
@@ -139,6 +150,7 @@ struct ConformalGrid2d : public dg::Grid2d
     ConformalGrid2d( const ConformalGrid3d<container>& g):
         dg::Grid2d( g.x0(), g.x1(), g.y0(), g.y1(), g.n(), g.Nx(), g.Ny(), g.bcx(), g.bcy())
     {
+        generator_ = g.generator();
         unsigned s = this->size();
         r_.resize( s), z_.resize(s), xr_.resize(s), xz_.resize(s), yr_.resize(s), yz_.resize(s);
         gradU2_.resize( s), vol2d_.resize(s);
@@ -148,6 +160,14 @@ struct ConformalGrid2d : public dg::Grid2d
         thrust::copy( g.perpVol().begin(), g.perpVol().begin()+s, vol2d_.begin());
     }
 
+    void set(unsigned new_n, unsigned new_Nx, unsigned new_Ny)
+    {
+        dg::Grid2d::set( new_n, new_Nx, new_Ny, new_Nz);
+        ConformalGrid3d<container> g( generator_, n,Nx,Ny,1,bcx);
+        r_=g.r(), z_=g.z(), xr_=g.xr(), xz_=g.xz(), yr_=g.yr(), yz_=g.yz();
+        gradU2_=g.g_xx(), vol2d_=g.perpVol();
+    }
+
 
     const thrust::host_vector<double>& r()const{return r_;}
     const thrust::host_vector<double>& z()const{return z_;}
@@ -159,9 +179,11 @@ struct ConformalGrid2d : public dg::Grid2d
     const container& g_yy()const{return gradU2_;}
     const container& vol()const{return vol2d_;}
     const container& perpVol()const{return vol2d_;}
+    aGenerator* const generator() const{return generator_;}
     private:
     thrust::host_vector<double> r_, z_, xr_, xz_, yr_, yz_;
     container gradU2_, vol2d_;
+    aGenerator* generator_;
 };
 
 ///@}
diff --git a/inc/geometries/curvilinear.h b/inc/geometries/curvilinear.h
index 19e5da28c..32e392847 100644
--- a/inc/geometries/curvilinear.h
+++ b/inc/geometries/curvilinear.h
@@ -3,6 +3,7 @@
 #include "dg/backend/grid.h"
 #include "dg/blas1.h"
 #include "dg/geometry/geometry_traits.h"
+#include "generator.h"
 
 namespace dg
 {
@@ -26,7 +27,6 @@ struct CurvilinearGrid3d : public dg::Grid3d
 
     /*!@brief Constructor
     
-     * @tparam Generator models aGenerator
      * @param generator must generate an orthogonal grid
      * @param n 
      * @param Nx
@@ -34,11 +34,21 @@ struct CurvilinearGrid3d : public dg::Grid3d
      @param Nz 
      @param bcx
      */
-    template< class Generator>
-    CurvilinearGrid3d( Generator generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx=dg::DIR):
-        dg::Grid3d( 0, generator.width(), 0., generator.height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, dg::PER, dg::PER)
+    CurvilinearGrid3d( aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx=dg::DIR):
+        dg::Grid3d( 0, generator->width(), 0., generator->height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, dg::PER, dg::PER)
     { 
-        construct( generator, n, Nx, Ny);
+        generator_ = generator;
+        construct( n, Nx, Ny);
+    }
+    /**
+    * @brief Reconstruct the grid coordinates
+    *
+    * @copydetails Grid3d::set()
+    * @attention the generator must still live when this function is called
+    */
+    void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny,unsigned new_Nz){
+        dg::Grid3d::set( new_n, new_Nx, new_Ny,new_Nz);
+        construct( new_n, new_Nx, new_Ny);
     }
 
     perpendicular_grid perp_grid() const { return perpendicular_grid(*this);}
@@ -54,16 +64,16 @@ struct CurvilinearGrid3d : public dg::Grid3d
     const container& g_pp()const{return g_pp_;}
     const container& vol()const{return vol_;}
     const container& perpVol()const{return vol2d_;}
+    aGenerator* const generator() const{return generator_;}
     private:
-    template< class Generator>
-    void construct( Generator generator, unsigned n, unsigned Nx, unsigned Ny)
+    void construct( unsigned n, unsigned Nx, unsigned Ny)
     {
-        dg::Grid1d gY1d( 0, generator.height(), n, Ny, dg::PER);
-        dg::Grid1d gX1d( 0., generator.width(), n, Nx);
+        dg::Grid1d gY1d( 0, generator_->height(), n, Ny, dg::PER);
+        dg::Grid1d gX1d( 0., generator_->width(), n, Nx);
         thrust::host_vector<double> x_vec = dg::evaluate( dg::cooX1d, gX1d);
         thrust::host_vector<double> y_vec = dg::evaluate( dg::cooX1d, gY1d);
-        generator( x_vec, y_vec, r_, z_, xr_, xz_, yr_, yz_);
-        init_X_boundaries( 0., generator.width());
+        *generator_( x_vec, y_vec, r_, z_, xr_, xz_, yr_, yz_);
+        init_X_boundaries( 0., generator->width());
         lift3d( ); //lift to 3D grid
         construct_metric();
     }
@@ -105,6 +115,7 @@ struct CurvilinearGrid3d : public dg::Grid3d
     }
     thrust::host_vector<double> r_, z_, xr_, xz_, yr_, yz_;
     container g_xx_, g_xy_, g_yy_, g_pp_, vol_, vol2d_;
+    aGenerator* generator_;
 };
 
 /**
@@ -123,9 +134,8 @@ struct CurvilinearGrid2d : public dg::Grid2d
      @param Ny number of cells in second coordinate
      @param bcx boundary condition in first coordinate
      */
-    template< class Generator>
-    CurvilinearGrid2d( Generator generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx=dg::DIR):
-        dg::Grid2d( 0, generator.width(), 0., generator.height(), n, Nx, Ny, bcx, dg::PER)
+    CurvilinearGrid2d( aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx=dg::DIR):
+        dg::Grid2d( 0, generator->width(), 0., generator->height(), n, Nx, Ny, bcx, dg::PER)
     {
         CurvilinearGrid3d<container> g( generator, n,Nx,Ny,1,bcx);
         init_X_boundaries( g.x0(), g.x1());
@@ -137,6 +147,7 @@ struct CurvilinearGrid2d : public dg::Grid2d
     CurvilinearGrid2d( const CurvilinearGrid3d<container>& g):
         dg::Grid2d( g.x0(), g.x1(), g.y0(), g.y1(), g.n(), g.Nx(), g.Ny(), g.bcx(), g.bcy())
     {
+        generator_ = g.generator();
         unsigned s = this->size();
         r_.resize( s), z_.resize(s), xr_.resize(s), xz_.resize(s), yr_.resize(s), yz_.resize(s);
         g_xx_.resize( s), g_xy_.resize(s), g_yy_.resize(s), vol2d_.resize(s);
@@ -148,6 +159,14 @@ struct CurvilinearGrid2d : public dg::Grid2d
         thrust::copy( g.perpVol().begin(), g.perpVol().begin()+s, vol2d_.begin());
     }
 
+    void set(unsigned new_n, unsigned new_Nx, unsigned new_Ny)
+    {
+        dg::Grid2d::set( new_n, new_Nx, new_Ny, new_Nz);
+        CurvilinearGrid3d<container> g( generator_, n,Nx,Ny,1,bcx);
+        r_=g.r(), z_=g.z(), xr_=g.xr(), xz_=g.xz(), yr_=g.yr(), yz_=g.yz();
+        g_xx_=g.g_xx(), g_xy_=g.g_xy(), g_yy_=g.g_yy();
+        vol2d_=g.perpVol();
+    }
     const thrust::host_vector<double>& r()const{return r_;}
     const thrust::host_vector<double>& z()const{return z_;}
     const thrust::host_vector<double>& xr()const{return xr_;}
@@ -159,9 +178,11 @@ struct CurvilinearGrid2d : public dg::Grid2d
     const container& g_xy()const{return g_xy_;}
     const container& vol()const{return vol2d_;}
     const container& perpVol()const{return vol2d_;}
+    aGenerator* const generator() const{return generator_;}
     private:
     thrust::host_vector<double> r_, z_, xr_, xz_, yr_, yz_;
     container g_xx_, g_xy_, g_yy_, vol2d_;
+    aGenerator* generator_;
 };
 
 ///@}
diff --git a/inc/geometries/flux.h b/inc/geometries/flux.h
index 7ad228f90..e604c0f50 100644
--- a/inc/geometries/flux.h
+++ b/inc/geometries/flux.h
@@ -10,6 +10,7 @@
 #include "dg/nullstelle.h"
 #include "dg/geometry.h"
 #include "ribeiro.h"
+#include "generator.h"
 
 
 namespace dg
@@ -135,7 +136,7 @@ struct Fpsi
  * @tparam IpolY models aBinaryOperator 
  */
 template< class Psi, class PsiX, class PsiY, class PsiXX, class PsiXY, class PsiYY, class Ipol, class IpolX, class IpolY>
-struct FluxGenerator
+struct FluxGenerator : public aGenerator
 {
     /**
      * @brief Construct a symmetry flux grid generator
@@ -194,10 +195,7 @@ struct FluxGenerator
      * @return 2pi 
      */
     double height() const{return 2.*M_PI;}
-    /**
-     * @copydoc aGenerator::operator()(const thrust::host_vector<double>&,const thrust::host_vector<double>&,thrust::host_vector<double>&,thrust::host_vector<double>&,thrust::host_vector<double>&,thrust::host_vector<double>&,thrust::host_vector<double>&,thrust::host_vector<double>&)
-     * @note All the resulting vectors are write-only and get properly resized
-     */
+
     void operator()( 
          const thrust::host_vector<double>& zeta1d, 
          const thrust::host_vector<double>& eta1d, 
@@ -265,7 +263,7 @@ struct FluxGenerator
      * @tparam PsiYY models aBinaryOperator 
  */
 template< class Psi, class PsiX, class PsiY, class PsiXX, class PsiXY, class PsiYY>
-struct RibeiroFluxGenerator
+struct RibeiroFluxGenerator : public aGenerator
 {
     /**
      * @brief Construct a flux aligned grid generator
@@ -311,10 +309,7 @@ struct RibeiroFluxGenerator
      * @return 2pi 
      */
     double height() const{return 2.*M_PI;}
-    /**
-     * @copydoc aGenerator::operator()(const thrust::host_vector<double>&,const thrust::host_vector<double>&,thrust::host_vector<double>&,thrust::host_vector<double>&,thrust::host_vector<double>&,thrust::host_vector<double>&,thrust::host_vector<double>&,thrust::host_vector<double>&)
-     * @note All the resulting vectors are write-only and get properly resized
-     */
+
     void operator()( 
          const thrust::host_vector<double>& zeta1d, 
          const thrust::host_vector<double>& eta1d, 
diff --git a/inc/geometries/generator.h b/inc/geometries/generator.h
index 9e0d7332b..bd0abda53 100644
--- a/inc/geometries/generator.h
+++ b/inc/geometries/generator.h
@@ -29,7 +29,8 @@ struct aGenerator
     * @param zetaY (output) the list of \f$ N_\eta N_\zeta\f$ elements \f$ \partial\zeta/\partial y (\zeta_i, \eta_j)\f$ 
     * @param etaX (output) the list of \f$ N_\eta N_\zeta\f$ elements \f$ \partial\eta/\partial x (\zeta_i, \eta_j)\f$ 
     * @param etaY (output) the list of \f$ N_\eta N_\zeta\f$ elements \f$ \partial\eta/\partial y (\zeta_i, \eta_j)\f$ 
-    @note the \f$ \zeta\f$ coordinate is contiguous in memory
+    * @note the \f$ \zeta\f$ coordinate is contiguous in memory
+    * @note All the resulting vectors are write-only and get properly resized
     */
     virtual void operator()( 
          const thrust::host_vector<double>& zeta1d, 
diff --git a/inc/geometries/hector.h b/inc/geometries/hector.h
index ca844074b..69cc3502d 100644
--- a/inc/geometries/hector.h
+++ b/inc/geometries/hector.h
@@ -10,6 +10,7 @@
 #include "orthogonal.h"
 #include "curvilinear.h"
 #include "adaption.h"
+#include "generator.h"
 
 
 
@@ -206,7 +207,7 @@ void transform(
  * @tparam container models aContainer 
  */
 template <class IMatrix = dg::IHMatrix, class Matrix = dg::HMatrix, class container = dg::HVec>
-struct Hector
+struct Hector : public aGenerator
 {
     /**
      * @brief Construct a conformal grid 
diff --git a/inc/geometries/mpi_conformal.h b/inc/geometries/mpi_conformal.h
index 4d56d87f7..aafc5e1b6 100644
--- a/inc/geometries/mpi_conformal.h
+++ b/inc/geometries/mpi_conformal.h
@@ -22,19 +22,48 @@ struct ConformalMPIGrid2d;
 /**
  * @tparam LocalContainer Vector class that holds metric coefficients
  */
-template<class LocalContainer>
+template<class MPIContainer>
 struct ConformalMPIGrid3d : public dg::MPIGrid3d
 {
     typedef dg::ConformalCylindricalTag metric_category; //!< metric tag
     typedef dg::ConformalMPIGrid2d<LocalContainer> perpendicular_grid; //!< the two-dimensional grid
+    typedef typename MPIContainer::container_type LocalContainer; //!< the local container type
 
     template< class Generator>
     ConformalMPIGrid3d( const Generator& generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx, MPI_Comm comm): 
-        dg::MPIGrid3d( 0, 1, 0., 2*M_PI, 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, dg::PER, dg::PER, comm),
-        r_(dg::evaluate( dg::one, *this)), z_(r_), xr_(r_), xz_(r_), yr_(r_), yz_(r_),
-        g_xx_(r_), g_xy_(g_xx_), g_yy_(g_xx_), g_pp_(g_xx_), vol_(g_xx_), vol2d_(g_xx_),
+        dg::MPIGrid3d( 0, generator->width(), 0., generator->height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, dg::PER, dg::PER, comm),
         g( generator, n,Nx, Ny, local().Nz(), bcx)
     {
+        divide_and_conquer();
+    }
+    void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz)
+    {
+        dg::MPIGrid3d::set(new_n, new_Nx, new_Ny, new_Nz);
+        g.set( new_n, new_Nx, new_Ny, new_Nz);//construct new grid
+        divide_and_conquer();//distribute to processes
+    }
+
+    perpendicular_grid perp_grid() const { return perpendicular_grid(*this);}
+
+    const dg::MPI_Vector<thrust::host_vector<double> >& r()const{return r_;}
+    const dg::MPI_Vector<thrust::host_vector<double> >& z()const{return z_;}
+    const dg::MPI_Vector<thrust::host_vector<double> >& xr()const{return xr_;}
+    const dg::MPI_Vector<thrust::host_vector<double> >& yr()const{return yr_;}
+    const dg::MPI_Vector<thrust::host_vector<double> >& xz()const{return xz_;}
+    const dg::MPI_Vector<thrust::host_vector<double> >& yz()const{return yz_;}
+    const MPIContainer& g_xx()const{return g_xx_;}
+    const MPIContainer& g_yy()const{return g_yy_;}
+    const MPIContainer& g_xy()const{return g_xy_;}
+    const MPIContainer& g_pp()const{return g_pp_;}
+    const MPIContainer& vol()const{return vol_;}
+    const MPIContainer& perpVol()const{return vol2d_;}
+    const dg::ConformalGrid3d<LocalContainer>& global() const {return g;}
+    aGenerator* const generator() const{return g.generator();}
+    private:
+    void divide_and_conquer(){
+        r_=dg::evaluate( dg::one, *this), z_=r_, xr_=r_, xz_=r_, yr_=r_, yz_=r_;
+        dg::blas1::transfer( r_, g_xx_);
+        g_xy_=g_xx_, g_yy_=g_xx_, g_pp_=g_xx_, vol_=g_xx_, vol2d_=g_xx_;
         //divide and conquer
         int dims[3], periods[3], coords[3];
         MPI_Cart_get( comm, 3, dims, periods, coords);
@@ -61,25 +90,6 @@ struct ConformalMPIGrid3d : public dg::MPIGrid3d
                             vol2d_.data()[idx1] = g.perpVol()[idx2];
                         }
     }
-
-    //these are for the Field class
-
-    perpendicular_grid perp_grid() const { return perpendicular_grid(*this);}
-
-    const dg::MPI_Vector<thrust::host_vector<double> >& r()const{return r_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& z()const{return z_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& xr()const{return xr_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& yr()const{return yr_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& xz()const{return xz_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& yz()const{return yz_;}
-    const dg::MPI_Vector<LocalContainer>& g_xx()const{return g_xx_;}
-    const dg::MPI_Vector<LocalContainer>& g_yy()const{return g_yy_;}
-    const dg::MPI_Vector<LocalContainer>& g_xy()const{return g_xy_;}
-    const dg::MPI_Vector<LocalContainer>& g_pp()const{return g_pp_;}
-    const dg::MPI_Vector<LocalContainer>& vol()const{return vol_;}
-    const dg::MPI_Vector<LocalContainer>& perpVol()const{return vol2d_;}
-    const dg::ConformalGrid3d<LocalContainer>& global() const {return g;}
-    private:
     dg::MPI_Vector<thrust::host_vector<double> > r_, z_, xr_, xz_, yr_, yz_; //3d vector
     dg::MPI_Vector<LocalContainer> g_xx_, g_xy_, g_yy_, g_pp_, vol_, vol2d_;
     dg::ConformalGrid3d<LocalContainer> g;
@@ -88,40 +98,19 @@ struct ConformalMPIGrid3d : public dg::MPIGrid3d
 /**
  * @tparam LocalContainer Vector class that holds metric coefficients
  */
-template<class LocalContainer>
+template<class MPIContainer>
 struct ConformalMPIGrid2d : public dg::MPIGrid2d
 {
     typedef dg::ConformalCylindricalTag metric_category; 
+    typedef typename MPIContainer::container_type LocalContainer; //!< the local container type
 
-    template< class Generator>
     ConformalMPIGrid2d( const Generator& generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx, MPI_Comm comm2d): 
         dg::MPIGrid2d( 0, 1, 0., 2*M_PI, n, Nx, Ny, bcx, dg::PER, comm2d),
         r_(dg::evaluate( dg::one, *this)), z_(r_), xr_(r_), xz_(r_), yr_(r_), yz_(r_), 
         g_xx_(r_), g_xy_(g_xx_), g_yy_(g_xx_), vol2d_(g_xx_),
         g_( generator, n,Nx, Ny, bcx)
     {
-        //divide and conquer
-        int dims[2], periods[2], coords[2];
-        MPI_Cart_get( communicator(), 2, dims, periods, coords);
-        init_X_boundaries( g_.x0(), g_.x1());
-            //for( unsigned py=0; py<dims[1]; py++)
-                for( unsigned i=0; i<this->n()*this->Ny(); i++)
-                    //for( unsigned px=0; px<dims[0]; px++)
-                        for( unsigned j=0; j<this->n()*this->Nx(); j++)
-                        {
-                            unsigned idx1 = i*this->n()*this->Nx() + j;
-                            unsigned idx2 = ((coords[1]*this->n()*this->Ny()+i)*dims[0] + coords[0])*this->n()*this->Nx() + j;
-                            r_.data()[idx1] = g_.r()[idx2];
-                            z_.data()[idx1] = g_.z()[idx2];
-                            xr_.data()[idx1] = g_.xr()[idx2];
-                            xz_.data()[idx1] = g_.xz()[idx2];
-                            yr_.data()[idx1] = g_.yr()[idx2];
-                            yz_.data()[idx1] = g_.yz()[idx2];
-                            g_xx_.data()[idx1] = g_.g_xx()[idx2];
-                            g_xy_.data()[idx1] = g_.g_xy()[idx2];
-                            g_yy_.data()[idx1] = g_.g_yy()[idx2];
-                            vol2d_.data()[idx1] = g_.perpVol()[idx2];
-                        }
+        divide_and_conquer();
     }
     ConformalMPIGrid2d( const ConformalMPIGrid3d<LocalContainer>& g):
         dg::MPIGrid2d( g.global().x0(), g.global().x1(), g.global().y0(), g.global().y1(), g.global().n(), g.global().Nx(), g.global().Ny(), g.global().bcx(), g.global().bcy(), get_reduced_comm( g.communicator() )),
@@ -145,6 +134,12 @@ struct ConformalMPIGrid2d : public dg::MPIGrid2d
         thrust::copy( g.perpVol().data().begin(), g.perpVol().data().begin()+s, vol2d_.data().begin());
         
     }
+    void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny)
+    {
+        dg::MPIGrid3d::set(new_n, new_Nx, new_Ny);
+        g.set( new_n, new_Nx, new_Ny);//construct new grid
+        divide_and_conquer();//distribute to processes
+    }
 
     const dg::MPI_Vector<thrust::host_vector<double> >& r()const{return r_;}
     const dg::MPI_Vector<thrust::host_vector<double> >& z()const{return z_;}
@@ -152,12 +147,13 @@ struct ConformalMPIGrid2d : public dg::MPIGrid2d
     const dg::MPI_Vector<thrust::host_vector<double> >& yr()const{return yr_;}
     const dg::MPI_Vector<thrust::host_vector<double> >& xz()const{return xz_;}
     const dg::MPI_Vector<thrust::host_vector<double> >& yz()const{return yz_;}
-    const dg::MPI_Vector<LocalContainer>& g_xx()const{return g_xx_;}
-    const dg::MPI_Vector<LocalContainer>& g_yy()const{return g_yy_;}
-    const dg::MPI_Vector<LocalContainer>& g_xy()const{return g_xy_;}
-    const dg::MPI_Vector<LocalContainer>& vol()const{return vol2d_;}
-    const dg::MPI_Vector<LocalContainer>& perpVol()const{return vol2d_;}
+    const MPIContainer& g_xx()const{return g_xx_;}
+    const MPIContainer& g_yy()const{return g_yy_;}
+    const MPIContainer& g_xy()const{return g_xy_;}
+    const MPIContainer& vol()const{return vol2d_;}
+    const MPIContainer& perpVol()const{return vol2d_;}
     const dg::ConformalGrid2d<LocalContainer>& global()const{return g_;}
+    aGenerator* const generator() const{return g.generator();}
     private:
     MPI_Comm get_reduced_comm( MPI_Comm src)
     {
@@ -166,6 +162,33 @@ struct ConformalMPIGrid2d : public dg::MPIGrid2d
         MPI_Cart_sub( src, remain_dims, &planeComm);
         return planeComm;
     }
+    void divide_and_conquer(){
+        r_=dg::evaluate( dg::one, *this), z_=r_, xr_=r_, xz_=r_, yr_=r_, yz_=r_;
+        dg::blas1::transfer( r_, g_xx_);
+        g_xy_=g_xx_, g_yy_=g_xx_, vol2d_=g_xx_;
+        //divide and conquer
+        int dims[2], periods[2], coords[2];
+        MPI_Cart_get( communicator(), 2, dims, periods, coords);
+        init_X_boundaries( g_.x0(), g_.x1());
+            //for( unsigned py=0; py<dims[1]; py++)
+                for( unsigned i=0; i<this->n()*this->Ny(); i++)
+                    //for( unsigned px=0; px<dims[0]; px++)
+                        for( unsigned j=0; j<this->n()*this->Nx(); j++)
+                        {
+                            unsigned idx1 = i*this->n()*this->Nx() + j;
+                            unsigned idx2 = ((coords[1]*this->n()*this->Ny()+i)*dims[0] + coords[0])*this->n()*this->Nx() + j;
+                            r_.data()[idx1] = g_.r()[idx2];
+                            z_.data()[idx1] = g_.z()[idx2];
+                            xr_.data()[idx1] = g_.xr()[idx2];
+                            xz_.data()[idx1] = g_.xz()[idx2];
+                            yr_.data()[idx1] = g_.yr()[idx2];
+                            yz_.data()[idx1] = g_.yz()[idx2];
+                            g_xx_.data()[idx1] = g_.g_xx()[idx2];
+                            g_xy_.data()[idx1] = g_.g_xy()[idx2];
+                            g_yy_.data()[idx1] = g_.g_yy()[idx2];
+                            vol2d_.data()[idx1] = g_.perpVol()[idx2];
+                        }
+    }
 
     dg::MPI_Vector<thrust::host_vector<double> > r_, z_, xr_, xz_, yr_, yz_; //2d vector
     dg::MPI_Vector<LocalContainer> g_xx_, g_xy_, g_yy_, vol2d_;
diff --git a/inc/geometries/mpi_curvilinear.h b/inc/geometries/mpi_curvilinear.h
index 827c43653..69dd9b039 100644
--- a/inc/geometries/mpi_curvilinear.h
+++ b/inc/geometries/mpi_curvilinear.h
@@ -23,19 +23,48 @@ struct CurvilinearMPIGrid2d;
 /**
  * @tparam LocalContainer Vector class that holds metric coefficients
  */
-template<class LocalContainer>
+template<class MPIContainer>
 struct CurvilinearMPIGrid3d : public dg::MPIGrid3d
 {
     typedef dg::CurvilinearCylindricalTag metric_category; //!< metric tag
     typedef dg::CurvilinearMPIGrid2d<LocalContainer> perpendicular_grid; //!< the two-dimensional grid
+    typedef typename MPIContainer::container_type LocalContainer; //!< the local container type
 
-    template< class Generator>
-    CurvilinearMPIGrid3d( const Generator& generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx, MPI_Comm comm): 
-        dg::MPIGrid3d( 0, generator.width(), 0., generator.height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, dg::PER, dg::PER, comm),
-        r_(dg::evaluate( dg::one, *this)), z_(r_), xr_(r_), xz_(r_), yr_(r_), yz_(r_),
-        g_xx_(r_), g_xy_(g_xx_), g_yy_(g_xx_), g_pp_(g_xx_), vol_(g_xx_), vol2d_(g_xx_),
+    CurvilinearMPIGrid3d( aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx, MPI_Comm comm): 
+        dg::MPIGrid3d( 0, generator->width(), 0., generator->height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, dg::PER, dg::PER, comm),
         g( generator, n,Nx, Ny, local().Nz(), bcx)
     {
+        divide_and_conquer();
+    }
+    void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz)
+    {
+        dg::MPIGrid3d::set(new_n, new_Nx, new_Ny, new_Nz);
+        g.set( new_n, new_Nx, new_Ny, new_Nz);//construct new grid
+        divide_and_conquer();//distribute to processes
+    }
+
+    perpendicular_grid perp_grid() const { return perpendicular_grid(*this);}
+
+    const dg::MPI_Vector<thrust::host_vector<double> >& r()const{return r_;}
+    const dg::MPI_Vector<thrust::host_vector<double> >& z()const{return z_;}
+    const dg::MPI_Vector<thrust::host_vector<double> >& xr()const{return xr_;}
+    const dg::MPI_Vector<thrust::host_vector<double> >& yr()const{return yr_;}
+    const dg::MPI_Vector<thrust::host_vector<double> >& xz()const{return xz_;}
+    const dg::MPI_Vector<thrust::host_vector<double> >& yz()const{return yz_;}
+    const MPIContainer& g_xx()const{return g_xx_;}
+    const MPIContainer& g_yy()const{return g_yy_;}
+    const MPIContainer& g_xy()const{return g_xy_;}
+    const MPIContainer& g_pp()const{return g_pp_;}
+    const MPIContainer& vol()const{return vol_;}
+    const MPIContainer& perpVol()const{return vol2d_;}
+    const dg::CurvilinearGrid3d<LocalContainer>& global() const {return g;}
+    aGenerator* const generator() const{return g.generator();}
+    private:
+    void divide_and_conquer( )
+    {
+        r_=dg::evaluate( dg::one, *this), z_=r_, xr_=r_, xz_=r_, yr_=r_, yz_=r_;
+        dg::blas1::transfer( r_, g_xx_);
+        g_xy_=g_xx_, g_yy_=g_xx_, g_pp_=g_xx_, vol_=g_xx_, vol2d_=g_xx_;
         //divide and conquer
         int dims[3], periods[3], coords[3];
         MPI_Cart_get( comm, 3, dims, periods, coords);
@@ -62,67 +91,25 @@ struct CurvilinearMPIGrid3d : public dg::MPIGrid3d
                             vol2d_.data()[idx1] = g.perpVol()[idx2];
                         }
     }
-
-    //these are for the Field class
-
-    perpendicular_grid perp_grid() const { return perpendicular_grid(*this);}
-
-    const dg::MPI_Vector<thrust::host_vector<double> >& r()const{return r_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& z()const{return z_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& xr()const{return xr_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& yr()const{return yr_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& xz()const{return xz_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& yz()const{return yz_;}
-    const dg::MPI_Vector<LocalContainer>& g_xx()const{return g_xx_;}
-    const dg::MPI_Vector<LocalContainer>& g_yy()const{return g_yy_;}
-    const dg::MPI_Vector<LocalContainer>& g_xy()const{return g_xy_;}
-    const dg::MPI_Vector<LocalContainer>& g_pp()const{return g_pp_;}
-    const dg::MPI_Vector<LocalContainer>& vol()const{return vol_;}
-    const dg::MPI_Vector<LocalContainer>& perpVol()const{return vol2d_;}
-    const dg::CurvilinearGrid3d<LocalContainer>& global() const {return g;}
-    private:
     dg::MPI_Vector<thrust::host_vector<double> > r_, z_, xr_, xz_, yr_, yz_; //3d vector
-    dg::MPI_Vector<LocalContainer> g_xx_, g_xy_, g_yy_, g_pp_, vol_, vol2d_;
+    MPIContainer g_xx_, g_xy_, g_yy_, g_pp_, vol_, vol2d_;
     dg::CurvilinearGrid3d<LocalContainer> g;
 };
 
 /**
  * @tparam LocalContainer Vector class that holds metric coefficients
  */
-template<class LocalContainer>
+template<class MPIContainer>
 struct CurvilinearMPIGrid2d : public dg::MPIGrid2d
 {
     typedef dg::CurvilinearCylindricalTag metric_category; 
+    typedef typename MPIContainer::container_type LocalContainer; //!< the local container type
 
-    template< class Generator>
-    CurvilinearMPIGrid2d( const Generator& generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx, MPI_Comm comm2d): 
-        dg::MPIGrid2d( 0, generator.width(), 0., generator.height(), n, Nx, Ny, bcx, dg::PER, comm2d),
-        r_(dg::evaluate( dg::one, *this)), z_(r_), xr_(r_), xz_(r_), yr_(r_), yz_(r_), 
-        g_xx_(r_), g_xy_(g_xx_), g_yy_(g_xx_), vol2d_(g_xx_),
+    CurvilinearMPIGrid2d( aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx, MPI_Comm comm2d): 
+        dg::MPIGrid2d( 0, generator->width(), 0., generator->height(), n, Nx, Ny, bcx, dg::PER, comm2d),
         g_( generator, n,Nx, Ny, bcx)
     {
-        //divide and conquer
-        int dims[2], periods[2], coords[2];
-        MPI_Cart_get( communicator(), 2, dims, periods, coords);
-        init_X_boundaries( g_.x0(), g_.x1());
-        //for( unsigned py=0; py<dims[1]; py++)
-            for( unsigned i=0; i<this->n()*this->Ny(); i++)
-                //for( unsigned px=0; px<dims[0]; px++)
-                    for( unsigned j=0; j<this->n()*this->Nx(); j++)
-                    {
-                        unsigned idx1 = i*this->n()*this->Nx() + j;
-                        unsigned idx2 = ((coords[1]*this->n()*this->Ny()+i)*dims[0] + coords[0])*this->n()*this->Nx() + j;
-                        r_.data()[idx1] = g_.r()[idx2];
-                        z_.data()[idx1] = g_.z()[idx2];
-                        xr_.data()[idx1] = g_.xr()[idx2];
-                        xz_.data()[idx1] = g_.xz()[idx2];
-                        yr_.data()[idx1] = g_.yr()[idx2];
-                        yz_.data()[idx1] = g_.yz()[idx2];
-                        g_xx_.data()[idx1] = g_.g_xx()[idx2];
-                        g_xy_.data()[idx1] = g_.g_xy()[idx2];
-                        g_yy_.data()[idx1] = g_.g_yy()[idx2];
-                        vol2d_.data()[idx1] = g_.perpVol()[idx2];
-                    }
+        divide_and_conquer();
     }
     CurvilinearMPIGrid2d( const CurvilinearMPIGrid3d<LocalContainer>& g):
         dg::MPIGrid2d( g.global().x0(), g.global().x1(), g.global().y0(), g.global().y1(), g.global().n(), g.global().Nx(), g.global().Ny(), g.global().bcx(), g.global().bcy(), get_reduced_comm( g.communicator() )),
@@ -146,6 +133,12 @@ struct CurvilinearMPIGrid2d : public dg::MPIGrid2d
         thrust::copy( g.perpVol().data().begin(), g.perpVol().data().begin()+s, vol2d_.data().begin());
         
     }
+    void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny)
+    {
+        dg::MPIGrid3d::set(new_n, new_Nx, new_Ny);
+        g.set( new_n, new_Nx, new_Ny);//construct new grid
+        divide_and_conquer();//distribute to processes
+    }
 
     const dg::MPI_Vector<thrust::host_vector<double> >& r()const{return r_;}
     const dg::MPI_Vector<thrust::host_vector<double> >& z()const{return z_;}
@@ -153,12 +146,13 @@ struct CurvilinearMPIGrid2d : public dg::MPIGrid2d
     const dg::MPI_Vector<thrust::host_vector<double> >& yr()const{return yr_;}
     const dg::MPI_Vector<thrust::host_vector<double> >& xz()const{return xz_;}
     const dg::MPI_Vector<thrust::host_vector<double> >& yz()const{return yz_;}
-    const dg::MPI_Vector<LocalContainer>& g_xx()const{return g_xx_;}
-    const dg::MPI_Vector<LocalContainer>& g_yy()const{return g_yy_;}
-    const dg::MPI_Vector<LocalContainer>& g_xy()const{return g_xy_;}
-    const dg::MPI_Vector<LocalContainer>& vol()const{return vol2d_;}
-    const dg::MPI_Vector<LocalContainer>& perpVol()const{return vol2d_;}
+    const MPIContainer& g_xx()const{return g_xx_;}
+    const MPIContainer& g_yy()const{return g_yy_;}
+    const MPIContainer& g_xy()const{return g_xy_;}
+    const MPIContainer& vol()const{return vol2d_;}
+    const MPIContainer& perpVol()const{return vol2d_;}
     const dg::CurvilinearGrid2d<LocalContainer>& global() const {return g_;}
+    aGenerator* const generator() const{return g.generator();}
     private:
     MPI_Comm get_reduced_comm( MPI_Comm src)
     {
@@ -167,9 +161,37 @@ struct CurvilinearMPIGrid2d : public dg::MPIGrid2d
         MPI_Cart_sub( src, remain_dims, &planeComm);
         return planeComm;
     }
+    void divide_and_conquer()
+    {
+        r_=dg::evaluate( dg::one, *this), z_=r_, xr_=r_, xz_=r_, yr_=r_, yz_=r_;
+        dg::blas1::transfer( r_, g_xx_);
+        g_xy_=g_xx_, g_yy_=g_xx_, vol2d_=g_xx_;
+        //divide and conquer
+        int dims[2], periods[2], coords[2];
+        MPI_Cart_get( communicator(), 2, dims, periods, coords);
+        init_X_boundaries( g_.x0(), g_.x1());
+            //for( unsigned py=0; py<dims[1]; py++)
+                for( unsigned i=0; i<this->n()*this->Ny(); i++)
+                    //for( unsigned px=0; px<dims[0]; px++)
+                        for( unsigned j=0; j<this->n()*this->Nx(); j++)
+                        {
+                            unsigned idx1 = i*this->n()*this->Nx() + j;
+                            unsigned idx2 = ((coords[1]*this->n()*this->Ny()+i)*dims[0] + coords[0])*this->n()*this->Nx() + j;
+                            r_.data()[idx1] = g_.r()[idx2];
+                            z_.data()[idx1] = g_.z()[idx2];
+                            xr_.data()[idx1] = g_.xr()[idx2];
+                            xz_.data()[idx1] = g_.xz()[idx2];
+                            yr_.data()[idx1] = g_.yr()[idx2];
+                            yz_.data()[idx1] = g_.yz()[idx2];
+                            g_xx_.data()[idx1] = g_.g_xx()[idx2];
+                            g_xy_.data()[idx1] = g_.g_xy()[idx2];
+                            g_yy_.data()[idx1] = g_.g_yy()[idx2];
+                            vol2d_.data()[idx1] = g_.perpVol()[idx2];
+                        }
+    }
 
     dg::MPI_Vector<thrust::host_vector<double> > r_, z_, xr_, xz_, yr_, yz_; //2d vector
-    dg::MPI_Vector<LocalContainer> g_xx_, g_xy_, g_yy_, vol2d_;
+    dg::MPIContainer g_xx_, g_xy_, g_yy_, vol2d_;
     dg::CurvilinearGrid2d<LocalContainer> g_;
 };
 ///@}
diff --git a/inc/geometries/mpi_orthogonal.h b/inc/geometries/mpi_orthogonal.h
index 44ec29af1..880a3ae2d 100644
--- a/inc/geometries/mpi_orthogonal.h
+++ b/inc/geometries/mpi_orthogonal.h
@@ -29,13 +29,43 @@ struct OrthogonalMPIGrid3d : public dg::MPIGrid3d
     typedef dg::OrthogonalMPIGrid2d<LocalContainer> perpendicular_grid; //!< the two-dimensional grid
     typedef typename MPIContainer::container_type LocalContainer; //!< the local container type
 
-    template< class Generator>
-    OrthogonalMPIGrid3d( const Generator& generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx, MPI_Comm comm): 
-        dg::MPIGrid3d( 0, generator.width(), 0., generator.height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, dg::PER, dg::PER, comm),
-        r_(dg::evaluate( dg::one, *this)), z_(r_), xr_(r_), xz_(r_), yr_(r_), yz_(r_),
-        g_xx_(r_), g_xy_(g_xx_), g_yy_(g_xx_), g_pp_(g_xx_), vol_(g_xx_), vol2d_(g_xx_),
+    OrthogonalMPIGrid3d( aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx, MPI_Comm comm): 
+        dg::MPIGrid3d( 0, generator->width(), 0., generator->height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, dg::PER, dg::PER, comm),
         g( generator, n, Nx, Ny, local().Nz(), bcx)
     {
+        divide_and_conquer();//distribute to processes
+    }
+    void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz)
+    {
+        dg::MPIGrid3d::set(new_n, new_Nx, new_Ny, new_Nz);
+        g.set( new_n, new_Nx, new_Ny, new_Nz);//construct new grid
+        divide_and_conquer();//distribute to processes
+    }
+
+    //these are for the Field class
+
+    perpendicular_grid perp_grid() const { return perpendicular_grid(*this);}
+
+    const dg::MPI_Vector<thrust::host_vector<double> >& r()const{return r_;}
+    const dg::MPI_Vector<thrust::host_vector<double> >& z()const{return z_;}
+    const dg::MPI_Vector<thrust::host_vector<double> >& xr()const{return xr_;}
+    const dg::MPI_Vector<thrust::host_vector<double> >& yr()const{return yr_;}
+    const dg::MPI_Vector<thrust::host_vector<double> >& xz()const{return xz_;}
+    const dg::MPI_Vector<thrust::host_vector<double> >& yz()const{return yz_;}
+    const MPIContainer& g_xx()const{return g_xx_;}
+    const MPIContainer& g_yy()const{return g_yy_;}
+    const MPIContainer& g_xy()const{return g_xy_;}
+    const MPIContainer& g_pp()const{return g_pp_;}
+    const MPIContainer& vol()const{return vol_;}
+    const MPIContainer& perpVol()const{return vol2d_;}
+    const dg::OrthogonalGrid3d<LocalContainer>& global() const{return g;}
+    aGenerator* const generator() const{return g.generator();}
+    private:
+    void divide_and_conquer( )
+    {
+        r_=dg::evaluate( dg::one, *this), z_=r_, xr_=r_, xz_=r_, yr_=r_, yz_=r_;
+        dg::blas1::transfer( r_, g_xx_);
+        g_xy_=g_xx_, g_yy_=g_xx_, g_pp_=g_xx_, vol_=g_xx_, vol2d_=g_xx_;
         //divide and conquer
         int dims[3], periods[3], coords[3];
         MPI_Cart_get( comm, 3, dims, periods, coords);
@@ -62,32 +92,13 @@ struct OrthogonalMPIGrid3d : public dg::MPIGrid3d
                             vol2d_.data()[idx1] = g.perpVol()[idx2];
                         }
     }
-
-    //these are for the Field class
-
-    perpendicular_grid perp_grid() const { return perpendicular_grid(*this);}
-
-    const dg::MPI_Vector<thrust::host_vector<double> >& r()const{return r_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& z()const{return z_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& xr()const{return xr_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& yr()const{return yr_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& xz()const{return xz_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& yz()const{return yz_;}
-    const dg::MPI_Vector<LocalContainer>& g_xx()const{return g_xx_;}
-    const dg::MPI_Vector<LocalContainer>& g_yy()const{return g_yy_;}
-    const dg::MPI_Vector<LocalContainer>& g_xy()const{return g_xy_;}
-    const dg::MPI_Vector<LocalContainer>& g_pp()const{return g_pp_;}
-    const dg::MPI_Vector<LocalContainer>& vol()const{return vol_;}
-    const dg::MPI_Vector<LocalContainer>& perpVol()const{return vol2d_;}
-    const dg::OrthogonalGrid3d<LocalContainer>& global() const{return g;}
-    private:
     dg::MPI_Vector<thrust::host_vector<double> > r_, z_, xr_, xz_, yr_, yz_; //3d vector
-    dg::MPI_Vector<LocalContainer> g_xx_, g_xy_, g_yy_, g_pp_, vol_, vol2d_;
+    MPIContainer g_xx_, g_xy_, g_yy_, g_pp_, vol_, vol2d_;
     dg::OrthogonalGrid3d<LocalContainer> g;
 };
 
 /**
- * @tparam LocalContainer Vector class that holds metric coefficients
+ * @tparam MPIContainer Vector class that holds metric coefficients
  */
 template<class MPIContainer>
 struct OrthogonalMPIGrid2d : public dg::MPIGrid2d
@@ -95,35 +106,11 @@ struct OrthogonalMPIGrid2d : public dg::MPIGrid2d
     typedef dg::OrthogonalTag metric_category; 
     typedef typename MPIContainer::container_type LocalContainer; //!< the local container type
 
-    template< class Generator>
-    OrthogonalMPIGrid2d( const Generator& generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx, MPI_Comm comm2d): 
-        dg::MPIGrid2d( 0, generator.width(), 0., generator.height(), n, Nx, Ny, bcx, dg::PER, comm2d),
-        r_(dg::evaluate( dg::one, *this)), z_(r_), xr_(r_), xz_(r_), yr_(r_), yz_(r_), 
-        g_xx_(r_), g_xy_(g_xx_), g_yy_(g_xx_), vol2d_(g_xx_),
+    OrthogonalMPIGrid2d( aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx, MPI_Comm comm2d): 
+        dg::MPIGrid2d( 0, generator->width(), 0., generator->height(), n, Nx, Ny, bcx, dg::PER, comm2d),
         g_( generator, n, Nx, Ny, bcx)
     {
-        //divide and conquer
-        int dims[2], periods[2], coords[2];
-        MPI_Cart_get( communicator(), 2, dims, periods, coords);
-        init_X_boundaries( g_.x0(), g_.x1());
-            //for( unsigned py=0; py<dims[1]; py++)
-                for( unsigned i=0; i<this->n()*this->Ny(); i++)
-                    //for( unsigned px=0; px<dims[0]; px++)
-                        for( unsigned j=0; j<this->n()*this->Nx(); j++)
-                        {
-                            unsigned idx1 = i*this->n()*this->Nx() + j;
-                            unsigned idx2 = ((coords[1]*this->n()*this->Ny()+i)*dims[0] + coords[0])*this->n()*this->Nx() + j;
-                            r_.data()[idx1] = g_.r()[idx2];
-                            z_.data()[idx1] = g_.z()[idx2];
-                            xr_.data()[idx1] = g_.xr()[idx2];
-                            xz_.data()[idx1] = g_.xz()[idx2];
-                            yr_.data()[idx1] = g_.yr()[idx2];
-                            yz_.data()[idx1] = g_.yz()[idx2];
-                            g_xx_.data()[idx1] = g_.g_xx()[idx2];
-                            g_xy_.data()[idx1] = g_.g_xy()[idx2];
-                            g_yy_.data()[idx1] = g_.g_yy()[idx2];
-                            vol2d_.data()[idx1] = g_.perpVol()[idx2];
-                        }
+        divide_and_conquer();
     }
     OrthogonalMPIGrid2d( const OrthogonalMPIGrid3d<LocalContainer>& g):
         dg::MPIGrid2d( g.global().x0(), g.global().x1(), g.global().y0(), g.global().y1(), g.global().n(), g.global().Nx(), g.global().Ny(), g.global().bcx(), g.global().bcy(), get_reduced_comm( g.communicator() )),
@@ -147,6 +134,12 @@ struct OrthogonalMPIGrid2d : public dg::MPIGrid2d
         thrust::copy( g.perpVol().data().begin(), g.perpVol().data().begin()+s, vol2d_.data().begin());
         
     }
+    void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny)
+    {
+        dg::MPIGrid3d::set(new_n, new_Nx, new_Ny);
+        g.set( new_n, new_Nx, new_Ny);//construct new grid
+        divide_and_conquer();//distribute to processes
+    }
 
     const dg::MPI_Vector<thrust::host_vector<double> >& r()const{return r_;}
     const dg::MPI_Vector<thrust::host_vector<double> >& z()const{return z_;}
@@ -154,12 +147,13 @@ struct OrthogonalMPIGrid2d : public dg::MPIGrid2d
     const dg::MPI_Vector<thrust::host_vector<double> >& yr()const{return yr_;}
     const dg::MPI_Vector<thrust::host_vector<double> >& xz()const{return xz_;}
     const dg::MPI_Vector<thrust::host_vector<double> >& yz()const{return yz_;}
-    const dg::MPI_Vector<LocalContainer>& g_xx()const{return g_xx_;}
-    const dg::MPI_Vector<LocalContainer>& g_yy()const{return g_yy_;}
-    const dg::MPI_Vector<LocalContainer>& g_xy()const{return g_xy_;}
-    const dg::MPI_Vector<LocalContainer>& vol()const{return vol2d_;}
-    const dg::MPI_Vector<LocalContainer>& perpVol()const{return vol2d_;}
+    const MPIContainer& g_xx()const{return g_xx_;}
+    const MPIContainer& g_yy()const{return g_yy_;}
+    const MPIContainer& g_xy()const{return g_xy_;}
+    const MPIContainer& vol()const{return vol2d_;}
+    const MPIContainer& perpVol()const{return vol2d_;}
     const dg::OrthogonalGrid2d<LocalContainer>& global() const {return g;}
+    aGenerator* const generator() const{return g.generator();}
     private:
     MPI_Comm get_reduced_comm( MPI_Comm src)
     {
@@ -168,9 +162,37 @@ struct OrthogonalMPIGrid2d : public dg::MPIGrid2d
         MPI_Cart_sub( src, remain_dims, &planeComm);
         return planeComm;
     }
+    void divide_and_conquer()
+    {
+        r_=dg::evaluate( dg::one, *this), z_=r_, xr_=r_, xz_=r_, yr_=r_, yz_=r_;
+        dg::blas1::transfer( r_, g_xx_);
+        g_xy_=g_xx_, g_yy_=g_xx_, vol2d_=g_xx_;
+        //divide and conquer
+        int dims[2], periods[2], coords[2];
+        MPI_Cart_get( communicator(), 2, dims, periods, coords);
+        init_X_boundaries( g_.x0(), g_.x1());
+            //for( unsigned py=0; py<dims[1]; py++)
+                for( unsigned i=0; i<this->n()*this->Ny(); i++)
+                    //for( unsigned px=0; px<dims[0]; px++)
+                        for( unsigned j=0; j<this->n()*this->Nx(); j++)
+                        {
+                            unsigned idx1 = i*this->n()*this->Nx() + j;
+                            unsigned idx2 = ((coords[1]*this->n()*this->Ny()+i)*dims[0] + coords[0])*this->n()*this->Nx() + j;
+                            r_.data()[idx1] = g_.r()[idx2];
+                            z_.data()[idx1] = g_.z()[idx2];
+                            xr_.data()[idx1] = g_.xr()[idx2];
+                            xz_.data()[idx1] = g_.xz()[idx2];
+                            yr_.data()[idx1] = g_.yr()[idx2];
+                            yz_.data()[idx1] = g_.yz()[idx2];
+                            g_xx_.data()[idx1] = g_.g_xx()[idx2];
+                            g_xy_.data()[idx1] = g_.g_xy()[idx2];
+                            g_yy_.data()[idx1] = g_.g_yy()[idx2];
+                            vol2d_.data()[idx1] = g_.perpVol()[idx2];
+                        }
+    }
 
     dg::MPI_Vector<thrust::host_vector<double> > r_, z_, xr_, xz_, yr_, yz_; //2d vector
-    dg::MPI_Vector<LocalContainer> g_xx_, g_xy_, g_yy_, vol2d_;
+    MPIContainer g_xx_, g_xy_, g_yy_, vol2d_;
     dg::OrthogonalGrid2d<LocalContainer> g_;
 };
 ///@}
diff --git a/inc/geometries/orthogonal.h b/inc/geometries/orthogonal.h
index ae1b4f49c..6a9638703 100644
--- a/inc/geometries/orthogonal.h
+++ b/inc/geometries/orthogonal.h
@@ -41,14 +41,15 @@ struct OrthogonalGrid3d : public dg::Grid3d
         generator_ = generator;
         construct( n, Nx, Ny);
     }
+
     /**
     * @brief Reconstruct the grid coordinates
     *
     * @copydetails Grid3d::set()
     * @attention the generator must still live when this function is called
     */
-    void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny){
-        dg::Grid3d::set( new_n, new_Nx, new_Ny);
+    void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz){
+        dg::Grid3d::set( new_n, new_Nx, new_Ny, new_Nz);
         construct( new_n, new_Nx, new_Ny);
     }
 
@@ -65,12 +66,12 @@ struct OrthogonalGrid3d : public dg::Grid3d
     const container& g_pp()const{return g_pp_;}
     const container& vol()const{return vol_;}
     const container& perpVol()const{return vol2d_;}
-    const OrthogonalGrid3d& global() const{return *this;}
+    aGenerator* const generator() const{return generator_;}
     private:
     void construct( unsigned n, unsigned Nx, unsigned Ny)
     {
         dg::Grid1d gY1d( 0, generator_->height(), n, Ny, dg::PER);
-        dg::Grid1d gX1d( 0., generator_->width(), n, Nx);
+        dg::Grid1d gX1d( 0, generator_->width(), n, Nx);
         thrust::host_vector<double> x_vec = dg::evaluate( dg::cooX1d, gX1d);
         thrust::host_vector<double> y_vec = dg::evaluate( dg::cooX1d, gY1d);
         *generator_( x_vec, y_vec, r_, z_, xr_, xz_, yr_, yz_);
@@ -129,18 +130,17 @@ struct OrthogonalGrid2d : public dg::Grid2d
     typedef dg::OrthogonalTag metric_category;
     /*!@brief Constructor
     
-     * @tparam Generator models aGenerator
      * @param generator must generate an orthogonal grid
      * @param n number of polynomial coefficients
      * @param Nx number of cells in first coordinate
      @param Ny number of cells in second coordinate
      @param bcx boundary condition in first coordinate
      */
-    template< class Generator>
-    OrthogonalGrid2d( const Generator& generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx=dg::DIR):
-        dg::Grid2d( 0, generator.width(), 0., generator.height(), n, Nx, Ny, bcx, dg::PER)
+    OrthogonalGrid2d( aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx=dg::DIR):
+        dg::Grid2d( 0, generator->width(), 0., generator->height(), n, Nx, Ny, bcx, dg::PER)
     {
-        OrthogonalGrid3d<container> g( generator, n,Nx,Ny,1,bcx);
+        generator_=generator;
+        OrthogonalGrid3d<container> g( generator_, n,Nx,Ny,1,bcx);
         init_X_boundaries( g.x0(), g.x1());
         r_=g.r(), z_=g.z(), xr_=g.xr(), xz_=g.xz(), yr_=g.yr(), yz_=g.yz();
         g_xx_=g.g_xx(), g_xy_=g.g_xy(), g_yy_=g.g_yy();
@@ -150,6 +150,7 @@ struct OrthogonalGrid2d : public dg::Grid2d
     OrthogonalGrid2d( const OrthogonalGrid3d<container>& g):
         dg::Grid2d( g.x0(), g.x1(), g.y0(), g.y1(), g.n(), g.Nx(), g.Ny(), g.bcx(), g.bcy())
     {
+        generator_ = g.generator();
         unsigned s = this->size();
         r_.resize( s), z_.resize(s), xr_.resize(s), xz_.resize(s), yr_.resize(s), yz_.resize(s);
         g_xx_.resize( s), g_xy_.resize(s), g_yy_.resize(s), vol2d_.resize(s);
@@ -160,6 +161,14 @@ struct OrthogonalGrid2d : public dg::Grid2d
         thrust::copy( g.g_yy().begin(), g.g_yy().begin()+s, g_yy_.begin());
         thrust::copy( g.perpVol().begin(), g.perpVol().begin()+s, vol2d_.begin());
     }
+    void set(unsigned new_n, unsigned new_Nx, unsigned new_Ny)
+    {
+        dg::Grid2d::set( new_n, new_Nx, new_Ny, new_Nz);
+        OrthogonalGrid3d<container> g( generator_, n,Nx,Ny,1,bcx);
+        r_=g.r(), z_=g.z(), xr_=g.xr(), xz_=g.xz(), yr_=g.yr(), yz_=g.yz();
+        g_xx_=g.g_xx(), g_xy_=g.g_xy(), g_yy_=g.g_yy();
+        vol2d_=g.perpVol();
+    }
 
     const thrust::host_vector<double>& r()const{return r_;}
     const thrust::host_vector<double>& z()const{return z_;}
@@ -172,10 +181,11 @@ struct OrthogonalGrid2d : public dg::Grid2d
     const container& g_xy()const{return g_xy_;}
     const container& vol()const{return vol2d_;}
     const container& perpVol()const{return vol2d_;}
-    const OrthogonalGrid2d& global() const{return *this;}
+    aGenerator* const generator() const{return generator_;}
     private:
     thrust::host_vector<double> r_, z_, xr_, xz_, yr_, yz_;
     container g_xx_, g_xy_, g_yy_, vol2d_;
+    aGenerator* generator_;
 };
 
 ///@}
diff --git a/inc/geometries/ribeiro.h b/inc/geometries/ribeiro.h
index 69a5da064..69e05d81e 100644
--- a/inc/geometries/ribeiro.h
+++ b/inc/geometries/ribeiro.h
@@ -10,6 +10,7 @@
 #include "dg/nullstelle.h"
 #include "dg/geometry.h"
 #include "utilities.h"
+#include "generator.h"
 
 
 
@@ -193,7 +194,7 @@ struct FieldFinv
      * @tparam PsiYY models aBinaryOperator 
  */
 template< class Psi, class PsiX, class PsiY, class PsiXX, class PsiXY, class PsiYY>
-struct Ribeiro
+struct Ribeiro : public aGenerator
 {
     /**
      * @brief Construct a near-conformal grid generator
@@ -243,20 +244,7 @@ struct Ribeiro
      * @return f(x)
      */
     thrust::host_vector<double> fx() const{ return fx_;}
-    /**
-    * @brief Generate grid points and elements of the Jacobian 
-    *
-    * @param zeta1d (input) a list of \f$ N_\zeta\f$ points \f$ 0<\zeta_i<\f$width() 
-    * @param eta1d (input) a list of \f$ N_\eta\f$ points \f$ 0<\eta_j<\f$height() 
-    * @param x (output) the list of \f$ N_\eta N_\zeta\f$ coordinates \f$ x(\zeta_i, \eta_j)\f$ 
-    * @param y (output) the list of \f$ N_\eta N_\zeta\f$ coordinates \f$ y(\zeta_i, \eta_j)\f$ 
-    * @param zetaX (output) the list of \f$ N_\eta N_\zeta\f$ elements \f$ \partial\zeta/\partial x (\zeta_i, \eta_j)\f$ 
-    * @param zetaY (output) the list of \f$ N_\eta N_\zeta\f$ elements \f$ \partial\zeta/\partial y (\zeta_i, \eta_j)\f$ 
-    * @param etaX (output) the list of \f$ N_\eta N_\zeta\f$ elements \f$ \partial\eta/\partial x (\zeta_i, \eta_j)\f$ 
-    * @param etaY (output) the list of \f$ N_\eta N_\zeta\f$ elements \f$ \partial\eta/\partial y (\zeta_i, \eta_j)\f$ 
-    @note the \f$ \zeta\f$ coordinate is contiguous in memory
-     * @note All the resulting vectors are write-only and get properly resized
-     */
+
     void operator()( 
          const thrust::host_vector<double>& zeta1d, 
          const thrust::host_vector<double>& eta1d, 
diff --git a/inc/geometries/simple_orthogonal.h b/inc/geometries/simple_orthogonal.h
index 28f4ff15f..47b772b68 100644
--- a/inc/geometries/simple_orthogonal.h
+++ b/inc/geometries/simple_orthogonal.h
@@ -10,6 +10,7 @@
 #include "dg/nullstelle.h"
 #include "dg/geometry.h"
 #include "utilities.h"
+#include "generator.h"
 
 
 namespace dg
@@ -343,10 +344,7 @@ struct SimpleOrthogonal : public aGenerator
      * @return false
      */
     bool isConformal()  const{return false;}
-    /**
-     * @copydoc aGenerator::operator()()
-     * @note All the resulting vectors are write-only and get properly resized
-     */
+
     void operator()( 
          const thrust::host_vector<double>& zeta1d, 
          const thrust::host_vector<double>& eta1d, 
diff --git a/inc/geometries/solovev_doc.h b/inc/geometries/solovev_doc.h
index 17636f9b9..18ea5595c 100644
--- a/inc/geometries/solovev_doc.h
+++ b/inc/geometries/solovev_doc.h
@@ -36,46 +36,6 @@
  */
 
 
-/**
-* @brief The generator template model
-
-A generator is there to construct coordinates from some coordinates
-\f$ x,y\f$ to the computational domain \f$\zeta, \eta\f$, which
-is a product space. 
- @attention this is not a real class it's there for documentation only
- @attention parameter names can be different
- @ingroup temp
-*/
-struct aGenerator
-{
-    double width() const; //!<length in \f$ \zeta\f$ 
-    double height() const; //!<length in \f$ \eta\f$
-    bool isOrthogonal() const; //!< true if coordinate system is orthogonal
-    bool isConformal() const; //!< true if coordinate system is conformal
-    /**
-    * @brief Generate grid points and elements of the Jacobian 
-    *
-    * @param zeta1d (input) a list of \f$ N_\zeta\f$ points \f$ 0<\zeta_i<\f$width() 
-    * @param eta1d (input) a list of \f$ N_\eta\f$ points \f$ 0<\eta_j<\f$height() 
-    * @param x (output) the list of \f$ N_\eta N_\zeta\f$ coordinates \f$ x(\zeta_i, \eta_j)\f$ 
-    * @param y (output) the list of \f$ N_\eta N_\zeta\f$ coordinates \f$ y(\zeta_i, \eta_j)\f$ 
-    * @param zetaX (output) the list of \f$ N_\eta N_\zeta\f$ elements \f$ \partial\zeta/\partial x (\zeta_i, \eta_j)\f$ 
-    * @param zetaY (output) the list of \f$ N_\eta N_\zeta\f$ elements \f$ \partial\zeta/\partial y (\zeta_i, \eta_j)\f$ 
-    * @param etaX (output) the list of \f$ N_\eta N_\zeta\f$ elements \f$ \partial\eta/\partial x (\zeta_i, \eta_j)\f$ 
-    * @param etaY (output) the list of \f$ N_\eta N_\zeta\f$ elements \f$ \partial\eta/\partial y (\zeta_i, \eta_j)\f$ 
-    @note the \f$ \zeta\f$ coordinate is contiguous in memory
-    */
-    void operator()( 
-         const thrust::host_vector<double>& zeta1d, 
-         const thrust::host_vector<double>& eta1d, 
-         thrust::host_vector<double>& x, 
-         thrust::host_vector<double>& y, 
-         thrust::host_vector<double>& zetaX, 
-         thrust::host_vector<double>& zetaY, 
-         thrust::host_vector<double>& etaX, 
-         thrust::host_vector<double>& etaY) ;
-};
-
 /**
 * @brief The generatorX  template model
 
-- 
GitLab


From bfebdd329e3e7db79c6dcc257e99a7778a41f90b Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 20 Jul 2017 18:01:20 +0200
Subject: [PATCH 057/453] ds and fieldaligned almost ready

---
 inc/dg/ds.h                    |  8 --------
 inc/dg/geometry/fieldaligned.h | 12 ++++++++----
 2 files changed, 8 insertions(+), 12 deletions(-)

diff --git a/inc/dg/ds.h b/inc/dg/ds.h
index e364ed829..e0ba42821 100644
--- a/inc/dg/ds.h
+++ b/inc/dg/ds.h
@@ -226,14 +226,6 @@ DS<G, I, M,container>::DS(MagneticField mag, Geometry grid, dg::norm no, dg::dir
         invB(dg::pullback(dg::geo::InvB<MagneticField>(mag),grid)), 
         no_(no), dir_(dir), apply_jumpX_(jumpX)
 {
-    //if geometry = cylindrical  (use traits)
-    //construct cylinder field Field
-    Field<Magneticfield> field(mag);
-    //else 
-    //construct interpolate field DSField
-    DSField<typename G::perpendicular_grid> field( mag, grid.perp_grid());
-    //if flux-aligned
-    //use only 1d refinement
     volume_ = dg::evaluate( dg::one, grid);
     dg::geo::multiplyVolume( volume_, grid);
 }
diff --git a/inc/dg/geometry/fieldaligned.h b/inc/dg/geometry/fieldaligned.h
index 431da8806..1973510d0 100644
--- a/inc/dg/geometry/fieldaligned.h
+++ b/inc/dg/geometry/fieldaligned.h
@@ -11,6 +11,7 @@
 #include "../functors.h"
 #include "../nullstelle.h"
 #include "../runge_kutta.h"
+#include "../../geometries/magnetic_field.h"
 
 namespace dg{
 
@@ -357,8 +358,8 @@ struct FieldAligned
 ////////////////////////////////////DEFINITIONS////////////////////////////////////////
 
 template<class I, class container>
-template <class Field, class Geometry, class Limiter>
-FieldAligned<I, container>::FieldAligned(Field field, Geometry grid, unsigned mx, unsigned my, double eps, Limiter limit, dg::bc globalbcz, double deltaPhi):
+template <class MagneticField, class Geometry, class Limiter>
+FieldAligned<I, container>::FieldAligned(MagneticField mag, Geometry grid, unsigned mx, unsigned my, double eps, Limiter limit, dg::bc globalbcz, double deltaPhi, bool integrateAll):
         hz_( dg::evaluate( dg::zero, grid)), hp_( hz_), hm_( hz_), 
         Nz_(grid.Nz()), bcz_(grid.bcz())
 {
@@ -370,17 +371,20 @@ FieldAligned<I, container>::FieldAligned(Field field, Geometry grid, unsigned mx
     right_ = left_ = dg::evaluate( zero, g2dCoarse);
     ghostM.resize( perp_size_); ghostP.resize( perp_size_);
     //Set starting points
-    typename Geometry::perpendicular_grid g2dFine = g2dCoarse.multiply( mX, mY);
+    typename Geometry::perpendicular_grid g2dFine = g2dCoarse.resize( (double)mX, (double)mY);
     
     std::vector<thrust::host_vector<double> > y( 3, dg::evaluate( dg::cooX2d, g2dFine)); // x
     y[1] = dg::evaluate( dg::cooY2d, g2dFine); //y
     y[2] = dg::evaluate( dg::zero, g2dFine);
-    //integrate field lines for all points
     std::vector<thrust::host_vector<double> > yp( 3, dg::evaluate(dg::zero, g2dFine)), ym(yp); 
     if( deltaPhi <=0) deltaPhi = grid.hz();
     else assert( grid.Nz() == 1 || grid.hz()==deltaPhi);
     dg::Timer t;
     t.tic();
+    //construct field on high polynomial grid, then integrate it
+    typename Geometry::perpendicular_grid g2dField = g2dCoarse;
+    g2dField.set( 11, g2dField.Nx(), g2dField.Ny());
+    dg::geo::DSField<typename Geometry::perpendicular_grid> field( mag, g2dField);
 #ifdef _OPENMP
 #pragma omp parallel for shared(field)
 #endif //_OPENMP
-- 
GitLab


From 1ba76b771f014d26e1c30539787e151abcc62193 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 20 Jul 2017 22:25:10 +0200
Subject: [PATCH 058/453] removed Geometry template parameter in ds.h

---
 inc/dg/ds.h                     |  63 +++++++++---------
 inc/dg/ds_b.cu                  |  38 -----------
 inc/dg/geometry/fieldaligned.h  | 111 ++++++++++++++++++++++++++++++--
 inc/geometries/magnetic_field.h | 105 ------------------------------
 4 files changed, 138 insertions(+), 179 deletions(-)

diff --git a/inc/dg/ds.h b/inc/dg/ds.h
index e0ba42821..b818e3d63 100644
--- a/inc/dg/ds.h
+++ b/inc/dg/ds.h
@@ -25,12 +25,11 @@ namespace dg{
 \mathbf{b}\cdot \nabla = b_R\partial_R + b_Z\partial_Z + b_\phi\partial_\phi \f$, \f$\nabla_\parallel^\dagger\f$ and \f$\Delta_\parallel=\nabla_\parallel^\dagger\cdot\nabla_\parallel\f$ in
 arbitrary coordinates
 * @ingroup fieldaligned
-* @tparam Geometry The grid geometry
 * @tparam IMatrix The type of the interpolation matrix
 * @tparam Matrix The matrix class of the jump matrix
 * @tparam container The container-class on which the interpolation matrix operates on (does not need to be dg::HVec)
 */
-template< class Geometry, class IMatrix, class Matrix, class container >
+template< class IMatrix, class Matrix, class container >
 struct DS
 {
     /**
@@ -43,8 +42,8 @@ struct DS
     * @param dir the direction affects both the operator() and the symv function
     @param jumpX determines if a jump matrix is added in X-direction
     */
-    template<class InvB, class Geometry>
-    DS(const FA& field, Geometry, InvB invB, dg::norm no=dg::normed, dg::direction dir = dg::centered, bool jumpX = true);
+    template<class MagneticField, class Geometry>
+    DS(const MagneticField& field, Geometry, dg::norm no=dg::normed, dg::direction dir = dg::centered, bool jumpX = true, unsigned mx=1, unsigned my=1);
 
     /**
     * @brief Apply the forward derivative on a 3d vector
@@ -215,12 +214,12 @@ struct DS
 ///@cond
 ////////////////////////////////////DEFINITIONS////////////////////////////////////////
 
-template<class G, class I, class M, class container>
-template <class MagneticField>
-DS<G, I, M,container>::DS(MagneticField mag, Geometry grid, dg::norm no, dg::direction dir, bool jumpX):
+template<class I, class M, class container>
+template <class MagneticField, class Geometry>
+DS< I, M,container>::DS(MagneticField mag, Geometry grid, dg::norm no, dg::direction dir, bool jumpX, unsigned mx, unsigned my):
         jumpX( dg::create::jumpX( grid)),
         jumpY( dg::create::jumpY( grid)),
-        tempP( dg::evaluate( dg::zero, field.grid())), temp0( tempP), tempM( tempP), 
+        tempP( dg::evaluate( dg::zero, grid())), temp0( tempP), tempM( tempP), 
         f(tempP), dsf(tempP),
         vol3d( dg::create::volume( grid)), inv3d( dg::create::inv_volume( grid)),
         invB(dg::pullback(dg::geo::InvB<MagneticField>(mag),grid)), 
@@ -230,8 +229,8 @@ DS<G, I, M,container>::DS(MagneticField mag, Geometry grid, dg::norm no, dg::dir
     dg::geo::multiplyVolume( volume_, grid);
 }
 
-template<class G, class I, class M, class container>
-inline void DS<G,I,M,container>::operator()( const container& f, container& dsf) { 
+template<class I, class M, class container>
+inline void DS<I,M,container>::operator()( const container& f, container& dsf) { 
     if( dir_ == dg::centered)
         return centered( f, dsf);
     else if( dir_ == dg::forward)
@@ -241,8 +240,8 @@ inline void DS<G,I,M,container>::operator()( const container& f, container& dsf)
 }
 
 
-template<class G, class I, class M, class container>
-void DS<G,I,M,container>::centered( const container& f, container& dsf)
+template<class I, class M, class container>
+void DS<I,M,container>::centered( const container& f, container& dsf)
 {
     //direct discretisation
     assert( &f != &dsf);
@@ -252,8 +251,8 @@ void DS<G,I,M,container>::centered( const container& f, container& dsf)
     dg::blas1::pointwiseDivide( tempM, f_.hz(), dsf);
 }
 
-template<class G, class I, class M, class container>
-void DS<G,I,M,container>::centeredAdj( const container& f, container& dsf)
+template<class I, class M, class container>
+void DS<I,M,container>::centeredAdj( const container& f, container& dsf)
 {               
     //adjoint discretisation
     assert( &f != &dsf);    
@@ -265,8 +264,8 @@ void DS<G,I,M,container>::centeredAdj( const container& f, container& dsf)
     dg::blas1::pointwiseDot( inv3d, dsf, dsf); 
 }
 
-template<class G, class I, class M, class container>
-void DS<G,I,M,container>::centeredAdjDir( const container& f, container& dsf)
+template<class I, class M, class container>
+void DS<I,M,container>::centeredAdjDir( const container& f, container& dsf)
 {       
 //     Direct discretisation
     assert( &f != &dsf);    
@@ -278,8 +277,8 @@ void DS<G,I,M,container>::centeredAdjDir( const container& f, container& dsf)
     dg::blas1::pointwiseDivide( dsf, invB, dsf);
 }
 
-template<class G, class I, class M, class container>
-void DS<G,I,M,container>::forward( const container& f, container& dsf)
+template<class I, class M, class container>
+void DS<I,M,container>::forward( const container& f, container& dsf)
 {
     //direct
     assert( &f != &dsf);
@@ -288,8 +287,8 @@ void DS<G,I,M,container>::forward( const container& f, container& dsf)
     dg::blas1::pointwiseDivide( tempP, f_.hp(), dsf);
 }
 
-template<class G, class I, class M, class container>
-void DS<G,I,M,container>::forwardAdj( const container& f, container& dsf)
+template<class I, class M, class container>
+void DS<I,M,container>::forwardAdj( const container& f, container& dsf)
 {    
     //adjoint discretisation
     assert( &f != &dsf);
@@ -300,8 +299,8 @@ void DS<G,I,M,container>::forwardAdj( const container& f, container& dsf)
     dg::blas1::pointwiseDot( inv3d, dsf, dsf); 
 }
 
-template<class G, class I, class M, class container>
-void DS<G,I,M,container>::forwardAdjDir( const container& f, container& dsf)
+template<class I, class M, class container>
+void DS<I,M,container>::forwardAdjDir( const container& f, container& dsf)
 {
     //direct discretisation
     assert( &f != &dsf);
@@ -312,8 +311,8 @@ void DS<G,I,M,container>::forwardAdjDir( const container& f, container& dsf)
     dg::blas1::pointwiseDivide( dsf, invB, dsf);
 }
 
-template<class G, class I, class M, class container>
-void DS<G,I,M,container>::backward( const container& f, container& dsf)
+template<class I, class M, class container>
+void DS<I,M,container>::backward( const container& f, container& dsf)
 {
     //direct
     assert( &f != &dsf);
@@ -322,8 +321,8 @@ void DS<G,I,M,container>::backward( const container& f, container& dsf)
     dg::blas1::pointwiseDivide( tempM, f_.hm(), dsf);
 }
 
-template<class G, class I, class M, class container>
-void DS<G,I,M,container>::backwardAdj( const container& f, container& dsf)
+template<class I, class M, class container>
+void DS<I,M,container>::backwardAdj( const container& f, container& dsf)
 {    
     //adjoint discretisation
     assert( &f != &dsf);
@@ -334,8 +333,8 @@ void DS<G,I,M,container>::backwardAdj( const container& f, container& dsf)
     dg::blas1::pointwiseDot( inv3d, dsf, dsf); 
 }
 
-template<class G, class I, class M, class container>
-void DS<G,I,M,container>::backwardAdjDir( const container& f, container& dsf)
+template<class I, class M, class container>
+void DS<I,M,container>::backwardAdjDir( const container& f, container& dsf)
 {
     //direct
     assert( &f != &dsf);
@@ -346,8 +345,8 @@ void DS<G,I,M,container>::backwardAdjDir( const container& f, container& dsf)
     dg::blas1::pointwiseDivide( dsf, invB, dsf);
 }
 
-template<class G, class I, class M, class container>
-void DS<G,I,M,container>::symv( const container& f, container& dsTdsf)
+template<class I, class M, class container>
+void DS<I,M,container>::symv( const container& f, container& dsTdsf)
 {
     if(dir_ == dg::centered)
     {
@@ -380,8 +379,8 @@ void DS<G,I,M,container>::symv( const container& f, container& dsTdsf)
 }
 
 //enables the use of the dg::blas2::symv function 
-template< class G, class I, class M, class V>
-struct MatrixTraits< DS<G,I,M, V> >
+template< class I, class M, class V>
+struct MatrixTraits< DS<I,M, V> >
 {
     typedef double value_type;
     typedef SelfMadeMatrixTag matrix_category;
diff --git a/inc/dg/ds_b.cu b/inc/dg/ds_b.cu
index 64066d15e..2e3288308 100644
--- a/inc/dg/ds_b.cu
+++ b/inc/dg/ds_b.cu
@@ -10,43 +10,6 @@
 #include "backend/timer.cuh"
 #include "geometry.h"
 
-struct Field
-{
-    Field( double R_0, double I_0):R_0(R_0), I_0(I_0){}
-    void operator()( const dg::HVec& y, dg::HVec& yp) const
-    {
-        double gradpsi = ((y[0]-R_0)*(y[0]-R_0) + y[1]*y[1])/I_0/I_0;
-        yp[2] = y[0]*sqrt(1 + gradpsi);
-        yp[0] = y[0]*y[1]/I_0;
-        yp[1] = y[0]/I_0*(R_0-y[0]) ;
-    }
-    double operator()( double x, double y, double z) const
-    {
-        double gradpsi = ((x-R_0)*(x-R_0) + y*y)/I_0/I_0;
-        return  x/sqrt( 1 + gradpsi)/R_0/I_0;
-    }
-    double error( const dg::HVec& x0, const dg::HVec& x1)
-    {
-        return sqrt( (x0[0]-x1[0])*(x0[0]-x1[0]) +(x0[1]-x1[1])*(x0[1]-x1[1])+(x0[2]-x1[2])*(x0[2]-x1[2]));
-    }
-    bool monitor( const dg::HVec& end){ 
-        if ( std::isnan(end[0]) || std::isnan(end[1]) || std::isnan(end[2]) ) 
-        {
-            return false;
-        }
-        //if new integrated point outside domain
-        if ((1e-5 > end[0]  ) || (1e10 < end[0])  ||(-1e10  > end[1]  ) || (1e10 < end[1])||(-1e10 > end[2]  ) || (1e10 < end[2])  )
-        {
-            return false;
-        }
-        return true;
-    }
-    private:
-    double R_0, I_0;
-};
-
-double R_0 = 10;
-double I_0 = 40;
 double func(double R, double Z, double phi)
 {
     double r2 = (R-R_0)*(R-R_0)+Z*Z;
@@ -58,7 +21,6 @@ double deri(double R, double Z, double phi)
     return I_0/R/sqrt(I_0*I_0 + r2)* r2*cos(phi);
 }
 
-
 int main()
 {
     Field field( R_0, I_0);
diff --git a/inc/dg/geometry/fieldaligned.h b/inc/dg/geometry/fieldaligned.h
index 1973510d0..91907195b 100644
--- a/inc/dg/geometry/fieldaligned.h
+++ b/inc/dg/geometry/fieldaligned.h
@@ -49,6 +49,109 @@ struct DefaultField
 
 };
 
+/**
+ * @brief Integrates the equations for a field line 
+ * @tparam MagneticField models aTokamakMagneticField
+ * @ingroup misc
+ */ 
+template<class MagneticField>
+struct Field
+{
+    Field( const MagneticField& c):c_(c), invB_(c), R_0_(c.R_0) { }
+    /**
+     * @brief \f[ \frac{d \hat{R} }{ d \varphi}  = \frac{\hat{R}}{\hat{I}} \frac{\partial\hat{\psi}_p}{\partial \hat{Z}}, \hspace {3 mm}
+     \frac{d \hat{Z} }{ d \varphi}  =- \frac{\hat{R}}{\hat{I}} \frac{\partial \hat{\psi}_p}{\partial \hat{R}} , \hspace {3 mm}
+     \frac{d \hat{l} }{ d \varphi}  =\frac{\hat{R}^2 \hat{B}}{\hat{I}  \hat{R}_0}  \f]
+     */ 
+    void operator()( const dg::HVec& y, dg::HVec& yp) const
+    {
+        double ipol = c_.ipol(y[0],y[1]);
+        yp[2] =  y[0]*y[0]/invB_(y[0],y[1])/ipol/R_0_;       //ds/dphi =  R^2 B/I/R_0_hat
+        yp[0] =  y[0]*c_.psipZ(y[0],y[1])/ipol;              //dR/dphi =  R/I Psip_Z
+        yp[1] = -y[0]*c_.psipR(y[0],y[1])/ipol ;             //dZ/dphi = -R/I Psip_R
+
+    }
+    double error( const dg::HVec& x0, const dg::HVec& x1)
+    {
+        return sqrt( (x0[0]-x1[0])*(x0[0]-x1[0]) +(x0[1]-x1[1])*(x0[1]-x1[1])+(x0[2]-x1[2])*(x0[2]-x1[2]));
+    }
+    bool monitor( const dg::HVec& end){ 
+        if ( std::isnan(end[0]) || std::isnan(end[1]) || std::isnan(end[2]) ) 
+        {
+            return false;
+        }
+        //if new integrated point outside domain
+        if ((1e-5 > end[0]  ) || (1e10 < end[0])  ||(-1e10  > end[1]  ) || (1e10 < end[1])||(-1e10 > end[2]  ) || (1e10 < end[2])  )
+        {
+            return false;
+        }
+        return true;
+    }
+    
+    private:
+    MagneticField c_;
+    InvB invB_;
+    double R_0_;
+};
+
+template< class GeometryPerp>
+struct DSField
+{
+    template<class MagneticField>
+    DSField( const MagneticField& c, const GeometryPerp& g)
+    {
+        InvB<MagneticField> invB(c);
+        FieldR<MagneticField> fieldR(c);
+        FieldZ<MagneticField> fieldZ(c);
+        thrust::host_vector<double> b_zeta, b_eta;
+        dg::geo::pushForwardPerp( fieldR, fieldZ, b_zeta, b_eta, g);
+        FieldP<MagneticField> fieldP(c);
+        thrust::host_vector<double> b_phi = dg::pullback( fieldP, g);
+        Bmodule<MagneticField> bmod( c);
+        thrust::host_vector<double> b_mod = dg::pullback( bmod, g);
+        dg::blas1::pointwiseDivide( b_zeta, b_phi, b_zeta);
+        dg::blas1::pointwiseDivide( b_eta,  b_phi, b_eta);
+        dg::blas1::pointwiseDivide( b_mod,  b_phi, b_mod);
+        dzetadphi_ = dg::forward_transform( b_zeta, g );
+        detadphi_  = dg::forward_transform( b_eta, g );
+        dsdphi_    = dg::forward_transform( b_mod, g );
+    }
+
+    void operator()(thrust::host_vector<double> y, thrust::host_vector<double>& yp)
+    {
+        g_.shift_topologic( y[0], y[1], y[0], y[1]); //shift points onto domain
+        if( !g_.contains( y[0], y[1])) yp[0] = yp[1]= yp[2] = 0;
+        else
+        {
+            //else interpolate
+            yp[0] = interpolate( y[0], y[1], dzetadphi_, g_);
+            yp[1] = interpolate( y[0], y[1], detadphi_, g_);
+            yp[2] = interpolate( y[0], y[1], dsphi_, g_);
+        }
+    }
+
+    double error( const dg::HVec& x0, const dg::HVec& x1)
+    {
+        return sqrt( (x0[0]-x1[0])*(x0[0]-x1[0]) +(x0[1]-x1[1])*(x0[1]-x1[1])+(x0[2]-x1[2])*(x0[2]-x1[2]));
+    }
+    bool monitor( const dg::HVec& end){ 
+        if ( std::isnan(end[0]) || std::isnan(end[1]) || std::isnan(end[2]) ) 
+        {
+            return false;
+        }
+        //if new integrated point outside domain
+        if ((1e-5 > end[0]  ) || (1e10 < end[0])  ||(-1e10  > end[1]  ) || (1e10 < end[1])||(-1e10 > end[2]  ) || (1e10 < end[2])  )
+        {
+            return false;
+        }
+        return true;
+    }
+    private:
+    thrust::host_vector<double> dzetadphi_, detadphi_, dsdphi_;
+    GeometryPerp g_;
+
+};
+
 /**
  * @brief Default Limiter means there is a limiter everywhere
  */
@@ -228,8 +331,8 @@ struct FieldAligned
         by the bcz variable from the grid and can be changed by the set_boundaries function. 
         If there is no limiter the boundary condition is periodic.
     */
-    template <class Field, class Geometry, class Limiter>
-    FieldAligned(Field field, Geometry grid, unsigned multiplyX, unsigned multiplyY, double eps = 1e-4, Limiter limit = DefaultLimiter(), dg::bc globalbcz = dg::DIR, double deltaPhi = -1);
+    template <class MagneticField, class Geometry, class Limiter>
+    FieldAligned(MagnetricField field, Geometry grid, unsigned multiplyX, unsigned multiplyY, double eps = 1e-4, Limiter limit = DefaultLimiter(), dg::bc globalbcz = dg::DIR, double deltaPhi = -1);
 
     /**
     * @brief Set boundary conditions in the limiter region
@@ -359,7 +462,7 @@ struct FieldAligned
 
 template<class I, class container>
 template <class MagneticField, class Geometry, class Limiter>
-FieldAligned<I, container>::FieldAligned(MagneticField mag, Geometry grid, unsigned mx, unsigned my, double eps, Limiter limit, dg::bc globalbcz, double deltaPhi, bool integrateAll):
+FieldAligned<I, container>::FieldAligned(MagneticField mag, Geometry grid, unsigned mx, unsigned my, double eps, Limiter limit, dg::bc globalbcz, double deltaPhi):
         hz_( dg::evaluate( dg::zero, grid)), hp_( hz_), hm_( hz_), 
         Nz_(grid.Nz()), bcz_(grid.bcz())
 {
@@ -384,7 +487,7 @@ FieldAligned<I, container>::FieldAligned(MagneticField mag, Geometry grid, unsig
     //construct field on high polynomial grid, then integrate it
     typename Geometry::perpendicular_grid g2dField = g2dCoarse;
     g2dField.set( 11, g2dField.Nx(), g2dField.Ny());
-    dg::geo::DSField<typename Geometry::perpendicular_grid> field( mag, g2dField);
+    dg::DSField<typename Geometry::perpendicular_grid> field( mag, g2dField);
 #ifdef _OPENMP
 #pragma omp parallel for shared(field)
 #endif //_OPENMP
diff --git a/inc/geometries/magnetic_field.h b/inc/geometries/magnetic_field.h
index 993f0b852..4437a1c20 100644
--- a/inc/geometries/magnetic_field.h
+++ b/inc/geometries/magnetic_field.h
@@ -489,111 +489,6 @@ struct BHatP
 
 ///@} 
 
-/**
- * @brief Integrates the equations for a field line 
- * @tparam MagneticField models aTokamakMagneticField
- * @ingroup misc
- */ 
-template<class MagneticField>
-struct Field
-{
-    Field( const MagneticField& c):c_(c), invB_(c), R_0_(c.R_0) { }
-    /**
-     * @brief \f[ \frac{d \hat{R} }{ d \varphi}  = \frac{\hat{R}}{\hat{I}} \frac{\partial\hat{\psi}_p}{\partial \hat{Z}}, \hspace {3 mm}
-     \frac{d \hat{Z} }{ d \varphi}  =- \frac{\hat{R}}{\hat{I}} \frac{\partial \hat{\psi}_p}{\partial \hat{R}} , \hspace {3 mm}
-     \frac{d \hat{l} }{ d \varphi}  =\frac{\hat{R}^2 \hat{B}}{\hat{I}  \hat{R}_0}  \f]
-     */ 
-    void operator()( const dg::HVec& y, dg::HVec& yp) const
-    {
-        double ipol = c_.ipol(y[0],y[1]);
-        yp[2] =  y[0]*y[0]/invB_(y[0],y[1])/ipol/R_0_;       //ds/dphi =  R^2 B/I/R_0_hat
-        yp[0] =  y[0]*c_.psipZ(y[0],y[1])/ipol;              //dR/dphi =  R/I Psip_Z
-        yp[1] = -y[0]*c_.psipR(y[0],y[1])/ipol ;             //dZ/dphi = -R/I Psip_R
-
-    }
-    double error( const dg::HVec& x0, const dg::HVec& x1)
-    {
-        return sqrt( (x0[0]-x1[0])*(x0[0]-x1[0]) +(x0[1]-x1[1])*(x0[1]-x1[1])+(x0[2]-x1[2])*(x0[2]-x1[2]));
-    }
-    bool monitor( const dg::HVec& end){ 
-        if ( std::isnan(end[0]) || std::isnan(end[1]) || std::isnan(end[2]) ) 
-        {
-            return false;
-        }
-        //if new integrated point outside domain
-        if ((1e-5 > end[0]  ) || (1e10 < end[0])  ||(-1e10  > end[1]  ) || (1e10 < end[1])||(-1e10 > end[2]  ) || (1e10 < end[2])  )
-        {
-            return false;
-        }
-        return true;
-    }
-    
-    private:
-    MagneticField c_;
-    InvB invB_;
-    double R_0_;
-};
-
-
-template< class GeometryPerp>
-struct DSField
-{
-    template<class MagneticField>
-    DSField( const MagneticField& c, const GeometryPerp& g)
-    {
-        InvB<MagneticField> invB(c);
-        FieldR<MagneticField> fieldR(c);
-        FieldZ<MagneticField> fieldZ(c);
-        thrust::host_vector<double> b_zeta, b_eta;
-        dg::geo::pushForwardPerp( fieldR, fieldZ, b_zeta, b_eta, g);
-        FieldP<MagneticField> fieldP(c);
-        thrust::host_vector<double> b_phi = dg::pullback( fieldP, g);
-        Bmodule<MagneticField> bmod( c);
-        thrust::host_vector<double> b_mod = dg::pullback( bmod, g);
-        dg::blas1::pointwiseDivide( b_zeta, b_phi, b_zeta);
-        dg::blas1::pointwiseDivide( b_eta,  b_phi, b_eta);
-        dg::blas1::pointwiseDivide( b_mod,  b_phi, b_mod);
-        dzetadphi_ = dg::forward_transform( b_zeta, g );
-        detadphi_  = dg::forward_transform( b_eta, g );
-        dsdphi_    = dg::forward_transform( b_mod, g );
-    }
-
-    void operator()(thrust::host_vector<double> y, thrust::host_vector<double>& yp)
-    {
-        g_.shift_topologic( y[0], y[1], y[0], y[1]); //shift points onto domain
-        if( !g_.contains( y[0], y[1])) yp[0] = yp[1]= yp[2] = 0;
-        else
-        {
-            //else interpolate
-            yp[0] = interpolate( y[0], y[1], dzetadphi_, g_);
-            yp[1] = interpolate( y[0], y[1], detadphi_, g_);
-            yp[2] = interpolate( y[0], y[1], dsphi_, g_);
-        }
-    }
-
-    double error( const dg::HVec& x0, const dg::HVec& x1)
-    {
-        return sqrt( (x0[0]-x1[0])*(x0[0]-x1[0]) +(x0[1]-x1[1])*(x0[1]-x1[1])+(x0[2]-x1[2])*(x0[2]-x1[2]));
-    }
-    bool monitor( const dg::HVec& end){ 
-        if ( std::isnan(end[0]) || std::isnan(end[1]) || std::isnan(end[2]) ) 
-        {
-            return false;
-        }
-        //if new integrated point outside domain
-        if ((1e-5 > end[0]  ) || (1e10 < end[0])  ||(-1e10  > end[1]  ) || (1e10 < end[1])||(-1e10 > end[2]  ) || (1e10 < end[2])  )
-        {
-            return false;
-        }
-        return true;
-    }
-    private:
-    thrust::host_vector<double> dzetadphi_, detadphi_, dsdphi_;
-    GeometryPerp g_;
-
-};
-
-
 } //namespace geo
 } //namespace dg
 
-- 
GitLab


From 3469da9c914af0ea20fcab4287e94dae467684ed Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 20 Jul 2017 22:50:10 +0200
Subject: [PATCH 059/453] moved ds algorithm into geometries folder

---
 inc/geometries/README                              | 3 +++
 inc/{dg => geometries}/ds.h                        | 0
 inc/{dg => geometries}/ds_b.cu                     | 0
 inc/{dg => geometries}/ds_mpib.cu                  | 0
 inc/{dg => geometries}/dz_mpit.cu                  | 0
 inc/{dg => geometries}/dz_t.cu                     | 0
 inc/{dg/geometry => geometries}/fieldaligned.h     | 0
 inc/{dg/geometry => geometries}/mpi_fieldaligned.h | 0
 8 files changed, 3 insertions(+)
 rename inc/{dg => geometries}/ds.h (100%)
 rename inc/{dg => geometries}/ds_b.cu (100%)
 rename inc/{dg => geometries}/ds_mpib.cu (100%)
 rename inc/{dg => geometries}/dz_mpit.cu (100%)
 rename inc/{dg => geometries}/dz_t.cu (100%)
 rename inc/{dg/geometry => geometries}/fieldaligned.h (100%)
 rename inc/{dg/geometry => geometries}/mpi_fieldaligned.h (100%)

diff --git a/inc/geometries/README b/inc/geometries/README
index dac07a2fd..218294d5f 100644
--- a/inc/geometries/README
+++ b/inc/geometries/README
@@ -41,4 +41,7 @@
     adaption.h
 
 #miscellaneous
+    ds.h
+    fieldaligned.h
+    mpi_fieldaligned.h
     average.h
diff --git a/inc/dg/ds.h b/inc/geometries/ds.h
similarity index 100%
rename from inc/dg/ds.h
rename to inc/geometries/ds.h
diff --git a/inc/dg/ds_b.cu b/inc/geometries/ds_b.cu
similarity index 100%
rename from inc/dg/ds_b.cu
rename to inc/geometries/ds_b.cu
diff --git a/inc/dg/ds_mpib.cu b/inc/geometries/ds_mpib.cu
similarity index 100%
rename from inc/dg/ds_mpib.cu
rename to inc/geometries/ds_mpib.cu
diff --git a/inc/dg/dz_mpit.cu b/inc/geometries/dz_mpit.cu
similarity index 100%
rename from inc/dg/dz_mpit.cu
rename to inc/geometries/dz_mpit.cu
diff --git a/inc/dg/dz_t.cu b/inc/geometries/dz_t.cu
similarity index 100%
rename from inc/dg/dz_t.cu
rename to inc/geometries/dz_t.cu
diff --git a/inc/dg/geometry/fieldaligned.h b/inc/geometries/fieldaligned.h
similarity index 100%
rename from inc/dg/geometry/fieldaligned.h
rename to inc/geometries/fieldaligned.h
diff --git a/inc/dg/geometry/mpi_fieldaligned.h b/inc/geometries/mpi_fieldaligned.h
similarity index 100%
rename from inc/dg/geometry/mpi_fieldaligned.h
rename to inc/geometries/mpi_fieldaligned.h
-- 
GitLab


From 550640704b42cdd0001e40bbbc3ccb6f639e01ef Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Fri, 21 Jul 2017 00:10:44 +0200
Subject: [PATCH 060/453] trouble in Hector with pointers

---
 inc/dg/geometry.h                  |  2 +-
 inc/geometries/conformal.h         | 20 ++++++++++----------
 inc/geometries/conformal_t.cu      | 11 ++++++-----
 inc/geometries/curvilinear.h       | 21 ++++++++++-----------
 inc/geometries/flux.h              |  8 ++++----
 inc/geometries/generator.h         | 12 +++++++-----
 inc/geometries/magnetic_field.h    |  4 ++--
 inc/geometries/mpi_conformal.h     |  9 ++++-----
 inc/geometries/mpi_curvilinear.h   |  8 ++++----
 inc/geometries/mpi_orthogonal.h    |  8 ++++----
 inc/geometries/orthogonal.h        | 20 ++++++++++----------
 inc/geometries/ribeiro.h           |  2 +-
 inc/geometries/simple_orthogonal.h |  2 +-
 13 files changed, 64 insertions(+), 63 deletions(-)

diff --git a/inc/dg/geometry.h b/inc/dg/geometry.h
index ffc70c54a..a45e2944d 100644
--- a/inc/dg/geometry.h
+++ b/inc/dg/geometry.h
@@ -206,7 +206,7 @@ void pushForwardPerp(
  * @param chiyy yy-component of tensor (gets properly resized)
  * @param g The geometry object
  */
-template<class FunctorRR, class FunctorRZ, class FunctorZZ, class Geometry> 
+template<class FunctorRR, class FunctorRZ, class FunctorZZ, class container, class Geometry> 
 void pushForwardPerp( FunctorRR chiRR, FunctorRZ chiRZ, FunctorZZ chiZZ,
         container& chixx, container& chixy, container& chiyy,
         const Geometry& g)
diff --git a/inc/geometries/conformal.h b/inc/geometries/conformal.h
index 682c745a1..013eab3ea 100644
--- a/inc/geometries/conformal.h
+++ b/inc/geometries/conformal.h
@@ -36,10 +36,10 @@ struct ConformalGrid3d : public dg::Grid3d
      * @param Nz
      * @param bcx
      */
-    ConformalGrid3d( aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx=dg::DIR) :
+    ConformalGrid3d( geo::aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx=dg::DIR) :
         dg::Grid3d( 0, generator->width(), 0., generator->height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, dg::PER, dg::PER)
     {
-        assert( generator.isConformal());
+        assert( generator->isConformal());
         generator_=generator;
         construct(n,Nx, Ny);
     }
@@ -66,7 +66,7 @@ struct ConformalGrid3d : public dg::Grid3d
     const container& g_pp()const{return g_pp_;}
     const container& vol()const{return vol_;}
     const container& perpVol()const{return vol2d_;}
-    aGenerator* const generator() const{return generator_;}
+    geo::aGenerator* const generator() const{return generator_;}
     private:
     void construct( unsigned n, unsigned Nx, unsigned Ny) 
     {
@@ -74,7 +74,7 @@ struct ConformalGrid3d : public dg::Grid3d
         dg::Grid1d gv( 0., generator_->height(), n, Ny);
         const thrust::host_vector<double> u1d = dg::evaluate( dg::cooX1d, gu);
         const thrust::host_vector<double> v1d = dg::evaluate( dg::cooX1d, gv);
-        *generator_( u1d, v1d, r_, z_, xr_, xz_, yr_, yz_);
+        (*generator_)( u1d, v1d, r_, z_, xr_, xz_, yr_, yz_);
         init_X_boundaries( 0., generator_->width());
         lift3d( ); //lift to 3D grid
         construct_metric();
@@ -117,7 +117,7 @@ struct ConformalGrid3d : public dg::Grid3d
     
     thrust::host_vector<double> r_, z_, xr_, xz_, yr_, yz_; //3d vector
     container gradU2_, g_pp_, vol_, vol2d_;
-    aGenerator* generator_;
+    geo::aGenerator* generator_;
 
 };
 
@@ -137,7 +137,7 @@ struct ConformalGrid2d : public dg::Grid2d
      * @param Ny
      * @param bcx
      */
-    ConformalGrid2d( aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx=dg::DIR):
+    ConformalGrid2d( geo::aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx=dg::DIR):
         dg::Grid2d( 0, generator->width(), 0., generator->height(), n,Nx,Ny, bcx, dg::PER)
     {
         generator_=generator;
@@ -162,8 +162,8 @@ struct ConformalGrid2d : public dg::Grid2d
 
     void set(unsigned new_n, unsigned new_Nx, unsigned new_Ny)
     {
-        dg::Grid2d::set( new_n, new_Nx, new_Ny, new_Nz);
-        ConformalGrid3d<container> g( generator_, n,Nx,Ny,1,bcx);
+        dg::Grid2d::set( new_n, new_Nx, new_Ny);
+        ConformalGrid3d<container> g( generator_, new_n,new_Nx,new_Ny,1,bcx());
         r_=g.r(), z_=g.z(), xr_=g.xr(), xz_=g.xz(), yr_=g.yr(), yz_=g.yz();
         gradU2_=g.g_xx(), vol2d_=g.perpVol();
     }
@@ -179,11 +179,11 @@ struct ConformalGrid2d : public dg::Grid2d
     const container& g_yy()const{return gradU2_;}
     const container& vol()const{return vol2d_;}
     const container& perpVol()const{return vol2d_;}
-    aGenerator* const generator() const{return generator_;}
+    geo::aGenerator* const generator() const{return generator_;}
     private:
     thrust::host_vector<double> r_, z_, xr_, xz_, yr_, yz_;
     container gradU2_, vol2d_;
-    aGenerator* generator_;
+    geo::aGenerator* generator_;
 };
 
 ///@}
diff --git a/inc/geometries/conformal_t.cu b/inc/geometries/conformal_t.cu
index 91d8bcc8f..d74c2e62d 100644
--- a/inc/geometries/conformal_t.cu
+++ b/inc/geometries/conformal_t.cu
@@ -16,7 +16,6 @@
 #include "curvilinear.h"
 #include "hector.h"
 //#include "refined_conformal.h"
-#include "dg/ds.h"
 #include "init.h"
 
 #include "file/nc_utilities.h"
@@ -77,8 +76,9 @@ int main( int argc, char* argv[])
     std::cout << "Constructing conformal grid ... \n";
     t.tic();
     dg::geo::solovev::MagneticField c( gp); 
+    Hector<dg::IDMatrix, dg::DMatrix, dg::DVec>* hector;
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-    Hector<dg::IDMatrix, dg::DMatrix, dg::DVec> hector( c.psip, c.psipR, c.psipZ, c.psipRR, c.psipRZ, c.psipZZ, psi_0, psi_1, gp.R_0, 0., nGrid, NxGrid, NyGrid, epsHector, true);
+    hector = new Hector<dg::IDMatrix, dg::DMatrix, dg::DVec>( c.psip, c.psipR, c.psipZ, c.psipRR, c.psipRZ, c.psipZZ, psi_0, psi_1, gp.R_0, 0., nGrid, NxGrid, NyGrid, epsHector, true);
     dg::ConformalGrid3d<dg::HVec> g3d(hector, n, Nx, Ny,Nz, dg::DIR);
     dg::ConformalGrid2d<dg::HVec> g2d = g3d.perp_grid();
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -96,7 +96,7 @@ int main( int argc, char* argv[])
     dg::Grid2d g2d_periodic(g2d.x0(), g2d.x1(), g2d.y0(), g2d.y1(), g2d.n(), g2d.Nx(), g2d.Ny()+1); 
     t.toc();
     std::cout << "Construction took "<<t.diff()<<"s"<<std::endl;
-    std::cout << "Length in u is    "<<hector.width()<<std::endl;
+    std::cout << "Length in u is    "<<hector->width()<<std::endl;
     int ncid;
     file::NC_Error_Handle err;
     err = nc_create( "conformal.nc", NC_NETCDF4|NC_CLOBBER, &ncid);
@@ -157,12 +157,13 @@ int main( int argc, char* argv[])
     dg::HVec ones2d = dg::evaluate( dg::one, g2d);
     double volumeUV = dg::blas1::dot( vol, ones2d);
 
-    vol = dg::create::volume( hector.internal_grid());
-    ones2d = dg::evaluate( dg::one, hector.internal_grid());
+    vol = dg::create::volume( hector->internal_grid());
+    ones2d = dg::evaluate( dg::one, hector->internal_grid());
     double volumeZE = dg::blas1::dot( vol, ones2d);
     std::cout << "volumeUV is "<< volumeUV<<std::endl;
     std::cout << "volumeZE is "<< volumeZE<<std::endl;
     std::cout << "relative difference in volume is "<<fabs(volumeUV - volumeZE)/volumeZE<<std::endl;
     err = nc_close( ncid);
+    delete hector;
     return 0;
 }
diff --git a/inc/geometries/curvilinear.h b/inc/geometries/curvilinear.h
index 32e392847..6e8f735aa 100644
--- a/inc/geometries/curvilinear.h
+++ b/inc/geometries/curvilinear.h
@@ -34,7 +34,7 @@ struct CurvilinearGrid3d : public dg::Grid3d
      @param Nz 
      @param bcx
      */
-    CurvilinearGrid3d( aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx=dg::DIR):
+    CurvilinearGrid3d( geo::aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx=dg::DIR):
         dg::Grid3d( 0, generator->width(), 0., generator->height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, dg::PER, dg::PER)
     { 
         generator_ = generator;
@@ -64,7 +64,7 @@ struct CurvilinearGrid3d : public dg::Grid3d
     const container& g_pp()const{return g_pp_;}
     const container& vol()const{return vol_;}
     const container& perpVol()const{return vol2d_;}
-    aGenerator* const generator() const{return generator_;}
+    geo::aGenerator* const generator() const{return generator_;}
     private:
     void construct( unsigned n, unsigned Nx, unsigned Ny)
     {
@@ -72,8 +72,8 @@ struct CurvilinearGrid3d : public dg::Grid3d
         dg::Grid1d gX1d( 0., generator_->width(), n, Nx);
         thrust::host_vector<double> x_vec = dg::evaluate( dg::cooX1d, gX1d);
         thrust::host_vector<double> y_vec = dg::evaluate( dg::cooX1d, gY1d);
-        *generator_( x_vec, y_vec, r_, z_, xr_, xz_, yr_, yz_);
-        init_X_boundaries( 0., generator->width());
+        (*generator_)( x_vec, y_vec, r_, z_, xr_, xz_, yr_, yz_);
+        init_X_boundaries( 0., generator_->width());
         lift3d( ); //lift to 3D grid
         construct_metric();
     }
@@ -115,7 +115,7 @@ struct CurvilinearGrid3d : public dg::Grid3d
     }
     thrust::host_vector<double> r_, z_, xr_, xz_, yr_, yz_;
     container g_xx_, g_xy_, g_yy_, g_pp_, vol_, vol2d_;
-    aGenerator* generator_;
+    geo::aGenerator* generator_;
 };
 
 /**
@@ -127,14 +127,13 @@ struct CurvilinearGrid2d : public dg::Grid2d
     typedef dg::CurvilinearCylindricalTag metric_category;
     /*!@brief Constructor
     
-     * @tparam Generator models aGenerator
      * @param generator must generate an orthogonal grid
      * @param n number of polynomial coefficients
      * @param Nx number of cells in first coordinate
      @param Ny number of cells in second coordinate
      @param bcx boundary condition in first coordinate
      */
-    CurvilinearGrid2d( aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx=dg::DIR):
+    CurvilinearGrid2d( geo::aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx=dg::DIR):
         dg::Grid2d( 0, generator->width(), 0., generator->height(), n, Nx, Ny, bcx, dg::PER)
     {
         CurvilinearGrid3d<container> g( generator, n,Nx,Ny,1,bcx);
@@ -161,8 +160,8 @@ struct CurvilinearGrid2d : public dg::Grid2d
 
     void set(unsigned new_n, unsigned new_Nx, unsigned new_Ny)
     {
-        dg::Grid2d::set( new_n, new_Nx, new_Ny, new_Nz);
-        CurvilinearGrid3d<container> g( generator_, n,Nx,Ny,1,bcx);
+        dg::Grid2d::set( new_n, new_Nx, new_Ny);
+        CurvilinearGrid3d<container> g( generator_, new_n,new_Nx,new_Ny,1,bcx());
         r_=g.r(), z_=g.z(), xr_=g.xr(), xz_=g.xz(), yr_=g.yr(), yz_=g.yz();
         g_xx_=g.g_xx(), g_xy_=g.g_xy(), g_yy_=g.g_yy();
         vol2d_=g.perpVol();
@@ -178,11 +177,11 @@ struct CurvilinearGrid2d : public dg::Grid2d
     const container& g_xy()const{return g_xy_;}
     const container& vol()const{return vol2d_;}
     const container& perpVol()const{return vol2d_;}
-    aGenerator* const generator() const{return generator_;}
+    geo::aGenerator* const generator() const{return generator_;}
     private:
     thrust::host_vector<double> r_, z_, xr_, xz_, yr_, yz_;
     container g_xx_, g_xy_, g_yy_, vol2d_;
-    aGenerator* generator_;
+    geo::aGenerator* generator_;
 };
 
 ///@}
diff --git a/inc/geometries/flux.h b/inc/geometries/flux.h
index e604c0f50..4e5649da5 100644
--- a/inc/geometries/flux.h
+++ b/inc/geometries/flux.h
@@ -204,7 +204,7 @@ struct FluxGenerator : public aGenerator
          thrust::host_vector<double>& zetaX, 
          thrust::host_vector<double>& zetaY, 
          thrust::host_vector<double>& etaX, 
-         thrust::host_vector<double>& etaY) 
+         thrust::host_vector<double>& etaY) const 
     {
         //compute psi(x) for a grid on x and call construct_rzy for all psi
         thrust::host_vector<double> psi_x(zeta1d);
@@ -219,6 +219,7 @@ struct FluxGenerator : public aGenerator
         unsigned size = zeta1d.size()*eta1d.size();
         x.resize(size), y.resize(size);
         zetaX = zetaY = etaX = etaY =x ;
+        thrust::host_vector<double> fx_;
         fx_.resize( zeta1d.size());
         thrust::host_vector<double> f_p(fx_);
         unsigned Nx = zeta1d.size(), Ny = eta1d.size();
@@ -247,7 +248,6 @@ struct FluxGenerator : public aGenerator
     Ipol ipol_;
     IpolX ipolR_;
     IpolY ipolZ_;
-    thrust::host_vector<double> fx_;
     double f0_, lx_, x0_, y0_, psi0_, psi1_;
     int mode_;
 };
@@ -318,7 +318,7 @@ struct RibeiroFluxGenerator : public aGenerator
          thrust::host_vector<double>& zetaX, 
          thrust::host_vector<double>& zetaY, 
          thrust::host_vector<double>& etaX, 
-         thrust::host_vector<double>& etaY) 
+         thrust::host_vector<double>& etaY) const
     {
         //compute psi(x) for a grid on x and call construct_rzy for all psi
         thrust::host_vector<double> psi_x(zeta1d);
@@ -331,6 +331,7 @@ struct RibeiroFluxGenerator : public aGenerator
         unsigned size = zeta1d.size()*eta1d.size();
         x.resize(size), y.resize(size);
         zetaX = zetaY = etaX = etaY =x ;
+        thrust::host_vector<double> fx_;
         fx_.resize( zeta1d.size());
         thrust::host_vector<double> f_p(fx_);
         unsigned Nx = zeta1d.size(), Ny = eta1d.size();
@@ -356,7 +357,6 @@ struct RibeiroFluxGenerator : public aGenerator
     PsiXX psiXX_;
     PsiXY psiXY_;
     PsiYY psiYY_;
-    thrust::host_vector<double> fx_;
     double f0_, lx_, x0_, y0_, psi0_, psi1_;
     int mode_;
 };
diff --git a/inc/geometries/generator.h b/inc/geometries/generator.h
index bd0abda53..5a0e603d1 100644
--- a/inc/geometries/generator.h
+++ b/inc/geometries/generator.h
@@ -14,10 +14,10 @@ is a product space.
 */
 struct aGenerator
 {
-    virtual double width() =0 const; //!<length in \f$ \zeta\f$ 
-    virtual double height()=0 const; //!<length in \f$ \eta\f$
-    virtual bool isOrthogonal()=0 const; //!< true if coordinate system is orthogonal
-    virtual bool isConformal()=0 const; //!< true if coordinate system is conformal
+    virtual double width()  const=0; //!<length in \f$ \zeta\f$ 
+    virtual double height() const=0; //!<length in \f$ \eta\f$
+    virtual bool isOrthogonal() const=0; //!< true if coordinate system is orthogonal
+    virtual bool isConformal()const=0; //!< true if coordinate system is conformal
     /**
     * @brief Generate grid points and elements of the Jacobian 
     *
@@ -40,7 +40,9 @@ struct aGenerator
          thrust::host_vector<double>& zetaX, 
          thrust::host_vector<double>& zetaY, 
          thrust::host_vector<double>& etaX, 
-         thrust::host_vector<double>& etaY)=0;
+         thrust::host_vector<double>& etaY) const =0;
+
+   virtual ~aGenerator(){}
 };
 
 }//namespace geo
diff --git a/inc/geometries/magnetic_field.h b/inc/geometries/magnetic_field.h
index 4437a1c20..62257e204 100644
--- a/inc/geometries/magnetic_field.h
+++ b/inc/geometries/magnetic_field.h
@@ -107,7 +107,7 @@ struct LnB
 template<class MagneticField>
 struct BR
 {
-    BR(const MagneticField& c):  R_0_(c.R_0), invB_(c, R0), c_(c) { }
+    BR(const MagneticField& c):  R_0_(c.R_0), invB_(c), c_(c) { }
 /**
  * @brief \f[  \frac{\partial \hat{B} }{ \partial \hat{R}} = 
       -\frac{1}{\hat B \hat R}   
@@ -142,7 +142,7 @@ template<class MagneticField>
 struct BZ
 {
 
-    BZ(const MagneticField& c ):  R_0_(c.R_0), c_(c), invB_(c, R0) { }
+    BZ(const MagneticField& c ):  R_0_(c.R_0), c_(c), invB_(c) { }
     /**
      * @brief \f[  \frac{\partial \hat{B} }{ \partial \hat{Z}} = 
      \frac{ \hat I \left(\frac{\partial \hat I}{\partial\hat Z}    \right)+
diff --git a/inc/geometries/mpi_conformal.h b/inc/geometries/mpi_conformal.h
index aafc5e1b6..043cca4a3 100644
--- a/inc/geometries/mpi_conformal.h
+++ b/inc/geometries/mpi_conformal.h
@@ -29,8 +29,7 @@ struct ConformalMPIGrid3d : public dg::MPIGrid3d
     typedef dg::ConformalMPIGrid2d<LocalContainer> perpendicular_grid; //!< the two-dimensional grid
     typedef typename MPIContainer::container_type LocalContainer; //!< the local container type
 
-    template< class Generator>
-    ConformalMPIGrid3d( const Generator& generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx, MPI_Comm comm): 
+    ConformalMPIGrid3d( geo::aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx, MPI_Comm comm): 
         dg::MPIGrid3d( 0, generator->width(), 0., generator->height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, dg::PER, dg::PER, comm),
         g( generator, n,Nx, Ny, local().Nz(), bcx)
     {
@@ -58,7 +57,7 @@ struct ConformalMPIGrid3d : public dg::MPIGrid3d
     const MPIContainer& vol()const{return vol_;}
     const MPIContainer& perpVol()const{return vol2d_;}
     const dg::ConformalGrid3d<LocalContainer>& global() const {return g;}
-    aGenerator* const generator() const{return g.generator();}
+    geo::aGenerator* const generator() const{return g.generator();}
     private:
     void divide_and_conquer(){
         r_=dg::evaluate( dg::one, *this), z_=r_, xr_=r_, xz_=r_, yr_=r_, yz_=r_;
@@ -104,7 +103,7 @@ struct ConformalMPIGrid2d : public dg::MPIGrid2d
     typedef dg::ConformalCylindricalTag metric_category; 
     typedef typename MPIContainer::container_type LocalContainer; //!< the local container type
 
-    ConformalMPIGrid2d( const Generator& generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx, MPI_Comm comm2d): 
+    ConformalMPIGrid2d( geo::aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx, MPI_Comm comm2d): 
         dg::MPIGrid2d( 0, 1, 0., 2*M_PI, n, Nx, Ny, bcx, dg::PER, comm2d),
         r_(dg::evaluate( dg::one, *this)), z_(r_), xr_(r_), xz_(r_), yr_(r_), yz_(r_), 
         g_xx_(r_), g_xy_(g_xx_), g_yy_(g_xx_), vol2d_(g_xx_),
@@ -153,7 +152,7 @@ struct ConformalMPIGrid2d : public dg::MPIGrid2d
     const MPIContainer& vol()const{return vol2d_;}
     const MPIContainer& perpVol()const{return vol2d_;}
     const dg::ConformalGrid2d<LocalContainer>& global()const{return g_;}
-    aGenerator* const generator() const{return g.generator();}
+    geo::aGenerator* const generator() const{return g.generator();}
     private:
     MPI_Comm get_reduced_comm( MPI_Comm src)
     {
diff --git a/inc/geometries/mpi_curvilinear.h b/inc/geometries/mpi_curvilinear.h
index 69dd9b039..a749c700a 100644
--- a/inc/geometries/mpi_curvilinear.h
+++ b/inc/geometries/mpi_curvilinear.h
@@ -30,7 +30,7 @@ struct CurvilinearMPIGrid3d : public dg::MPIGrid3d
     typedef dg::CurvilinearMPIGrid2d<LocalContainer> perpendicular_grid; //!< the two-dimensional grid
     typedef typename MPIContainer::container_type LocalContainer; //!< the local container type
 
-    CurvilinearMPIGrid3d( aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx, MPI_Comm comm): 
+    CurvilinearMPIGrid3d( geo::aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx, MPI_Comm comm): 
         dg::MPIGrid3d( 0, generator->width(), 0., generator->height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, dg::PER, dg::PER, comm),
         g( generator, n,Nx, Ny, local().Nz(), bcx)
     {
@@ -58,7 +58,7 @@ struct CurvilinearMPIGrid3d : public dg::MPIGrid3d
     const MPIContainer& vol()const{return vol_;}
     const MPIContainer& perpVol()const{return vol2d_;}
     const dg::CurvilinearGrid3d<LocalContainer>& global() const {return g;}
-    aGenerator* const generator() const{return g.generator();}
+    geo::aGenerator* const generator() const{return g.generator();}
     private:
     void divide_and_conquer( )
     {
@@ -105,7 +105,7 @@ struct CurvilinearMPIGrid2d : public dg::MPIGrid2d
     typedef dg::CurvilinearCylindricalTag metric_category; 
     typedef typename MPIContainer::container_type LocalContainer; //!< the local container type
 
-    CurvilinearMPIGrid2d( aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx, MPI_Comm comm2d): 
+    CurvilinearMPIGrid2d( geo::aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx, MPI_Comm comm2d): 
         dg::MPIGrid2d( 0, generator->width(), 0., generator->height(), n, Nx, Ny, bcx, dg::PER, comm2d),
         g_( generator, n,Nx, Ny, bcx)
     {
@@ -152,7 +152,7 @@ struct CurvilinearMPIGrid2d : public dg::MPIGrid2d
     const MPIContainer& vol()const{return vol2d_;}
     const MPIContainer& perpVol()const{return vol2d_;}
     const dg::CurvilinearGrid2d<LocalContainer>& global() const {return g_;}
-    aGenerator* const generator() const{return g.generator();}
+    geo::aGenerator* const generator() const{return g.generator();}
     private:
     MPI_Comm get_reduced_comm( MPI_Comm src)
     {
diff --git a/inc/geometries/mpi_orthogonal.h b/inc/geometries/mpi_orthogonal.h
index 880a3ae2d..d3d55f864 100644
--- a/inc/geometries/mpi_orthogonal.h
+++ b/inc/geometries/mpi_orthogonal.h
@@ -29,7 +29,7 @@ struct OrthogonalMPIGrid3d : public dg::MPIGrid3d
     typedef dg::OrthogonalMPIGrid2d<LocalContainer> perpendicular_grid; //!< the two-dimensional grid
     typedef typename MPIContainer::container_type LocalContainer; //!< the local container type
 
-    OrthogonalMPIGrid3d( aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx, MPI_Comm comm): 
+    OrthogonalMPIGrid3d( geo::aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx, MPI_Comm comm): 
         dg::MPIGrid3d( 0, generator->width(), 0., generator->height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, dg::PER, dg::PER, comm),
         g( generator, n, Nx, Ny, local().Nz(), bcx)
     {
@@ -59,7 +59,7 @@ struct OrthogonalMPIGrid3d : public dg::MPIGrid3d
     const MPIContainer& vol()const{return vol_;}
     const MPIContainer& perpVol()const{return vol2d_;}
     const dg::OrthogonalGrid3d<LocalContainer>& global() const{return g;}
-    aGenerator* const generator() const{return g.generator();}
+    geo::aGenerator* const generator() const{return g.generator();}
     private:
     void divide_and_conquer( )
     {
@@ -106,7 +106,7 @@ struct OrthogonalMPIGrid2d : public dg::MPIGrid2d
     typedef dg::OrthogonalTag metric_category; 
     typedef typename MPIContainer::container_type LocalContainer; //!< the local container type
 
-    OrthogonalMPIGrid2d( aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx, MPI_Comm comm2d): 
+    OrthogonalMPIGrid2d( geo::aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx, MPI_Comm comm2d): 
         dg::MPIGrid2d( 0, generator->width(), 0., generator->height(), n, Nx, Ny, bcx, dg::PER, comm2d),
         g_( generator, n, Nx, Ny, bcx)
     {
@@ -153,7 +153,7 @@ struct OrthogonalMPIGrid2d : public dg::MPIGrid2d
     const MPIContainer& vol()const{return vol2d_;}
     const MPIContainer& perpVol()const{return vol2d_;}
     const dg::OrthogonalGrid2d<LocalContainer>& global() const {return g;}
-    aGenerator* const generator() const{return g.generator();}
+    geo::aGenerator* const generator() const{return g.generator();}
     private:
     MPI_Comm get_reduced_comm( MPI_Comm src)
     {
diff --git a/inc/geometries/orthogonal.h b/inc/geometries/orthogonal.h
index 6a9638703..cf6fb4688 100644
--- a/inc/geometries/orthogonal.h
+++ b/inc/geometries/orthogonal.h
@@ -3,7 +3,7 @@
 #include "dg/backend/grid.h"
 #include "dg/blas1.h"
 #include "dg/geometry/geometry_traits.h"
-#include "dg/generator.h"
+#include "generator.h"
 
 namespace dg
 {
@@ -34,7 +34,7 @@ struct OrthogonalGrid3d : public dg::Grid3d
      @param Nz 
      @param bcx
      */
-    OrthogonalGrid3d( aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx=dg::DIR):
+    OrthogonalGrid3d( geo::aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx=dg::DIR):
         dg::Grid3d( 0, generator->width(), 0., generator->height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, dg::PER, dg::PER)
     { 
         assert( generator->isOrthogonal());
@@ -66,7 +66,7 @@ struct OrthogonalGrid3d : public dg::Grid3d
     const container& g_pp()const{return g_pp_;}
     const container& vol()const{return vol_;}
     const container& perpVol()const{return vol2d_;}
-    aGenerator* const generator() const{return generator_;}
+    geo::aGenerator* const generator() const{return generator_;}
     private:
     void construct( unsigned n, unsigned Nx, unsigned Ny)
     {
@@ -74,7 +74,7 @@ struct OrthogonalGrid3d : public dg::Grid3d
         dg::Grid1d gX1d( 0, generator_->width(), n, Nx);
         thrust::host_vector<double> x_vec = dg::evaluate( dg::cooX1d, gX1d);
         thrust::host_vector<double> y_vec = dg::evaluate( dg::cooX1d, gY1d);
-        *generator_( x_vec, y_vec, r_, z_, xr_, xz_, yr_, yz_);
+        (*generator_)( x_vec, y_vec, r_, z_, xr_, xz_, yr_, yz_);
         init_X_boundaries( 0., generator->width());
         lift3d( ); //lift to 3D grid
         construct_metric();
@@ -117,7 +117,7 @@ struct OrthogonalGrid3d : public dg::Grid3d
     }
     thrust::host_vector<double> r_, z_, xr_, xz_, yr_, yz_;
     container g_xx_, g_xy_, g_yy_, g_pp_, vol_, vol2d_;
-    aGenerator* generator_;
+    geo::aGenerator* generator_;
 };
 
 /**
@@ -136,7 +136,7 @@ struct OrthogonalGrid2d : public dg::Grid2d
      @param Ny number of cells in second coordinate
      @param bcx boundary condition in first coordinate
      */
-    OrthogonalGrid2d( aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx=dg::DIR):
+    OrthogonalGrid2d( geo::aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx=dg::DIR):
         dg::Grid2d( 0, generator->width(), 0., generator->height(), n, Nx, Ny, bcx, dg::PER)
     {
         generator_=generator;
@@ -163,8 +163,8 @@ struct OrthogonalGrid2d : public dg::Grid2d
     }
     void set(unsigned new_n, unsigned new_Nx, unsigned new_Ny)
     {
-        dg::Grid2d::set( new_n, new_Nx, new_Ny, new_Nz);
-        OrthogonalGrid3d<container> g( generator_, n,Nx,Ny,1,bcx);
+        dg::Grid2d::set( new_n, new_Nx, new_Ny);
+        OrthogonalGrid3d<container> g( generator_, new_n,new_Nx,new_Ny,1,bcx());
         r_=g.r(), z_=g.z(), xr_=g.xr(), xz_=g.xz(), yr_=g.yr(), yz_=g.yz();
         g_xx_=g.g_xx(), g_xy_=g.g_xy(), g_yy_=g.g_yy();
         vol2d_=g.perpVol();
@@ -181,11 +181,11 @@ struct OrthogonalGrid2d : public dg::Grid2d
     const container& g_xy()const{return g_xy_;}
     const container& vol()const{return vol2d_;}
     const container& perpVol()const{return vol2d_;}
-    aGenerator* const generator() const{return generator_;}
+    geo::aGenerator* const generator() const{return generator_;}
     private:
     thrust::host_vector<double> r_, z_, xr_, xz_, yr_, yz_;
     container g_xx_, g_xy_, g_yy_, vol2d_;
-    aGenerator* generator_;
+    geo::aGenerator* generator_;
 };
 
 ///@}
diff --git a/inc/geometries/ribeiro.h b/inc/geometries/ribeiro.h
index 69e05d81e..cb37fb542 100644
--- a/inc/geometries/ribeiro.h
+++ b/inc/geometries/ribeiro.h
@@ -253,7 +253,7 @@ struct Ribeiro : public aGenerator
          thrust::host_vector<double>& zetaX, 
          thrust::host_vector<double>& zetaY, 
          thrust::host_vector<double>& etaX, 
-         thrust::host_vector<double>& etaY) 
+         thrust::host_vector<double>& etaY) const
     {
         //compute psi(x) for a grid on x and call construct_rzy for all psi
         ribeiro::detail::FieldFinv<Psi, PsiX, PsiY> fpsiMinv_(psi_, psiX_, psiY_, x0_,y0_, 500, mode_);
diff --git a/inc/geometries/simple_orthogonal.h b/inc/geometries/simple_orthogonal.h
index 47b772b68..aacb33609 100644
--- a/inc/geometries/simple_orthogonal.h
+++ b/inc/geometries/simple_orthogonal.h
@@ -353,7 +353,7 @@ struct SimpleOrthogonal : public aGenerator
          thrust::host_vector<double>& zetaX, 
          thrust::host_vector<double>& zetaY, 
          thrust::host_vector<double>& etaX, 
-         thrust::host_vector<double>& etaY) 
+         thrust::host_vector<double>& etaY) const
     {
         thrust::host_vector<double> r_init, z_init;
         orthogonal::detail::compute_rzy( psiX_, psiY_, eta1d, r_init, z_init, R0_, Z0_, f0_, firstline_);
-- 
GitLab


From c5a019a7596691c1ce8141d74acc8bf9886c1efe Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Fri, 21 Jul 2017 11:42:34 +0200
Subject: [PATCH 061/453] removed orthogonal and conformal grids: performance
 gain too low to justiy so many grids

---
 inc/dg/geometry/geometry_traits.h             |  45 ++--
 inc/geometries/README                         |   9 +-
 inc/geometries/conformal.h                    | 190 -----------------
 inc/geometries/curvilinear.h                  |  12 +-
 inc/geometries/mpi_conformal.h                | 199 -----------------
 inc/geometries/mpi_orthogonal.h               | 200 ------------------
 inc/geometries/orthogonal.h                   | 192 -----------------
 inc/geometries/orthogonalX.h                  | 176 ---------------
 inc/geometries/refined_conformal.h            | 157 --------------
 ...d_orthogonalX.h => refined_curvilinearX.h} |  30 +--
 inc/geometries/refined_orthogonal.h           | 160 --------------
 11 files changed, 45 insertions(+), 1325 deletions(-)
 delete mode 100644 inc/geometries/conformal.h
 delete mode 100644 inc/geometries/mpi_conformal.h
 delete mode 100644 inc/geometries/mpi_orthogonal.h
 delete mode 100644 inc/geometries/orthogonal.h
 delete mode 100644 inc/geometries/orthogonalX.h
 delete mode 100644 inc/geometries/refined_conformal.h
 rename inc/geometries/{refined_orthogonalX.h => refined_curvilinearX.h} (83%)
 delete mode 100644 inc/geometries/refined_orthogonal.h

diff --git a/inc/dg/geometry/geometry_traits.h b/inc/dg/geometry/geometry_traits.h
index 8f6cb9127..3fb227216 100644
--- a/inc/dg/geometry/geometry_traits.h
+++ b/inc/dg/geometry/geometry_traits.h
@@ -8,10 +8,7 @@ namespace dg
 //
 struct CurvilinearTag{};  //! 3d curvilinear
 struct CurvilinearCylindricalTag: public CurvilinearTag{}; //! perpVol, vol(), g_xx, g_xy, g_yy
-struct OrthogonalTag:public CurvilinearCylindricalTag{}; //! perpVol, vol(), g_xx, g_yy
-struct ConformalCylindricalTag:public OrthogonalTag{}; //! perpVol, vol(), g_xx, g_yy
-struct ConformalTag:public ConformalCylindricalTag{}; //! A 2d conformal 
-struct OrthonormalCylindricalTag:public ConformalCylindricalTag{}; //! vol(), cylindrical grid
+struct OrthonormalCylindricalTag:public CurvilinearCylindricalTag{}; //! vol(), cylindrical grid
 struct OrthonormalTag: public OrthonormalCylindricalTag{}; //! Cartesian grids
 
 //memory_category and dimensionality Tags are already defined in grid.h
@@ -94,37 +91,37 @@ void doRaisePerpIndex( container& in1, container& in2, container& out1, containe
     in2.swap( out2);
 };
 template <class container, class Geometry>
-void doRaisePerpIndex( container& in1, container& in2, container& out1, container& out2, const Geometry& g, OrthogonalTag)
-{
-    dg::blas1::pointwiseDot( g.g_xx(), in1, out1); //gxx*v_x
-    dg::blas1::pointwiseDot( g.g_yy(), in2, out2); //gyy*v_y
-};
-template <class container, class Geometry>
 void doRaisePerpIndex( container& in1, container& in2, container& out1, container& out2, const Geometry& g, CurvilinearCylindricalTag)
 {
+    if( g.isOrthogonal())
+    {
+        dg::blas1::pointwiseDot( g.g_xx(), in1, out1); //gxx*v_x
+        dg::blas1::pointwiseDot( g.g_yy(), in2, out2); //gyy*v_y
+        return;
+    }
     dg::blas1::pointwiseDot( g.g_xx(), in1, out1); //gxx*v_x
     dg::blas1::pointwiseDot( g.g_xy(), in1, out2); //gyx*v_x
     dg::blas1::pointwiseDot( 1., g.g_xy(), in2, 1., out1);//gxy*v_y
     dg::blas1::pointwiseDot( 1., g.g_yy(), in2, 1., out2); //gyy*v_y
 };
 
-template <class container, class Geometry>
-void doVolRaisePerpIndex( container& in1, container& in2, container& out1, container& out2, const Geometry& g, ConformalCylindricalTag)
-{
-    in1.swap( out1);
-    in2.swap( out2);
-};
-template <class container, class Geometry>
-void doVolRaisePerpIndex( container& in1, container& in2, container& out1, container& out2, const Geometry& g, OrthogonalTag)
-{
-    dg::blas1::pointwiseDot( g.g_xx(), in1, out1); //gxx*v_x
-    dg::blas1::pointwiseDot( g.g_yy(), in2, out2); //gyy*v_y
-    dg::blas1::pointwiseDot( out1, g.perpVol(), out1);
-    dg::blas1::pointwiseDot( out2, g.perpVol(), out2);
-};
 template <class container, class Geometry>
 void doVolRaisePerpIndex( container& in1, container& in2, container& out1, container& out2, const Geometry& g, CurvilinearCylindricalTag)
 {
+    if( g.isConformal())
+    {
+        in1.swap( out1);
+        in2.swap( out2);
+        return;
+    }
+    if( g.isOrthogonal())
+    {
+        dg::blas1::pointwiseDot( g.g_xx(), in1, out1); //gxx*v_x
+        dg::blas1::pointwiseDot( g.g_yy(), in2, out2); //gyy*v_y
+        dg::blas1::pointwiseDot( out1, g.perpVol(), out1);
+        dg::blas1::pointwiseDot( out2, g.perpVol(), out2);
+        return;
+    }
     dg::blas1::pointwiseDot( g.g_xx(), in1, out1); //gxx*v_x
     dg::blas1::pointwiseDot( g.g_xy(), in1, out2); //gyx*v_x
     dg::blas1::pointwiseDot( 1., g.g_xy(), in2, 1., out1);//gxy*v_y
diff --git a/inc/geometries/README b/inc/geometries/README
index 218294d5f..10720c2df 100644
--- a/inc/geometries/README
+++ b/inc/geometries/README
@@ -1,18 +1,11 @@
 
 #grids
-    conformal.h
-    orthogonal.h
     curvilinear.h
-    mpi_conformal.h
-    mpi_orthogonal.h
     mpi_curvilinear.h
-    refined_conformal.h
-    refined_orthogonal.h
     refined_curvilinear.h
 
-    orthogonalX.h
     curvilinearX.h
-    refined_orthogonalX.h
+    refined_curvilinearX.h
 
 #generators
     generator.h
diff --git a/inc/geometries/conformal.h b/inc/geometries/conformal.h
deleted file mode 100644
index 013eab3ea..000000000
--- a/inc/geometries/conformal.h
+++ /dev/null
@@ -1,190 +0,0 @@
-#pragma once
-
-#include "dg/backend/grid.h"
-#include "dg/blas1.h"
-#include "dg/geometry/geometry_traits.h"
-#include "generator.h"
-
-namespace dg
-{
-///@addtogroup grids
-///@{
-
-///@cond
-template< class container>
-struct ConformalGrid2d; 
-///@endcond
-
-/**
- * @brief A three-dimensional grid based on conformal coordintes 
- *
- * @tparam container Vector class that holds metric coefficients (models aContainer)
- */
-template< class container>
-struct ConformalGrid3d : public dg::Grid3d
-{
-    typedef dg::ConformalCylindricalTag metric_category; //!< metric tag
-    typedef ConformalGrid2d<container> perpendicular_grid; //!< the two-dimensional grid type
-
-    /**
-     * @brief 
-     *
-     * @param generator must generate a conformal grid
-     * @param n
-     * @param Nx
-     * @param Ny
-     * @param Nz
-     * @param bcx
-     */
-    ConformalGrid3d( geo::aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx=dg::DIR) :
-        dg::Grid3d( 0, generator->width(), 0., generator->height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, dg::PER, dg::PER)
-    {
-        assert( generator->isConformal());
-        generator_=generator;
-        construct(n,Nx, Ny);
-    }
-    /**
-    * @brief Reconstruct the grid coordinates
-    *
-    * @copydetails dg::Grid3d::set()
-    * @attention the generator must still live when this function is called
-    */
-    void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz){
-        dg::Grid3d::set( new_n, new_Nx, new_Ny, new_Nz);
-        construct( new_n, new_Nx, new_Ny);
-    }
-
-    perpendicular_grid perp_grid() const { return ConformalGrid2d<container>(*this);}
-    const thrust::host_vector<double>& r()const{return r_;}
-    const thrust::host_vector<double>& z()const{return z_;}
-    const thrust::host_vector<double>& xr()const{return xr_;}
-    const thrust::host_vector<double>& yr()const{return yr_;}
-    const thrust::host_vector<double>& xz()const{return xz_;}
-    const thrust::host_vector<double>& yz()const{return yz_;}
-    const container& g_xx()const{return gradU2_;}
-    const container& g_yy()const{return gradU2_;}
-    const container& g_pp()const{return g_pp_;}
-    const container& vol()const{return vol_;}
-    const container& perpVol()const{return vol2d_;}
-    geo::aGenerator* const generator() const{return generator_;}
-    private:
-    void construct( unsigned n, unsigned Nx, unsigned Ny) 
-    {
-        dg::Grid1d gu( 0., generator_->width(), n, Nx);
-        dg::Grid1d gv( 0., generator_->height(), n, Ny);
-        const thrust::host_vector<double> u1d = dg::evaluate( dg::cooX1d, gu);
-        const thrust::host_vector<double> v1d = dg::evaluate( dg::cooX1d, gv);
-        (*generator_)( u1d, v1d, r_, z_, xr_, xz_, yr_, yz_);
-        init_X_boundaries( 0., generator_->width());
-        lift3d( ); //lift to 3D grid
-        construct_metric();
-    }
-    void lift3d( )
-    {
-        //lift to 3D grid
-        unsigned size = this->size();
-        r_.resize( size), z_.resize(size), xr_.resize(size), yr_.resize( size), xz_.resize( size), yz_.resize(size);
-        unsigned Nx = this->n()*this->Nx(), Ny = this->n()*this->Ny();
-        for( unsigned k=1; k<this->Nz(); k++)
-            for( unsigned i=0; i<Nx*Ny; i++)
-            {
-                r_[k*Nx*Ny+i] = r_[(k-1)*Nx*Ny+i];
-                z_[k*Nx*Ny+i] = z_[(k-1)*Nx*Ny+i];
-                xr_[k*Nx*Ny+i] = xr_[(k-1)*Nx*Ny+i];
-                xz_[k*Nx*Ny+i] = xz_[(k-1)*Nx*Ny+i];
-                yr_[k*Nx*Ny+i] = yr_[(k-1)*Nx*Ny+i];
-                yz_[k*Nx*Ny+i] = yz_[(k-1)*Nx*Ny+i];
-            }
-    }
-    //compute metric elements from xr, xz, yr, yz, r and z
-    void construct_metric( )
-    {
-        thrust::host_vector<double> tempxx( r_), tempvol(r_);
-        for( unsigned i = 0; i<this->size(); i++)
-        {
-            tempxx[i] = (xr_[i]*xr_[i]+xz_[i]*xz_[i]);
-            tempvol[i] = r_[i]/ tempxx[i];
-        }
-        dg::blas1::transfer( tempxx, gradU2_);
-        dg::blas1::transfer( tempvol, vol_);
-        dg::blas1::pointwiseDivide( tempvol, r_, tempvol);
-        dg::blas1::transfer( tempvol, vol2d_);
-        thrust::host_vector<double> ones = dg::evaluate( dg::one, *this);
-        dg::blas1::pointwiseDivide( ones, r_, tempxx);
-        dg::blas1::pointwiseDivide( tempxx, r_, tempxx); //1/R^2
-        g_pp_=tempxx;
-    }
-    
-    thrust::host_vector<double> r_, z_, xr_, xz_, yr_, yz_; //3d vector
-    container gradU2_, g_pp_, vol_, vol2d_;
-    geo::aGenerator* generator_;
-
-};
-
-/**
- * @brief A two-dimensional grid based on the conformal coordinates  (models aContainer)
- */
-template< class container>
-struct ConformalGrid2d : public dg::Grid2d
-{
-    typedef dg::ConformalCylindricalTag metric_category;
-    /**
-     * @brief 
-     *
-     * @param generator must generate a conformal grid
-     * @param n
-     * @param Nx
-     * @param Ny
-     * @param bcx
-     */
-    ConformalGrid2d( geo::aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx=dg::DIR):
-        dg::Grid2d( 0, generator->width(), 0., generator->height(), n,Nx,Ny, bcx, dg::PER)
-    {
-        generator_=generator;
-        ConformalGrid3d<container> g( generator, n,Nx,Ny,1,bcx);
-        init_X_boundaries( g.x0(), g.x1());
-        r_=g.r(), z_=g.z(), xr_=g.xr(), xz_=g.xz(), yr_=g.yr(), yz_=g.yz();
-        gradU2_=g.g_xx();
-        vol2d_=g.perpVol();
-    }
-    ConformalGrid2d( const ConformalGrid3d<container>& g):
-        dg::Grid2d( g.x0(), g.x1(), g.y0(), g.y1(), g.n(), g.Nx(), g.Ny(), g.bcx(), g.bcy())
-    {
-        generator_ = g.generator();
-        unsigned s = this->size();
-        r_.resize( s), z_.resize(s), xr_.resize(s), xz_.resize(s), yr_.resize(s), yz_.resize(s);
-        gradU2_.resize( s), vol2d_.resize(s);
-        for( unsigned i=0; i<s; i++)
-        { r_[i]=g.r()[i], z_[i]=g.z()[i], xr_[i]=g.xr()[i], xz_[i]=g.xz()[i], yr_[i]=g.yr()[i], yz_[i]=g.yz()[i];}
-        thrust::copy( g.g_xx().begin(), g.g_xx().begin()+s, gradU2_.begin());
-        thrust::copy( g.perpVol().begin(), g.perpVol().begin()+s, vol2d_.begin());
-    }
-
-    void set(unsigned new_n, unsigned new_Nx, unsigned new_Ny)
-    {
-        dg::Grid2d::set( new_n, new_Nx, new_Ny);
-        ConformalGrid3d<container> g( generator_, new_n,new_Nx,new_Ny,1,bcx());
-        r_=g.r(), z_=g.z(), xr_=g.xr(), xz_=g.xz(), yr_=g.yr(), yz_=g.yz();
-        gradU2_=g.g_xx(), vol2d_=g.perpVol();
-    }
-
-
-    const thrust::host_vector<double>& r()const{return r_;}
-    const thrust::host_vector<double>& z()const{return z_;}
-    const thrust::host_vector<double>& xr()const{return xr_;}
-    const thrust::host_vector<double>& yr()const{return yr_;}
-    const thrust::host_vector<double>& xz()const{return xz_;}
-    const thrust::host_vector<double>& yz()const{return yz_;}
-    const container& g_xx()const{return gradU2_;}
-    const container& g_yy()const{return gradU2_;}
-    const container& vol()const{return vol2d_;}
-    const container& perpVol()const{return vol2d_;}
-    geo::aGenerator* const generator() const{return generator_;}
-    private:
-    thrust::host_vector<double> r_, z_, xr_, xz_, yr_, yz_;
-    container gradU2_, vol2d_;
-    geo::aGenerator* generator_;
-};
-
-///@}
-}//namespace
diff --git a/inc/geometries/curvilinear.h b/inc/geometries/curvilinear.h
index 6e8f735aa..0e1fcc15f 100644
--- a/inc/geometries/curvilinear.h
+++ b/inc/geometries/curvilinear.h
@@ -34,7 +34,7 @@ struct CurvilinearGrid3d : public dg::Grid3d
      @param Nz 
      @param bcx
      */
-    CurvilinearGrid3d( geo::aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx=dg::DIR):
+    CurvilinearGrid3d( const geo::aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx=dg::DIR):
         dg::Grid3d( 0, generator->width(), 0., generator->height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, dg::PER, dg::PER)
     { 
         generator_ = generator;
@@ -64,7 +64,9 @@ struct CurvilinearGrid3d : public dg::Grid3d
     const container& g_pp()const{return g_pp_;}
     const container& vol()const{return vol_;}
     const container& perpVol()const{return vol2d_;}
-    geo::aGenerator* const generator() const{return generator_;}
+    geo::aGenerator const * generator() const{return generator_;}
+    bool isOrthogonal() const { return generator_.isOrthogonal();}
+    bool isConformal() const { return generator_.isConformal();}
     private:
     void construct( unsigned n, unsigned Nx, unsigned Ny)
     {
@@ -115,7 +117,7 @@ struct CurvilinearGrid3d : public dg::Grid3d
     }
     thrust::host_vector<double> r_, z_, xr_, xz_, yr_, yz_;
     container g_xx_, g_xy_, g_yy_, g_pp_, vol_, vol2d_;
-    geo::aGenerator* generator_;
+    const geo::aGenerator* generator_;
 };
 
 /**
@@ -177,7 +179,9 @@ struct CurvilinearGrid2d : public dg::Grid2d
     const container& g_xy()const{return g_xy_;}
     const container& vol()const{return vol2d_;}
     const container& perpVol()const{return vol2d_;}
-    geo::aGenerator* const generator() const{return generator_;}
+    geo::aGenerator const * generator() const{return generator_;}
+    bool isOrthogonal() const { return generator_.isOrthogonal();}
+    bool isConformal() const { return generator_.isConformal();}
     private:
     thrust::host_vector<double> r_, z_, xr_, xz_, yr_, yz_;
     container g_xx_, g_xy_, g_yy_, vol2d_;
diff --git a/inc/geometries/mpi_conformal.h b/inc/geometries/mpi_conformal.h
deleted file mode 100644
index 043cca4a3..000000000
--- a/inc/geometries/mpi_conformal.h
+++ /dev/null
@@ -1,199 +0,0 @@
-#pragma once
-
-#include <mpi.h>
-
-#include "dg/backend/mpi_grid.h"
-#include "dg/backend/mpi_vector.h"
-#include "conformal.h"
-
-
-
-namespace dg
-{
-
-///@cond
-template< class container>
-struct ConformalMPIGrid2d; 
-///@endcond
-//
-///@addtogroup grids
-///@{
-
-/**
- * @tparam LocalContainer Vector class that holds metric coefficients
- */
-template<class MPIContainer>
-struct ConformalMPIGrid3d : public dg::MPIGrid3d
-{
-    typedef dg::ConformalCylindricalTag metric_category; //!< metric tag
-    typedef dg::ConformalMPIGrid2d<LocalContainer> perpendicular_grid; //!< the two-dimensional grid
-    typedef typename MPIContainer::container_type LocalContainer; //!< the local container type
-
-    ConformalMPIGrid3d( geo::aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx, MPI_Comm comm): 
-        dg::MPIGrid3d( 0, generator->width(), 0., generator->height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, dg::PER, dg::PER, comm),
-        g( generator, n,Nx, Ny, local().Nz(), bcx)
-    {
-        divide_and_conquer();
-    }
-    void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz)
-    {
-        dg::MPIGrid3d::set(new_n, new_Nx, new_Ny, new_Nz);
-        g.set( new_n, new_Nx, new_Ny, new_Nz);//construct new grid
-        divide_and_conquer();//distribute to processes
-    }
-
-    perpendicular_grid perp_grid() const { return perpendicular_grid(*this);}
-
-    const dg::MPI_Vector<thrust::host_vector<double> >& r()const{return r_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& z()const{return z_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& xr()const{return xr_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& yr()const{return yr_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& xz()const{return xz_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& yz()const{return yz_;}
-    const MPIContainer& g_xx()const{return g_xx_;}
-    const MPIContainer& g_yy()const{return g_yy_;}
-    const MPIContainer& g_xy()const{return g_xy_;}
-    const MPIContainer& g_pp()const{return g_pp_;}
-    const MPIContainer& vol()const{return vol_;}
-    const MPIContainer& perpVol()const{return vol2d_;}
-    const dg::ConformalGrid3d<LocalContainer>& global() const {return g;}
-    geo::aGenerator* const generator() const{return g.generator();}
-    private:
-    void divide_and_conquer(){
-        r_=dg::evaluate( dg::one, *this), z_=r_, xr_=r_, xz_=r_, yr_=r_, yz_=r_;
-        dg::blas1::transfer( r_, g_xx_);
-        g_xy_=g_xx_, g_yy_=g_xx_, g_pp_=g_xx_, vol_=g_xx_, vol2d_=g_xx_;
-        //divide and conquer
-        int dims[3], periods[3], coords[3];
-        MPI_Cart_get( comm, 3, dims, periods, coords);
-        init_X_boundaries( g.x0(), g.x1());
-        for( unsigned s=0; s<this->Nz(); s++)
-            //for( unsigned py=0; py<dims[1]; py++)
-                for( unsigned i=0; i<this->n()*this->Ny(); i++)
-                    //for( unsigned px=0; px<dims[0]; px++)
-                        for( unsigned j=0; j<this->n()*this->Nx(); j++)
-                        {
-                            unsigned idx1 = (s*this->n()*this->Ny()+i)*this->n()*this->Nx() + j;
-                            unsigned idx2 = (((s*dims[1]+coords[1])*this->n()*this->Ny()+i)*dims[0] + coords[0])*this->n()*this->Nx() + j;
-                            r_.data()[idx1] = g.r()[idx2];
-                            z_.data()[idx1] = g.z()[idx2];
-                            xr_.data()[idx1] = g.xr()[idx2];
-                            xz_.data()[idx1] = g.xz()[idx2];
-                            yr_.data()[idx1] = g.yr()[idx2];
-                            yz_.data()[idx1] = g.yz()[idx2];
-                            g_xx_.data()[idx1] = g.g_xx()[idx2];
-                            g_xy_.data()[idx1] = g.g_xy()[idx2];
-                            g_yy_.data()[idx1] = g.g_yy()[idx2];
-                            g_pp_.data()[idx1] = g.g_pp()[idx2];
-                            vol_.data()[idx1] = g.vol()[idx2];
-                            vol2d_.data()[idx1] = g.perpVol()[idx2];
-                        }
-    }
-    dg::MPI_Vector<thrust::host_vector<double> > r_, z_, xr_, xz_, yr_, yz_; //3d vector
-    dg::MPI_Vector<LocalContainer> g_xx_, g_xy_, g_yy_, g_pp_, vol_, vol2d_;
-    dg::ConformalGrid3d<LocalContainer> g;
-};
-
-/**
- * @tparam LocalContainer Vector class that holds metric coefficients
- */
-template<class MPIContainer>
-struct ConformalMPIGrid2d : public dg::MPIGrid2d
-{
-    typedef dg::ConformalCylindricalTag metric_category; 
-    typedef typename MPIContainer::container_type LocalContainer; //!< the local container type
-
-    ConformalMPIGrid2d( geo::aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx, MPI_Comm comm2d): 
-        dg::MPIGrid2d( 0, 1, 0., 2*M_PI, n, Nx, Ny, bcx, dg::PER, comm2d),
-        r_(dg::evaluate( dg::one, *this)), z_(r_), xr_(r_), xz_(r_), yr_(r_), yz_(r_), 
-        g_xx_(r_), g_xy_(g_xx_), g_yy_(g_xx_), vol2d_(g_xx_),
-        g_( generator, n,Nx, Ny, bcx)
-    {
-        divide_and_conquer();
-    }
-    ConformalMPIGrid2d( const ConformalMPIGrid3d<LocalContainer>& g):
-        dg::MPIGrid2d( g.global().x0(), g.global().x1(), g.global().y0(), g.global().y1(), g.global().n(), g.global().Nx(), g.global().Ny(), g.global().bcx(), g.global().bcy(), get_reduced_comm( g.communicator() )),
-        r_(dg::evaluate( dg::one, *this)), z_(r_), xr_(r_), xz_(r_), yr_(r_), yz_(r_), 
-        g_xx_(r_), g_xy_(g_xx_), g_yy_(g_xx_), vol2d_(g_xx_),
-        g_(g.global())
-    {
-        unsigned s = this->size();
-        for( unsigned i=0; i<s; i++)
-        {
-            r_.data()[i]=g.r().data()[i]; 
-            z_.data()[i]=g.z().data()[i]; 
-            xr_.data()[i]=g.xr().data()[i]; 
-            xz_.data()[i]=g.xz().data()[i]; 
-            yr_.data()[i]=g.yr().data()[i]; 
-            yz_.data()[i]=g.yz().data()[i];
-        }
-        thrust::copy( g.g_xx().data().begin(), g.g_xx().data().begin()+s, g_xx_.data().begin());
-        thrust::copy( g.g_xy().data().begin(), g.g_xy().data().begin()+s, g_xy_.data().begin());
-        thrust::copy( g.g_yy().data().begin(), g.g_yy().data().begin()+s, g_yy_.data().begin());
-        thrust::copy( g.perpVol().data().begin(), g.perpVol().data().begin()+s, vol2d_.data().begin());
-        
-    }
-    void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny)
-    {
-        dg::MPIGrid3d::set(new_n, new_Nx, new_Ny);
-        g.set( new_n, new_Nx, new_Ny);//construct new grid
-        divide_and_conquer();//distribute to processes
-    }
-
-    const dg::MPI_Vector<thrust::host_vector<double> >& r()const{return r_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& z()const{return z_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& xr()const{return xr_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& yr()const{return yr_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& xz()const{return xz_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& yz()const{return yz_;}
-    const MPIContainer& g_xx()const{return g_xx_;}
-    const MPIContainer& g_yy()const{return g_yy_;}
-    const MPIContainer& g_xy()const{return g_xy_;}
-    const MPIContainer& vol()const{return vol2d_;}
-    const MPIContainer& perpVol()const{return vol2d_;}
-    const dg::ConformalGrid2d<LocalContainer>& global()const{return g_;}
-    geo::aGenerator* const generator() const{return g.generator();}
-    private:
-    MPI_Comm get_reduced_comm( MPI_Comm src)
-    {
-        MPI_Comm planeComm;
-        int remain_dims[] = {true,true,false}; //true true false
-        MPI_Cart_sub( src, remain_dims, &planeComm);
-        return planeComm;
-    }
-    void divide_and_conquer(){
-        r_=dg::evaluate( dg::one, *this), z_=r_, xr_=r_, xz_=r_, yr_=r_, yz_=r_;
-        dg::blas1::transfer( r_, g_xx_);
-        g_xy_=g_xx_, g_yy_=g_xx_, vol2d_=g_xx_;
-        //divide and conquer
-        int dims[2], periods[2], coords[2];
-        MPI_Cart_get( communicator(), 2, dims, periods, coords);
-        init_X_boundaries( g_.x0(), g_.x1());
-            //for( unsigned py=0; py<dims[1]; py++)
-                for( unsigned i=0; i<this->n()*this->Ny(); i++)
-                    //for( unsigned px=0; px<dims[0]; px++)
-                        for( unsigned j=0; j<this->n()*this->Nx(); j++)
-                        {
-                            unsigned idx1 = i*this->n()*this->Nx() + j;
-                            unsigned idx2 = ((coords[1]*this->n()*this->Ny()+i)*dims[0] + coords[0])*this->n()*this->Nx() + j;
-                            r_.data()[idx1] = g_.r()[idx2];
-                            z_.data()[idx1] = g_.z()[idx2];
-                            xr_.data()[idx1] = g_.xr()[idx2];
-                            xz_.data()[idx1] = g_.xz()[idx2];
-                            yr_.data()[idx1] = g_.yr()[idx2];
-                            yz_.data()[idx1] = g_.yz()[idx2];
-                            g_xx_.data()[idx1] = g_.g_xx()[idx2];
-                            g_xy_.data()[idx1] = g_.g_xy()[idx2];
-                            g_yy_.data()[idx1] = g_.g_yy()[idx2];
-                            vol2d_.data()[idx1] = g_.perpVol()[idx2];
-                        }
-    }
-
-    dg::MPI_Vector<thrust::host_vector<double> > r_, z_, xr_, xz_, yr_, yz_; //2d vector
-    dg::MPI_Vector<LocalContainer> g_xx_, g_xy_, g_yy_, vol2d_;
-    dg::ConformalGrid2d<LocalContainer> g_;
-};
-///@}
-}//namespace dg
-
-
diff --git a/inc/geometries/mpi_orthogonal.h b/inc/geometries/mpi_orthogonal.h
deleted file mode 100644
index d3d55f864..000000000
--- a/inc/geometries/mpi_orthogonal.h
+++ /dev/null
@@ -1,200 +0,0 @@
-#pragma once
-
-#include <mpi.h>
-
-#include "dg/backend/mpi_grid.h"
-#include "dg/backend/mpi_vector.h"
-#include "orthogonal.h"
-
-
-
-namespace dg
-{
-
-///@cond
-template< class container>
-struct OrthogonalMPIGrid2d; 
-///@endcond
-//
-///@addtogroup grids
-///@{
-
-/**
- * @tparam LocalContainer Vector class that holds metric coefficients
- */
-template<class MPIContainer>
-struct OrthogonalMPIGrid3d : public dg::MPIGrid3d
-{
-    typedef dg::OrthogonalTag metric_category; //!< metric tag
-    typedef dg::OrthogonalMPIGrid2d<LocalContainer> perpendicular_grid; //!< the two-dimensional grid
-    typedef typename MPIContainer::container_type LocalContainer; //!< the local container type
-
-    OrthogonalMPIGrid3d( geo::aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx, MPI_Comm comm): 
-        dg::MPIGrid3d( 0, generator->width(), 0., generator->height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, dg::PER, dg::PER, comm),
-        g( generator, n, Nx, Ny, local().Nz(), bcx)
-    {
-        divide_and_conquer();//distribute to processes
-    }
-    void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz)
-    {
-        dg::MPIGrid3d::set(new_n, new_Nx, new_Ny, new_Nz);
-        g.set( new_n, new_Nx, new_Ny, new_Nz);//construct new grid
-        divide_and_conquer();//distribute to processes
-    }
-
-    //these are for the Field class
-
-    perpendicular_grid perp_grid() const { return perpendicular_grid(*this);}
-
-    const dg::MPI_Vector<thrust::host_vector<double> >& r()const{return r_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& z()const{return z_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& xr()const{return xr_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& yr()const{return yr_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& xz()const{return xz_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& yz()const{return yz_;}
-    const MPIContainer& g_xx()const{return g_xx_;}
-    const MPIContainer& g_yy()const{return g_yy_;}
-    const MPIContainer& g_xy()const{return g_xy_;}
-    const MPIContainer& g_pp()const{return g_pp_;}
-    const MPIContainer& vol()const{return vol_;}
-    const MPIContainer& perpVol()const{return vol2d_;}
-    const dg::OrthogonalGrid3d<LocalContainer>& global() const{return g;}
-    geo::aGenerator* const generator() const{return g.generator();}
-    private:
-    void divide_and_conquer( )
-    {
-        r_=dg::evaluate( dg::one, *this), z_=r_, xr_=r_, xz_=r_, yr_=r_, yz_=r_;
-        dg::blas1::transfer( r_, g_xx_);
-        g_xy_=g_xx_, g_yy_=g_xx_, g_pp_=g_xx_, vol_=g_xx_, vol2d_=g_xx_;
-        //divide and conquer
-        int dims[3], periods[3], coords[3];
-        MPI_Cart_get( comm, 3, dims, periods, coords);
-        init_X_boundaries( g.x0(), g.x1());
-        for( unsigned s=0; s<this->Nz(); s++)
-            //for( unsigned py=0; py<dims[1]; py++)
-                for( unsigned i=0; i<this->n()*this->Ny(); i++)
-                    //for( unsigned px=0; px<dims[0]; px++)
-                        for( unsigned j=0; j<this->n()*this->Nx(); j++)
-                        {
-                            unsigned idx1 = (s*this->n()*this->Ny()+i)*this->n()*this->Nx() + j;
-                            unsigned idx2 = (((s*dims[1]+coords[1])*this->n()*this->Ny()+i)*dims[0] + coords[0])*this->n()*this->Nx() + j;
-                            r_.data()[idx1] = g.r()[idx2];
-                            z_.data()[idx1] = g.z()[idx2];
-                            xr_.data()[idx1] = g.xr()[idx2];
-                            xz_.data()[idx1] = g.xz()[idx2];
-                            yr_.data()[idx1] = g.yr()[idx2];
-                            yz_.data()[idx1] = g.yz()[idx2];
-                            g_xx_.data()[idx1] = g.g_xx()[idx2];
-                            g_xy_.data()[idx1] = g.g_xy()[idx2];
-                            g_yy_.data()[idx1] = g.g_yy()[idx2];
-                            g_pp_.data()[idx1] = g.g_pp()[idx2];
-                            vol_.data()[idx1] = g.vol()[idx2];
-                            vol2d_.data()[idx1] = g.perpVol()[idx2];
-                        }
-    }
-    dg::MPI_Vector<thrust::host_vector<double> > r_, z_, xr_, xz_, yr_, yz_; //3d vector
-    MPIContainer g_xx_, g_xy_, g_yy_, g_pp_, vol_, vol2d_;
-    dg::OrthogonalGrid3d<LocalContainer> g;
-};
-
-/**
- * @tparam MPIContainer Vector class that holds metric coefficients
- */
-template<class MPIContainer>
-struct OrthogonalMPIGrid2d : public dg::MPIGrid2d
-{
-    typedef dg::OrthogonalTag metric_category; 
-    typedef typename MPIContainer::container_type LocalContainer; //!< the local container type
-
-    OrthogonalMPIGrid2d( geo::aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx, MPI_Comm comm2d): 
-        dg::MPIGrid2d( 0, generator->width(), 0., generator->height(), n, Nx, Ny, bcx, dg::PER, comm2d),
-        g_( generator, n, Nx, Ny, bcx)
-    {
-        divide_and_conquer();
-    }
-    OrthogonalMPIGrid2d( const OrthogonalMPIGrid3d<LocalContainer>& g):
-        dg::MPIGrid2d( g.global().x0(), g.global().x1(), g.global().y0(), g.global().y1(), g.global().n(), g.global().Nx(), g.global().Ny(), g.global().bcx(), g.global().bcy(), get_reduced_comm( g.communicator() )),
-        r_(dg::evaluate( dg::one, *this)), z_(r_), xr_(r_), xz_(r_), yr_(r_), yz_(r_), 
-        g_xx_(r_), g_xy_(g_xx_), g_yy_(g_xx_), vol2d_(g_xx_),
-        g_( g.global())
-    {
-        unsigned s = this->size();
-        for( unsigned i=0; i<s; i++)
-        {
-            r_.data()[i]=g.r().data()[i]; 
-            z_.data()[i]=g.z().data()[i]; 
-            xr_.data()[i]=g.xr().data()[i]; 
-            xz_.data()[i]=g.xz().data()[i]; 
-            yr_.data()[i]=g.yr().data()[i]; 
-            yz_.data()[i]=g.yz().data()[i];
-        }
-        thrust::copy( g.g_xx().data().begin(), g.g_xx().data().begin()+s, g_xx_.data().begin());
-        thrust::copy( g.g_xy().data().begin(), g.g_xy().data().begin()+s, g_xy_.data().begin());
-        thrust::copy( g.g_yy().data().begin(), g.g_yy().data().begin()+s, g_yy_.data().begin());
-        thrust::copy( g.perpVol().data().begin(), g.perpVol().data().begin()+s, vol2d_.data().begin());
-        
-    }
-    void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny)
-    {
-        dg::MPIGrid3d::set(new_n, new_Nx, new_Ny);
-        g.set( new_n, new_Nx, new_Ny);//construct new grid
-        divide_and_conquer();//distribute to processes
-    }
-
-    const dg::MPI_Vector<thrust::host_vector<double> >& r()const{return r_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& z()const{return z_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& xr()const{return xr_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& yr()const{return yr_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& xz()const{return xz_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& yz()const{return yz_;}
-    const MPIContainer& g_xx()const{return g_xx_;}
-    const MPIContainer& g_yy()const{return g_yy_;}
-    const MPIContainer& g_xy()const{return g_xy_;}
-    const MPIContainer& vol()const{return vol2d_;}
-    const MPIContainer& perpVol()const{return vol2d_;}
-    const dg::OrthogonalGrid2d<LocalContainer>& global() const {return g;}
-    geo::aGenerator* const generator() const{return g.generator();}
-    private:
-    MPI_Comm get_reduced_comm( MPI_Comm src)
-    {
-        MPI_Comm planeComm;
-        int remain_dims[] = {true,true,false}; //true true false
-        MPI_Cart_sub( src, remain_dims, &planeComm);
-        return planeComm;
-    }
-    void divide_and_conquer()
-    {
-        r_=dg::evaluate( dg::one, *this), z_=r_, xr_=r_, xz_=r_, yr_=r_, yz_=r_;
-        dg::blas1::transfer( r_, g_xx_);
-        g_xy_=g_xx_, g_yy_=g_xx_, vol2d_=g_xx_;
-        //divide and conquer
-        int dims[2], periods[2], coords[2];
-        MPI_Cart_get( communicator(), 2, dims, periods, coords);
-        init_X_boundaries( g_.x0(), g_.x1());
-            //for( unsigned py=0; py<dims[1]; py++)
-                for( unsigned i=0; i<this->n()*this->Ny(); i++)
-                    //for( unsigned px=0; px<dims[0]; px++)
-                        for( unsigned j=0; j<this->n()*this->Nx(); j++)
-                        {
-                            unsigned idx1 = i*this->n()*this->Nx() + j;
-                            unsigned idx2 = ((coords[1]*this->n()*this->Ny()+i)*dims[0] + coords[0])*this->n()*this->Nx() + j;
-                            r_.data()[idx1] = g_.r()[idx2];
-                            z_.data()[idx1] = g_.z()[idx2];
-                            xr_.data()[idx1] = g_.xr()[idx2];
-                            xz_.data()[idx1] = g_.xz()[idx2];
-                            yr_.data()[idx1] = g_.yr()[idx2];
-                            yz_.data()[idx1] = g_.yz()[idx2];
-                            g_xx_.data()[idx1] = g_.g_xx()[idx2];
-                            g_xy_.data()[idx1] = g_.g_xy()[idx2];
-                            g_yy_.data()[idx1] = g_.g_yy()[idx2];
-                            vol2d_.data()[idx1] = g_.perpVol()[idx2];
-                        }
-    }
-
-    dg::MPI_Vector<thrust::host_vector<double> > r_, z_, xr_, xz_, yr_, yz_; //2d vector
-    MPIContainer g_xx_, g_xy_, g_yy_, vol2d_;
-    dg::OrthogonalGrid2d<LocalContainer> g_;
-};
-///@}
-}//namespace dg
-
diff --git a/inc/geometries/orthogonal.h b/inc/geometries/orthogonal.h
deleted file mode 100644
index cf6fb4688..000000000
--- a/inc/geometries/orthogonal.h
+++ /dev/null
@@ -1,192 +0,0 @@
-#pragma once
-
-#include "dg/backend/grid.h"
-#include "dg/blas1.h"
-#include "dg/geometry/geometry_traits.h"
-#include "generator.h"
-
-namespace dg
-{
-///@addtogroup grids
-///@{
-
-///@cond
-template< class container>
-struct OrthogonalGrid2d; 
-///@endcond
-
-/**
- * @brief A three-dimensional grid based on orthogonal coordinates
- * @tparam container models aContainer
- */
-template< class container>
-struct OrthogonalGrid3d : public dg::Grid3d
-{
-    typedef dg::OrthogonalTag metric_category;
-    typedef OrthogonalGrid2d<container> perpendicular_grid;
-
-    /*!@brief Constructor
-    
-     * @param generator the pointer must not be deleted as long as you intend to call the set() function
-     * @param n 
-     * @param Nx
-     @param Ny
-     @param Nz 
-     @param bcx
-     */
-    OrthogonalGrid3d( geo::aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx=dg::DIR):
-        dg::Grid3d( 0, generator->width(), 0., generator->height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, dg::PER, dg::PER)
-    { 
-        assert( generator->isOrthogonal());
-        generator_ = generator;
-        construct( n, Nx, Ny);
-    }
-
-    /**
-    * @brief Reconstruct the grid coordinates
-    *
-    * @copydetails Grid3d::set()
-    * @attention the generator must still live when this function is called
-    */
-    void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz){
-        dg::Grid3d::set( new_n, new_Nx, new_Ny, new_Nz);
-        construct( new_n, new_Nx, new_Ny);
-    }
-
-    perpendicular_grid perp_grid() const { return OrthogonalGrid2d<container>(*this);}
-    const thrust::host_vector<double>& r()const{return r_;}
-    const thrust::host_vector<double>& z()const{return z_;}
-    const thrust::host_vector<double>& xr()const{return xr_;}
-    const thrust::host_vector<double>& yr()const{return yr_;}
-    const thrust::host_vector<double>& xz()const{return xz_;}
-    const thrust::host_vector<double>& yz()const{return yz_;}
-    const container& g_xx()const{return g_xx_;}
-    const container& g_yy()const{return g_yy_;}
-    const container& g_xy()const{return g_xy_;}
-    const container& g_pp()const{return g_pp_;}
-    const container& vol()const{return vol_;}
-    const container& perpVol()const{return vol2d_;}
-    geo::aGenerator* const generator() const{return generator_;}
-    private:
-    void construct( unsigned n, unsigned Nx, unsigned Ny)
-    {
-        dg::Grid1d gY1d( 0, generator_->height(), n, Ny, dg::PER);
-        dg::Grid1d gX1d( 0, generator_->width(), n, Nx);
-        thrust::host_vector<double> x_vec = dg::evaluate( dg::cooX1d, gX1d);
-        thrust::host_vector<double> y_vec = dg::evaluate( dg::cooX1d, gY1d);
-        (*generator_)( x_vec, y_vec, r_, z_, xr_, xz_, yr_, yz_);
-        init_X_boundaries( 0., generator->width());
-        lift3d( ); //lift to 3D grid
-        construct_metric();
-    }
-    void lift3d( )
-    {
-        //lift to 3D grid
-        unsigned size = this->size();
-        r_.resize( size), z_.resize(size), xr_.resize(size), yr_.resize( size), xz_.resize( size), yz_.resize(size);
-        unsigned Nx = this->n()*this->Nx(), Ny = this->n()*this->Ny();
-        for( unsigned k=1; k<this->Nz(); k++)
-            for( unsigned i=0; i<Nx*Ny; i++)
-            {
-                r_[k*Nx*Ny+i] = r_[(k-1)*Nx*Ny+i];
-                z_[k*Nx*Ny+i] = z_[(k-1)*Nx*Ny+i];
-                xr_[k*Nx*Ny+i] = xr_[(k-1)*Nx*Ny+i];
-                xz_[k*Nx*Ny+i] = xz_[(k-1)*Nx*Ny+i];
-                yr_[k*Nx*Ny+i] = yr_[(k-1)*Nx*Ny+i];
-                yz_[k*Nx*Ny+i] = yz_[(k-1)*Nx*Ny+i];
-            }
-    }
-    //compute metric elements from xr, xz, yr, yz, r and z
-    void construct_metric( )
-    {
-        thrust::host_vector<double> tempxx( r_), tempxy(r_), tempyy(r_), tempvol(r_);
-        for( unsigned i = 0; i<this->size(); i++)
-        {
-            tempxx[i] = (xr_[i]*xr_[i]+xz_[i]*xz_[i]);
-            tempxy[i] = (yr_[i]*xr_[i]+yz_[i]*xz_[i]);
-            tempyy[i] = (yr_[i]*yr_[i]+yz_[i]*yz_[i]);
-            tempvol[i] = r_[i]/sqrt( tempxx[i]*tempyy[i] );
-        }
-        g_xx_=tempxx, g_xy_=tempxy, g_yy_=tempyy, vol_=tempvol;
-        dg::blas1::pointwiseDivide( tempvol, r_, tempvol);
-        vol2d_ = tempvol;
-        thrust::host_vector<double> ones = dg::evaluate( dg::one, *this);
-        dg::blas1::pointwiseDivide( ones, r_, tempxx);
-        dg::blas1::pointwiseDivide( tempxx, r_, tempxx); //1/R^2
-        g_pp_=tempxx;
-    }
-    thrust::host_vector<double> r_, z_, xr_, xz_, yr_, yz_;
-    container g_xx_, g_xy_, g_yy_, g_pp_, vol_, vol2d_;
-    geo::aGenerator* generator_;
-};
-
-/**
- * @brief A three-dimensional grid based on orthogonal coordinates
- @tparam container models aContainer
- */
-template< class container>
-struct OrthogonalGrid2d : public dg::Grid2d
-{
-    typedef dg::OrthogonalTag metric_category;
-    /*!@brief Constructor
-    
-     * @param generator must generate an orthogonal grid
-     * @param n number of polynomial coefficients
-     * @param Nx number of cells in first coordinate
-     @param Ny number of cells in second coordinate
-     @param bcx boundary condition in first coordinate
-     */
-    OrthogonalGrid2d( geo::aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx=dg::DIR):
-        dg::Grid2d( 0, generator->width(), 0., generator->height(), n, Nx, Ny, bcx, dg::PER)
-    {
-        generator_=generator;
-        OrthogonalGrid3d<container> g( generator_, n,Nx,Ny,1,bcx);
-        init_X_boundaries( g.x0(), g.x1());
-        r_=g.r(), z_=g.z(), xr_=g.xr(), xz_=g.xz(), yr_=g.yr(), yz_=g.yz();
-        g_xx_=g.g_xx(), g_xy_=g.g_xy(), g_yy_=g.g_yy();
-        vol2d_=g.perpVol();
-
-    }
-    OrthogonalGrid2d( const OrthogonalGrid3d<container>& g):
-        dg::Grid2d( g.x0(), g.x1(), g.y0(), g.y1(), g.n(), g.Nx(), g.Ny(), g.bcx(), g.bcy())
-    {
-        generator_ = g.generator();
-        unsigned s = this->size();
-        r_.resize( s), z_.resize(s), xr_.resize(s), xz_.resize(s), yr_.resize(s), yz_.resize(s);
-        g_xx_.resize( s), g_xy_.resize(s), g_yy_.resize(s), vol2d_.resize(s);
-        for( unsigned i=0; i<s; i++)
-        { r_[i]=g.r()[i], z_[i]=g.z()[i], xr_[i]=g.xr()[i], xz_[i]=g.xz()[i], yr_[i]=g.yr()[i], yz_[i]=g.yz()[i];}
-        thrust::copy( g.g_xx().begin(), g.g_xx().begin()+s, g_xx_.begin());
-        thrust::copy( g.g_xy().begin(), g.g_xy().begin()+s, g_xy_.begin());
-        thrust::copy( g.g_yy().begin(), g.g_yy().begin()+s, g_yy_.begin());
-        thrust::copy( g.perpVol().begin(), g.perpVol().begin()+s, vol2d_.begin());
-    }
-    void set(unsigned new_n, unsigned new_Nx, unsigned new_Ny)
-    {
-        dg::Grid2d::set( new_n, new_Nx, new_Ny);
-        OrthogonalGrid3d<container> g( generator_, new_n,new_Nx,new_Ny,1,bcx());
-        r_=g.r(), z_=g.z(), xr_=g.xr(), xz_=g.xz(), yr_=g.yr(), yz_=g.yz();
-        g_xx_=g.g_xx(), g_xy_=g.g_xy(), g_yy_=g.g_yy();
-        vol2d_=g.perpVol();
-    }
-
-    const thrust::host_vector<double>& r()const{return r_;}
-    const thrust::host_vector<double>& z()const{return z_;}
-    const thrust::host_vector<double>& xr()const{return xr_;}
-    const thrust::host_vector<double>& yr()const{return yr_;}
-    const thrust::host_vector<double>& xz()const{return xz_;}
-    const thrust::host_vector<double>& yz()const{return yz_;}
-    const container& g_xx()const{return g_xx_;}
-    const container& g_yy()const{return g_yy_;}
-    const container& g_xy()const{return g_xy_;}
-    const container& vol()const{return vol2d_;}
-    const container& perpVol()const{return vol2d_;}
-    geo::aGenerator* const generator() const{return generator_;}
-    private:
-    thrust::host_vector<double> r_, z_, xr_, xz_, yr_, yz_;
-    container g_xx_, g_xy_, g_yy_, vol2d_;
-    geo::aGenerator* generator_;
-};
-
-///@}
-}//namespace dg
diff --git a/inc/geometries/orthogonalX.h b/inc/geometries/orthogonalX.h
deleted file mode 100644
index 291b0d1c8..000000000
--- a/inc/geometries/orthogonalX.h
+++ /dev/null
@@ -1,176 +0,0 @@
-#pragma once
-
-#include "dg/backend/gridX.h"
-#include "dg/backend/evaluationX.cuh"
-#include "dg/blas1.h"
-#include "dg/geometry/geometry_traits.h"
-
-namespace dg
-{
-
-///@cond
-template< class container>
-struct OrthogonalGridX2d; 
-///@endcond
-
-/**
- * @brief Orthogonal X-point grid in three dimensions
- * @ingroup grids
- */
-template< class container>
-struct OrthogonalGridX3d : public dg::GridX3d
-{
-    typedef dg::OrthogonalTag metric_category;
-    typedef OrthogonalGridX2d<container> perpendicular_grid;
-
-    /*!@brief Constructor
-    
-     * @tparam GeneratorX models aGeneratorX
-     * @param generator isOrthogonal() must return true
-     * @param psi_0
-     * @param fx
-     * @param fy
-     * @param n 
-     * @param Nx
-     @param Ny
-     @param Nz 
-     @param bcx
-     @param bcy
-     */
-    template< class GeneratorX>
-    OrthogonalGridX3d( GeneratorX generator, double psi_0, double fx, double fy, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx, dg::bc bcy):
-        dg::GridX3d( 0,1, -2.*M_PI*fy/(1.-2.*fy), 2.*M_PI*(1.+fy/(1.-2.*fy)), 0., 2*M_PI, fx, fy, n, Nx, Ny, Nz, bcx, bcy, dg::PER),
-        r_(this->size()), z_(r_), xr_(r_), xz_(r_), yr_(r_), yz_(r_)
-    {
-        assert( generator.isOrthogonal());
-        construct( generator, psi_0, fx, n, Nx, Ny);
-    }
-
-    const thrust::host_vector<double>& r()const{return r_;}
-    const thrust::host_vector<double>& z()const{return z_;}
-    const thrust::host_vector<double>& xr()const{return xr_;}
-    const thrust::host_vector<double>& yr()const{return yr_;}
-    const thrust::host_vector<double>& xz()const{return xz_;}
-    const thrust::host_vector<double>& yz()const{return yz_;}
-    const container& g_xx()const{return g_xx_;}
-    const container& g_yy()const{return g_yy_;}
-    const container& g_xy()const{return g_xy_;}
-    const container& g_pp()const{return g_pp_;}
-    const container& vol()const{return vol_;}
-    const container& perpVol()const{return vol2d_;}
-    perpendicular_grid perp_grid() const { return OrthogonalGridX2d<container>(*this);}
-    private:
-    template<class GeneratorX>
-    void construct( GeneratorX generator, double psi_0, double fx, unsigned n, unsigned Nx, unsigned Ny )
-    {
-        const double x_0 = generator.f0()*psi_0;
-        const double x_1 = -fx/(1.-fx)*x_0;
-        init_X_boundaries( x_0, x_1);
-        dg::Grid1d gX1d( this->x0(), this->x1(), n, Nx, dg::DIR);
-        thrust::host_vector<double> x_vec = dg::evaluate( dg::cooX1d, gX1d);
-        dg::GridX1d gY1d( -this->fy()*2.*M_PI/(1.-2.*this->fy()), 2*M_PI+this->fy()*2.*M_PI/(1.-2.*this->fy()), this->fy(), this->n(), this->Ny(), dg::DIR);
-        thrust::host_vector<double> y_vec = dg::evaluate( dg::cooX1d, gY1d);
-        thrust::host_vector<double> rvec, zvec, yrvec, yzvec, xrvec, xzvec;
-        generator( x_vec, y_vec, gY1d.n()*gY1d.outer_N(), gY1d.n()*(gY1d.inner_N()+gY1d.outer_N()), rvec, zvec, xrvec, xzvec, yrvec, yzvec);
-
-        unsigned Mx = this->n()*this->Nx(), My = this->n()*this->Ny();
-        //now lift to 3D grid
-        for( unsigned k=0; k<this->Nz(); k++)
-            for( unsigned i=0; i<Mx*My; i++)
-            {
-                r_[k*Mx*My+i] = rvec[i];
-                z_[k*Mx*My+i] = zvec[i];
-                yr_[k*Mx*My+i] = yrvec[i];
-                yz_[k*Mx*My+i] = yzvec[i];
-                xr_[k*Mx*My+i] = xrvec[i];
-                xz_[k*Mx*My+i] = xzvec[i];
-            }
-        construct_metric();
-    }
-    //compute metric elements from xr, xz, yr, yz, r and z
-    void construct_metric()
-    {
-        //std::cout << "CONSTRUCTING METRIC\n";
-        thrust::host_vector<double> tempxx( r_), tempxy(r_), tempyy(r_), tempvol(r_);
-        for( unsigned idx=0; idx<this->size(); idx++)
-        {
-            tempxx[idx] = (xr_[idx]*xr_[idx]+xz_[idx]*xz_[idx]);
-            tempxy[idx] = (yr_[idx]*xr_[idx]+yz_[idx]*xz_[idx]);
-            tempyy[idx] = (yr_[idx]*yr_[idx]+yz_[idx]*yz_[idx]);
-            tempvol[idx] = r_[idx]/sqrt(tempxx[idx]*tempyy[idx]-tempxy[idx]*tempxy[idx]);
-        }
-        g_xx_=tempxx, g_xy_=tempxy, g_yy_=tempyy, vol_=tempvol;
-        dg::blas1::pointwiseDivide( tempvol, r_, tempvol);
-        vol2d_ = tempvol;
-        thrust::host_vector<double> ones = dg::evaluate( dg::one, *this);
-        dg::blas1::pointwiseDivide( ones, r_, tempxx);
-        dg::blas1::pointwiseDivide( tempxx, r_, tempxx); //1/R^2
-        g_pp_=tempxx;
-    }
-    thrust::host_vector<double> r_, z_, xr_, xz_, yr_, yz_;
-    container g_xx_, g_xy_, g_yy_, g_pp_, vol_, vol2d_;
-};
-
-/**
- * @brief Orthogonal X-point grid in two dimensions
- * @ingroup grids
- */
-template< class container>
-struct OrthogonalGridX2d : public dg::GridX2d
-{
-    typedef dg::OrthogonalTag metric_category;
-    /*!@brief Constructor
-    
-     * @tparam GeneratorX models aGeneratorX
-     * @param generator isOrthogonal() must return true
-     * @param psi_0 left flux surface
-     * @param fx
-     * @param fy
-     * @param n 
-     * @param Nx
-     @param Ny
-     @param bcx
-     @param bcy
-     */
-    template<class Generator>
-    OrthogonalGridX2d(Generator generator, double psi_0, double fx, double fy, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx, dg::bc bcy):
-        dg::GridX2d( 0, 1,-fy*2.*M_PI/(1.-2.*fy), 2*M_PI+fy*2.*M_PI/(1.-2.*fy), fx, fy, n, Nx, Ny, bcx, bcy)
-    {
-        OrthogonalGridX3d<container> g( generator, psi_0, fx,fy, n,Nx,Ny,1,bcx,bcy);
-        init_X_boundaries( g.x0(),g.x1());
-        r_=g.r(), z_=g.z(), xr_=g.xr(), xz_=g.xz(), yr_=g.yr(), yz_=g.yz();
-        g_xx_=g.g_xx(), g_xy_=g.g_xy(), g_yy_=g.g_yy();
-        vol2d_=g.perpVol();
-    }
-    OrthogonalGridX2d( const OrthogonalGridX3d<container>& g):
-        dg::GridX2d( g.x0(), g.x1(), g.y0(), g.y1(), g.fx(), g.fy(), g.n(), g.Nx(), g.Ny(), g.bcx(), g.bcy())
-    {
-        unsigned s = this->size();
-        r_.resize( s), z_.resize(s), xr_.resize(s), xz_.resize(s), yr_.resize(s), yz_.resize(s);
-        g_xx_.resize( s), g_xy_.resize(s), g_yy_.resize(s), vol2d_.resize(s);
-        for( unsigned i=0; i<s; i++)
-        { r_[i]=g.r()[i], z_[i]=g.z()[i], xr_[i]=g.xr()[i], xz_[i]=g.xz()[i], yr_[i]=g.yr()[i], yz_[i]=g.yz()[i];}
-        thrust::copy( g.g_xx().begin(), g.g_xx().begin()+s, g_xx_.begin());
-        thrust::copy( g.g_xy().begin(), g.g_xy().begin()+s, g_xy_.begin());
-        thrust::copy( g.g_yy().begin(), g.g_yy().begin()+s, g_yy_.begin());
-        thrust::copy( g.perpVol().begin(), g.perpVol().begin()+s, vol2d_.begin());
-    }
-    const thrust::host_vector<double>& r()const{return r_;}
-    const thrust::host_vector<double>& z()const{return z_;}
-    const thrust::host_vector<double>& xr()const{return xr_;}
-    const thrust::host_vector<double>& yr()const{return yr_;}
-    const thrust::host_vector<double>& xz()const{return xz_;}
-    const thrust::host_vector<double>& yz()const{return yz_;}
-
-    const container& g_xx()const{return g_xx_;}
-    const container& g_yy()const{return g_yy_;}
-    const container& g_xy()const{return g_xy_;}
-    const container& vol()const{return vol2d_;}
-    const container& perpVol()const{return vol2d_;}
-    private:
-    thrust::host_vector<double> r_, z_, xr_, xz_, yr_, yz_;
-    container g_xx_, g_xy_, g_yy_, vol2d_;
-};
-
-}//namespace dg
-
diff --git a/inc/geometries/refined_conformal.h b/inc/geometries/refined_conformal.h
deleted file mode 100644
index a851a7716..000000000
--- a/inc/geometries/refined_conformal.h
+++ /dev/null
@@ -1,157 +0,0 @@
-#pragma once
-
-#include "dg/geometry/refined_grid.h"
-#include "conformal.h"
-
-namespace dg
-{
-
-///@cond
-template< class container>
-struct ConformalRefinedGrid2d; 
-///@endcond
-
-///@addtogroup grids
-///@{
-
-/**
- * @brief A conformal refined grid
- */
-template< class container>
-struct ConformalRefinedGrid3d : public dg::RefinedGrid3d
-{
-    typedef dg::ConformalCylindricalTag metric_category;
-    typedef ConformalRefinedGrid2d<container> perpendicular_grid;
-
-    template<class Generator>
-    ConformalRefinedGrid3d( unsigned multiple_x, unsigned multiple_y, const Generator& generator, unsigned n, unsigned n_old, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx): 
-        dg::RefinedGrid3d( multiple_x, multiple_y, 0, 1, 0., 2.*M_PI, 0., 2.*M_PI, n, n_old, Nx, Ny, Nz, bcx, dg::PER, dg::PER),
-        g_assoc_( generator, n_old, Nx, Ny, Nz, bcx)
-    { 
-        assert( generator.isConformal());
-        construct( generator);
-    }
-
-
-    perpendicular_grid perp_grid() const { return perpendicular_grid(*this);}
-    const dg::ConformalGrid3d<container>& associated() const{ return g_assoc_;}
-
-    const thrust::host_vector<double>& r()const{return r_;}
-    const thrust::host_vector<double>& z()const{return z_;}
-    const thrust::host_vector<double>& xr()const{return xr_;}
-    const thrust::host_vector<double>& yr()const{return yr_;}
-    const thrust::host_vector<double>& xz()const{return xz_;}
-    const thrust::host_vector<double>& yz()const{return yz_;}
-    const container& g_xx()const{return g_xx_;}
-    const container& g_yy()const{return g_xx_;}
-    const container& g_pp()const{return g_pp_;}
-    const container& vol()const{return vol_;}
-    const container& perpVol()const{return vol2d_;}
-    private:
-    template< class Generator>
-    void construct( Generator generator)
-    {
-        init_X_boundaries( 0., generator.width());
-        unsigned sizeY = this->n()*this->Ny();
-        unsigned sizeX = this->n()*this->Nx();
-        thrust::host_vector<double> y_vec(sizeY), x_vec(sizeX);
-        for(unsigned i=0; i<sizeY; i++) y_vec[i] = this->abscissasY()[i*sizeX];
-        for(unsigned i=0; i<sizeX; i++) x_vec[i] = this->abscissasX()[i];
-        generator( x_vec, y_vec, r_, z_, xr_, xz_, yr_, yz_);
-        lift3d( ); //lift to 3D grid
-        construct_metric();
-    }
-    void lift3d( )
-    {
-        //lift to 3D grid
-        unsigned size = this->size();
-        r_.resize( size), z_.resize(size), xr_.resize(size), yr_.resize( size), xz_.resize( size), yz_.resize(size);
-        unsigned Nx = this->n()*this->Nx(), Ny = this->n()*this->Ny();
-        thrust::host_vector<double> wx = this->weightsX();
-        thrust::host_vector<double> wy = this->weightsY();
-        for( unsigned k=1; k<this->Nz(); k++)
-            for( unsigned i=0; i<Nx*Ny; i++)
-            {
-                r_[k*Nx*Ny+i] = r_[(k-1)*Nx*Ny+i];
-                z_[k*Nx*Ny+i] = z_[(k-1)*Nx*Ny+i];
-                yr_[k*Nx*Ny+i] = yr_[(k-1)*Nx*Ny+i]*wy[k*Nx*Ny+i];
-                yz_[k*Nx*Ny+i] = yz_[(k-1)*Nx*Ny+i]*wy[k*Nx*Ny+i];
-                xr_[k*Nx*Ny+i] = xr_[(k-1)*Nx*Ny+i]*wx[k*Nx*Ny+i];
-                xz_[k*Nx*Ny+i] = xz_[(k-1)*Nx*Ny+i]*wx[k*Nx*Ny+i];
-            }
-    }
-    //compute metric elements from xr, xz, yr, yz, r and z
-    void construct_metric( )
-    {
-        thrust::host_vector<double> tempxx( r_), tempvol(r_);
-        for( unsigned i = 0; i<this->size(); i++)
-        {
-            tempxx[i] = (xr_[i]*xr_[i]+xz_[i]*xz_[i]);
-            tempvol[i] = r_[i]/ tempxx[i];
-        }
-        dg::blas1::transfer( tempxx, g_xx_);
-        dg::blas1::transfer( tempvol, vol_);
-        dg::blas1::pointwiseDivide( tempvol, r_, tempvol);
-        dg::blas1::transfer( tempvol, vol2d_);
-        thrust::host_vector<double> ones = dg::evaluate( dg::one, *this);
-        dg::blas1::pointwiseDivide( ones, r_, tempxx);
-        dg::blas1::pointwiseDivide( tempxx, r_, tempxx); //1/R^2
-        g_pp_=tempxx;
-    }
-    thrust::host_vector<double> r_, z_, xr_, xz_, yr_, yz_; 
-    container g_xx_, g_pp_, vol_, vol2d_;
-    dg::ConformalGrid3d<container> g_assoc_;
-
-};
-
-/**
- * @brief A conformal refined grid
- */
-template< class container>
-struct ConformalRefinedGrid2d : public dg::RefinedGrid2d
-{
-    typedef dg::ConformalCylindricalTag metric_category;
-    template< class Generator>
-    ConformalRefinedGrid2d( unsigned multiple_x, unsigned multiple_y, const Generator& generator, unsigned n, unsigned n_old, unsigned Nx, unsigned Ny, dg::bc bcx):
-        dg::RefinedGrid2d( multiple_x, multiple_y, 0, 1., 0., 2*M_PI, n,n_old,Nx,Ny, bcx, dg::PER),
-        g_assoc_( generator, n_old, Nx, Ny, bcx) 
-    {
-        dg::ConformalRefinedGrid3d<container> g( multiple_x, multiple_y, generator, n,n_old,Nx,Ny,1,bcx);
-        init_X_boundaries( g.x0(), g.x1());
-        r_=g.r(), z_=g.z(), xr_=g.xr(), xz_=g.xz(), yr_=g.yr(), yz_=g.yz();
-        g_xx_=g.g_xx();
-        vol2d_=g.perpVol();
-    }
-
-    ConformalRefinedGrid2d( const ConformalRefinedGrid3d<container>& g):
-        dg::RefinedGrid2d( g ), g_assoc_(g.associated())
-    {
-        unsigned s = this->size();
-        r_.resize( s), z_.resize(s), xr_.resize(s), xz_.resize(s), yr_.resize(s), yz_.resize(s),
-        g_xx_.resize( s), vol2d_.resize(s);
-        for( unsigned i=0; i<s; i++)
-        {r_[i]=g.r()[i], z_[i]=g.z()[i], xr_[i]=g.xr()[i], xz_[i]=g.xz()[i], yr_[i]=g.yr()[i], yz_[i]=g.yz()[i];}
-        thrust::copy( g.g_xx().begin(), g.g_xx().begin()+s, g_xx_.begin());
-        thrust::copy( g.perpVol().begin(), g.perpVol().begin()+s, vol2d_.begin());
-    }
-
-    const dg::ConformalGrid2d<container>& associated()const{return g_assoc_;}
-
-    const thrust::host_vector<double>& r()const{return r_;}
-    const thrust::host_vector<double>& z()const{return z_;}
-    const thrust::host_vector<double>& xr()const{return xr_;}
-    const thrust::host_vector<double>& yr()const{return yr_;}
-    const thrust::host_vector<double>& xz()const{return xz_;}
-    const thrust::host_vector<double>& yz()const{return yz_;}
-    const container& g_xx()const{return g_xx_;}
-    const container& g_yy()const{return g_xx_;}
-    const container& vol()const{return vol2d_;}
-    const container& perpVol()const{return vol2d_;}
-    private:
-    thrust::host_vector<double> r_, z_, xr_, xz_, yr_, yz_;
-    container g_xx_, vol2d_;
-    dg::ConformalGrid2d<container> g_assoc_;
-};
-
-///@}
-}//namespace dg
diff --git a/inc/geometries/refined_orthogonalX.h b/inc/geometries/refined_curvilinearX.h
similarity index 83%
rename from inc/geometries/refined_orthogonalX.h
rename to inc/geometries/refined_curvilinearX.h
index 14ad2463a..1209a3a51 100644
--- a/inc/geometries/refined_orthogonalX.h
+++ b/inc/geometries/refined_curvilinearX.h
@@ -8,7 +8,7 @@ namespace dg
 
 ///@cond
 template< class container>
-struct OrthogonalRefinedGridX2d;
+struct CurvilinearRefinedGridX2d;
 ///@endcond
 
 ///@addtogroup grids
@@ -18,23 +18,23 @@ struct OrthogonalRefinedGridX2d;
  * @brief A three-dimensional refined X-Grid
  */
 template< class container>
-struct OrthogonalRefinedGridX3d : public dg::RefinedGridX3d
+struct CurvilinearRefinedGridX3d : public dg::RefinedGridX3d
 {
-    typedef dg::OrthogonalTag metric_category;
-    typedef dg::OrthogonalRefinedGridX2d<container> perpendicular_grid;
+    typedef dg::CurvilinearCylindricalTag metric_category;
+    typedef dg::CurvilinearRefinedGridX2d<container> perpendicular_grid;
 
     template <class Generator>
-    OrthogonalRefinedGridX3d( unsigned add_x, unsigned add_y, unsigned howmanyX, unsigned howmanyY, const Generator& generator, double psi_0, double fx, double fy, unsigned n, unsigned n_old, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx, dg::bc bcy): 
+    CurvilinearRefinedGridX3d( unsigned add_x, unsigned add_y, unsigned howmanyX, unsigned howmanyY, const Generator& generator, double psi_0, double fx, double fy, unsigned n, unsigned n_old, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx, dg::bc bcy): 
         dg::RefinedGridX3d( add_x, add_y, howmanyX, howmanyY, 0,1, -2.*M_PI*fy/(1.-2.*fy), 2.*M_PI*(1.+fy/(1.-2.*fy)), 0., 2*M_PI, fx, fy, n, n_old, Nx, Ny, Nz, bcx, bcy, dg::PER),
         r_(this->size()), z_(r_), xr_(r_), xz_(r_), yr_(r_), yz_(r_),
         g_assoc_( generator, psi_0, fx, fy, n_old, Nx, Ny, Nz, bcx, bcy)
     {
-        assert( generator.isOrthogonal());
+        assert( generator.isCurvilinear());
         construct( generator, psi_0, fx);
     }
 
     perpendicular_grid perp_grid() const { return perpendicular_grid(*this);}
-    const dg::OrthogonalGridX3d<container>& associated() const{ return g_assoc_;}
+    const dg::CurvilinearGridX3d<container>& associated() const{ return g_assoc_;}
 
     const thrust::host_vector<double>& r()const{return r_;}
     const thrust::host_vector<double>& z()const{return z_;}
@@ -114,29 +114,29 @@ struct OrthogonalRefinedGridX3d : public dg::RefinedGridX3d
     }
     thrust::host_vector<double> r_, z_, xr_, xz_, yr_, yz_; //3d vector
     container g_xx_, g_xy_, g_yy_, g_pp_, vol_, vol2d_;
-    dg::OrthogonalGridX3d<container> g_assoc_;
+    dg::CurvilinearGridX3d<container> g_assoc_;
 };
 
 /**
  * @brief A two-dimensional refined X-Grid
  */
 template< class container>
-struct OrthogonalRefinedGridX2d : public dg::RefinedGridX2d
+struct CurvilinearRefinedGridX2d : public dg::RefinedGridX2d
 {
-    typedef dg::OrthogonalTag metric_category;
+    typedef dg::CurvilinearCylindricalTag metric_category;
 
     template<class Generator>
-    OrthogonalRefinedGridX2d( unsigned add_x, unsigned add_y, unsigned howmanyX, unsigned howmanyY, const Generator& generator, double psi_0, double fx, double fy, unsigned n, unsigned n_old, unsigned Nx, unsigned Ny, dg::bc bcx, dg::bc bcy): 
+    CurvilinearRefinedGridX2d( unsigned add_x, unsigned add_y, unsigned howmanyX, unsigned howmanyY, const Generator& generator, double psi_0, double fx, double fy, unsigned n, unsigned n_old, unsigned Nx, unsigned Ny, dg::bc bcx, dg::bc bcy): 
         dg::RefinedGridX2d( add_x, add_y, howmanyX, howmanyY, 0, 1,-fy*2.*M_PI/(1.-2.*fy), 2*M_PI+fy*2.*M_PI/(1.-2.*fy), fx, fy, n, n_old, Nx, Ny, bcx, bcy),
         g_assoc_( generator, psi_0, fx, fy, n_old, Nx, Ny, bcx, bcy) 
     {
-        dg::OrthogonalRefinedGridX3d<container> g(add_x, add_y, howmanyX, howmanyY, generator, psi_0, fx,fy, n,n_old,Nx,Ny,1,bcx,bcy);
+        dg::CurvilinearRefinedGridX3d<container> g(add_x, add_y, howmanyX, howmanyY, generator, psi_0, fx,fy, n,n_old,Nx,Ny,1,bcx,bcy);
         init_X_boundaries( g.x0(), g.x1());
         r_=g.r(), z_=g.z(), xr_=g.xr(), xz_=g.xz(), yr_=g.yr(), yz_=g.yz();
         g_xx_=g.g_xx(), g_xy_=g.g_xy(), g_yy_=g.g_yy();
         vol2d_=g.perpVol();
     }
-    OrthogonalRefinedGridX2d( const OrthogonalRefinedGridX3d<container>& g):
+    CurvilinearRefinedGridX2d( const CurvilinearRefinedGridX3d<container>& g):
         dg::RefinedGridX2d(g), g_assoc_(g.associated())
     {
         unsigned s = this->size();
@@ -149,7 +149,7 @@ struct OrthogonalRefinedGridX2d : public dg::RefinedGridX2d
         thrust::copy( g.g_yy().begin(), g.g_yy().begin()+s, g_yy_.begin());
         thrust::copy( g.perpVol().begin(), g.perpVol().begin()+s, vol2d_.begin());
     }
-    const dg::OrthogonalGridX2d<container>& associated()const{return g_assoc_;}
+    const dg::CurvilinearGridX2d<container>& associated()const{return g_assoc_;}
     const thrust::host_vector<double>& r()const{return r_;}
     const thrust::host_vector<double>& z()const{return z_;}
     const thrust::host_vector<double>& xr()const{return xr_;}
@@ -167,7 +167,7 @@ struct OrthogonalRefinedGridX2d : public dg::RefinedGridX2d
     private:
     thrust::host_vector<double> r_, z_, xr_, xz_, yr_, yz_; //2d vector
     container g_xx_, g_xy_, g_yy_, vol2d_;
-    dg::OrthogonalGridX2d<container> g_assoc_;
+    dg::CurvilinearGridX2d<container> g_assoc_;
 };
 ///@}
 
diff --git a/inc/geometries/refined_orthogonal.h b/inc/geometries/refined_orthogonal.h
deleted file mode 100644
index b1e903995..000000000
--- a/inc/geometries/refined_orthogonal.h
+++ /dev/null
@@ -1,160 +0,0 @@
-#pragma once
-
-#include "dg/geometry/refined_grid.h"
-#include "orthogonal.h"
-
-namespace dg
-{
-
-///@cond
-template< class container>
-struct OrthogonalRefinedGrid2d; 
-///@endcond
-
-///@addtogroup grids
-///@{
-
-/**
- * @brief An orthogonal refined grid
- */
-template< class container>
-struct OrthogonalRefinedGrid3d : public dg::RefinedGrid3d
-{
-    typedef dg::OrthogonalTag metric_category;
-    typedef OrthogonalRefinedGrid2d<container> perpendicular_grid;
-
-    template<class Generator>
-    OrthogonalRefinedGrid3d( unsigned multiple_x, unsigned multiple_y, const Generator& generator, unsigned n, unsigned n_old, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx): 
-        dg::RefinedGrid3d( multiple_x, multiple_y, 0, 1, 0., 2.*M_PI, 0., 2.*M_PI, n, n_old, Nx, Ny, Nz, bcx, dg::PER, dg::PER),
-        g_assoc_( generator, n_old, Nx, Ny, Nz, bcx)
-    { 
-        assert( generator.isOrthogonal());
-        construct( generator);
-    }
-
-    perpendicular_grid perp_grid() const { return perpendicular_grid(*this);}
-    const dg::OrthogonalGrid3d<container>& associated() const{ return g_assoc_;}
-
-    const thrust::host_vector<double>& r()const{return r_;}
-    const thrust::host_vector<double>& z()const{return z_;}
-    const thrust::host_vector<double>& xr()const{return xr_;}
-    const thrust::host_vector<double>& yr()const{return yr_;}
-    const thrust::host_vector<double>& xz()const{return xz_;}
-    const thrust::host_vector<double>& yz()const{return yz_;}
-    const container& g_xx()const{return g_xx_;}
-    const container& g_yy()const{return g_yy_;}
-    const container& g_xy()const{return g_xy_;}
-    const container& g_pp()const{return g_pp_;}
-    const container& vol()const{return vol_;}
-    const container& perpVol()const{return vol2d_;}
-    private:
-    template< class Generator>
-    void construct( Generator generator)
-    {
-        init_X_boundaries( 0., generator.width());
-        unsigned sizeY = this->n()*this->Ny();
-        unsigned sizeX = this->n()*this->Nx();
-        thrust::host_vector<double> y_vec(sizeY), x_vec(sizeX);
-        for(unsigned i=0; i<sizeY; i++) y_vec[i] = this->abscissasY()[i*sizeX];
-        for(unsigned i=0; i<sizeX; i++) x_vec[i] = this->abscissasX()[i];
-        generator( x_vec, y_vec, r_, z_, xr_, xz_, yr_, yz_);
-        lift3d( ); //lift to 3D grid
-        construct_metric();
-    }
-    void lift3d( )
-    {
-        //lift to 3D grid
-        unsigned size = this->size();
-        r_.resize( size), z_.resize(size), xr_.resize(size), yr_.resize( size), xz_.resize( size), yz_.resize(size);
-        unsigned Nx = this->n()*this->Nx(), Ny = this->n()*this->Ny();
-        thrust::host_vector<double> wx = this->weightsX();
-        thrust::host_vector<double> wy = this->weightsY();
-        for( unsigned k=1; k<this->Nz(); k++)
-            for( unsigned i=0; i<Nx*Ny; i++)
-            {
-                r_[k*Nx*Ny+i] = r_[(k-1)*Nx*Ny+i];
-                z_[k*Nx*Ny+i] = z_[(k-1)*Nx*Ny+i];
-                yr_[k*Nx*Ny+i] = yr_[(k-1)*Nx*Ny+i]*wy[k*Nx*Ny+i];
-                yz_[k*Nx*Ny+i] = yz_[(k-1)*Nx*Ny+i]*wy[k*Nx*Ny+i];
-                xr_[k*Nx*Ny+i] = xr_[(k-1)*Nx*Ny+i]*wx[k*Nx*Ny+i];
-                xz_[k*Nx*Ny+i] = xz_[(k-1)*Nx*Ny+i]*wx[k*Nx*Ny+i];
-            }
-    }
-    //compute metric elements from xr, xz, yr, yz, r and z
-    void construct_metric( )
-    {
-        thrust::host_vector<double> tempxx( r_), tempxy(r_), tempyy(r_), tempvol(r_);
-        for( unsigned i = 0; i<this->size(); i++)
-        {
-            tempxx[i] = (xr_[i]*xr_[i]+xz_[i]*xz_[i]);
-            tempxy[i] = (yr_[i]*xr_[i]+yz_[i]*xz_[i]);
-            tempyy[i] = (yr_[i]*yr_[i]+yz_[i]*yz_[i]);
-            tempvol[i] = r_[i]/sqrt( tempxx[i]*tempyy[i] );
-        }
-        g_xx_=tempxx, g_xy_=tempxy, g_yy_=tempyy, vol_=tempvol;
-        dg::blas1::pointwiseDivide( tempvol, r_, tempvol);
-        vol2d_ = tempvol;
-        thrust::host_vector<double> ones = dg::evaluate( dg::one, *this);
-        dg::blas1::pointwiseDivide( ones, r_, tempxx);
-        dg::blas1::pointwiseDivide( tempxx, r_, tempxx); //1/R^2
-        g_pp_=tempxx;
-    }
-    thrust::host_vector<double> r_, z_, xr_, xz_, yr_, yz_;
-    container g_xx_, g_xy_, g_yy_, g_pp_, vol_, vol2d_;
-    dg::OrthogonalGrid3d<container> g_assoc_;
-
-};
-
-/**
- * @brief An orthogonal refined grid
- */
-template< class container>
-struct OrthogonalRefinedGrid2d : public dg::RefinedGrid2d
-{
-    typedef dg::OrthogonalTag metric_category;
-    template< class Generator>
-    OrthogonalRefinedGrid2d( unsigned multiple_x, unsigned multiple_y, const Generator& generator, unsigned n, unsigned n_old, unsigned Nx, unsigned Ny, dg::bc bcx):
-        dg::RefinedGrid2d( multiple_x, multiple_y, 0, 1., 0., 2*M_PI, n,n_old,Nx,Ny, bcx, dg::PER),
-        g_assoc_( generator, n_old, Nx, Ny, bcx) 
-    {
-        dg::OrthogonalRefinedGrid3d<container> g( multiple_x, multiple_y, generator, n,n_old,Nx,Ny,1,bcx);
-        init_X_boundaries( g.x0(), g.x1());
-        r_=g.r(), z_=g.z(), xr_=g.xr(), xz_=g.xz(), yr_=g.yr(), yz_=g.yz();
-        g_xx_=g.g_xx(), g_xy_=g.g_xy(), g_yy_=g.g_yy();
-        vol2d_=g.perpVol();
-    }
-    OrthogonalRefinedGrid2d( const OrthogonalRefinedGrid3d<container>& g):
-        dg::RefinedGrid2d( g ), g_assoc_(g.associated())
-    {
-        unsigned s = this->size();
-        r_.resize( s), z_.resize(s), xr_.resize(s), xz_.resize(s), yr_.resize(s), yz_.resize(s);
-        g_xx_.resize( s), g_xy_.resize(s), g_yy_.resize(s), vol2d_.resize(s);
-        for( unsigned i=0; i<s; i++)
-        {r_[i]=g.r()[i], z_[i]=g.z()[i], xr_[i]=g.xr()[i], xz_[i]=g.xz()[i], yr_[i]=g.yr()[i], yz_[i]=g.yz()[i];}
-        thrust::copy( g.g_xx().begin(), g.g_xx().begin()+s, g_xx_.begin());
-        thrust::copy( g.g_xy().begin(), g.g_xy().begin()+s, g_xy_.begin());
-        thrust::copy( g.g_yy().begin(), g.g_yy().begin()+s, g_yy_.begin());
-        thrust::copy( g.perpVol().begin(), g.perpVol().begin()+s, vol2d_.begin());
-    }
-
-    const dg::OrthogonalGrid2d<container>& associated()const{return g_assoc_;}
-
-    const thrust::host_vector<double>& r()const{return r_;}
-    const thrust::host_vector<double>& z()const{return z_;}
-    const thrust::host_vector<double>& xr()const{return xr_;}
-    const thrust::host_vector<double>& yr()const{return yr_;}
-    const thrust::host_vector<double>& xz()const{return xz_;}
-    const thrust::host_vector<double>& yz()const{return yz_;}
-    const container& g_xx()const{return g_xx_;}
-    const container& g_yy()const{return g_yy_;}
-    const container& g_xy()const{return g_xy_;}
-    const container& vol()const{return vol2d_;}
-    const container& perpVol()const{return vol2d_;}
-    private:
-    thrust::host_vector<double> r_, z_, xr_, xz_, yr_, yz_;
-    container g_xx_, g_xy_, g_yy_, vol2d_;
-    dg::OrthogonalGrid2d<container> g_assoc_;
-};
-
-///@}
-}//namespace dg
-- 
GitLab


From 2de89d9a405e5023ab30fff11acebfc48ec0a7a5 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Fri, 21 Jul 2017 12:18:17 +0200
Subject: [PATCH 062/453] made Hector unique (no copy / assignment) and
 conformal_t to hector_t

---
 inc/dg/geometry/geometry_traits.h             | 37 +++++++++----------
 inc/geometries/curvilinear.h                  | 16 ++++----
 inc/geometries/hector.h                       | 23 +++++++-----
 .../{conformal_t.cu => hector_t.cu}           | 32 ++++++++--------
 inc/geometries/mpi_curvilinear.h              | 12 ++++--
 5 files changed, 63 insertions(+), 57 deletions(-)
 rename inc/geometries/{conformal_t.cu => hector_t.cu} (78%)

diff --git a/inc/dg/geometry/geometry_traits.h b/inc/dg/geometry/geometry_traits.h
index 3fb227216..03c099c04 100644
--- a/inc/dg/geometry/geometry_traits.h
+++ b/inc/dg/geometry/geometry_traits.h
@@ -136,9 +136,9 @@ void doPushForwardPerp( TernaryOp1 f1, TernaryOp2 f2,
         container& vx, container& vy,
         const Geometry& g, OrthonormalCylindricalTag)
 {
-    typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector& out1, out2; 
-    out1 = evaluate( f1, g);
-    out2 = evaluate( f2, g);
+    typedef typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector host_vec;
+    host_vec out1 = evaluate( f1, g);
+    host_vec out2 = evaluate( f2, g);
     dg::blas1::transfer( out1, vx);
     dg::blas1::transfer( out2, vy);
 }
@@ -148,9 +148,9 @@ void doPushForwardPerp( TernaryOp1 f1, TernaryOp2 f2,
         container& vx, container& vy,
         const Geometry& g, CurvilinearCylindricalTag)
 {
-    typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector& out1, out2, temp1, temp2; 
-    temp1 = out1 = pullback( f1, g);
-    temp2 = out2 = pullback( f2, g);
+    typedef typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector host_vec;
+    host_vec out1 = pullback( f1, g), temp1(out1);
+    host_vec out2 = pullback( f2, g), temp2(out2);
     dg::blas1::pointwiseDot( g.xr(), temp1, out1);
     dg::blas1::pointwiseDot( 1., g.xz(), temp2, 1., out1);
     dg::blas1::pointwiseDot( g.yr(), temp1, out2);
@@ -164,10 +164,10 @@ void doPushForwardPerp( FunctorRR chiRR, FunctorRZ chiRZ, FunctorZZ chiZZ,
         container& chixx, container& chixy, container& chiyy,
         const Geometry& g, OrthonormalCylindricalTag)
 {
-    typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector& chixx_, chixy_, chiyy_; 
-    chixx_ = evaluate( chiRR, g);
-    chixy_ = evaluate( chiRZ, g);
-    chiyy_ = evaluate( chiZZ, g);
+    typedef typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector host_vec;
+    host_vec chixx_ = evaluate( chiRR, g);
+    host_vec chixy_ = evaluate( chiRZ, g);
+    host_vec chiyy_ = evaluate( chiZZ, g);
     dg::blas1::transfer( chixx_, chixx);
     dg::blas1::transfer( chixy_, chixy);
     dg::blas1::transfer( chiyy_, chiyy);
@@ -179,13 +179,12 @@ void doPushForwardPerp( FunctorRR chiRR_, FunctorRZ chiRZ_, FunctorZZ chiZZ_,
         const Geometry& g, CurvilinearCylindricalTag)
 {
     //compute the rhs
-    typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector chiRR, chiRZ, chiZZ;
-    typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector& chixx_, chixy_, chiyy_; 
-    chixx_ = chiRR = pullback( chiRR_, g);
-    chixy_ = chiRZ = pullback( chiRZ_, g);
-    chiyy_ = chiZZ = pullback( chiZZ_, g);
+    typedef typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector host_vec;
+    host_vec chixx_ = pullback( chiRR_, g), chiRR(chixx_);
+    host_vec chixy_ = pullback( chiRZ_, g), chiRZ(chixy_);
+    host_vec chiyy_ = pullback( chiZZ_, g), chiZZ(chiyy_);
     //compute the transformation matrix
-    typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector t00(chixx), t01(t00), t02(t00), t10(t00), t11(t00), t12(t00), t20(t00), t21(t00), t22(t00);
+    host_vec t00(chixx), t01(t00), t02(t00), t10(t00), t11(t00), t12(t00), t20(t00), t21(t00), t22(t00);
     dg::blas1::pointwiseDot( g.xr(), g.xr(), t00);
     dg::blas1::pointwiseDot( g.xr(), g.xz(), t01);
     dg::blas1::scal( t01, 2.);
@@ -238,8 +237,7 @@ template< class Geometry>
 typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector doCreateVolume( const Geometry& g, CurvilinearTag)
 {
     typedef typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector host_vector;
-    host_vector temp, vol;
-    dg::blas1::transfer( dg::create::weights( g), temp);
+    host_vector temp = dg::create::weights( g), vol(temp);
     dg::blas1::transfer( g.vol(), vol); //g.vol might be on device
     dg::blas1::pointwiseDot( vol, temp, temp);
     return temp;
@@ -249,8 +247,7 @@ template< class Geometry>
 typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector doCreateInvVolume( const Geometry& g, CurvilinearTag)
 {
     typedef typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector host_vector;
-    host_vector temp, vol;
-    dg::blas1::transfer( dg::create::inv_weights( g), temp);
+    host_vector temp = dg::create::inv_weights(g), vol(temp);
     dg::blas1::transfer( g.vol(), vol); //g.vol might be on device
     dg::blas1::pointwiseDivide( temp, vol, temp);
     return temp;
diff --git a/inc/geometries/curvilinear.h b/inc/geometries/curvilinear.h
index 0e1fcc15f..17bdc82e6 100644
--- a/inc/geometries/curvilinear.h
+++ b/inc/geometries/curvilinear.h
@@ -64,9 +64,9 @@ struct CurvilinearGrid3d : public dg::Grid3d
     const container& g_pp()const{return g_pp_;}
     const container& vol()const{return vol_;}
     const container& perpVol()const{return vol2d_;}
-    geo::aGenerator const * generator() const{return generator_;}
-    bool isOrthogonal() const { return generator_.isOrthogonal();}
-    bool isConformal() const { return generator_.isConformal();}
+    const geo::aGenerator * generator() const{return generator_;}
+    bool isOrthogonal() const { return generator_->isOrthogonal();}
+    bool isConformal() const { return generator_->isConformal();}
     private:
     void construct( unsigned n, unsigned Nx, unsigned Ny)
     {
@@ -135,7 +135,7 @@ struct CurvilinearGrid2d : public dg::Grid2d
      @param Ny number of cells in second coordinate
      @param bcx boundary condition in first coordinate
      */
-    CurvilinearGrid2d( geo::aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx=dg::DIR):
+    CurvilinearGrid2d( const geo::aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx=dg::DIR):
         dg::Grid2d( 0, generator->width(), 0., generator->height(), n, Nx, Ny, bcx, dg::PER)
     {
         CurvilinearGrid3d<container> g( generator, n,Nx,Ny,1,bcx);
@@ -179,13 +179,13 @@ struct CurvilinearGrid2d : public dg::Grid2d
     const container& g_xy()const{return g_xy_;}
     const container& vol()const{return vol2d_;}
     const container& perpVol()const{return vol2d_;}
-    geo::aGenerator const * generator() const{return generator_;}
-    bool isOrthogonal() const { return generator_.isOrthogonal();}
-    bool isConformal() const { return generator_.isConformal();}
+    const geo::aGenerator * generator() const{return generator_;}
+    bool isOrthogonal() const { return generator_->isOrthogonal();}
+    bool isConformal() const { return generator_->isConformal();}
     private:
     thrust::host_vector<double> r_, z_, xr_, xz_, yr_, yz_;
     container g_xx_, g_xy_, g_yy_, vol2d_;
-    geo::aGenerator* generator_;
+    const geo::aGenerator* generator_;
 };
 
 ///@}
diff --git a/inc/geometries/hector.h b/inc/geometries/hector.h
index 69cc3502d..a0e263fd5 100644
--- a/inc/geometries/hector.h
+++ b/inc/geometries/hector.h
@@ -6,8 +6,6 @@
 #include "dg/elliptic.h"
 #include "dg/cg.h"
 #include "flux.h"
-#include "conformal.h"
-#include "orthogonal.h"
 #include "curvilinear.h"
 #include "adaption.h"
 #include "generator.h"
@@ -236,7 +234,8 @@ struct Hector : public aGenerator
      */
     template< class Psi, class PsiX, class PsiY, class PsiXX ,class PsiXY, class PsiYY >
     Hector( Psi psi, PsiX psiX, PsiY psiY, PsiXX psiXX, PsiXY psiXY, PsiYY psiYY, double psi0, double psi1, double X0, double Y0, unsigned n = 13, unsigned Nx = 2, unsigned Ny = 10, double eps_u = 1e-10, bool verbose=false) : 
-        g2d_(dg::geo::RibeiroFluxGenerator<Psi,PsiX,PsiY,PsiXX, PsiXY, PsiYY>(psi, psiX, psiY, psiXX, psiXY, psiYY, psi0, psi1, X0, Y0,1), n, Nx, Ny, dg::DIR)
+        generator_( new dg::geo::RibeiroFluxGenerator<Psi,PsiX,PsiY,PsiXX, PsiXY, PsiYY>(psi, psiX, psiY, psiXX, psiXY, psiYY, psi0, psi1, X0, Y0,1)),
+        g2d_(generator_, n, Nx, Ny, dg::DIR)
     {
         //first construct u_
         container u = construct_grid_and_u( psi, psiX, psiY, psiXX, psiXY, psiYY, dg::ONE(), dg::geo::detail::LaplacePsi<PsiXX, PsiYY>(psiXX, psiYY), psi0, psi1, X0, Y0, n, Nx, Ny, eps_u , verbose);
@@ -284,7 +283,8 @@ struct Hector : public aGenerator
      */
     template< class Psi, class PsiX, class PsiY, class PsiXX, class PsiXY, class PsiYY, class Chi, class ChiX, class ChiY>
     Hector( Psi psi, PsiX psiX, PsiY psiY, PsiXX psiXX, PsiXY psiXY, PsiYY psiYY, Chi chi, ChiX chiX, ChiY chiY, double psi0, double psi1, double X0, double Y0, unsigned n = 13, unsigned Nx = 2, unsigned Ny = 10, double eps_u = 1e-10, bool verbose=false) : 
-        g2d_(dg::geo::RibeiroFluxGenerator<Psi,PsiX,PsiY,PsiXX,PsiXY,PsiYY>(psi, psiX, psiY, psiXX, psiXY, psiYY, psi0, psi1, X0, Y0,1), n, Nx, Ny, dg::DIR)
+        generator_( new dg::geo::RibeiroFluxGenerator<Psi,PsiX,PsiY,PsiXX, PsiXY, PsiYY>(psi, psiX, psiY, psiXX, psiXY, psiYY, psi0, psi1, X0, Y0,1)),
+        g2d_(generator_, n, Nx, Ny, dg::DIR)
     {
         dg::geo::detail::LaplaceAdaptPsi<PsiX, PsiY, dg::geo::detail::LaplacePsi<PsiXX, PsiYY>, Chi, ChiX, ChiY> lapAdaPsi( psiX, psiY, dg::geo::detail::LaplacePsi<PsiXX, PsiYY>(psiXX, psiYY), chi, chiX, chiY);
         //first construct u_
@@ -340,8 +340,8 @@ struct Hector : public aGenerator
             Chi_XX chi_XX, Chi_XY chi_XY, Chi_YY chi_YY, 
             DivChiX divChiX, DivChiY divChiY,
             double psi0, double psi1, double X0, double Y0, unsigned n = 13, unsigned Nx = 2, unsigned Ny = 10, double eps_u = 1e-10, bool verbose=false) : 
-        g2d_(dg::geo::RibeiroFluxGenerator<Psi,PsiX,PsiY,PsiXX,PsiXY,PsiYY> (
-                    psi, psiX, psiY, psiXX,psiXY,psiYY, psi0, psi1, X0, Y0,1), n, Nx, Ny, dg::DIR)
+        generator_( new dg::geo::RibeiroFluxGenerator<Psi,PsiX,PsiY,PsiXX, PsiXY, PsiYY>(psi, psiX, psiY, psiXX, psiXY, psiYY, psi0, psi1, X0, Y0,1)),
+        g2d_(generator_, n, Nx, Ny, dg::DIR)
     {
         dg::geo::detail::LaplaceChiPsi<PsiX, PsiY, PsiXX, PsiXY, PsiYY, Chi_XX, Chi_XY, Chi_YY, DivChiX, DivChiY>
             lapChiPsi( psiX, psiY, psiXX, psiXY, psiYY, 
@@ -449,13 +449,16 @@ struct Hector : public aGenerator
      * @return  orthogonal zeta, eta grid
      */
     const dg::CurvilinearGrid2d<container>& internal_grid() const {return g2d_;}
+    ~Hector() { delete generator_;}
     private:
+    //make Hector non-copyable
+    Hector( const Hector& src);
+    Hector& operator=( const Hector& src);
     template< class Psi, class PsiX, class PsiY, class PsiXX, class PsiXY, class PsiYY, class Chi, class LaplaceChiPsi>
     container construct_grid_and_u( Psi psi, PsiX psiX, PsiY psiY, PsiXX psiXX, PsiXY psiXY, PsiYY psiYY, Chi chi, LaplaceChiPsi lapCP, double psi0, double psi1, double X0, double Y0, unsigned n, unsigned Nx, unsigned Ny, double eps_u , bool verbose) 
     {
         //first find u( \zeta, \eta)
         double eps = 1e10, eps_old = 2e10;
-        dg::geo::RibeiroFluxGenerator<Psi,PsiX,PsiY,PsiXX, PsiXY, PsiYY> generator(psi, psiX, psiY, psiXX, psiXY, psiYY, psi0, psi1, X0, Y0,1);
         dg::CurvilinearGrid2d<container> g2d_old = g2d_;
         container adapt = dg::pullback(chi, g2d_old);
         dg::Elliptic<dg::CurvilinearGrid2d<container>, Matrix, container> ellipticD_old( g2d_old, dg::DIR, dg::PER, dg::not_normed, dg::centered);
@@ -469,7 +472,7 @@ struct Hector : public aGenerator
         {
             eps = eps_old;
             Nx*=2, Ny*=2;
-            dg::CurvilinearGrid2d<container> g2d(generator, n, Nx, Ny, dg::DIR);
+            dg::CurvilinearGrid2d<container> g2d(generator_, n, Nx, Ny, dg::DIR);
             if(verbose) std::cout << "Nx "<<Nx<<" Ny "<<Ny<<std::flush;
             dg::Elliptic<dg::CurvilinearGrid2d<container>, Matrix, container> ellipticD( g2d, dg::DIR, dg::PER, dg::not_normed, dg::centered);
             adapt = dg::pullback(chi, g2d);
@@ -500,7 +503,6 @@ struct Hector : public aGenerator
     {
         //first find u( \zeta, \eta)
         double eps = 1e10, eps_old = 2e10;
-        dg::geo::RibeiroFluxGenerator<Psi,PsiX,PsiY,PsiXX, PsiXY, PsiYY> generator(psi, psiX, psiY, psiXX, psiXY, psiYY, psi0, psi1, X0, Y0,1);
         dg::CurvilinearGrid2d<container> g2d_old = g2d_;
         dg::TensorElliptic<dg::CurvilinearGrid2d<container>, Matrix, container> ellipticD_old( g2d_old, dg::DIR, dg::PER, dg::not_normed, dg::centered);
         ellipticD_old.set( chi_XX, chi_XY, chi_YY);
@@ -513,7 +515,7 @@ struct Hector : public aGenerator
         {
             eps = eps_old;
             Nx*=2, Ny*=2;
-            dg::CurvilinearGrid2d<container> g2d(generator, n, Nx, Ny, dg::DIR);
+            dg::CurvilinearGrid2d<container> g2d(generator_, n, Nx, Ny, dg::DIR);
             if(verbose)std::cout << "Nx "<<Nx<<" Ny "<<Ny<<std::flush;
             dg::TensorElliptic<dg::CurvilinearGrid2d<container>, Matrix, container> ellipticD( g2d, dg::DIR, dg::PER, dg::not_normed, dg::centered);
             ellipticD.set( chi_XX, chi_XY, chi_YY );
@@ -605,6 +607,7 @@ struct Hector : public aGenerator
     double c0_, lu_;
     thrust::host_vector<double> u_, ux_, uy_, vx_, vy_;
     thrust::host_vector<double> etaV_, zetaU_, etaU_;
+    aGenerator* generator_;
     dg::CurvilinearGrid2d<container> g2d_;
 
 };
diff --git a/inc/geometries/conformal_t.cu b/inc/geometries/hector_t.cu
similarity index 78%
rename from inc/geometries/conformal_t.cu
rename to inc/geometries/hector_t.cu
index d74c2e62d..e32b97b53 100644
--- a/inc/geometries/conformal_t.cu
+++ b/inc/geometries/hector_t.cu
@@ -11,8 +11,6 @@
 #include "dg/backend/timer.cuh"
 //#include "guenther.h"
 #include "solovev.h"
-#include "conformal.h"
-#include "orthogonal.h"
 #include "curvilinear.h"
 #include "hector.h"
 //#include "refined_conformal.h"
@@ -78,20 +76,24 @@ int main( int argc, char* argv[])
     dg::geo::solovev::MagneticField c( gp); 
     Hector<dg::IDMatrix, dg::DMatrix, dg::DVec>* hector;
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-    hector = new Hector<dg::IDMatrix, dg::DMatrix, dg::DVec>( c.psip, c.psipR, c.psipZ, c.psipRR, c.psipRZ, c.psipZZ, psi_0, psi_1, gp.R_0, 0., nGrid, NxGrid, NyGrid, epsHector, true);
-    dg::ConformalGrid3d<dg::HVec> g3d(hector, n, Nx, Ny,Nz, dg::DIR);
-    dg::ConformalGrid2d<dg::HVec> g2d = g3d.perp_grid();
-    //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-    //dg::geo::NablaPsiInvCollective<solovev::PsipR, solovev::PsipZ, solovev::PsipRR, solovev::PsipRZ, solovev::PsipZZ> nc( c.psipR, c.psipZ, c.psipRR, c.psipRZ, c.psipZZ);
-    //dg::Hector<dg::IDMatrix, dg::DMatrix, dg::DVec> hector( c.psip, c.psipR, c.psipZ, c.psipRR, c.psipRZ, c.psipZZ, nc.nablaPsiInv, nc.nablaPsiInvX, nc.nablaPsiInvY, psi_0, psi_1, gp.R_0, 0., nGrid, NxGrid, NyGrid, epsHector, true);
-    //dg::OrthogonalGrid3d<dg::HVec> g3d(hector, n, Nx, Ny,Nz, dg::DIR);
-    //dg::OrthogonalGrid2d<dg::HVec> g2d = g3d.perp_grid();
-    //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-    //dg::geo::LiseikinCollective<solovev::PsipR, solovev::PsipZ, solovev::PsipRR, solovev::PsipRZ, solovev::PsipZZ> lc( c.psipR, c.psipZ, c.psipRR, c.psipRZ, c.psipZZ, 0.1, 0.001);
-    //dg::Hector<dg::IDMatrix, dg::DMatrix, dg::DVec> hector( c.psip, c.psipR, c.psipZ, c.psipRR, c.psipRZ, c.psipZZ, lc.chi_XX, lc.chi_XY, lc.chi_YY, lc.divChiX, lc.divChiY, psi_0, psi_1, gp.R_0, 0., nGrid, NxGrid, NyGrid, epsHector, true);
-    //dg::CurvilinearGrid3d<dg::HVec> g3d(hector, n, Nx, Ny,Nz, dg::DIR);
-    //dg::CurvilinearGrid2d<dg::HVec> g2d = g3d.perp_grid();
+    int construction = 0;
+    if( construction == 0)
+    {
+        hector = new Hector<dg::IDMatrix, dg::DMatrix, dg::DVec>( c.psip, c.psipR, c.psipZ, c.psipRR, c.psipRZ, c.psipZZ, psi_0, psi_1, gp.R_0, 0., nGrid, NxGrid, NyGrid, epsHector, true);
+    }
+    else if( construction == 1)
+    {
+        dg::geo::NablaPsiInvCollective<solovev::PsipR, solovev::PsipZ, solovev::PsipRR, solovev::PsipRZ, solovev::PsipZZ> nc( c.psipR, c.psipZ, c.psipRR, c.psipRZ, c.psipZZ);
+        hector = new Hector<dg::IDMatrix, dg::DMatrix, dg::DVec>( c.psip, c.psipR, c.psipZ, c.psipRR, c.psipRZ, c.psipZZ, nc.nablaPsiInv, nc.nablaPsiInvX, nc.nablaPsiInvY, psi_0, psi_1, gp.R_0, 0., nGrid, NxGrid, NyGrid, epsHector, true);
+    }
+    else
+    {
+        dg::geo::LiseikinCollective<solovev::PsipR, solovev::PsipZ, solovev::PsipRR, solovev::PsipRZ, solovev::PsipZZ> lc( c.psipR, c.psipZ, c.psipRR, c.psipRZ, c.psipZZ, 0.1, 0.001);
+        hector = new Hector<dg::IDMatrix, dg::DMatrix, dg::DVec>( c.psip, c.psipR, c.psipZ, c.psipRR, c.psipRZ, c.psipZZ, lc.chi_XX, lc.chi_XY, lc.chi_YY, lc.divChiX, lc.divChiY, psi_0, psi_1, gp.R_0, 0., nGrid, NxGrid, NyGrid, epsHector, true);
+    }
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+    dg::CurvilinearGrid3d<dg::HVec> g3d(hector, n, Nx, Ny,Nz, dg::DIR);
+    dg::CurvilinearGrid2d<dg::HVec> g2d = g3d.perp_grid();
 
     dg::Grid2d g2d_periodic(g2d.x0(), g2d.x1(), g2d.y0(), g2d.y1(), g2d.n(), g2d.Nx(), g2d.Ny()+1); 
     t.toc();
diff --git a/inc/geometries/mpi_curvilinear.h b/inc/geometries/mpi_curvilinear.h
index a749c700a..5ae41ce45 100644
--- a/inc/geometries/mpi_curvilinear.h
+++ b/inc/geometries/mpi_curvilinear.h
@@ -30,7 +30,7 @@ struct CurvilinearMPIGrid3d : public dg::MPIGrid3d
     typedef dg::CurvilinearMPIGrid2d<LocalContainer> perpendicular_grid; //!< the two-dimensional grid
     typedef typename MPIContainer::container_type LocalContainer; //!< the local container type
 
-    CurvilinearMPIGrid3d( geo::aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx, MPI_Comm comm): 
+    CurvilinearMPIGrid3d( const geo::aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx, MPI_Comm comm): 
         dg::MPIGrid3d( 0, generator->width(), 0., generator->height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, dg::PER, dg::PER, comm),
         g( generator, n,Nx, Ny, local().Nz(), bcx)
     {
@@ -58,7 +58,9 @@ struct CurvilinearMPIGrid3d : public dg::MPIGrid3d
     const MPIContainer& vol()const{return vol_;}
     const MPIContainer& perpVol()const{return vol2d_;}
     const dg::CurvilinearGrid3d<LocalContainer>& global() const {return g;}
-    geo::aGenerator* const generator() const{return g.generator();}
+    const geo::aGenerator* generator() const{return g.generator();}
+    bool isOrthogonal() const { return g.isOrthogonal();}
+    bool isConformal() const { return g.isConformal();}
     private:
     void divide_and_conquer( )
     {
@@ -105,7 +107,7 @@ struct CurvilinearMPIGrid2d : public dg::MPIGrid2d
     typedef dg::CurvilinearCylindricalTag metric_category; 
     typedef typename MPIContainer::container_type LocalContainer; //!< the local container type
 
-    CurvilinearMPIGrid2d( geo::aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx, MPI_Comm comm2d): 
+    CurvilinearMPIGrid2d( const geo::aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx, MPI_Comm comm2d): 
         dg::MPIGrid2d( 0, generator->width(), 0., generator->height(), n, Nx, Ny, bcx, dg::PER, comm2d),
         g_( generator, n,Nx, Ny, bcx)
     {
@@ -152,7 +154,9 @@ struct CurvilinearMPIGrid2d : public dg::MPIGrid2d
     const MPIContainer& vol()const{return vol2d_;}
     const MPIContainer& perpVol()const{return vol2d_;}
     const dg::CurvilinearGrid2d<LocalContainer>& global() const {return g_;}
-    geo::aGenerator* const generator() const{return g.generator();}
+    const geo::aGenerator* generator() const{return g.generator();}
+    bool isOrthogonal() const { return g.isOrthogonal();}
+    bool isConformal() const { return g.isConformal();}
     private:
     MPI_Comm get_reduced_comm( MPI_Comm src)
     {
-- 
GitLab


From f943127d5f684e192d00df4446c4e271f3a5f18c Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Fri, 21 Jul 2017 13:22:52 +0200
Subject: [PATCH 063/453] made some programs compile again in geometries

---
 inc/dg/geometry/refined_grid.h                 |  4 +++-
 inc/geometries/ds.h                            | 12 ++++++------
 inc/geometries/fieldaligned.h                  | 18 +++++++++---------
 inc/geometries/flux_t.cu                       |  7 ++-----
 inc/geometries/mpi_fieldaligned.h              | 18 +++++++++---------
 inc/geometries/orthogonalX_t.cu                | 10 +++++-----
 inc/geometries/refined_curvilinear.h           | 15 +++++++--------
 inc/geometries/ribeiro.h                       |  9 +--------
 inc/geometries/ribeiro_mpit.cu                 |  6 ++----
 inc/geometries/ribeiro_t.cu                    |  6 ++----
 ...{orthogonal_t.cu => simple_orthogonal_t.cu} |  6 +++---
 11 files changed, 49 insertions(+), 62 deletions(-)
 rename inc/geometries/{orthogonal_t.cu => simple_orthogonal_t.cu} (99%)

diff --git a/inc/dg/geometry/refined_grid.h b/inc/dg/geometry/refined_grid.h
index c132f766f..a69d7fa9d 100644
--- a/inc/dg/geometry/refined_grid.h
+++ b/inc/dg/geometry/refined_grid.h
@@ -542,7 +542,7 @@ RefinedGrid2d::RefinedGrid2d( const dg::RefinedGrid3d& g) :
 template<class container>
 struct CartesianRefinedGrid2d : public dg::RefinedGrid2d
 {
-    typedef OrthogonalTag metric_category; 
+    typedef CurvilinearCylindricalTag metric_category; 
     CartesianRefinedGrid2d( unsigned multiple_x, unsigned multiple_y, double x0, double x1, double y0, double y1, unsigned n, unsigned n_old, unsigned Nx, unsigned Ny, bc bcx = PER, bc bcy = PER): dg::RefinedGrid2d(multiple_x, multiple_y,x0,x1,y0,y1,n,n_old,Nx,Ny,bcx,bcy), g_assoc_(x0,x1,y0,y1,n_old,Nx,Ny,bcx,bcy){ 
         dg::blas1::transfer( weightsX(), g_xx_);
         dg::blas1::transfer( weightsY(), g_yy_);
@@ -560,6 +560,8 @@ struct CartesianRefinedGrid2d : public dg::RefinedGrid2d
     const container& g_yy()const{return g_yy_;}
     const container& vol()const{return vol2d_;}
     const container& perpVol()const{return vol2d_;}
+    bool isOrthogonal()const{return true;}
+    bool isConformal()const{return false;}
     private:
     container g_xx_, g_yy_, vol2d_;
     dg::CartesianGrid2d g_assoc_;
diff --git a/inc/geometries/ds.h b/inc/geometries/ds.h
index b818e3d63..e0a506ba2 100644
--- a/inc/geometries/ds.h
+++ b/inc/geometries/ds.h
@@ -1,14 +1,14 @@
 #pragma once
 
-#include "blas.h"
-#include "geometry.h"
-#include "backend/derivatives.h"
-#include "geometry/fieldaligned.h"
+#include "dg/blas.h"
+#include "dg/geometry.h"
+#include "dg/backend/derivatives.h"
+#include "fieldaligned.h"
 #ifdef MPI_VERSION
 #include "backend/mpi_derivatives.h"
-#include "geometry/mpi_fieldaligned.h"
+#include "mpi_fieldaligned.h"
 #endif //MPI_VERSION
-#include "../geometries/magnetic_field.h"
+#include "magnetic_field.h"
 
 /*!@file 
  *
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index 91907195b..8e255503a 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -3,15 +3,15 @@
 #include <cusp/transpose.h>
 #include <cusp/csr_matrix.h>
 
-#include "../backend/grid.h"
-#include "../blas.h"
-#include "../backend/interpolation.cuh"
-#include "../backend/functions.h"
-
-#include "../functors.h"
-#include "../nullstelle.h"
-#include "../runge_kutta.h"
-#include "../../geometries/magnetic_field.h"
+#include "dg/backend/grid.h"
+#include "dg/blas.h"
+#include "dg/backend/interpolation.cuh"
+#include "dg/backend/functions.h"
+
+#include "dg/functors.h"
+#include "dg/nullstelle.h"
+#include "dg/runge_kutta.h"
+#include "magnetic_field.h"
 
 namespace dg{
 
diff --git a/inc/geometries/flux_t.cu b/inc/geometries/flux_t.cu
index b4d3583bd..be4234c13 100644
--- a/inc/geometries/flux_t.cu
+++ b/inc/geometries/flux_t.cu
@@ -9,12 +9,11 @@
 #include "dg/functors.h"
 
 #include "dg/backend/timer.cuh"
-#include "orthogonal.h"
 #include "curvilinear.h"
 //#include "guenther.h"
 #include "solovev.h"
 #include "flux.h"
-#include "dg/ds.h"
+//#include "ds.h"
 #include "init.h"
 
 #include "file/nc_utilities.h"
@@ -43,8 +42,6 @@ double sineX( double x, double y) {return sin(x)*sin(y);}
 double cosineX( double x, double y) {return cos(x)*sin(y);}
 double sineY( double x, double y) {return sin(x)*sin(y);}
 double cosineY( double x, double y) {return sin(x)*cos(y);}
-typedef dg::FieldAligned< dg::CurvilinearGrid3d<dg::HVec> , dg::IHMatrix, dg::HVec> DFA;
-//typedef dg::FieldAligned< OrthogonalGrid3d<dg::HVec> , dg::IHMatrix, dg::HVec> DFA;
 
 int main( int argc, char* argv[])
 {
@@ -78,7 +75,7 @@ int main( int argc, char* argv[])
     MagneticField c( gp);
     dg::geo::FluxGenerator<Psip, PsipR, PsipZ, PsipRR, PsipRZ, PsipZZ, Ipol, IpolR, IpolZ>
         flux( c.psip, c.psipR, c.psipZ, c.psipRR, c.psipRZ, c.psipZZ, c.ipol, c.ipolR, c.ipolZ, psi_0, psi_1, gp.R_0, 0., 1);
-    dg::CurvilinearGrid3d<dg::HVec> g3d(flux, n, Nx, Ny,Nz, dg::DIR);
+    dg::CurvilinearGrid3d<dg::HVec> g3d(&flux, n, Nx, Ny,Nz, dg::DIR);
     dg::CurvilinearGrid2d<dg::HVec> g2d = g3d.perp_grid();
     //OrthogonalGrid3d<dg::HVec> g3d(gp, psi_0, psi_1, n, Nx, Ny,Nz, dg::DIR);
     //OrthogonalGrid2d<dg::HVec> g2d = g3d.perp_grid();
diff --git a/inc/geometries/mpi_fieldaligned.h b/inc/geometries/mpi_fieldaligned.h
index ba1ea9d6b..ab501d5d4 100644
--- a/inc/geometries/mpi_fieldaligned.h
+++ b/inc/geometries/mpi_fieldaligned.h
@@ -1,15 +1,15 @@
 #pragma once
 
 #include "fieldaligned.h"
-#include "../backend/grid.h"
-#include "../backend/mpi_evaluation.h"
-#include "../backend/mpi_matrix.h"
-#include "../backend/mpi_matrix_blas.h"
-#include "../backend/mpi_collective.h"
-#include "../backend/mpi_grid.h"
-#include "../backend/interpolation.cuh"
-#include "../backend/functions.h"
-#include "../runge_kutta.h"
+#include "dg/backend/grid.h"
+#include "dg/backend/mpi_evaluation.h"
+#include "dg/backend/mpi_matrix.h"
+#include "dg/backend/mpi_matrix_blas.h"
+#include "dg/backend/mpi_collective.h"
+#include "dg/backend/mpi_grid.h"
+#include "dg/backend/interpolation.cuh"
+#include "dg/backend/functions.h"
+#include "dg/runge_kutta.h"
 
 namespace dg{
  
diff --git a/inc/geometries/orthogonalX_t.cu b/inc/geometries/orthogonalX_t.cu
index 03a5493a9..bf298bd73 100644
--- a/inc/geometries/orthogonalX_t.cu
+++ b/inc/geometries/orthogonalX_t.cu
@@ -12,7 +12,7 @@
 #include "solovev.h"
 #include "taylor.h"
 //#include "guenther.h"
-#include "orthogonalX.h"
+#include "curilinearX.h"
 #include "refined_orthogonalX.h"
 #include "separatrix_orthogonal.h"
 #include "dg/ds.h"
@@ -34,7 +34,7 @@ struct ZCutter
 };
 double sine( double x) {return sin(x);}
 double cosine( double x) {return cos(x);}
-typedef dg::FieldAligned< dg::OrthogonalGridX3d<dg::HVec> , dg::IHMatrix, dg::HVec> HFA;
+typedef dg::FieldAligned< dg::CurvilinearGridX3d<dg::HVec> , dg::IHMatrix, dg::HVec> HFA;
 
 //using namespace dg::geo::solovev;
 using namespace dg::geo::taylor;
@@ -130,9 +130,9 @@ int main( int argc, char* argv[])
     dg::geo::SeparatrixOrthogonal<Psip,PsipR,PsipZ,LaplacePsip> generator(c.psip, c.psipR, c.psipZ, c.laplacePsip, psi_0, R_X,Z_X, R0, Z0,0);
     //dg::geo::SimpleOrthogonalX<Psip,PsipR,PsipZ,LaplacePsip> generator(c.psip, c.psipR, c.psipZ, c.laplacePsip, psi_0, R_X,Z_X, R0, Z0,0);
     //dg::OrthogonalGridX3d<dg::HVec> g3d(generator, psi_0, fx_0, fy_0, n, Nx, Ny,Nz, dg::DIR, dg::NEU);
-    //dg::OrthogonalGridX2d<dg::HVec> g2d = g3d.perp_grid();
-    dg::OrthogonalRefinedGridX3d<dg::HVec> g3d(add_x, add_y, 1,1, generator, psi_0, fx_0, fy_0, n, n, Nx, Ny,Nz, dg::DIR, dg::NEU);
-    dg::OrthogonalRefinedGridX2d<dg::HVec> g2d = g3d.perp_grid();
+    //dg::CurvilinearGridX2d<dg::HVec> g2d = g3d.perp_grid();
+    dg::CurvilinearRefinedGridX3d<dg::HVec> g3d(add_x, add_y, 1,1, generator, psi_0, fx_0, fy_0, n, n, Nx, Ny,Nz, dg::DIR, dg::NEU);
+    dg::CurvilinearRefinedGridX2d<dg::HVec> g2d = g3d.perp_grid();
     t.toc();
     dg::GridX3d g3d_periodic(g3d.x0(), g3d.x1(), g3d.y0(), g3d.y1(), g3d.z0(), g3d.z1(), g3d.fx(), g3d.fy(), g3d.n(), g3d.Nx(), g3d.Ny(), 2); 
     std::cout << "Construction took "<<t.diff()<<"s"<<std::endl;
diff --git a/inc/geometries/refined_curvilinear.h b/inc/geometries/refined_curvilinear.h
index 68860d8b5..062b7899b 100644
--- a/inc/geometries/refined_curvilinear.h
+++ b/inc/geometries/refined_curvilinear.h
@@ -23,8 +23,7 @@ struct CurvilinearRefinedGrid3d : public dg::RefinedGrid3d
     typedef dg::CurvilinearCylindricalTag metric_category; //!< metric tag
     typedef CurvilinearRefinedGrid2d<container> perpendicular_grid; //!< the two-dimensional grid type
 
-    template<class Generator>
-    CurvilinearRefinedGrid3d( unsigned multiple_x, unsigned multiple_y, const Generator& generator, unsigned n, unsigned n_old, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx): 
+    CurvilinearRefinedGrid3d( unsigned multiple_x, unsigned multiple_y, const geo::aGenerator* generator, unsigned n, unsigned n_old, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx): 
         dg::RefinedGrid3d( multiple_x, multiple_y, 0, 1, 0., 2.*M_PI, 0., 2.*M_PI, n, n_old, Nx, Ny, Nz, bcx, dg::PER, dg::PER),
         g_assoc_( generator, n_old, Nx, Ny, Nz, bcx)
     { 
@@ -46,16 +45,15 @@ struct CurvilinearRefinedGrid3d : public dg::RefinedGrid3d
     const container& vol()const{return vol_;}
     const container& perpVol()const{return vol2d_;}
     private:
-    template< class Generator>
-    void construct( Generator generator)
+    void construct( const geo::aGenerator* generator)
     {
-        init_X_boundaries( 0., generator.width());
+        init_X_boundaries( 0., generator->width());
         unsigned sizeY = this->n()*this->Ny();
         unsigned sizeX = this->n()*this->Nx();
         thrust::host_vector<double> y_vec(sizeY), x_vec(sizeX);
         for(unsigned i=0; i<sizeY; i++) y_vec[i] = this->abscissasY()[i*sizeX];
         for(unsigned i=0; i<sizeX; i++) x_vec[i] = this->abscissasX()[i];
-        generator( x_vec, y_vec, r_, z_, xr_, xz_, yr_, yz_);
+        (*generator)( x_vec, y_vec, r_, z_, xr_, xz_, yr_, yz_);
         lift3d( ); //lift to 3D grid
         construct_metric();
     }
@@ -110,8 +108,7 @@ struct CurvilinearRefinedGrid2d : public dg::RefinedGrid2d
 {
     typedef dg::CurvilinearCylindricalTag metric_category;
 
-    template< class Generator>
-    CurvilinearRefinedGrid2d( unsigned multiple_x, unsigned multiple_y, const Generator& generator, unsigned n, unsigned n_old, unsigned Nx, unsigned Ny, dg::bc bcx):
+    CurvilinearRefinedGrid2d( unsigned multiple_x, unsigned multiple_y, const geo::aGenerator* generator, unsigned n, unsigned n_old, unsigned Nx, unsigned Ny, dg::bc bcx):
         dg::RefinedGrid2d( multiple_x, multiple_y, 0, 1., 0., 2*M_PI, n,n_old,Nx,Ny, bcx, dg::PER),
         g_assoc_( generator, n_old, Nx, Ny, bcx) 
     {
@@ -149,6 +146,8 @@ struct CurvilinearRefinedGrid2d : public dg::RefinedGrid2d
     const container& g_xy()const{return g_xy_;}
     const container& vol()const{return vol2d_;}
     const container& perpVol()const{return vol2d_;}
+    bool isOrthogonal() const{return false;}
+    bool isConformal() const{return false;}
     private:
     thrust::host_vector<double> r_, z_, xr_, xz_, yr_, yz_; 
     container g_xx_, g_xy_, g_yy_, vol2d_;
diff --git a/inc/geometries/ribeiro.h b/inc/geometries/ribeiro.h
index cb37fb542..8a826d2a8 100644
--- a/inc/geometries/ribeiro.h
+++ b/inc/geometries/ribeiro.h
@@ -238,12 +238,6 @@ struct Ribeiro : public aGenerator
      * @return 2pi 
      */
     double height() const{return 2.*M_PI;}
-    /**
-     * @brief The vector f(x)
-     *
-     * @return f(x)
-     */
-    thrust::host_vector<double> fx() const{ return fx_;}
 
     void operator()( 
          const thrust::host_vector<double>& zeta1d, 
@@ -257,7 +251,7 @@ struct Ribeiro : public aGenerator
     {
         //compute psi(x) for a grid on x and call construct_rzy for all psi
         ribeiro::detail::FieldFinv<Psi, PsiX, PsiY> fpsiMinv_(psi_, psiX_, psiY_, x0_,y0_, 500, mode_);
-        thrust::host_vector<double> psi_x;
+        thrust::host_vector<double> psi_x, fx_;
         dg::geo::detail::construct_psi_values( fpsiMinv_, psi0_, psi1_, 0., zeta1d, lx_, psi_x, fx_);
 
         //std::cout << "In grid function:\n";
@@ -291,7 +285,6 @@ struct Ribeiro : public aGenerator
     PsiXX psiXX_;
     PsiXY psiXY_;
     PsiYY psiYY_;
-    thrust::host_vector<double> fx_;
     double lx_, x0_, y0_, psi0_, psi1_;
     int mode_; //0 = ribeiro, 1 = equalarc
 };
diff --git a/inc/geometries/ribeiro_mpit.cu b/inc/geometries/ribeiro_mpit.cu
index 467b255f5..80ca7dc9b 100644
--- a/inc/geometries/ribeiro_mpit.cu
+++ b/inc/geometries/ribeiro_mpit.cu
@@ -15,11 +15,9 @@
 //#include "guenther.h"
 #include "solovev.h"
 #include "mpi_curvilinear.h"
-#include "mpi_orthogonal.h"
-#include "mpi_conformal.h"
 #include "ribeiro.h"
 #include "simple_orthogonal.h"
-#include "dg/ds.h"
+//#include "ds.h"
 #include "init.h"
 
 #include <netcdf_par.h>
@@ -70,7 +68,7 @@ int main( int argc, char* argv[])
     MagneticField c( gp);
     dg::geo::Ribeiro<Psip, PsipR, PsipZ, PsipRR, PsipRZ, PsipZZ>
         ribeiro( c.psip, c.psipR, c.psipZ, c.psipRR, c.psipRZ, c.psipZZ, psi_0, psi_1, gp.R_0, 0., 1);
-    dg::CurvilinearMPIGrid3d<dg::HVec> g3d(ribeiro, n, Nx, Ny,Nz, dg::DIR,comm);
+    dg::CurvilinearMPIGrid3d<dg::HVec> g3d(&ribeiro, n, Nx, Ny,Nz, dg::DIR,comm);
     dg::CurvilinearMPIGrid2d<dg::HVec> g2d = g3d.perp_grid();
     //dg::SimpleOrthogonal<Psip, PsipR, PsipZ, LaplacePsip> 
     //    orthogonal( c.psip, c.psipR, c.psipZ, c.laplacePsip, psi_0, psi_1, gp.R_0, 0., 1);
diff --git a/inc/geometries/ribeiro_t.cu b/inc/geometries/ribeiro_t.cu
index 8ed436dae..29f601833 100644
--- a/inc/geometries/ribeiro_t.cu
+++ b/inc/geometries/ribeiro_t.cu
@@ -14,7 +14,7 @@
 #include "ribeiro.h"
 #include "curvilinear.h"
 #include "refined_curvilinear.h"
-#include "dg/ds.h"
+//#include "ds.h"
 #include "init.h"
 
 #include "file/nc_utilities.h"
@@ -42,8 +42,6 @@ double sineX( double x, double y) {return sin(x)*sin(y);}
 double cosineX( double x, double y) {return cos(x)*sin(y);}
 double sineY( double x, double y) {return sin(x)*sin(y);}
 double cosineY( double x, double y) {return sin(x)*cos(y);}
-typedef dg::FieldAligned< dg::CurvilinearGrid3d<dg::HVec> , dg::IHMatrix, dg::HVec> DFA;
-//typedef dg::FieldAligned< dg::ribeiro::RefinedGrid3d<dg::HVec> , dg::IHMatrix, dg::HVec> DFA;
 
 int main( int argc, char* argv[])
 {
@@ -82,7 +80,7 @@ int main( int argc, char* argv[])
         ribeiro( c.psip, c.psipR, c.psipZ, c.psipRR, c.psipRZ, c.psipZZ, psi_0, psi_1, gp.R_0, 0., 1);
     //dg::CurvilinearGrid3d<dg::HVec> g3d(ribeiro, n, Nx, Ny,Nz, dg::DIR);
     //dg::CurvilinearGrid2d<dg::HVec> g2d = g3d.perp_grid();
-    dg::CurvilinearRefinedGrid3d<dg::HVec> g3d(multiple_x, multiple_y, ribeiro, n_ref, n, Nx, Ny,Nz, dg::DIR);
+    dg::CurvilinearRefinedGrid3d<dg::HVec> g3d(multiple_x, multiple_y, &ribeiro, n_ref, n, Nx, Ny,Nz, dg::DIR);
     dg::CurvilinearRefinedGrid2d<dg::HVec> g2d = g3d.perp_grid();
     dg::Grid2d g2d_periodic(g2d.x0(), g2d.x1(), g2d.y0(), g2d.y1(), g2d.n(), g2d.Nx(), g2d.Ny()+1); 
     t.toc();
diff --git a/inc/geometries/orthogonal_t.cu b/inc/geometries/simple_orthogonal_t.cu
similarity index 99%
rename from inc/geometries/orthogonal_t.cu
rename to inc/geometries/simple_orthogonal_t.cu
index 4ec323358..2bb3b05ae 100644
--- a/inc/geometries/orthogonal_t.cu
+++ b/inc/geometries/simple_orthogonal_t.cu
@@ -11,11 +11,11 @@
 #include "dg/backend/timer.cuh"
 //#include "guenther.h"
 #include "solovev.h"
-#include "orthogonal.h"
-#include "refined_orthogonal.h"
+#include "curvilinear.h"
+#include "refined_curvilinear.h"
 
 #include "simple_orthogonal.h"
-#include "dg/ds.h"
+#include "ds.h"
 #include "init.h"
 #include "testfunctors.h"
 
-- 
GitLab


From b76807319e15c987b93a5b649a3ff9d6d70c8152 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Fri, 21 Jul 2017 13:24:12 +0200
Subject: [PATCH 064/453] renamed orthogonalX_t.cu to
 separatrix_orthogonal_t.cu

---
 inc/geometries/{orthogonalX_t.cu => separatrix_orthogonal_t.cu} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename inc/geometries/{orthogonalX_t.cu => separatrix_orthogonal_t.cu} (100%)

diff --git a/inc/geometries/orthogonalX_t.cu b/inc/geometries/separatrix_orthogonal_t.cu
similarity index 100%
rename from inc/geometries/orthogonalX_t.cu
rename to inc/geometries/separatrix_orthogonal_t.cu
-- 
GitLab


From 93b82139fc50db571c2bcfafda70661460a01192 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Fri, 21 Jul 2017 19:37:28 +0200
Subject: [PATCH 065/453] implemented missing geometry_traits function to avoid
 isConformal in Cylindrical

---
 inc/dg/geometry/cylindrical.h     | 2 +-
 inc/dg/geometry/geometry_traits.h | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/inc/dg/geometry/cylindrical.h b/inc/dg/geometry/cylindrical.h
index 4e8735b7f..28839ab2c 100644
--- a/inc/dg/geometry/cylindrical.h
+++ b/inc/dg/geometry/cylindrical.h
@@ -45,7 +45,7 @@ struct CylindricalGrid3d : public dg::Grid3d
      */
     const container& vol()const {return R_;}
     void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz){
-        dg::Grid3d::set(new_n,new_Ny,new_Nz);
+        dg::Grid3d::set(new_n,new_Nx,new_Ny,new_Nz);
         R_=dg::evaluate(dg::cooX3d, *this);
     }
     private:
diff --git a/inc/dg/geometry/geometry_traits.h b/inc/dg/geometry/geometry_traits.h
index 03c099c04..a19c2c962 100644
--- a/inc/dg/geometry/geometry_traits.h
+++ b/inc/dg/geometry/geometry_traits.h
@@ -105,6 +105,12 @@ void doRaisePerpIndex( container& in1, container& in2, container& out1, containe
     dg::blas1::pointwiseDot( 1., g.g_yy(), in2, 1., out2); //gyy*v_y
 };
 
+template <class container, class Geometry>
+void doVolRaisePerpIndex( container& in1, container& in2, container& out1, container& out2, const Geometry& g, OrthonormalCylindricalTag)
+{
+    in1.swap( out1);
+    in2.swap( out2);
+};
 template <class container, class Geometry>
 void doVolRaisePerpIndex( container& in1, container& in2, container& out1, container& out2, const Geometry& g, CurvilinearCylindricalTag)
 {
-- 
GitLab


From b60ca4dd2022a257e1ec4bd77545bfe135025220 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sun, 23 Jul 2017 17:55:59 +0200
Subject: [PATCH 066/453] made magnetic_field polymorphic

---
 inc/geometries/magnetic_field.h | 449 ++++++++++++++------------------
 inc/geometries/solovev.h        |  59 +++--
 2 files changed, 224 insertions(+), 284 deletions(-)

diff --git a/inc/geometries/magnetic_field.h b/inc/geometries/magnetic_field.h
index 62257e204..51f91a4af 100644
--- a/inc/geometries/magnetic_field.h
+++ b/inc/geometries/magnetic_field.h
@@ -9,45 +9,116 @@ namespace dg
 {
 namespace geo
 {
+
+struct aBinaryFunctor
+{
+    virtual double operator()(double R, double Z) const=0;
+    double operator()(double R, double Z, double phi)
+    {
+        return this->operator()(R,Z);
+    }
+    virtual aBinaryFunctor* clone()const=0;
+    virtual ~aBinaryFunctor(){}
+};
+
+//https://katyscode.wordpress.com/2013/08/22/c-polymorphic-cloning-and-the-crtp-curiously-recurring-template-pattern/
+template<class Derived<
+struct aCloneableBinaryFunctor : public a BinaryFunctor
+{
+    virtual aBinaryFunctor* clone() const
+    {
+        return new Derived(statimag_cast<Derived const &>(*this));
+    }
+};
+/**
+ * @brief Contains all solovev fields (models aTokamakMagneticField)
+ */
+struct aTokamakMagneticField
+{
+    aTokamakMagneticField( const aTokamakMagneticField& mag)
+    {
+        R0_ = mag.R0_;
+        for( unsigned i=0; i<p_.size(); i++)
+            p_[i] = mag.p_[i]->clone();
+    }
+    aTokamakMagneticField& operator=( const aTokamakMagneticField& mag)
+    {
+        aTokamakMagneticField temp(mag);
+        std::swap( temp.p_, p_);
+        return *this;
+    }
+    double R0()const {return R0_;}
+    const aBinaryFunctor& psip()const{return *p_[0];}
+    const aBinaryFunctor& psipR()const{return *p_[1];}
+    const aBinaryFunctor& psipZ()const{return *p_[2];}
+    const aBinaryFunctor& psipRR()const{return *p_[3];}
+    const aBinaryFunctor& psipRZ()const{return *p_[4];}
+    const aBinaryFunctor& psipZZ()const{return *p_[5];}
+    const aBinaryFunctor& laplacePsip()const{return *p_[6];}
+    const aBinaryFunctor& ipol()const{return *p_[7];}
+    const aBinaryFunctor& ipolR()const{return *p_[8];}
+    const aBinaryFunctor& ipolZ()const{return *p_[9];}
+
+    protected:
+    ~aTokamakMagneticField(){
+        for( unsigned i=0; i<p_.size(); i++)
+            delete p_[i];
+    }
+    aTokamakMagneticField( double R0,
+        aBinaryFunctor* psip,
+        aBinaryFunctor* psipR,
+        aBinaryFunctor* psipZ,
+        aBinaryFunctor* psipRR,
+        aBinaryFunctor* psipRZ,
+        aBinaryFunctor* psipZZ,
+        aBinaryFunctor* laplacePsip,
+        aBinaryFunctor* ipol,
+        aBinaryFunctor* ipolR,
+        aBinaryFunctor* ipolZ
+        ):p_(10){ 
+            p_[0] = psip,
+            p_[1] = psipR,
+            p_[2] = psipZ,
+            p_[3] = psipRR,
+            p_[4] = psipRZ,
+            p_[5] = psipZZ,
+            p_[6] = laplacePsip,
+            p_[7] = psip,
+            p_[8] = ipolR,
+            p_[9] = ipolZ,
+        }
+    private:
+    double R0_;
+    std::vector<aBinaryFunctor*> p_;
+};
+
 ///@addtogroup magnetic
 ///@{
 
 /**
  * @brief \f[   |B| = R_0\sqrt{I^2+(\nabla\psi)^2}/R   \f]
- @tparam MagneticField models aTokamakMagneticField
  */ 
-template<class MagneticField>
-struct Bmodule
+struct Bmodule : public aCloneableBinaryFunctor<Bmodule>
 {
-    Bmodule( const MagneticField& c):  R_0_(c.R_0), c_(c)  { }
+    Bmodule( const aTokamakMagneticField& mag): mag_(mag)  { }
     /**
     * @brief \f[   \hat{B} \f]
     */ 
     double operator()(double R, double Z) const
     {    
-        double psipR = c_.psipR(R,Z), psipZ = c_.psipZ(R,Z), ipol = c_.ipol(R,Z);
-        return R_0_/R*sqrt(ipol*ipol+psipR*psipR +psipZ*psipZ);
-    }
-    /**
-     * @brief == operator()(R,Z)
-     */ 
-    double operator()(double R, double Z, double phi) const
-    {    
-        return operator()(R,Z);
+        double psipR = mag_.psipR()(R,Z), psipZ = mag_.psipZ()(R,Z), ipol = mag_.ipol()(R,Z);
+        return mag_.R0()/R*sqrt(ipol*ipol+psipR*psipR +psipZ*psipZ);
     }
   private:
-    double R_0_;
-    MagneticField c_;
+    aTokamakMagneticField mag_;
 };
 
 /**
  * @brief \f[  |B|^{-1} = R/R_0\sqrt{I^2+(\nabla\psi)^2}    \f]
- @tparam MagneticField models aTokamakMagneticField
  */ 
-template<class MagneticField>
-struct InvB
+struct InvB : public aCloneableBinaryFunctor<InvB>
 {
-    InvB(  const MagneticField& c):  R_0_(c.R_0), c_(c)  { }
+    InvB(  const aTokamakMagneticField& mag): mag_(mag){ }
     /**
     * @brief \f[   \frac{1}{\hat{B}} = 
         \frac{\hat{R}}{\hat{R}_0}\frac{1}{ \sqrt{ \hat{I}^2  + \left(\frac{\partial \hat{\psi}_p }{ \partial \hat{R}}\right)^2
@@ -55,59 +126,40 @@ struct InvB
     */ 
     double operator()(double R, double Z) const
     {    
-        double psipR = c_.psipR(R,Z), psipZ = c_.psipZ(R,Z), ipol = c_.ipol(R,Z);
-        return R/(R_0_*sqrt(ipol*ipol + psipR*psipR +psipZ*psipZ)) ;
-    }
-    /**
-     * @brief == operator()(R,Z)
-     */ 
-    double operator()(double R, double Z, double phi) const
-    {    
-        return operator()(R,Z);
+        double psipR = mag_.psipR()(R,Z), psipZ = mag_.psipZ()(R,Z), ipol = mag_.ipol()(R,Z);
+        return R/(mag_.R0()*sqrt(ipol*ipol + psipR*psipR +psipZ*psipZ)) ;
     }
   private:
-    double R_0_;
-    MagneticField c_;
+    aTokamakMagneticField mag_;
 };
 
 /**
  * @brief \f[   \ln{|B|}  \f]
- @tparam MagneticField models aTokamakMagneticField
  */ 
-template<class MagneticField>
-struct LnB
+struct LnB : public aCloneableBinaryFunctor<LnB>
 {
-    LnB(  const MagneticField& c):  R_0_(c.R_0), c_(c)  { }
-/**
- * @brief \f[   \ln{(   \hat{B})} = \ln{\left[
-      \frac{\hat{R}_0}{\hat{R}} \sqrt{ \hat{I}^2  + \left(\frac{\partial \hat{\psi}_p }{ \partial \hat{R}}\right)^2
-      + \left(\frac{\partial \hat{\psi}_p }{ \partial \hat{Z}}\right)^2} \right] } \f]
- */ 
-    double operator()(double R, double Z) const
-    {    
-        double psipR = c_.psipR(R,Z), psipZ = c_.psipZ(R,Z), ipol = c_.ipol(R,Z);
-        return log(R_0_/R*sqrt(ipol*ipol + psipR*psipR +psipZ*psipZ)) ;
-    }
+    LnB(const aTokamakMagneticField& mag): mag_(mag) { }
     /**
-     * @brief == operator()(R,Z)
+     * @brief \f[   \ln{(   \hat{B})} = \ln{\left[
+          \frac{\hat{R}_0}{\hat{R}} \sqrt{ \hat{I}^2  + \left(\frac{\partial \hat{\psi}_p }{ \partial \hat{R}}\right)^2
+          + \left(\frac{\partial \hat{\psi}_p }{ \partial \hat{Z}}\right)^2} \right] } \f]
      */ 
-    double operator()(double R, double Z, double phi) const
+    double operator()(double R, double Z) const
     {    
-        return operator()(R,Z);
+        double psipR = mag_.psipR()(R,Z), psipZ = mag_.psipZ()(R,Z), ipol = mag_.ipol()(R,Z);
+        return log(mag_.R0()/R*sqrt(ipol*ipol + psipR*psipR +psipZ*psipZ)) ;
     }
   private:
-    double R_0_;
-    MagneticField c_;
+    aTokamakMagneticField mag_;
 };
 
 /**
  * @brief \f[  \frac{\partial |\hat{B}| }{ \partial \hat{R}}  \f]
- @tparam MagneticField models aTokamakMagneticField
  */ 
-template<class MagneticField>
-struct BR
+struct BR: public aCloneableBinaryFunctor<BR>
+
 {
-    BR(const MagneticField& c):  R_0_(c.R_0), invB_(c), c_(c) { }
+    BR(const aTokamakMagneticField& mag): invB_(mag), mag_(mag) { }
 /**
  * @brief \f[  \frac{\partial \hat{B} }{ \partial \hat{R}} = 
       -\frac{1}{\hat B \hat R}   
@@ -119,30 +171,22 @@ struct BR
     double operator()(double R, double Z) const
     { 
         double Rn;
-        Rn = R/R_0_;
+        Rn = R/mag.R0();
         //sign before A changed to +
         //return -( Rn*Rn/invB_(R,Z)/invB_(R,Z)+ qampl_*qampl_*Rn *A_*psipR_(R,Z) - R  *(psipZ_(R,Z)*psipRZ_(R,Z)+psipR_(R,Z)*psipRR_(R,Z)))/(R*Rn*Rn/invB_(R,Z));
-        return -1./R/invB_(R,Z) + invB_(R,Z)/Rn/Rn*(c_.ipol(R,Z)*c_.ipolR(R,Z) + c_.psipR(R,Z)*c_.psipRR(R,Z) + c_.psipZ(R,Z)*c_.psipRZ(R,Z));
+        return -1./R/invB_(R,Z) + invB_(R,Z)/Rn/Rn*(mag_.ipol()(R,Z)*mag_.ipolR()(R,Z) + mag_.psipR()(R,Z)*mag_.psipRR()(R,Z) + mag_.psipZ()(R,Z)*mag_.psipRZ()(R,Z));
     }
-      /**
-       * @brief == operator()(R,Z)
-       */ 
-    double operator()(double R, double Z, double phi)const{return operator()(R,Z);}
   private:
-    double R_0_;
-    InvB<MagneticField> invB_;
-    MagneticField c_;
+    InvB invB_;
+    aTokamakMagneticField mag_;
 };
 
 /**
  * @brief \f[  \frac{\partial \hat{B} }{ \partial \hat{Z}}  \f]
- @tparam MagneticField models aTokamakMagneticField
  */ 
-template<class MagneticField>
-struct BZ
+struct BZ: public aCloneableBinaryFunctor<BZ>
 {
-
-    BZ(const MagneticField& c ):  R_0_(c.R_0), c_(c), invB_(c) { }
+    BZ(const aTokamakMagneticField& mag ): mag_(mag), invB_(mag) { }
     /**
      * @brief \f[  \frac{\partial \hat{B} }{ \partial \hat{Z}} = 
      \frac{ \hat I \left(\frac{\partial \hat I}{\partial\hat Z}    \right)+
@@ -152,29 +196,22 @@ struct BZ
     double operator()(double R, double Z) const
     { 
         double Rn;
-        Rn = R/R_0_;
+        Rn = R/mag_.R0();
         //sign before A changed to -
         //return (-qampl_*qampl_*A_/R_0_*psipZ_(R,Z) + psipR_(R,Z)*psipRZ_(R,Z)+psipZ_(R,Z)*psipZZ_(R,Z))/(Rn*Rn/invB_(R,Z));
-        return (invB_(R,Z)/Rn/Rn)*(c_.ipol(R,Z)*c_.ipolZ(R,Z) + c_.psipR(R,Z)*c_.psipRZ(R,Z) + c_.psipZ(R,Z)*c_.psipZZ(R,Z));
+        return (invB_(R,Z)/Rn/Rn)*(mag_.ipol()(R,Z)*mag_.ipolZ(R,Z) + mag_.psipR()(R,Z)*mag_.psipRZ()(R,Z) + mag_.psipZ()(R,Z)*mag_.psipZZ()(R,Z));
     }
-    /**
-     * @brief == operator()(R,Z)
-     */ 
-    double operator()(double R, double Z, double phi)const{return operator()(R,Z);}
   private:
-    double R_0_;
-    MagneticField c_;
-    InvB<MagneticField> invB_; 
+    aTokamakMagneticField mag_;
+    InvB invB_; 
 };
 
 /**
  * @brief \f[ \mathcal{\hat{K}}^{\hat{R}}_{\nabla B} \f]
- @tparam MagneticField models aTokamakMagneticField
  */ 
-template<class MagneticField>
-struct CurvatureNablaBR
+struct CurvatureNablaBR: public aCloneableBinaryFunctor<CurvatureNablaBR>
 {
-    CurvatureNablaBR(const MagneticField& c): invB_(c), bZ_(c) { }
+    CurvatureNablaBR(const aTokamakMagneticField& mag): invB_(mag), bZ_(mag) { }
     /**
      * @brief \f[ \mathcal{\hat{K}}^{\hat{R}}_{\nabla B} =-\frac{1}{ \hat{B}^2}  \frac{\partial \hat{B}}{\partial \hat{Z}}  \f]
      */ 
@@ -182,51 +219,36 @@ struct CurvatureNablaBR
     {
         return -invB_(R,Z)*invB_(R,Z)*bZ_(R,Z); 
     }
-    
-    /**
-     * @brief == operator()(R,Z)
-     */ 
-    double operator()( double R, double Z, double phi) const
-    {
-        return -invB_(R,Z,phi)*invB_(R,Z,phi)*bZ_(R,Z,phi); 
-    }
     private:    
-    InvB<MagneticField>   invB_;
-    BZ<MagneticField> bZ_;    
+    InvB invB_;
+    BZ bZ_;    
 };
 
 /**
  * @brief \f[  \mathcal{\hat{K}}^{\hat{Z}}_{\nabla B}  \f]
- @tparam MagneticField models aTokamakMagneticField
  */ 
-template<class MagneticField>
-struct CurvatureNablaBZ
+struct CurvatureNablaBZ: public aCloneableBinaryFunctor<CurvatureNablaBZ>
 {
-    CurvatureNablaBZ( const MagneticField& c): invB_(c), bR_(c) { }
- /**
- * @brief \f[  \mathcal{\hat{K}}^{\hat{Z}}_{\nabla B} =\frac{1}{ \hat{B}^2}   \frac{\partial \hat{B}}{\partial \hat{R}} \f]
- */    
+    CurvatureNablaBZ( const aTokamakMagneticField& mag): invB_(mag), bR_(mag) { }
+    /**
+     * @brief \f[  \mathcal{\hat{K}}^{\hat{Z}}_{\nabla B} =\frac{1}{ \hat{B}^2}   \frac{\partial \hat{B}}{\partial \hat{R}} \f]
+     */    
     double operator()( double R, double Z) const
     {
         return invB_(R,Z)*invB_(R,Z)*bR_(R,Z);
     }
-    /**
-     * @brief == operator()(R,Z)
-     */ 
-    double operator()( double R, double Z, double phi) const
-    {
-        return invB_(R,Z,phi)*invB_(R,Z,phi)*bR_(R,Z,phi);
-    }
     private:    
-    InvB<MagneticField> invB_;
-    BR<MagneticField> bR_;   
+    InvB invB_;
+    BR bR_;   
 };
 
 /**
  * @brief \f[ \mathcal{\hat{K}}^{\hat{R}}_{\vec{\kappa}}=0 \f]
  */ 
-struct CurvatureKappaR
+struct CurvatureKappaR: public aCloneableBinaryFunctor<CurvatureKappaR>
 {
+    CurvatureKappaR( ){ }
+    CurvatureKappaR( const aTokamakMagneticField& mag){ }
     /**
      * @brief \f[ \mathcal{\hat{K}}^{\hat{R}}_{\vec{\kappa}} =0  \f]
      */ 
@@ -234,257 +256,176 @@ struct CurvatureKappaR
     {
         return  0.;
     }
-    
-    /**
-     * @brief == operator()(R,Z)
-     */ 
-    double operator()( double R, double Z, double phi) const
-    {
-        return 0.;
-    }
 };
 
 /**
  * @brief \f[  \mathcal{\hat{K}}^{\hat{Z}}_{\vec{\kappa}}  \f]
- * @tparam MagneticField models aTokamakMagneticField
  */ 
-template<class MagneticField>
-struct CurvatureKappaZ
+struct CurvatureKappaZ: public aCloneableBinaryFunctor<CurvatureKappaZ>
 {
-    CurvatureKappaZ( const MagneticField c):
-        invB_(c) { }
- /**
- * @brief \f[  \mathcal{\hat{K}}^{\hat{Z}}_{\vec{\kappa}} = - \frac{1}{\hat{R} \hat{B}} \f]
- */    
+    CurvatureKappaZ( const aTokamakMagneticField& mag): invB_(mag) { }
+    /**
+     * @brief \f[  \mathcal{\hat{K}}^{\hat{Z}}_{\vec{\kappa}} = - \frac{1}{\hat{R} \hat{B}} \f]
+     */    
     double operator()( double R, double Z) const
     {
         return -invB_(R,Z)/R;
     }
-    /**
-     * @brief == operator()(R,Z)
-     */ 
-    double operator()( double R, double Z, double phi) const
-    {
-        return -invB_(R,Z,phi)/R;
-    }
     private:    
-    InvB<MagneticField>   invB_;
+    InvB invB_;
 };
 
 /**
  * @brief \f[  \vec{\hat{\nabla}}\cdot \mathcal{\hat{K}}_{\vec{\kappa}}  \f]
- * @tparam MagneticField models aTokamakMagneticField
  */ 
-template<class MagneticField>
-struct DivCurvatureKappa
+struct DivCurvatureKappa: public aCloneableBinaryFunctor<DivCurvatureKappa>
 {
-    DivCurvatureKappa( const MagneticField& c):
-        invB_(c),
-        bZ_(c){ }
- /**
- * @brief \f[  \vec{\hat{\nabla}}\cdot \mathcal{\hat{K}}_{\vec{\kappa}}  = \frac{1}{\hat{R}  \hat{B}^2 } \partial_{\hat{Z}} \hat{B}\f]
- */    
+    DivCurvatureKappa( const aTokamakMagneticField& mag): invB_(mag), bZ_(mag){ }
+    /**
+     * @brief \f[  \vec{\hat{\nabla}}\cdot \mathcal{\hat{K}}_{\vec{\kappa}}  = \frac{1}{\hat{R}  \hat{B}^2 } \partial_{\hat{Z}} \hat{B}\f]
+     */    
     double operator()( double R, double Z) const
     {
         return bZ_(R,Z)*invB_(R,Z)*invB_(R,Z)/R;
     }
-    /**
-     * @brief == operator()(R,Z)
-     */ 
-    double operator()( double R, double Z, double phi) const
-    {
-        return  bZ_(R,Z,phi)*invB_(R,Z,phi)*invB_(R,Z,phi)/R;
-    }
     private:    
-    InvB<MagneticField>   invB_;
-    BZ<MagneticField> bZ_;    
+    InvB invB_;
+    BZ bZ_;    
 };
 
 /**
  * @brief \f[  \hat{\nabla}_\parallel \ln{(\hat{B})} \f]
- @tparam MagneticField models aTokamakMagneticField
  */ 
-template<class MagneticField>
-struct GradLnB
+struct GradLnB: public aCloneableBinaryFunctor<GradLnB>
 {
-    GradLnB( const MagneticField& c): R_0_(c.R_0), c_(c), invB_(c), bR_(c), bZ_(c) { } 
+    GradLnB( const aTokamakMagneticField& mag): mag_(mag), invB_(mag), bR_(mag), bZ_(mag) { } 
     /**
- * @brief \f[  \hat{\nabla}_\parallel \ln{(\hat{B})} = \frac{1}{\hat{R}\hat{B}^2 } \left[ \hat{B}, \hat{\psi}_p\right]_{\hat{R}\hat{Z}} \f]
- */ 
+     * @brief \f[  \hat{\nabla}_\parallel \ln{(\hat{B})} = \frac{1}{\hat{R}\hat{B}^2 } \left[ \hat{B}, \hat{\psi}_p\right]_{\hat{R}\hat{Z}} \f]
+     */ 
     double operator()( double R, double Z) const
     {
         double invB = invB_(R,Z);
-        return R_0_*invB*invB*(bR_(R,Z)*c_.psipZ(R,Z)-bZ_(R,Z)*c_.psipR(R,Z))/R ;
+        return mag_.R0()*invB*invB*(bR_(R,Z)*mag_.psipZ()(R,Z)-bZ_(R,Z)*mag_.psipR()(R,Z))/R ;
     }
-    /**
-     * @brief == operator()(R,Z)
-     */ 
-    double operator()( double R, double Z, double phi)const{return operator()(R,Z);}
     private:
-    double R_0_;
-    MagneticField c_;
-    InvB<MagneticField>   invB_;
-    BR<MagneticField> bR_;
-    BZ<MagneticField> bZ_;   
+    aTokamakMagneticField mag_;
+    InvB invB_;
+    BR bR_;
+    BZ bZ_;   
 };
 
 /**
  * @brief \f[ B_\varphi = R_0I/R^2\f]
- @tparam MagneticField models aTokamakMagneticField
 */
-template<class MagneticField>
-struct FieldP
+struct FieldP: public aCloneableBinaryFunctor<LnB>
 {
-    FieldP( const MagneticField& c): R_0(c.R_0), c_(c){}
+    FieldP( const aTokamakMagneticField& mag): mag_(mag){}
     double operator()( double R, double Z, double phi) const
     {
-        return R_0*c_.ipol(R,Z)/R/R;
+        return mag.R0()*mag_.ipol()(R,Z)/R/R;
     }
     
     private:
-    double R_0;
-    MagneticField c_;
+    aTokamakMagneticField mag_;
 }; 
 
 /**
  * @brief \f[ B_R = R_0\psi_Z /R\f]
- @tparam MagneticField models aTokamakMagneticField
  */
-template<class MagneticField>
-struct FieldR
+struct FieldR: public aCloneableBinaryFunctor<FieldR>
 {
-    FieldR( const MagneticField& c): R_0(c.R_0), c_(c){}
+    FieldR( const aTokamakMagneticField& mag): mag_(mag){}
     double operator()( double R, double Z) const
     {
-        return  R_0/R*c_.psipZ(R,Z);
-    }
-    /**
-     * @brief == operator()(R,Z)
-     */ 
-    double operator()( double R, double Z, double phi) const
-    {
-        return  this->operator()(R,Z);
+        return  mag.R0()/R*mag_.psipZ()(R,Z);
     }
     private:
-    double R_0;
-    MagneticField c_;
+    aTokamakMagneticField mag_;
    
 };
 
 /**
  * @brief \f[ B_Z = -R_0\psi_R /R\f]
- @tparam MagneticField models aTokamakMagneticField
  */
-template<class MagneticField>
-struct FieldZ
+struct FieldZ: public aCloneableBinaryFunctor<FieldZ>
 {
-    FieldZ( const MagneticField& c): R_0(c.R_0), c_(c){}
+    FieldZ( const aTokamakMagneticField& mag): mag_(mag){}
     double operator()( double R, double Z) const
     {
-        return  -R_0/R*c_.psipR(R,Z);
-    }
-    /**
-     * @brief == operator()(R,Z)
-     */ 
-    double operator()( double R, double Z, double phi) const
-    {
-        return  this->operator()(R,Z);
+        return -mag_.R0()/R*mag_.psipR()(R,Z);
     }
     private:
-    double R_0;
-    MagneticField c_;
-   
+    aTokamakMagneticField mag_;
 };
 
 /**
  * @brief \f[  B^{\theta} = B^R\partial_R\theta + B^Z\partial_Z\theta\f]
- @tparam MagneticField models aTokamakMagneticField
  */ 
-template<class MagneticField>
-struct FieldT
+struct FieldT: public aCloneableBinaryFunctor<FieldT>
+
 {
-    FieldT( const MagneticField& c):  R_0_(c.R_0), fieldR_(c), fieldZ_(c){}
-  /**
- * @brief \f[  B^{\theta} = 
- * B^R\partial_R\theta + B^Z\partial_Z\theta\f]
- * where \f$ \theta \f$ is the geometrical poloidal angle.
- */ 
-  double operator()(double R, double Z) const
-  { 
-      double r2 = (R-R_0_)*(R-R_0_) + Z*Z;
-      return fieldR_(R,Z)*(-Z/r2) + fieldZ_(R,Z)*(R-R_0_)/r2;
-  }
+    FieldT( const aTokamakMagneticField& mag):  R_0_(mag.R0()), fieldR_(mag), fieldZ_(mag){}
     /**
-     * @brief == operator()(R,Z)
+     * @brief \f[  B^{\theta} = 
+     * B^R\partial_R\theta + B^Z\partial_Z\theta\f]
+     * where \f$ \theta \f$ is the geometrical poloidal angle.
      */ 
-  double operator()(double R, double Z, double phi) const
-  { 
-      return this->operator()(R,Z);
-  }
-  private:
+    double operator()(double R, double Z) const
+    { 
+        double r2 = (R-R_0_)*(R-R_0_) + Z*Z;
+        return fieldR_(R,Z)*(-Z/r2) + fieldZ_(R,Z)*(R-R_0_)/r2;
+    }
+    private:
     double R_0_;
-    FieldR<MagneticField> fieldR_;
-    FieldZ<MagneticField> fieldZ_;
-
+    FieldR fieldR_;
+    FieldZ fieldZ_;
 };
 
 /**
  * @brief \f[ b_R = B_R/|B|\f]
- @tparam MagneticField models aTokamakMagneticField
  */
-template<class MagneticField>
-struct BHatR
+struct BHatR: public aCloneableBinaryFunctor<BHatR>
 {
-    BHatR( const MagneticField& c): c_(c), R_0(c.R_0), invB_(c){ }
-    double operator()( double R, double Z, double phi) const
+    BHatR( const aTokamakMagneticField& mag): mag_(mag), invB_(mag){ }
+    double operator()( double R, double Z) const
     {
-        return  invB_(R,Z)*R_0/R*c_.psipZ(R,Z);
+        return  invB_(R,Z)*mag_.R0()/R*mag_.psipZ()(R,Z);
     }
     private:
-    MagneticField c_;
-    double R_0;
-    InvB<MagneticField>   invB_;
+    aTokamakMagneticField mag_;
+    InvB invB_;
 
 };
 
 /**
  * @brief \f[ b_Z = B_Z/|B|\f]
- @tparam MagneticField models aTokamakMagneticField
  */
-template<class MagneticField>
-struct BHatZ
+struct BHatZ: public aCloneableBinaryFunctor<BHatZ>
 {
-    BHatZ( const MagneticField& c): c_(c), R_0(c.R_0), invB_(c){ }
-
-    double operator()( double R, double Z, double phi) const
+    BHatZ( const aTokamakMagneticField& mag): mag_(mag), invB_(mag){ }
+    double operator()( double R, double Z) const
     {
-        return  -invB_(R,Z)*R_0/R*c_.psipR(R,Z);
+        return  -invB_(R,Z)*mag_.R0()/R*mag_.psipR()(R,Z);
     }
     private:
-    MagneticField c_;
-    double R_0;
-    InvB<MagneticField>   invB_;
-
+    aTokamakMagneticField mag_;
+    InvB invB_;
 };
 
 /**
  * @brief \f[ b_\varphi = B_\varphi/|B|\f]
- @tparam MagneticField models aTokamakMagneticField
  */
-template<class MagneticField>
-struct BHatP
+struct BHatP: public aCloneableBinaryFunctor<BHatP>
 {
-    BHatP( const MagneticField& c): c_(c), R_0(c.R_0), invB_(c){ }
-    double operator()( double R, double Z, double phi) const
+    BHatP( const aTokamakMagneticField& mag): mag_(mag), invB_(mag){ }
+    double operator()( double R, double Z) const
     {
-        return invB_(R,Z)*R_0*c_.ipol(R,Z)/R/R;
+        return invB_(R,Z)*mag_.R0()*mag_.ipol()(R,Z)/R/R;
     }
     
     private:
-    MagneticField c_;
-    double R_0;
-    InvB<MagneticField>   invB_;
-  
+    aTokamakMagneticField mag_;
+    InvB invB_;
 }; 
 
 ///@} 
diff --git a/inc/geometries/solovev.h b/inc/geometries/solovev.h
index 3a6a61037..6b2c1584d 100644
--- a/inc/geometries/solovev.h
+++ b/inc/geometries/solovev.h
@@ -57,7 +57,7 @@ namespace solovev
       with \f$ \bar R := \frac{ R}{R_0} \f$ and \f$\bar Z := \frac{Z}{R_0}\f$
  *
  */    
-struct Psip
+struct Psip: public aBinaryFunctor
 {
     /**
      * @brief Construct from given geometric parameters
@@ -167,7 +167,7 @@ struct Psip
       \ln{(\bar{R}   )})\Bigg\} \f]
       with \f$ \bar R := \frac{ R}{R_0} \f$ and \f$\bar Z := \frac{Z}{R_0}\f$
  */ 
-struct PsipR
+struct PsipR: public aBinaryFunctor
 {
     /**
      * @brief Construct from given geometric parameters
@@ -230,7 +230,7 @@ struct PsipR
 /**
  * @brief \f[ \frac{\partial^2  \hat{\psi}_p }{ \partial \hat{R}^2}\f]
  */ 
-struct PsipRR
+struct PsipRR: public aBinaryFunctor
 {
     /**
     * @brief Constructor
@@ -302,7 +302,7 @@ struct PsipRR
 /**
  * @brief \f[\frac{\partial \hat{\psi}_p }{ \partial \hat{Z}}\f]
  */ 
-struct PsipZ
+struct PsipZ: public aBinaryFunctor
 {
     PsipZ( GeomParameters gp ): R_0_(gp.R_0), A_(gp.A), c_(gp.c) { }
 /**
@@ -359,7 +359,7 @@ struct PsipZ
 /**
  * @brief \f[ \frac{\partial^2  \hat{\psi}_p }{ \partial \hat{Z}^2}\f]
  */ 
-struct PsipZZ
+struct PsipZZ: public aBinaryFunctor
 {
   PsipZZ( GeomParameters gp): R_0_(gp.R_0), A_(gp.A), c_(gp.c) { }
 /**
@@ -403,7 +403,7 @@ struct PsipZZ
 /**
  * @brief  \f[\frac{\partial^2  \hat{\psi}_p }{ \partial \hat{R} \partial\hat{Z}}\f] 
  */ 
-struct PsipRZ
+struct PsipRZ: public aBinaryFunctor
 {
     PsipRZ( GeomParameters gp ): R_0_(gp.R_0), A_(gp.A), c_(gp.c) { }
 /**
@@ -454,7 +454,7 @@ struct PsipRZ
 /**
  * @brief  \f[\frac{\partial^2  \hat{\psi}_p }{ \partial \hat{R}^2 } + \frac{\partial^2  \hat{\psi}_p }{ \partial \hat{Z}^2 } \f] 
  */
-struct LaplacePsip
+struct LaplacePsip: public aBinaryFunctor
 {
     LaplacePsip( GeomParameters gp ): psipRR_(gp), psipZZ_(gp){}
     /**
@@ -481,7 +481,7 @@ struct LaplacePsip
 /**
  * @brief \f[\hat{I}\f] 
  */ 
-struct Ipol
+struct Ipol: public aBinaryFunctor
 {
     Ipol(  GeomParameters gp ):  R_0_(gp.R_0), A_(gp.A), qampl_(gp.qampl), psip_(gp) { }
     /**
@@ -506,7 +506,7 @@ struct Ipol
 /**
  * @brief \f[\hat I_R\f]
  */
-struct IpolR
+struct IpolR: public aBinaryFunctor
 {
     IpolR(  GeomParameters gp ):  R_0_(gp.R_0), A_(gp.A), qampl_(gp.qampl), psip_(gp), psipR_(gp) { }
     double operator()(double R, double Z) const
@@ -528,7 +528,7 @@ struct IpolR
 /**
  * @brief \f[\hat I_Z\f]
  */
-struct IpolZ
+struct IpolZ: public aBinaryFunctor
 {
     IpolZ(  GeomParameters gp ):  R_0_(gp.R_0), A_(gp.A), qampl_(gp.qampl), psip_(gp), psipZ_(gp) { }
     double operator()(double R, double Z) const
@@ -551,20 +551,19 @@ struct IpolZ
 /**
  * @brief Contains all solovev fields (models aTokamakMagneticField)
  */
-struct MagneticField
+struct MagneticField : dg::geo::aToakamakMagneticField
 {
-    MagneticField( GeomParameters gp): R_0(gp.R_0), psip(gp), psipR(gp), psipZ(gp), psipRR(gp), psipRZ(gp), psipZZ(gp), laplacePsip(gp), ipol(gp), ipolR(gp), ipolZ(gp){}
-    double R_0;
-    Psip psip;
-    PsipR psipR;
-    PsipZ psipZ;
-    PsipRR psipRR;
-    PsipRZ psipRZ;
-    PsipZZ psipZZ;
-    LaplacePsip laplacePsip;
-    Ipol ipol;
-    IpolR ipolR;
-    IpolZ ipolZ;
+    MagneticField( GeomParameters gp): aTokamakMagneticField(gp.R_0, 
+        new Psip(gp), 
+        new PsipR(gp), 
+        new PsipZ(gp), 
+        new PsipRR(gp), 
+        new PsipRZ(gp), 
+        new PsipZZ(gp), 
+        new LaplacePsip(gp), 
+        new Ipol(gp), 
+        new IpolR(gp), 
+        new IpolZ(gp)){}
 };
 ///@}
 
@@ -572,7 +571,7 @@ struct MagneticField
 namespace mod
 {
 
-struct Psip
+struct Psip: public aBinaryFunctor
 {
     Psip( GeomParameters gp): R_X( gp.R_0-1.1*gp.triangularity*gp.a), Z_X(-1.1*gp.elongation*gp.a),
         psip_(gp), psipRR_(gp), psipRZ_(gp), psipZZ_(gp), cauchy_( R_X, Z_X, 50, 50,1.)
@@ -602,7 +601,7 @@ struct Psip
     solovev::PsipZZ psipZZ_;
     dg::Cauchy cauchy_;
 };
-struct PsipR
+struct PsipR: public aBinaryFunctor
 {
     PsipR( GeomParameters gp): R_X( gp.R_0-1.1*gp.triangularity*gp.a), Z_X(-1.1*gp.elongation*gp.a),
         psip_(gp), psipR_(gp), psipRR_(gp), psipRZ_(gp), psipZZ_(gp), cauchy_( R_X, Z_X, 50, 50,1.)
@@ -634,7 +633,7 @@ struct PsipR
     solovev::PsipZZ psipZZ_;
     dg::Cauchy cauchy_;
 };
-struct PsipZ
+struct PsipZ: public aBinaryFunctor
 {
     PsipZ( GeomParameters gp): R_X( gp.R_0-1.1*gp.triangularity*gp.a), Z_X(-1.1*gp.elongation*gp.a),
         psip_(gp), psipZ_(gp), psipRR_(gp), psipRZ_(gp), psipZZ_(gp), cauchy_( R_X, Z_X, 50, 50, 1)
@@ -667,7 +666,7 @@ struct PsipZ
     dg::Cauchy cauchy_;
 };
 
-struct PsipZZ
+struct PsipZZ: public aBinaryFunctor
 {
     PsipZZ( GeomParameters gp): R_X( gp.R_0-1.1*gp.triangularity*gp.a), Z_X(-1.1*gp.elongation*gp.a),
         psip_(gp), psipZ_(gp), psipRR_(gp), psipRZ_(gp), psipZZ_(gp), cauchy_( R_X, Z_X, 50, 50, 1)
@@ -700,7 +699,7 @@ struct PsipZZ
     solovev::PsipZZ psipZZ_;
     dg::Cauchy cauchy_;
 };
-struct PsipRR
+struct PsipRR: public aBinaryFunctor
 {
     PsipRR( GeomParameters gp): R_X( gp.R_0-1.1*gp.triangularity*gp.a), Z_X(-1.1*gp.elongation*gp.a),
         psip_(gp), psipR_(gp), psipRR_(gp), psipRZ_(gp), psipZZ_(gp), cauchy_( R_X, Z_X, 50, 50, 1)
@@ -733,7 +732,7 @@ struct PsipRR
     solovev::PsipZZ psipZZ_;
     dg::Cauchy cauchy_;
 };
-struct PsipRZ
+struct PsipRZ: public aBinaryFunctor
 {
     PsipRZ( GeomParameters gp): R_X( gp.R_0-1.1*gp.triangularity*gp.a), Z_X(-1.1*gp.elongation*gp.a),
         psip_(gp), psipR_(gp), psipZ_(gp), psipRR_(gp), psipRZ_(gp), psipZZ_(gp), cauchy_( R_X, Z_X, 50, 50, 1)
@@ -769,7 +768,7 @@ struct PsipRZ
     dg::Cauchy cauchy_;
 };
 
-struct LaplacePsip
+struct LaplacePsip: public aBinaryFunctor
 {
     LaplacePsip( GeomParameters gp ): psipRR_(gp), psipZZ_(gp){}
     double operator()(double R, double Z) const
-- 
GitLab


From 08df5b9d7634af237ca1f2e572e68913c7e07fcc Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sun, 23 Jul 2017 18:23:27 +0200
Subject: [PATCH 067/453] rewrite solovev, taylor, guenther and toroidal

---
 inc/geometries/guenther.h       |  61 +++++-------
 inc/geometries/magnetic_field.h |  34 ++++++-
 inc/geometries/solovev.h        | 161 +++++---------------------------
 inc/geometries/taylor.h         | 144 ++++++----------------------
 inc/geometries/toroidal.h       |  32 ++++---
 5 files changed, 127 insertions(+), 305 deletions(-)

diff --git a/inc/geometries/guenther.h b/inc/geometries/guenther.h
index 7cf14f824..cb5573124 100644
--- a/inc/geometries/guenther.h
+++ b/inc/geometries/guenther.h
@@ -31,84 +31,79 @@ namespace guenther
 /**
  * @brief \f[\cos(\pi(R-R_0)/2)\cos(\pi Z/2)\f]
  */
-struct Psip
+struct Psip : public aCloneableBinaryFunctor<Psip>
 {
     Psip(double R_0 ):   R_0(R_0) {}
     double operator()(double R, double Z) const
     {    
         return cos(M_PI*0.5*(R-R_0))*cos(M_PI*Z*0.5);
     }
-    double operator()(double R, double Z, double phi)const{return operator()(R,Z);}
   private:
     double R_0;
 };
 /**
  * @brief \f[-\pi\sin(\pi(R-R_0)/2)\cos(\pi Z/2)/2\f]
  */
-struct PsipR
+struct PsipR : public aCloneableBinaryFunctor<PsipR>
 {
     PsipR(double R_0 ):   R_0(R_0) {}
     double operator()(double R, double Z) const
     {    
         return -M_PI*0.5*sin(M_PI*0.5*(R-R_0))*cos(M_PI*Z*0.5);
     }
-    double operator()(double R, double Z, double phi)const{return operator()(R,Z);}
   private:
     double R_0;
 };
 /**
  * @brief \f[-\pi^2\cos(\pi(R-R_0)/2)\cos(\pi Z/2)/4\f]
  */
-struct PsipRR
+struct PsipRR : public aCloneableBinaryFunctor<PsipRR>
 {
     PsipRR(double R_0 ):   R_0(R_0) {}
     double operator()(double R, double Z) const
     {    
         return -M_PI*M_PI*0.25*cos(M_PI*0.5*(R-R_0))*cos(M_PI*Z*0.5);
     }
-    double operator()(double R, double Z, double phi)const{return operator()(R,Z);}
   private:
     double R_0;
 };
 /**
  * @brief \f[-\pi\cos(\pi(R-R_0)/2)\sin(\pi Z/2)/2\f]
  */
-struct PsipZ
+struct PsipZ : public aCloneableBinaryFunctor<PsipZ>
+
 {
     PsipZ(double R_0 ):   R_0(R_0) {}
     double operator()(double R, double Z) const
     {    
         return -M_PI*0.5*cos(M_PI*0.5*(R-R_0))*sin(M_PI*Z*0.5);
     }
-    double operator()(double R, double Z, double phi)const{return operator()(R,Z);}
   private:
     double R_0;
 };
 /**
  * @brief \f[-\pi^2\cos(\pi(R-R_0)/2)\cos(\pi Z/2)/4\f]
  */
-struct PsipZZ
+struct PsipZZ : public aCloneableBinaryFunctor<PsipZZ>
 {
     PsipZZ(double R_0 ):   R_0(R_0){}
     double operator()(double R, double Z) const
     {    
         return -M_PI*M_PI*0.25*cos(M_PI*0.5*(R-R_0))*cos(M_PI*Z*0.5);
     }
-    double operator()(double R, double Z, double phi)const{return operator()(R,Z);}
   private:
     double R_0;
 };
 /**
  * @brief \f[ \pi^2\sin(\pi(R-R_0)/2)\sin(\pi Z/2)/4\f]
  */
-struct PsipRZ
+struct PsipRZ : public aCloneableBinaryFunctor<PsipRZ>
 {
     PsipRZ(double R_0 ):   R_0(R_0) {}
     double operator()(double R, double Z) const
     {    
         return M_PI*M_PI*0.25*sin(M_PI*0.5*(R-R_0))*sin(M_PI*Z*0.5);
     }
-    double operator()(double R, double Z, double phi)const{return operator()(R,Z);}
   private:
     double R_0;
 };
@@ -116,17 +111,13 @@ struct PsipRZ
 /**
  * @brief \f[-\pi^2\cos(\pi(R-R_0)/2)\cos(\pi Z/2)/2\f]
  */
-struct LaplacePsip
+struct LaplacePsip : public aCloneableBinaryFunctor<LaplacePsip>
 {
     LaplacePsip( double R_0 ): psipRR_(R_0), psipZZ_(R_0){}
     double operator()(double R, double Z) const
     {    
         return psipRR_(R,Z) + psipZZ_(R,Z);
     }
-    double operator()(double R, double Z, double phi) const
-    {    
-        return operator()(R,Z);
-    }
   private:
     PsipRR psipRR_;
     PsipZZ psipZZ_;
@@ -135,50 +126,46 @@ struct LaplacePsip
 /**
  * @brief \f[ I_0\f]
  */
-struct Ipol
+struct Ipol : public aCloneableBinaryFunctor<Ipol>
 {
     Ipol( double I_0):   I_0(I_0) {}
     double operator()(double R, double Z) const { return I_0; }
-    double operator()(double R, double Z, double phi) const { return I_0; }
   private:
     double I_0;
 };
 /**
  * @brief \f[0\f]
  */
-struct IpolR
+struct IpolR : public aCloneableBinaryFunctor<IpolR>
 {
     IpolR(  ) {}
     double operator()(double R, double Z) const { return 0; }
-    double operator()(double R, double Z, double phi) const { return 0; }
 };
 /**
  * @brief \f[0\f]
  */
-struct IpolZ
+struct IpolZ : public aCloneableBinaryFunctor<IpolZ>
 {
     IpolZ(  ) {}
     double operator()(double R, double Z) const { return 0; }
-    double operator()(double R, double Z, double phi) const { return 0; }
 };
 
 /**
- * @brief Contains all guenther fields (models aTokamakMagneticField)
+ * @brief Contains all guenther fields
  */
-struct MagneticField
+struct MagneticField : public dg::geo::aTokamakMagneticField
 {
-    MagneticField( double R_0, double I_0): R_0(R_0), psip(R_0), psipR(R_0), psipZ(R_0), psipRR(R_0), psipRZ(R_0), psipZZ(R_0), laplacePsip(R_0), ipol(I_0), ipolR(), ipolZ(){}
-    double R_0;
-    Psip psip;
-    PsipR psipR;
-    PsipZ psipZ;
-    PsipRR psipRR;
-    PsipRZ psipRZ;
-    PsipZZ psipZZ;
-    LaplacePsip laplacePsip;
-    Ipol ipol;
-    IpolR ipolR;
-    IpolZ ipolZ;
+    MagneticField( GeomParameters gp): aTokamakMagneticField(gp.R_0, 
+        new Psip(gp), 
+        new PsipR(gp), 
+        new PsipZ(gp), 
+        new PsipRR(gp), 
+        new PsipRZ(gp), 
+        new PsipZZ(gp), 
+        new LaplacePsip(gp), 
+        new Ipol(gp), 
+        new IpolR(gp), 
+        new IpolZ(gp)){}
 };
 ///@}
 
diff --git a/inc/geometries/magnetic_field.h b/inc/geometries/magnetic_field.h
index 51f91a4af..3a278723b 100644
--- a/inc/geometries/magnetic_field.h
+++ b/inc/geometries/magnetic_field.h
@@ -10,26 +10,56 @@ namespace dg
 namespace geo
 {
 
+/**
+* @brief This functor represents functions written in cylindrical coordinates
+        that are independent of the angle phi
+*/
 struct aBinaryFunctor
 {
+    /**
+    * @brief The function value
+    *
+    * @param R radius (cylindrical coordinate)
+    * @param Z height (cylindrical coordinate)
+    *
+    * @return f(R,Z)
+    */
     virtual double operator()(double R, double Z) const=0;
+    /**
+    * @brief Redirects to the 2D version
+    *
+    * @param R radius (cylindrical coordinate)
+    * @param Z height (cylindrical coordinate)
+    * @param phi angle (cylindrical coordinate)
+    *
+    * @return f(R,Z)
+    */
     double operator()(double R, double Z, double phi)
     {
         return this->operator()(R,Z);
     }
     virtual aBinaryFunctor* clone()const=0;
+    protected:
     virtual ~aBinaryFunctor(){}
+    //do not allow object slicing
+    aBinaryFunctor(const aBinaryFunctor&){}
+    aBinaryFunctor& operator=(const aBinaryFunctor&){return *this}
 };
 
-//https://katyscode.wordpress.com/2013/08/22/c-polymorphic-cloning-and-the-crtp-curiously-recurring-template-pattern/
+/**
+* @brief Implementation helper for the clone pattern
+
+https://katyscode.wordpress.com/2013/08/22/c-polymorphic-cloning-and-the-crtp-curiously-recurring-template-pattern/
+*/
 template<class Derived<
 struct aCloneableBinaryFunctor : public a BinaryFunctor
 {
     virtual aBinaryFunctor* clone() const
     {
-        return new Derived(statimag_cast<Derived const &>(*this));
+        return new Derived(static_cast<Derived const &>(*this));
     }
 };
+
 /**
  * @brief Contains all solovev fields (models aTokamakMagneticField)
  */
diff --git a/inc/geometries/solovev.h b/inc/geometries/solovev.h
index 6b2c1584d..81802d169 100644
--- a/inc/geometries/solovev.h
+++ b/inc/geometries/solovev.h
@@ -57,7 +57,7 @@ namespace solovev
       with \f$ \bar R := \frac{ R}{R_0} \f$ and \f$\bar Z := \frac{Z}{R_0}\f$
  *
  */    
-struct Psip: public aBinaryFunctor
+struct Psip: public aCloneableBinaryFunctor<Psip>
 {
     /**
      * @brief Construct from given geometric parameters
@@ -76,19 +76,6 @@ struct Psip: public aBinaryFunctor
     {    
         return psi_alt( R, Z);
     }
-    /**
-     * @brief \f$ \psi_p(R,Z,\phi) \equiv \psi_p(R,Z)\f$
-     *
-      @param R radius (cylindrical coordinates)
-      @param Z height (cylindrical coordinates)
-      @param phi angle (cylindrical coordinates)
-     *
-     * @return \f$ \hat \psi_p(R,Z,\phi) \f$
-     */
-    double operator()(double R, double Z, double phi) const
-    {    
-        return operator()(R,Z);
-    }
     /**
      * @brief Show parameters to std::cout
      */
@@ -167,7 +154,7 @@ struct Psip: public aBinaryFunctor
       \ln{(\bar{R}   )})\Bigg\} \f]
       with \f$ \bar R := \frac{ R}{R_0} \f$ and \f$\bar Z := \frac{Z}{R_0}\f$
  */ 
-struct PsipR: public aBinaryFunctor
+struct PsipR: public aCloneableBinaryFunctor<PsipR>
 {
     /**
      * @brief Construct from given geometric parameters
@@ -186,18 +173,6 @@ struct PsipR: public aBinaryFunctor
     {    
         return psipR_alt( R, Z);
     }
-    /**
-     * @brief \f$ \frac{\partial  \hat{\psi}_p }{ \partial \hat{R}}(R,Z,\phi) \equiv \frac{\partial  \hat{\psi}_p }{ \partial \hat{R}}(R,Z)\f$
-      @param R radius (cylindrical coordinates)
-      @param Z height (cylindrical coordinates)
-      @param phi angle (cylindrical coordinates)
-    * @return \f$ \frac{\partial  \hat{\psi}_p}{ \partial \hat{R}}(R,Z,\phi)  \f$
- */ 
-    double operator()(double R, double Z, double phi) const
-    {    
-        return operator()(R,Z);
-    }
-
 
     /**
      * @brief Print parameters to std::cout
@@ -230,7 +205,7 @@ struct PsipR: public aBinaryFunctor
 /**
  * @brief \f[ \frac{\partial^2  \hat{\psi}_p }{ \partial \hat{R}^2}\f]
  */ 
-struct PsipRR: public aBinaryFunctor
+struct PsipRR: public aCloneableBinaryFunctor<PsipRR>
 {
     /**
     * @brief Constructor
@@ -261,19 +236,6 @@ struct PsipRR: public aBinaryFunctor
         return psipRR_alt( R, Z);
     }
     /**
-    * @brief return operator()(R,Z)
-    *
-      @param R radius (cylindrical coordinates)
-      @param Z height (cylindrical coordinates)
-      @param phi angle (cylindrical coordinates)
-    *
-    * @return value
-    */
-    double operator()(double R, double Z, double phi) const
-    {    
-        return operator()(R,Z);
-    }
-    /**
     * @brief Display the internal parameters to std::cout
     */
     void display()
@@ -302,7 +264,7 @@ struct PsipRR: public aBinaryFunctor
 /**
  * @brief \f[\frac{\partial \hat{\psi}_p }{ \partial \hat{Z}}\f]
  */ 
-struct PsipZ: public aBinaryFunctor
+struct PsipZ: public aCloneableBinaryFunctor<PsipZ>
 {
     PsipZ( GeomParameters gp ): R_0_(gp.R_0), A_(gp.A), c_(gp.c) { }
 /**
@@ -340,13 +302,6 @@ struct PsipZ: public aBinaryFunctor
     {    
         return psipZ_alt(R, Z);
     }
-    /**
-     * @brief == operator()(R,Z)
-     */ 
-    double operator()(double R, double Z, double phi) const
-    {    
-        return operator()(R,Z);
-    }
     void display() const
     {
         std::cout << R_0_ <<"  " <<A_ <<"\n";
@@ -359,29 +314,22 @@ struct PsipZ: public aBinaryFunctor
 /**
  * @brief \f[ \frac{\partial^2  \hat{\psi}_p }{ \partial \hat{Z}^2}\f]
  */ 
-struct PsipZZ: public aBinaryFunctor
+struct PsipZZ: public aCloneableBinaryFunctor<PsipZZ>
 {
-  PsipZZ( GeomParameters gp): R_0_(gp.R_0), A_(gp.A), c_(gp.c) { }
-/**
- * @brief \f[ \frac{\partial^2  \hat{\psi}_p }{ \partial \hat{Z}^2}=
+    PsipZZ( GeomParameters gp): R_0_(gp.R_0), A_(gp.A), c_(gp.c) { }
+  /**
+   * @brief \f[ \frac{\partial^2  \hat{\psi}_p }{ \partial \hat{Z}^2}=
       \hat{R}_0^{-1} \Bigg\{2 c_3 -8 c_4 \bar{R}^2 +6 c_{10}  \bar{Z}-24 c_{11}
       \bar{R}^2  \bar{Z}+c_6 (-24 \bar{R}^4 +96 \bar{R}^2  \bar{Z}^2)
       +c_5 (-18 \bar{R}^2 +24  \bar{Z}^2-24 \bar{R}^2  \ln{(\bar{R}   )})+
       c_{12} (160  \bar{Z}^3-480 \bar{R}^2  \bar{Z} \ln{(\bar{R}   )})
       +c_7 (150 \bar{R}^4 -1680 \bar{R}^2  \bar{Z}^2+240  \bar{Z}^4+360 \bar{R}^4 
       \ln{(\bar{R}   )}-1440 \bar{R}^2  \bar{Z}^2 \ln{(\bar{R}   )})\Bigg\} \f]
- */ 
+    */ 
     double operator()(double R, double Z) const
     {    
         return psipZZ_alt( R, Z);
     }
-    /**
-     * @brief == operator()(R,Z)
-     */ 
-    double operator()(double R, double Z, double phi) const
-    {    
-        return operator()(R,Z);
-    }
     void display() const
     {
         std::cout << R_0_ <<"  " <<A_ <<"\n";
@@ -403,7 +351,7 @@ struct PsipZZ: public aBinaryFunctor
 /**
  * @brief  \f[\frac{\partial^2  \hat{\psi}_p }{ \partial \hat{R} \partial\hat{Z}}\f] 
  */ 
-struct PsipRZ: public aBinaryFunctor
+struct PsipRZ: public aCloneableBinaryFunctor<PsipRZ>
 {
     PsipRZ( GeomParameters gp ): R_0_(gp.R_0), A_(gp.A), c_(gp.c) { }
 /**
@@ -421,13 +369,6 @@ struct PsipRZ: public aBinaryFunctor
     {    
         return psipRZ_alt( R, Z);
     }
-    /**
-     * @brief == operator()(R,Z)
-     */ 
-    double operator()(double R, double Z, double phi) const
-    {    
-        return operator()(R,Z);
-    }
     void display() const
     {
         std::cout << R_0_ <<"  " <<A_ <<"\n";
@@ -454,7 +395,7 @@ struct PsipRZ: public aBinaryFunctor
 /**
  * @brief  \f[\frac{\partial^2  \hat{\psi}_p }{ \partial \hat{R}^2 } + \frac{\partial^2  \hat{\psi}_p }{ \partial \hat{Z}^2 } \f] 
  */
-struct LaplacePsip: public aBinaryFunctor
+struct LaplacePsip: public aCloneableBinaryFunctor<LaplacePsip>
 {
     LaplacePsip( GeomParameters gp ): psipRR_(gp), psipZZ_(gp){}
     /**
@@ -464,13 +405,6 @@ struct LaplacePsip: public aBinaryFunctor
     {    
         return psipRR_(R,Z) + psipZZ_(R,Z);
     }
-    /**
-     * @brief == operator()(R,Z)
-     */ 
-    double operator()(double R, double Z, double phi) const
-    {    
-        return operator()(R,Z);
-    }
   private:
     PsipRR psipRR_;
     PsipZZ psipZZ_;
@@ -481,7 +415,7 @@ struct LaplacePsip: public aBinaryFunctor
 /**
  * @brief \f[\hat{I}\f] 
  */ 
-struct Ipol: public aBinaryFunctor
+struct Ipol: public aCloneableBinaryFunctor<Ipol>
 {
     Ipol(  GeomParameters gp ):  R_0_(gp.R_0), A_(gp.A), qampl_(gp.qampl), psip_(gp) { }
     /**
@@ -492,13 +426,6 @@ struct Ipol: public aBinaryFunctor
         //sign before A changed to -
         return qampl_*sqrt(-2.*A_* psip_(R,Z) /R_0_ + 1.);
     }
-    /**
-     * @brief == operator()(R,Z)
-     */ 
-    double operator()(double R, double Z, double phi) const
-    {    
-        return operator()( R,Z);
-    }
   private:
     double R_0_, A_,qampl_;
     Psip psip_;
@@ -506,20 +433,13 @@ struct Ipol: public aBinaryFunctor
 /**
  * @brief \f[\hat I_R\f]
  */
-struct IpolR: public aBinaryFunctor
+struct IpolR: public aCloneableBinaryFunctor<IpolR>
 {
     IpolR(  GeomParameters gp ):  R_0_(gp.R_0), A_(gp.A), qampl_(gp.qampl), psip_(gp), psipR_(gp) { }
     double operator()(double R, double Z) const
     {    
         return -qampl_/sqrt(-2.*A_* psip_(R,Z) /R_0_ + 1.)*(A_*psipR_(R,Z)/R_0_);
     }
-    /**
-     * @brief == operator()(R,Z)
-     */ 
-    double operator()(double R, double Z, double phi) const
-    {    
-        return operator()( R,Z);
-    }
   private:
     double R_0_, A_,qampl_;
     Psip psip_;
@@ -528,20 +448,13 @@ struct IpolR: public aBinaryFunctor
 /**
  * @brief \f[\hat I_Z\f]
  */
-struct IpolZ: public aBinaryFunctor
+struct IpolZ: public aCloneableBinaryFunctor<IpolZ>
 {
     IpolZ(  GeomParameters gp ):  R_0_(gp.R_0), A_(gp.A), qampl_(gp.qampl), psip_(gp), psipZ_(gp) { }
     double operator()(double R, double Z) const
     {    
         return -qampl_/sqrt(-2.*A_* psip_(R,Z) /R_0_ + 1.)*(A_*psipZ_(R,Z)/R_0_);
     }
-    /**
-     * @brief == operator()(R,Z)
-     */ 
-    double operator()(double R, double Z, double phi) const
-    {    
-        return operator()( R,Z);
-    }
   private:
     double R_0_, A_,qampl_;
     Psip psip_;
@@ -549,9 +462,9 @@ struct IpolZ: public aBinaryFunctor
 };
 
 /**
- * @brief Contains all solovev fields (models aTokamakMagneticField)
+ * @brief Contains all solovev fields
  */
-struct MagneticField : dg::geo::aToakamakMagneticField
+struct MagneticField : public dg::geo::aTokamakMagneticField
 {
     MagneticField( GeomParameters gp): aTokamakMagneticField(gp.R_0, 
         new Psip(gp), 
@@ -571,7 +484,7 @@ struct MagneticField : dg::geo::aToakamakMagneticField
 namespace mod
 {
 
-struct Psip: public aBinaryFunctor
+struct Psip: public aCloneableBinaryFunctor
 {
     Psip( GeomParameters gp): R_X( gp.R_0-1.1*gp.triangularity*gp.a), Z_X(-1.1*gp.elongation*gp.a),
         psip_(gp), psipRR_(gp), psipRZ_(gp), psipZZ_(gp), cauchy_( R_X, Z_X, 50, 50,1.)
@@ -588,10 +501,6 @@ struct Psip: public aBinaryFunctor
         double psip_2 =  0.5*(- psipZZ_X_*Rbar*Rbar + 2.*psipRZ_X_*Rbar*Zbar - psipRR_X_*Zbar*Zbar) - psip_RZ ; 
         return  psip_RZ + 0.5*psip_2*cauchy_(R,Z);
     }
-    double operator()(double R, double Z, double phi) const
-    {    
-        return operator()( R,Z);
-    }
     private:
     double R_X, Z_X; 
     double psipZZ_X_, psipRZ_X_, psipRR_X_;
@@ -601,7 +510,7 @@ struct Psip: public aBinaryFunctor
     solovev::PsipZZ psipZZ_;
     dg::Cauchy cauchy_;
 };
-struct PsipR: public aBinaryFunctor
+struct PsipR: public aCloneableBinaryFunctor
 {
     PsipR( GeomParameters gp): R_X( gp.R_0-1.1*gp.triangularity*gp.a), Z_X(-1.1*gp.elongation*gp.a),
         psip_(gp), psipR_(gp), psipRR_(gp), psipRZ_(gp), psipZZ_(gp), cauchy_( R_X, Z_X, 50, 50,1.)
@@ -619,10 +528,6 @@ struct PsipR: public aBinaryFunctor
         double psip_2R =  - psipZZ_X_*Rbar + psipRZ_X_*Zbar - psipR_RZ;
         return psipR_RZ + 0.5*(psip_2R*cauchy_(R,Z) + psip_2*cauchy_.dx(R,Z)  );
     }
-    double operator()(double R, double Z, double phi) const
-    {    
-        return operator()( R,Z);
-    }
     private:
     double R_X, Z_X; 
     double psipZZ_X_, psipRZ_X_, psipRR_X_;
@@ -633,7 +538,7 @@ struct PsipR: public aBinaryFunctor
     solovev::PsipZZ psipZZ_;
     dg::Cauchy cauchy_;
 };
-struct PsipZ: public aBinaryFunctor
+struct PsipZ: public aCloneableBinaryFunctor
 {
     PsipZ( GeomParameters gp): R_X( gp.R_0-1.1*gp.triangularity*gp.a), Z_X(-1.1*gp.elongation*gp.a),
         psip_(gp), psipZ_(gp), psipRR_(gp), psipRZ_(gp), psipZZ_(gp), cauchy_( R_X, Z_X, 50, 50, 1)
@@ -651,10 +556,6 @@ struct PsipZ: public aBinaryFunctor
         double psip_2Z =  - psipRR_X_*Zbar + psipRZ_X_*Rbar - psipZ_RZ;
         return psipZ_RZ + 0.5*(psip_2Z*cauchy_(R,Z) + psip_2*cauchy_.dy(R,Z));
     }
-    double operator()(double R, double Z, double phi) const
-    {    
-        return operator()( R,Z);
-    }
     private:
     double R_X, Z_X; 
     double psipZZ_X_, psipRZ_X_, psipRR_X_;
@@ -666,7 +567,7 @@ struct PsipZ: public aBinaryFunctor
     dg::Cauchy cauchy_;
 };
 
-struct PsipZZ: public aBinaryFunctor
+struct PsipZZ: public aCloneableBinaryFunctor
 {
     PsipZZ( GeomParameters gp): R_X( gp.R_0-1.1*gp.triangularity*gp.a), Z_X(-1.1*gp.elongation*gp.a),
         psip_(gp), psipZ_(gp), psipRR_(gp), psipRZ_(gp), psipZZ_(gp), cauchy_( R_X, Z_X, 50, 50, 1)
@@ -685,10 +586,6 @@ struct PsipZZ: public aBinaryFunctor
         double psip_2ZZ =  - psipRR_X_ - psipZZ_RZ;
         return psipZZ_RZ + 0.5*(psip_2ZZ*cauchy_(R,Z) + 2.*cauchy_.dy(R,Z)*psip_2Z +  psip_2*cauchy_.dyy(R,Z));
     }
-    double operator()(double R, double Z, double phi) const
-    {    
-        return operator()( R,Z);
-    }
     private:
     double R_X, Z_X; 
     double psipZZ_X_, psipRZ_X_, psipRR_X_;
@@ -699,7 +596,7 @@ struct PsipZZ: public aBinaryFunctor
     solovev::PsipZZ psipZZ_;
     dg::Cauchy cauchy_;
 };
-struct PsipRR: public aBinaryFunctor
+struct PsipRR: public aCloneableBinaryFunctor
 {
     PsipRR( GeomParameters gp): R_X( gp.R_0-1.1*gp.triangularity*gp.a), Z_X(-1.1*gp.elongation*gp.a),
         psip_(gp), psipR_(gp), psipRR_(gp), psipRZ_(gp), psipZZ_(gp), cauchy_( R_X, Z_X, 50, 50, 1)
@@ -718,10 +615,6 @@ struct PsipRR: public aBinaryFunctor
         double psip_2RR =  - psipZZ_X_ - psipRR_RZ;
         return psipRR_RZ + 0.5*(psip_2RR*cauchy_(R,Z) + 2.*cauchy_.dx(R,Z)*psip_2R +  psip_2*cauchy_.dxx(R,Z));
     }
-    double operator()(double R, double Z, double phi) const
-    {    
-        return operator()( R,Z);
-    }
     private:
     double R_X, Z_X; 
     double psipZZ_X_, psipRZ_X_, psipRR_X_;
@@ -732,7 +625,7 @@ struct PsipRR: public aBinaryFunctor
     solovev::PsipZZ psipZZ_;
     dg::Cauchy cauchy_;
 };
-struct PsipRZ: public aBinaryFunctor
+struct PsipRZ: public aCloneableBinaryFunctor<PsipRZ>
 {
     PsipRZ( GeomParameters gp): R_X( gp.R_0-1.1*gp.triangularity*gp.a), Z_X(-1.1*gp.elongation*gp.a),
         psip_(gp), psipR_(gp), psipZ_(gp), psipRR_(gp), psipRZ_(gp), psipZZ_(gp), cauchy_( R_X, Z_X, 50, 50, 1)
@@ -752,10 +645,6 @@ struct PsipRZ: public aBinaryFunctor
         double psip_2RZ =  - psipRZ_X_ - psipRZ_RZ;
         return psipRZ_RZ + 0.5*(psip_2RZ*cauchy_(R,Z) + cauchy_.dx(R,Z)*psip_2Z + cauchy_.dy(R,Z)*psip_2R  +  psip_2*cauchy_.dxy(R,Z));
     }
-    double operator()(double R, double Z, double phi) const
-    {    
-        return operator()( R,Z);
-    }
     private:
     double R_X, Z_X; 
     double psipZZ_X_, psipRZ_X_, psipRR_X_;
@@ -768,17 +657,13 @@ struct PsipRZ: public aBinaryFunctor
     dg::Cauchy cauchy_;
 };
 
-struct LaplacePsip: public aBinaryFunctor
+struct LaplacePsip: public aCloneableBinaryFunctor<LaplacePsip>
 {
     LaplacePsip( GeomParameters gp ): psipRR_(gp), psipZZ_(gp){}
     double operator()(double R, double Z) const
     {    
         return psipRR_(R,Z) + psipZZ_(R,Z);
     }
-    double operator()(double R, double Z, double phi) const
-    {    
-        return operator()(R,Z);
-    }
   private:
     solovev::mod::PsipRR psipRR_;
     solovev::mod::PsipZZ psipZZ_;
diff --git a/inc/geometries/taylor.h b/inc/geometries/taylor.h
index 1a29b88ff..579adaf8d 100644
--- a/inc/geometries/taylor.h
+++ b/inc/geometries/taylor.h
@@ -41,7 +41,7 @@ typedef dg::geo::solovev::GeomParameters GeomParameters; //!< bring GeomParamete
  * This is taken from A. J. Cerfon and M. O'Neil: Exact axisymmetric Taylor states for shaped plasmas, Physics of Plasmas 21, 064501 (2014)
  * @attention When the taylor field is used we need the boost library for special functions
  */
-struct Psip
+struct Psip : public aCloneableBinaryFunctor<Psip>
 {
     /**
      * @brief Construct from given geometric parameters
@@ -79,19 +79,6 @@ struct Psip
                + c_[9]*sin(c_[11]*Zn));
 
     }
-    /**
-     * @brief \f$ \psi_p(R,Z,\phi) \equiv \psi_p(R,Z)\f$
-     *
-      @param R radius (boost::math::cylindrical coordinates)
-      @param Z height (boost::math::cylindrical coordinates)
-      @param phi angle (boost::math::cylindrical coordinates)
-     *
-     * @return \f$ \hat \psi_p(R,Z,\phi) \f$
-     */
-    double operator()(double R, double Z, double phi) const
-    {    
-        return operator()(R,Z);
-    }
   private:
     double R0_, cs_;
     std::vector<double> c_;
@@ -101,7 +88,7 @@ struct Psip
  * @brief \f[\psi_R\f]
  * @attention When the taylor field is used we need the boost library for special functions
  */
-struct PsipR
+struct PsipR: public aCloneableBinaryFunctor<PsipR>
 {
     /**
      * @brief Construct from given geometric parameters
@@ -112,13 +99,13 @@ struct PsipR
         cs_=sqrt(c_[11]*c_[11]-c_[10]*c_[10]);
     
     }
-/**
- * @brief \f$ \frac{\partial  \hat{\psi}_p }{ \partial \hat{R}}(R,Z)  \f$
+    /**
+     * @brief \f$ \frac{\partial  \hat{\psi}_p }{ \partial \hat{R}}(R,Z)  \f$
 
-      @param R radius (boost::math::cylindrical coordinates)
-      @param Z height (boost::math::cylindrical coordinates)
-    * @return \f$ \frac{\partial  \hat{\psi}_p}{ \partial \hat{R}}(R,Z)  \f$
- */ 
+          @param R radius (boost::math::cylindrical coordinates)
+          @param Z height (boost::math::cylindrical coordinates)
+        * @return \f$ \frac{\partial  \hat{\psi}_p}{ \partial \hat{R}}(R,Z)  \f$
+     */ 
     double operator()(double R, double Z) const
     {    
         double Rn=R/R0_, Zn=Z/R0_;
@@ -143,19 +130,6 @@ struct PsipR
                + c_[7]*j1_csR*sin(c_[10]*Zn)
                + c_[8]*y1_csR*sin(c_[10]*Zn) );
     }
-    /**
-     * @brief \f$ \frac{\partial  \hat{\psi}_p }{ \partial \hat{R}}(R,Z,\phi) \equiv \frac{\partial  \hat{\psi}_p }{ \partial \hat{R}}(R,Z)\f$
-      @param R radius (boost::math::cylindrical coordinates)
-      @param Z height (boost::math::cylindrical coordinates)
-      @param phi angle (boost::math::cylindrical coordinates)
-    * @return \f$ \frac{\partial  \hat{\psi}_p}{ \partial \hat{R}}(R,Z,\phi)  \f$
- */ 
-    double operator()(double R, double Z, double phi) const
-    {    
-        return operator()(R,Z);
-    }
-
-
   private:
     double R0_, cs_;
     std::vector<double> c_;
@@ -163,7 +137,7 @@ struct PsipR
 /**
  * @brief \f[ \frac{\partial^2  \hat{\psi}_p }{ \partial \hat{R}^2}\f]
  */ 
-struct PsipRR
+struct PsipRR: public aCloneableBinaryFunctor<PsipRR>
 {
     /**
     * @brief Constructor
@@ -193,19 +167,6 @@ struct PsipRR
                + c_[7]*j1_csR*sin(c_[10]*Zn)
                + c_[8]*y1_csR*sin(c_[10]*Zn) );
     }
-    /**
-    * @brief return operator()(R,Z)
-    *
-      @param R radius (boost::math::cylindrical coordinates)
-      @param Z height (boost::math::cylindrical coordinates)
-      @param phi angle (boost::math::cylindrical coordinates)
-    *
-    * @return value
-    */
-    double operator()(double R, double Z, double phi) const
-    {    
-        return operator()(R,Z);
-    }
   private:
     double R0_, cs_;
     std::vector<double> c_;
@@ -213,7 +174,7 @@ struct PsipRR
 /**
  * @brief \f[\frac{\partial \hat{\psi}_p }{ \partial \hat{Z}}\f]
  */ 
-struct PsipZ
+struct PsipZ: public aCloneableBinaryFunctor<PsipZ>
 {
     PsipZ( solovev::GeomParameters gp ): R0_(gp.R_0), c_(gp.c) { 
         cs_ = sqrt( c_[11]*c_[11]-c_[10]*c_[10]);
@@ -236,13 +197,6 @@ struct PsipZ
                + c_[8]*Rn*y1_cs*c_[10]*cos(c_[10]*Zn)
                + c_[9]*c_[11]*cos(c_[11]*Zn));
     }
-    /**
-     * @brief == operator()(R,Z)
-     */ 
-    double operator()(double R, double Z, double phi) const
-    {    
-        return operator()(R,Z);
-    }
   private:
     double R0_,cs_; 
     std::vector<double> c_;
@@ -250,7 +204,7 @@ struct PsipZ
 /**
  * @brief \f[ \frac{\partial^2  \hat{\psi}_p }{ \partial \hat{Z}^2}\f]
  */ 
-struct PsipZZ
+struct PsipZZ: public aCloneableBinaryFunctor<PsipZZ>
 {
   PsipZZ( solovev::GeomParameters gp): R0_(gp.R_0), c_(gp.c) { 
         cs_ = sqrt( c_[11]*c_[11]-c_[10]*c_[10]);
@@ -271,13 +225,6 @@ struct PsipZZ
                - c_[8]*Rn*y1_cs*c_[10]*c_[10]*sin(c_[10]*Zn)
                - c_[9]*c_[11]*c_[11]*sin(c_[11]*Zn));
     }
-    /**
-     * @brief == operator()(R,Z)
-     */ 
-    double operator()(double R, double Z, double phi) const
-    {    
-        return operator()(R,Z);
-    }
   private:
     double R0_, cs_;
     std::vector<double> c_;
@@ -285,7 +232,7 @@ struct PsipZZ
 /**
  * @brief  \f[\frac{\partial^2  \hat{\psi}_p }{ \partial \hat{R} \partial\hat{Z}}\f] 
  */ 
-struct PsipRZ
+struct PsipRZ: public aCloneableBinaryFunctor<PsipRZ>
 {
     PsipRZ( solovev::GeomParameters gp ): R0_(gp.R_0), c_(gp.c) { 
         cs_ = sqrt( c_[11]*c_[11]-c_[10]*c_[10]);
@@ -312,29 +259,18 @@ struct PsipRZ
                + c_[7]*j1_csR*c_[10]*cos(c_[10]*Zn)
                + c_[8]*y1_csR*c_[10]*cos(c_[10]*Zn) );
     }
-    /**
-     * @brief == operator()(R,Z)
-     */ 
-    double operator()(double R, double Z, double phi) const
-    {    
-        return operator()(R,Z);
-    }
   private:
     double R0_, cs_;
     std::vector<double> c_;
 };
 
-struct LaplacePsip
+struct LaplacePsip: public aCloneableBinaryFunctor<LaplacePsip>
 {
     LaplacePsip( solovev::GeomParameters gp ): psipRR_(gp), psipZZ_(gp){}
     double operator()(double R, double Z) const
     {    
         return psipRR_(R,Z) + psipZZ_(R,Z);
     }
-    double operator()(double R, double Z, double phi) const
-    {    
-        return operator()(R,Z);
-    }
   private:
     PsipRR psipRR_;
     PsipZZ psipZZ_;
@@ -344,7 +280,7 @@ struct LaplacePsip
 /**
  * @brief \f[\hat{I} = c_{12}\psi\f] 
  */ 
-struct Ipol
+struct Ipol: public aCloneableBinaryFunctor<Ipol>
 {
     Ipol(  solovev::GeomParameters gp ): c12_(gp.c[11]), psip_(gp) { }
     /**
@@ -355,13 +291,6 @@ struct Ipol
         return c12_*psip_(R,Z);
         
     }
-    /**
-     * @brief == operator()(R,Z)
-     */ 
-    double operator()(double R, double Z, double phi) const
-    {    
-        return operator()( R,Z);
-    }
   private:
     double c12_;
     Psip psip_;
@@ -369,20 +298,13 @@ struct Ipol
 /**
  * @brief \f[\hat I_R\f]
  */
-struct IpolR
+struct IpolR: public aCloneableBinaryFunctor<IpolR>
 {
     IpolR(  solovev::GeomParameters gp ): c12_(gp.c[11]), psipR_(gp) { }
     double operator()(double R, double Z) const
     {    
         return c12_*psipR_(R,Z);
     }
-    /**
-     * @brief == operator()(R,Z)
-     */ 
-    double operator()(double R, double Z, double phi) const
-    {    
-        return operator()( R,Z);
-    }
   private:
     double c12_;
     PsipR psipR_;
@@ -390,42 +312,34 @@ struct IpolR
 /**
  * @brief \f[\hat I_Z\f]
  */
-struct IpolZ
+struct IpolZ: public aCloneableBinaryFunctor<IpolZ>
 {
     IpolZ(  solovev::GeomParameters gp ): c12_(gp.c[11]), psipZ_(gp) { }
     double operator()(double R, double Z) const
     {    
         return c12_*psipZ_(R,Z);
     }
-    /**
-     * @brief == operator()(R,Z)
-     */ 
-    double operator()(double R, double Z, double phi) const
-    {    
-        return operator()( R,Z);
-    }
   private:
     double c12_;
     PsipZ psipZ_;
 };
 
 /**
- * @brief Contains all taylor fields (models aTokamakMagneticField)
+ * @brief Contains all taylor fields 
  */
-struct MagneticField
+struct MagneticField : public dg::geo::aTokamakMagneticField
 {
-    MagneticField( solovev::GeomParameters gp): R_0(gp.R_0), psip(gp), psipR(gp), psipZ(gp), psipRR(gp), psipRZ(gp), psipZZ(gp), laplacePsip(gp), ipol(gp), ipolR(gp), ipolZ(gp){}
-    double R_0;
-    Psip psip;
-    PsipR psipR;
-    PsipZ psipZ;
-    PsipRR psipRR;
-    PsipRZ psipRZ;
-    PsipZZ psipZZ;
-    LaplacePsip laplacePsip;
-    Ipol ipol;
-    IpolR ipolR;
-    IpolZ ipolZ;
+    MagneticField( solovev::GeomParameters gp): aTokamakMagneticField(gp.R_0, 
+        new Psip(gp), 
+        new PsipR(gp), 
+        new PsipZ(gp), 
+        new PsipRR(gp), 
+        new PsipRZ(gp), 
+        new PsipZZ(gp), 
+        new LaplacePsip(gp), 
+        new Ipol(gp), 
+        new IpolR(gp), 
+        new IpolZ(gp)){}
 };
 
 ///@}
diff --git a/inc/geometries/toroidal.h b/inc/geometries/toroidal.h
index 8d4bf2088..ddffc5998 100644
--- a/inc/geometries/toroidal.h
+++ b/inc/geometries/toroidal.h
@@ -8,26 +8,32 @@ namespace dg{
 namespace geo{
 namespace toroidal{
 
+struct Constant:public aCloneableBinaryOperator<Constant> 
+{ 
+    Constant(double c):c_(c){}
+    double operator()(double R,double Z)const{return c_;}
+    private:
+    double c_;
+};
 /**
  * @brief Models a slab toroidal field (models aTokamakMagneticField)
  *
  * \f$ B=\frac{R_0}{R}\f$, \f$ \psi_p = 1\f$ and \f$ I = 1\f$.
  @note The solovev field can also be made to model a todoidal slab field
  */
-struct MagneticField
+struct MagneticField : public dg::geo::aTokamakMagneticField
 {
-    MagneticField(double R0): R_0(R0), psip(1), psipR(0), psipZ(0), psipRR(0), psipRZ(0), psipZZ(0), laplacePsip(0), ipol(1), ipolR(0), ipolZ(0){}
-    double R_0;
-    dg::CONSTANT psip;
-    dg::CONSTANT psipR;
-    dg::CONSTANT psipZ;
-    dg::CONSTANT psipRR;
-    dg::CONSTANT psipRZ;
-    dg::CONSTANT psipZZ;
-    dg::CONSTANT laplacePsip;
-    dg::CONSTANT ipol;
-    dg::CONSTANT ipolR;
-    dg::CONSTANT ipolZ;
+    MagneticField( double R0): aTokamakMagneticField(R0, 
+        new Constant(1), 
+        new Constant(0), 
+        new Constant(0), 
+        new Constant(0), 
+        new Constant(0), 
+        new Constant(0), 
+        new Constant(0), 
+        new Constant(1), 
+        new Constant(0), 
+        new Constant(0)){}
 };
 
 }//namespace toroidal
-- 
GitLab


From a923e097220f2e77952159ec3691b809f5e03bb7 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sun, 23 Jul 2017 19:15:23 +0200
Subject: [PATCH 068/453] update documentation

---
 inc/geometries/magnetic_field.h | 55 ++++++++++++++++++++++++++++-----
 inc/geometries/solovev_doc.h    | 30 ------------------
 2 files changed, 47 insertions(+), 38 deletions(-)

diff --git a/inc/geometries/magnetic_field.h b/inc/geometries/magnetic_field.h
index 3a278723b..1aeb46aba 100644
--- a/inc/geometries/magnetic_field.h
+++ b/inc/geometries/magnetic_field.h
@@ -13,6 +13,7 @@ namespace geo
 /**
 * @brief This functor represents functions written in cylindrical coordinates
         that are independent of the angle phi
+  @ingroup fluxfunctions
 */
 struct aBinaryFunctor
 {
@@ -38,31 +39,60 @@ struct aBinaryFunctor
     {
         return this->operator()(R,Z);
     }
+    /**
+    * @brief abstract copy of a binary functor
+    *
+    * @return a functor on the heap
+    */
     virtual aBinaryFunctor* clone()const=0;
     protected:
     virtual ~aBinaryFunctor(){}
-    //do not allow object slicing
+    /**
+    * @brief We do not allow object slicing so the copy is protected
+    */
     aBinaryFunctor(const aBinaryFunctor&){}
+    /**
+    * @brief We do not allow object slicing so the assignment is protected
+    */
     aBinaryFunctor& operator=(const aBinaryFunctor&){return *this}
 };
 
 /**
 * @brief Implementation helper for the clone pattern
 
-https://katyscode.wordpress.com/2013/08/22/c-polymorphic-cloning-and-the-crtp-curiously-recurring-template-pattern/
+    https://katyscode.wordpress.com/2013/08/22/c-polymorphic-cloning-and-the-crtp-curiously-recurring-template-pattern/
+  @ingroup fluxfunctions
 */
-template<class Derived<
-struct aCloneableBinaryFunctor : public a BinaryFunctor
+template<class Derived>
+struct dg::geo::aCloneableBinaryFunctor : public dg::geo::aBinaryFunctor
 {
+    /**
+    * @brief Returns a copy of the functor dynamically allocated on the heap
+    *
+    * @return new copy of the functor
+    */
     virtual aBinaryFunctor* clone() const
     {
         return new Derived(static_cast<Derived const &>(*this));
     }
 };
 
+///@addtogroup magnetic
+///@{
 /**
- * @brief Contains all solovev fields (models aTokamakMagneticField)
- */
+* @brief Base class of a tokamak magnetic geometry model
+
+ This is the representation of magnetic fields that can be modeled in the form
+ \f[
+ \vec B = \frac{R_0}{R} \left( I \hat e_\varphi + \nabla \psi_p \times \hat e_\varphi\right)
+ \f]
+ where \f$ R_0\f$ is a normalization constant, \f$ I\f$ the current 
+ and \f$ \psi_p\f$ the poloidal flux function.
+ 
+ This class holds and controls instances of aBinaryFunctor dynamically 
+ on the heap.
+ @note an instance of this class cannot be constructed nor deleted directly but it can be copied and assigned
+*/
 struct aTokamakMagneticField
 {
     aTokamakMagneticField( const aTokamakMagneticField& mag)
@@ -77,16 +107,27 @@ struct aTokamakMagneticField
         std::swap( temp.p_, p_);
         return *this;
     }
+    /// \f$ R_0 \f$ 
     double R0()const {return R0_;}
+    /// \f$ \psi_p(R,Z)\f$, where R, Z are cylindrical coordinates
     const aBinaryFunctor& psip()const{return *p_[0];}
+    /// \f$ \partial_R \psi_p(R,Z)\f$, where R, Z are cylindrical coordinates
     const aBinaryFunctor& psipR()const{return *p_[1];}
+    /// \f$ \partial_Z \psi_p(R,Z)\f$, where R, Z are cylindrical coordinates
     const aBinaryFunctor& psipZ()const{return *p_[2];}
+    /// \f$ \partial_R\partial_R \psi_p(R,Z)\f$, where R, Z are cylindrical coordinates
     const aBinaryFunctor& psipRR()const{return *p_[3];}
+    /// \f$ \partial_R\partial_Z \psi_p(R,Z)\f$, where R, Z are cylindrical coordinates
     const aBinaryFunctor& psipRZ()const{return *p_[4];}
+    /// \f$ \partial_Z\partial_Z \psi_p(R,Z)\f$, where R, Z are cylindrical coordinates
     const aBinaryFunctor& psipZZ()const{return *p_[5];}
+    /// 2d Laplacian of \f$ \psi_p\f$ 
     const aBinaryFunctor& laplacePsip()const{return *p_[6];}
+    /// \f$ I(\psi_p) \f$ the current
     const aBinaryFunctor& ipol()const{return *p_[7];}
+    /// \f$ \partial_R I(\psi_p) \f$ 
     const aBinaryFunctor& ipolR()const{return *p_[8];}
+    /// \f$ \partial_Z I(\psi_p) \f$ 
     const aBinaryFunctor& ipolZ()const{return *p_[9];}
 
     protected:
@@ -122,8 +163,6 @@ struct aTokamakMagneticField
     std::vector<aBinaryFunctor*> p_;
 };
 
-///@addtogroup magnetic
-///@{
 
 /**
  * @brief \f[   |B| = R_0\sqrt{I^2+(\nabla\psi)^2}/R   \f]
diff --git a/inc/geometries/solovev_doc.h b/inc/geometries/solovev_doc.h
index 18ea5595c..f8f6a30e8 100644
--- a/inc/geometries/solovev_doc.h
+++ b/inc/geometries/solovev_doc.h
@@ -116,33 +116,3 @@ struct aContainer
 {
 };
 
-/**
-* @brief The tokamak magnetic geometry template model
-
-This is the representation of magnetic fields that can be modeled in the form
-\f[
-\vec B = \frac{R_0}{R} \left( I \hat e_\varphi + \nabla \psi_p \times \hat e_\varphi\right)
-\f]
-where \f$ R_0\f$ is a normalization constant, \f$ I\f$ the current 
-and \f$ \psi_p\f$ the poloidal flux function.
-
-A tokamak geometry must contain the functors named as follows, all of
-which must model aBinaryOperator.
- @attention this is not a real class it's there for documentation only
- @attention parameter names cannot be different
- @ingroup temp
-*/
-struct aTokamakMagneticField
-{
-    double R_0; //!< \f$ R_0 \f$ 
-    Psip psip; //!< \f$ \psi_p(R,Z)\f$, where R, Z are cylindrical coordinates
-    PsipR psipR;//!< \f$ \partial_R \psi_p(R,Z)\f$, where R, Z are cylindrical coordinates
-    PsipZ psipZ;//!< \f$ \partial_Z \psi_p(R,Z)\f$, where R, Z are cylindrical coordinates
-    PsipRR psipRR;//!< \f$ \partial_R\partial_R \psi_p(R,Z)\f$, where R, Z are cylindrical coordinates
-    PsipRZ psipRZ;//!< \f$ \partial_R\partial_Z \psi_p(R,Z)\f$, where R, Z are cylindrical coordinates
-    PsipZZ psipZZ;//!< \f$ \partial_Z\partial_Z \psi_p(R,Z)\f$, where R, Z are cylindrical coordinates
-    LaplacePsip laplacePsip; //!< 2d Laplacian of \f$ \psi_p\f$ 
-    Ipol ipol; //!< \f$ I(\psi_p) \f$ the current
-    IpolR ipolR; //!< \f$ \partial_R I(\psi_p) \f$ 
-    IpolZ ipolZ; //!< \f$ \partial_Z I(\psi_p) \f$ 
-};
-- 
GitLab


From e273df795f3ddb4d7f1bb0f8b1cf4fdf2819d24f Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sun, 23 Jul 2017 19:42:55 +0200
Subject: [PATCH 069/453] make generators follow clone pattern

---
 inc/geometries/flux.h              | 32 +++++++++---------------
 inc/geometries/generator.h         | 37 +++++++++++++++++++++++-----
 inc/geometries/hector.h            | 39 +++++++++---------------------
 inc/geometries/ribeiro.h           | 14 ++++-------
 inc/geometries/simple_orthogonal.h | 21 ++++++----------
 5 files changed, 67 insertions(+), 76 deletions(-)

diff --git a/inc/geometries/flux.h b/inc/geometries/flux.h
index 4e5649da5..a323dc0ed 100644
--- a/inc/geometries/flux.h
+++ b/inc/geometries/flux.h
@@ -136,7 +136,7 @@ struct Fpsi
  * @tparam IpolY models aBinaryOperator 
  */
 template< class Psi, class PsiX, class PsiY, class PsiXX, class PsiXY, class PsiYY, class Ipol, class IpolX, class IpolY>
-struct FluxGenerator : public aGenerator
+struct FluxGenerator : public aGridGenerator
 {
     /**
      * @brief Construct a symmetry flux grid generator
@@ -178,8 +178,6 @@ struct FluxGenerator : public aGenerator
         //std::cout << "lx_ = "<<lx_<<"\n";
     }
 
-    bool isOrthogonal()const{return false;}
-    bool isConformal()const{return false;}
     /**
      * @brief The length of the zeta-domain
      *
@@ -187,16 +185,18 @@ struct FluxGenerator : public aGenerator
      * @return length of zeta-domain (f0*(psi_1-psi_0))
      * @note the length is always positive
      */
-    double width() const{return lx_;}
+    virtual double width() const{return lx_;}
     /**
      * @brief 2pi (length of the eta domain)
      *
      * Always returns 2pi
      * @return 2pi 
      */
-    double height() const{return 2.*M_PI;}
+    virtual double height() const{return 2.*M_PI;}
+    virtual FluxGenerator* clone() const{return new FluxGenerator(*this);}
 
-    void operator()( 
+    private:
+    virtual void generate( 
          const thrust::host_vector<double>& zeta1d, 
          const thrust::host_vector<double>& eta1d, 
          thrust::host_vector<double>& x, 
@@ -216,9 +216,6 @@ struct FluxGenerator : public aGenerator
         dg::geo::flux::FieldRZYRYZY<PsiX, PsiY, PsiXX, PsiXY, PsiYY, Ipol, IpolX, IpolY> fieldRZYRYZY(psiX_, psiY_, psiXX_, psiXY_, psiYY_, ipol_, ipolR_, ipolZ_);
         ribeiro::detail::Fpsi<Psi, PsiX, PsiY> fpsiRibeiro(psi_, psiX_, psiY_, x0_, y0_, mode_);
         dg::geo::equalarc::FieldRZYRYZY<PsiX, PsiY, PsiXX, PsiXY, PsiYY> fieldRZYRYZYequalarc(psiX_, psiY_, psiXX_, psiXY_, psiYY_);
-        unsigned size = zeta1d.size()*eta1d.size();
-        x.resize(size), y.resize(size);
-        zetaX = zetaY = etaX = etaY =x ;
         thrust::host_vector<double> fx_;
         fx_.resize( zeta1d.size());
         thrust::host_vector<double> f_p(fx_);
@@ -238,7 +235,6 @@ struct FluxGenerator : public aGenerator
             }
         }
     }
-    private:
     Psi psi_;
     PsiX psiX_;
     PsiY psiY_;
@@ -263,7 +259,7 @@ struct FluxGenerator : public aGenerator
      * @tparam PsiYY models aBinaryOperator 
  */
 template< class Psi, class PsiX, class PsiY, class PsiXX, class PsiXY, class PsiYY>
-struct RibeiroFluxGenerator : public aGenerator
+struct RibeiroFluxGenerator : public aGridGenerator
 {
     /**
      * @brief Construct a flux aligned grid generator
@@ -292,8 +288,6 @@ struct RibeiroFluxGenerator : public aGenerator
         x0_=x0, y0_=y0, psi0_=psi_0, psi1_=psi_1;
         //std::cout << "lx_ = "<<lx_<<"\n";
     }
-    bool isOrthogonal()const{return false;}
-    bool isConformal()const{return false;}
     /**
      * @brief The length of the zeta-domain
      *
@@ -301,16 +295,18 @@ struct RibeiroFluxGenerator : public aGenerator
      * @return length of zeta-domain (f0*(psi_1-psi_0))
      * @note the length is always positive
      */
-    double width() const{return lx_;}
+    virtual double width() const{return lx_;}
     /**
      * @brief 2pi (length of the eta domain)
      *
      * Always returns 2pi
      * @return 2pi 
      */
-    double height() const{return 2.*M_PI;}
+    virtual double height() const{return 2.*M_PI;}
+    virtual RibeiroFluxGenerator* clone() const{return new RibeiroFluxGenerator(*this);}
 
-    void operator()( 
+    private:
+    virtual void generate( 
          const thrust::host_vector<double>& zeta1d, 
          const thrust::host_vector<double>& eta1d, 
          thrust::host_vector<double>& x, 
@@ -328,9 +324,6 @@ struct RibeiroFluxGenerator : public aGenerator
         ribeiro::detail::Fpsi<Psi, PsiX, PsiY> fpsi(psi_, psiX_, psiY_, x0_, y0_, mode_);
         dg::geo::ribeiro::FieldRZYRYZY<PsiX, PsiY, PsiXX, PsiXY, PsiYY> fieldRZYRYZYribeiro(psiX_, psiY_, psiXX_, psiXY_, psiYY_);
         dg::geo::equalarc::FieldRZYRYZY<PsiX, PsiY, PsiXX, PsiXY, PsiYY> fieldRZYRYZYequalarc(psiX_, psiY_, psiXX_, psiXY_, psiYY_);
-        unsigned size = zeta1d.size()*eta1d.size();
-        x.resize(size), y.resize(size);
-        zetaX = zetaY = etaX = etaY =x ;
         thrust::host_vector<double> fx_;
         fx_.resize( zeta1d.size());
         thrust::host_vector<double> f_p(fx_);
@@ -350,7 +343,6 @@ struct RibeiroFluxGenerator : public aGenerator
             }
         }
     }
-    private:
     Psi psi_;
     PsiX psiX_;
     PsiY psiY_;
diff --git a/inc/geometries/generator.h b/inc/geometries/generator.h
index 5a0e603d1..6114a7970 100644
--- a/inc/geometries/generator.h
+++ b/inc/geometries/generator.h
@@ -12,12 +12,13 @@ A generator is there to construct coordinate transformations from physical coord
 is a product space. 
  @ingroup generators
 */
-struct aGenerator
+struct aGridGenerator
 {
     virtual double width()  const=0; //!<length in \f$ \zeta\f$ 
     virtual double height() const=0; //!<length in \f$ \eta\f$
-    virtual bool isOrthogonal() const=0; //!< true if coordinate system is orthogonal
-    virtual bool isConformal()const=0; //!< true if coordinate system is conformal
+    virtual bool isOrthonormal() const{return false;} //!< true if coordinate system is orthonormal (false by default)
+    virtual bool isOrthogonal() const{return false;} //!< true if coordinate system is orthogonal (false by default)
+    virtual bool isConformal()const{return false;} //!< true if coordinate system is conformal (false by default)
     /**
     * @brief Generate grid points and elements of the Jacobian 
     *
@@ -32,7 +33,7 @@ struct aGenerator
     * @note the \f$ \zeta\f$ coordinate is contiguous in memory
     * @note All the resulting vectors are write-only and get properly resized
     */
-    virtual void operator()( 
+    void operator()( 
          const thrust::host_vector<double>& zeta1d, 
          const thrust::host_vector<double>& eta1d, 
          thrust::host_vector<double>& x, 
@@ -40,9 +41,33 @@ struct aGenerator
          thrust::host_vector<double>& zetaX, 
          thrust::host_vector<double>& zetaY, 
          thrust::host_vector<double>& etaX, 
-         thrust::host_vector<double>& etaY) const =0;
+         thrust::host_vector<double>& etaY) const
+    {
+        unsigned size = zeta1d.size()*eta1d.size();
+        x.resize(size), y.resize(size);
+        zetaX = zetaY = etaX = etaY =x ;
+        generate( zeta1d, eta1d, x,y,zetaX,zetaY,etaX,etaY);
+    }
+    /**
+    * @brief Abstract clone method that returns a copy on the heap
+    *
+    * @return a copy of *this on the heap
+    */
+    virtual aGridGenerator* clone() const=0;
 
-   virtual ~aGenerator(){}
+    protected:
+    ///@copydoc operator()()
+    virtual void generate(
+         const thrust::host_vector<double>& zeta1d, 
+         const thrust::host_vector<double>& eta1d, 
+         thrust::host_vector<double>& x, 
+         thrust::host_vector<double>& y, 
+         thrust::host_vector<double>& zetaX, 
+         thrust::host_vector<double>& zetaY, 
+         thrust::host_vector<double>& etaX, 
+         thrust::host_vector<double>& etaY) const = 0;
+    
+   virtual ~aGridGenerator(){}
 };
 
 }//namespace geo
diff --git a/inc/geometries/hector.h b/inc/geometries/hector.h
index a0e263fd5..f8c2ebe23 100644
--- a/inc/geometries/hector.h
+++ b/inc/geometries/hector.h
@@ -370,43 +370,37 @@ struct Hector : public aGenerator
      * @return length of u-domain
      * @note the length is always positive
      */
-    double width() const {return lu_;}
+    virtual double width() const {return lu_;}
     /**
      * @brief 2pi
      *
      * Always returns 2pi
      * @return 2pi 
      */
-    double height() const {return 2.*M_PI;}
+    virtual double height() const {return 2.*M_PI;}
     /**
      * @brief True if conformal constructor was used
      *
      * @return true if conformal constructor was used
      */
-    bool isConformal() const {return conformal_;}
+    virtual bool isConformal() const {return conformal_;}
     /**
      * @brief True if orthogonal constructor was used
      *
      * @return true if orthogonal constructor was used
      */
-    bool isOrthogonal() const {return orthogonal_;}
+    virtual bool isOrthogonal() const {return orthogonal_;}
 
     /**
-     * @brief Generate the points and the elements of the Jacobian
+     * @brief Return the internally used orthogonal grid
      *
-     * Call the width() and height() function before calling this function!
-     * @param u1d one-dimensional list of points inside the u-domain (0<u<width())
-     * @param v1d one-dimensional list of points inside the v-domain (0<v<height())
-     * @param x  \f$= x(u,v)\f$
-     * @param y  \f$= y(u,v)\f$
-     * @param ux \f$= u_x(u,v)\f$
-     * @param uy \f$= u_y(u,v)\f$
-     * @param vx \f$= v_x(u,v)\f$
-     * @param vy \f$= v_y(u,v)\f$
-     * @note All the resulting vectors are write-only and get properly resized
-     * @note The \f$ u\f$ direction is continuous in memory
+     * @return  orthogonal zeta, eta grid
      */
-    void operator()( const thrust::host_vector<double>& u1d, 
+    const dg::CurvilinearGrid2d<container>& internal_grid() const {return g2d_;}
+    virtual ~Hector() { delete generator_;}
+    virtual Hector* clone() const{return new Hector(*this);}
+    private:
+    virtual void generate( const thrust::host_vector<double>& u1d, 
                      const thrust::host_vector<double>& v1d, 
                      thrust::host_vector<double>& x, 
                      thrust::host_vector<double>& y, 
@@ -424,8 +418,6 @@ struct Hector : public aGenerator
             eta[i] = fmod(eta[i]+2.*M_PI, 2.*M_PI); 
         dg::IHMatrix Q = dg::create::interpolation( zeta, eta, g2d_);
 
-        thrust::host_vector<double> u(u1d.size()*v1d.size());
-        x = y = ux = uy = vx = vy = u;  //resize 
         dg::blas2::symv( Q, g2d_.r(), x);
         dg::blas2::symv( Q, g2d_.z(), y);
         dg::blas2::symv( Q, ux_, ux);
@@ -433,6 +425,7 @@ struct Hector : public aGenerator
         dg::blas2::symv( Q, vx_, vx);
         dg::blas2::symv( Q, vy_, vy);
         ////Test if u1d is u
+        //thrust::host_vector<double> u(u1d.size()*v1d.size());
         //dg::blas2::symv( Q, u_, u);
         //dg::HVec u2d(u1d.size()*v1d.size());
         //for( unsigned i=0; i<v1d.size(); i++)
@@ -443,14 +436,6 @@ struct Hector : public aGenerator
         //std::cout << "Error in u is "<<eps<<std::endl;
     }
 
-    /**
-     * @brief Return the internally used orthogonal grid
-     *
-     * @return  orthogonal zeta, eta grid
-     */
-    const dg::CurvilinearGrid2d<container>& internal_grid() const {return g2d_;}
-    ~Hector() { delete generator_;}
-    private:
     //make Hector non-copyable
     Hector( const Hector& src);
     Hector& operator=( const Hector& src);
diff --git a/inc/geometries/ribeiro.h b/inc/geometries/ribeiro.h
index 8a826d2a8..1873321db 100644
--- a/inc/geometries/ribeiro.h
+++ b/inc/geometries/ribeiro.h
@@ -221,8 +221,6 @@ struct Ribeiro : public aGenerator
         x0_=x0, y0_=y0, psi0_=psi_0, psi1_=psi_1;
         //std::cout << "lx_ = "<<lx_<<"\n";
     }
-    bool isOrthogonal()const{return false;}
-    bool isConformal()const{return false;}
     /**
      * @brief The length of the zeta-domain
      *
@@ -230,16 +228,18 @@ struct Ribeiro : public aGenerator
      * @return length of zeta-domain (f0*(psi_1-psi_0))
      * @note the length is always positive
      */
-    double width() const{return lx_;}
+    virtual double width() const{return lx_;}
     /**
      * @brief 2pi (length of the eta domain)
      *
      * Always returns 2pi
      * @return 2pi 
      */
-    double height() const{return 2.*M_PI;}
+    virtual double height() const{return 2.*M_PI;}
+    virtual Ribeiro* clone() const{return new Ribeiro(*this);}
 
-    void operator()( 
+    private:
+    virtual void generate( 
          const thrust::host_vector<double>& zeta1d, 
          const thrust::host_vector<double>& eta1d, 
          thrust::host_vector<double>& x, 
@@ -258,9 +258,6 @@ struct Ribeiro : public aGenerator
         ribeiro::detail::Fpsi<Psi, PsiX, PsiY> fpsi(psi_, psiX_, psiY_, x0_, y0_, mode_);
         dg::geo::ribeiro::FieldRZYRYZY<PsiX, PsiY, PsiXX, PsiXY, PsiYY> fieldRZYRYZYribeiro(psiX_, psiY_, psiXX_, psiXY_, psiYY_);
         dg::geo::equalarc::FieldRZYRYZY<PsiX, PsiY, PsiXX, PsiXY, PsiYY> fieldRZYRYZYequalarc(psiX_, psiY_, psiXX_, psiXY_, psiYY_);
-        unsigned size = zeta1d.size()*eta1d.size();
-        x.resize(size), y.resize(size);
-        zetaX = zetaY = etaX = etaY =x ;
         thrust::host_vector<double> f_p(fx_);
         unsigned Nx = zeta1d.size(), Ny = eta1d.size();
         for( unsigned i=0; i<zeta1d.size(); i++)
@@ -278,7 +275,6 @@ struct Ribeiro : public aGenerator
             }
         }
     }
-    private:
     Psi psi_;
     PsiX psiX_;
     PsiY psiY_;
diff --git a/inc/geometries/simple_orthogonal.h b/inc/geometries/simple_orthogonal.h
index aacb33609..dae70342f 100644
--- a/inc/geometries/simple_orthogonal.h
+++ b/inc/geometries/simple_orthogonal.h
@@ -285,7 +285,7 @@ void construct_rz( Nemov nemov,
  * @tparam Psi All the template parameters must model a Binary-operator i.e. the bracket operator() must be callable with two arguments and return a double. 
  */
 template< class Psi, class PsiX, class PsiY, class LaplacePsi>
-struct SimpleOrthogonal : public aGenerator
+struct SimpleOrthogonal : public aGridGenerator
 {
     /**
      * @brief Construct a simple orthogonal grid 
@@ -324,28 +324,24 @@ struct SimpleOrthogonal : public aGenerator
      * @return length of zeta-domain (f0*(psi_1-psi_0))
      * @note the length is always positive
      */
-    double width() const{return lz_;}
+    virtual double width() const{return lz_;}
     /**
      * @brief 2pi (length of the eta domain)
      *
      * Always returns 2pi
      * @return 2pi 
      */
-    double height() const{return 2.*M_PI;}
+    virtual double height() const{return 2.*M_PI;}
     /**
      * @brief Indicate orthogonality
      *
      * @return true
      */
-    bool isOrthogonal() const{return true;}
-    /**
-     * @brief Indicate conformity
-     *
-     * @return false
-     */
-    bool isConformal()  const{return false;}
+    virtual bool isOrthogonal() const{return true;}
+    virtual SimpleOrthogonal* clone() const{return new SimpleOrthogonal(*this);}
 
-    void operator()( 
+    private:
+    virtual void generate( 
          const thrust::host_vector<double>& zeta1d, 
          const thrust::host_vector<double>& eta1d, 
          thrust::host_vector<double>& x, 
@@ -361,8 +357,6 @@ struct SimpleOrthogonal : public aGenerator
         thrust::host_vector<double> h;
         orthogonal::detail::construct_rz(nemov, 0., zeta1d, r_init, z_init, x, y, h);
         unsigned size = x.size();
-        zetaX.resize(size), zetaY.resize(size), 
-        etaX.resize(size), etaY.resize(size);
         for( unsigned idx=0; idx<size; idx++)
         {
             double psipR = psiX_(x[idx], y[idx]);
@@ -373,7 +367,6 @@ struct SimpleOrthogonal : public aGenerator
             etaY[idx] = +h[idx]*psipR;
         }
     }
-    private:
     PsiX psiX_;
     PsiY psiY_;
     LaplacePsi laplacePsi_;
-- 
GitLab


From 30f7f8645d537f7badb84e15ebe4e96b61e78087 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sun, 23 Jul 2017 20:05:36 +0200
Subject: [PATCH 070/453] protect virtual methods in grid classes

---
 inc/dg/backend/grid.h        | 35 +++++++++++++-----
 inc/dg/backend/mpi_grid.h    | 71 +++++++++++++++++++++---------------
 inc/geometries/curvilinear.h |  2 +-
 inc/geometries/generator.h   |  2 +
 4 files changed, 70 insertions(+), 40 deletions(-)

diff --git a/inc/dg/backend/grid.h b/inc/dg/backend/grid.h
index 0fdbd49bc..7edae7b6a 100644
--- a/inc/dg/backend/grid.h
+++ b/inc/dg/backend/grid.h
@@ -305,13 +305,12 @@ struct Grid2d
     * @param new_Nx new number of cells in x 
     * @param new_Ny new number of cells in y
     */
-    virtual void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny) {
-        n_ = new_n, Nx_ = new_Nx, Ny_ = new_Ny;
-        hx_ = lx_/(double)Nx_, hy_ = ly_/(double)Ny_;
-        dlt_ = DLT<double>( new_n);
-        assert( n_> 0 && Nx_ > 0  && Ny_ > 0);
+    void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny) {
+        assert( new_n> 0 && new_Nx > 0  && new_Ny > 0);
+        do_set(new_n,new_Nx,new_Ny);
     }
 
+
     /**
      * @brief discrete legendre trafo
      *
@@ -389,6 +388,15 @@ struct Grid2d
         return false;
     }
   protected:
+    virtual ~Grid2d(){}
+    Grid2d(const Grid2d& src){}
+    Grid2d& operator=(const Grid2d& src){}
+    ///@copydoc set()
+    virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny) {
+        n_ = new_n, Nx_ = new_Nx, Ny_ = new_Ny;
+        hx_ = lx_/(double)Nx_, hy_ = ly_/(double)Ny_;
+        dlt_ = DLT<double>( new_n);
+    }
     virtual void init_X_boundaries( double x0, double x1)
     {
         x0_ = x0, x1_ = x1;
@@ -476,11 +484,9 @@ struct Grid3d
     * @param new_Ny new number of cells in y
     * @param new_Nz new number of cells in z
     */
-    virtual void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz) {
-        n_ = new_n, Nx_ = new_Nx, Ny_ = new_Ny, Nz_ = new_Nz;
-        hx_ = lx_/(double)Nx_, hy_ = ly_/(double)Ny_, hz_ =lz_/(double)Nz_;
-        dlt_ = DLT<double>( new_n);
-        assert( n_>0); assert( Nx_ > 0  && Ny_ > 0); assert( Nz_ > 0);
+    void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz) {
+        assert( new_n>0); assert( new_Nx > 0  && new_Ny > 0); assert( new_Nz > 0);
+        do_set(new_n, new_Nx,new_Ny,new_Nz);
     }
 
     /**
@@ -696,6 +702,15 @@ struct Grid3d
         return false;
     }
   protected:
+    virtual ~Grid3d(){}
+    Grid3d(const Grid3d& src){}
+    Grid3d& operator=(const Grid3d& src){}
+    virtual void do_set(unsigned new_n, unsigned new_Ny, unsigned new_Nz)
+    {
+        n_ = new_n, Nx_ = new_Nx, Ny_ = new_Ny, Nz_ = new_Nz;
+        hx_ = lx_/(double)Nx_, hy_ = ly_/(double)Ny_, hz_ =lz_/(double)Nz_;
+        dlt_ = DLT<double>( new_n);
+    }
     virtual void init_X_boundaries( double x0, double x1)
     {
         x0_ = x0, x1_ = x1;
diff --git a/inc/dg/backend/mpi_grid.h b/inc/dg/backend/mpi_grid.h
index 424561e6d..3d18295d5 100644
--- a/inc/dg/backend/mpi_grid.h
+++ b/inc/dg/backend/mpi_grid.h
@@ -65,8 +65,8 @@ struct MPIGrid2d
     * @attention these are global parameters, i.e. set( g.n(), 2*g.Nx(), 2*g.Ny()) is NOT(!) what you want
     *           use the resize function instead, or set( g.n(), 2*g.global().Nx(), 2*g.global().Ny())
     */
-    virtual void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny) {
-        g.set(new_n,new_Nx,new_Ny);
+    void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny) {
+        do_set( new_n,new_Nx,new_Ny);
         check_division( new_Nx, new_Ny, g.bcx(), g.bcy());
     }
 
@@ -224,23 +224,6 @@ struct MPIGrid2d
 
     }
 
-    /**
-     * @brief Return a non-MPI grid local for the calling process
-     *
-     * The local grid returns the same values for x0(), x1(), ..., Nx(), Ny(), ... as the grid
-     * class itself 
-     * @return Grid object
-     * @note the boundary conditions in the local grid are not well defined since there might not actually be any boundaries
-     */
-    Grid2d local() const {return Grid2d(x0(), x1(), y0(), y1(), n(), Nx(), Ny(), bcx(), bcy());}
-
-    /**
-     * @brief Return the global non-MPI grid 
-     *
-     * The global grid contains the global boundaries and cell numbers. This is the grid that we would have to use in a non-MPI implementation.
-     * @return non-MPI Grid object
-     */
-    virtual const Grid2d& global() const {return g;}
     /**
      * @brief Returns the pid of the process that holds the local grid surrounding the given point
      *
@@ -301,7 +284,30 @@ struct MPIGrid2d
         }
     }
 
+    /**
+     * @brief Return a non-MPI grid local for the calling process
+     *
+     * The local grid returns the same values for x0(), x1(), ..., Nx(), Ny(), ... as the grid
+     * class itself 
+     * @return Grid object
+     * @note the boundary conditions in the local grid are not well defined since there might not actually be any boundaries
+     */
+    Grid2d local() const {return Grid2d(x0(), x1(), y0(), y1(), n(), Nx(), Ny(), bcx(), bcy());}
+
+    /**
+     * @brief Return the global non-MPI grid 
+     *
+     * The global grid contains the global boundaries and cell numbers. This is the grid that we would have to use in a non-MPI implementation.
+     * @return non-MPI Grid object
+     */
+    virtual const Grid2d& global() const {return g;}
     protected:
+    virtual ~MPIGrid2d(){}
+    MPIGrid2d(const MPIGrid2d& src){}
+    MPIGrid2d& operator=(const MPIGrid2d& src){}
+    virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny) {
+        g.set(new_n,new_Nx,new_Ny);
+    }
     void init_X_boundaries( double global_x0, double global_x1)
     {
         g.init_X_boundaries(global_x0, global_x1);
@@ -375,11 +381,12 @@ struct MPIGrid3d
      * @attention these are global parameters, i.e. set( g.n(), 2*g.Nx(), 2*g.Ny(), 2*g.Nz()) is NOT(!) what you want
      *           use the resize function instead, or set( g.n(), 2*g.global().Nx(), 2*g.global().Ny(), 2*g.global().Nz())
      */
-    virtual void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz) {
-        g.set(new_n,new_Nx,new_Ny,new_Nz);
+    void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz) {
+        do_set(new_n,new_Nx,new_Ny,new_Nz);
         check_division( new_Nx,new_Ny,new_Nz,g.bcx(),g.bcy(),g.bcz());
     }
 
+
     /**
      * @brief Return local x0
      *
@@ -569,14 +576,6 @@ struct MPIGrid3d
         grid.display();
 
     }
-    /**
-     *@copydoc MPIGrid2d::local()const
-     */
-    Grid3d local() const {return Grid3d(x0(), x1(), y0(), y1(), z0(), z1(), n(), Nx(), Ny(), Nz(), bcx(), bcy(), bcz());}
-    /**
-     *@copydoc MPIGrid2d::global()const
-     */
-    virtual const Grid3d& global() const {return g;}
     /**
      * @brief Returns the pid of the process that holds the local grid surrounding the given point
      *
@@ -627,7 +626,21 @@ struct MPIGrid3d
         else
             return false;
     }
+    /**
+     *@copydoc MPIGrid2d::local()const
+     */
+    Grid3d local() const {return Grid3d(x0(), x1(), y0(), y1(), z0(), z1(), n(), Nx(), Ny(), Nz(), bcx(), bcy(), bcz());}
+    /**
+     *@copydoc MPIGrid2d::global()const
+     */
+    virtual const Grid3d& global() const {return g;}
     protected:
+    virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz) {
+        g.set(new_n,new_Nx,new_Ny,new_Nz);
+    }
+    virtual ~MPIGrid3d(){}
+    MPIGrid3d(const MPIGrid3d& src){}
+    MPIGrid3d& operator=(const MPIGrid3d& src){}
     void init_X_boundaries( double global_x0, double global_x1)
     {
         g.init_X_boundaries(global_x0, global_x1);
diff --git a/inc/geometries/curvilinear.h b/inc/geometries/curvilinear.h
index 17bdc82e6..6e8b9cfda 100644
--- a/inc/geometries/curvilinear.h
+++ b/inc/geometries/curvilinear.h
@@ -27,7 +27,7 @@ struct CurvilinearGrid3d : public dg::Grid3d
 
     /*!@brief Constructor
     
-     * @param generator must generate an orthogonal grid
+     * @param generator must generate a grid
      * @param n 
      * @param Nx
      @param Ny
diff --git a/inc/geometries/generator.h b/inc/geometries/generator.h
index 6114a7970..268859476 100644
--- a/inc/geometries/generator.h
+++ b/inc/geometries/generator.h
@@ -56,6 +56,8 @@ struct aGridGenerator
     virtual aGridGenerator* clone() const=0;
 
     protected:
+    aGridGenerator(const aGridGenerator& src){}
+    aGridGenerator& operator=(const aGridGenerator& src){}
     ///@copydoc operator()()
     virtual void generate(
          const thrust::host_vector<double>& zeta1d, 
-- 
GitLab


From b1c12d7e5151ffb5afe6c8183ed080395e1edd17 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sun, 23 Jul 2017 20:12:14 +0200
Subject: [PATCH 071/453] virtual functions in standard grids

---
 inc/dg/geometry/cylindrical.h  |  4 ++--
 inc/dg/geometry/mpi_grids.h    |  4 ++--
 inc/dg/geometry/refined_grid.h | 12 ++++++++++++
 3 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/inc/dg/geometry/cylindrical.h b/inc/dg/geometry/cylindrical.h
index 28839ab2c..52c86f026 100644
--- a/inc/dg/geometry/cylindrical.h
+++ b/inc/dg/geometry/cylindrical.h
@@ -44,11 +44,11 @@ struct CylindricalGrid3d : public dg::Grid3d
      * @return the volume element R
      */
     const container& vol()const {return R_;}
-    void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz){
+    private:
+    virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz){
         dg::Grid3d::set(new_n,new_Nx,new_Ny,new_Nz);
         R_=dg::evaluate(dg::cooX3d, *this);
     }
-    private:
     container R_;
 };
 
diff --git a/inc/dg/geometry/mpi_grids.h b/inc/dg/geometry/mpi_grids.h
index da9503143..ead8d5ee6 100644
--- a/inc/dg/geometry/mpi_grids.h
+++ b/inc/dg/geometry/mpi_grids.h
@@ -104,11 +104,11 @@ struct CylindricalMPIGrid3d : public MPIGrid3d
         MPI_Cart_sub( communicator(), remain_dims, &planeComm);
         return dg::CartesianMPIGrid2d( global().x0(), global().x1(), global().y0(), global().y1(), global().n(), global().Nx(), global().Ny(), global().bcx(), global().bcy(), planeComm);
     }
-    void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz){
+    private:
+    virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz){
         MPIGrid3d::set(new_n,new_Ny,new_Nz);
         R_=dg::evaluate(dg::cooX3d, *this);
     }
-    private:
     MPIContainer R_;
 };
 ///@}
diff --git a/inc/dg/geometry/refined_grid.h b/inc/dg/geometry/refined_grid.h
index a69d7fa9d..28b178d72 100644
--- a/inc/dg/geometry/refined_grid.h
+++ b/inc/dg/geometry/refined_grid.h
@@ -12,6 +12,7 @@
 
 namespace dg
 {
+    //actually there shouldn't be a reason to manually refine a grid when we can use our generators instead
 
 ///@cond
 namespace detail
@@ -255,6 +256,7 @@ struct RefinedGrid3d;
 ///@endcond
 /**
  * @brief Refined grid 
+ * @deprecated
  * @ingroup grid
  */
 struct RefinedGrid2d : public dg::Grid2d
@@ -389,6 +391,7 @@ struct RefinedGrid2d : public dg::Grid2d
 
 /**
  * @brief Refined grid 
+ * @deprecated
  * @ingroup grid
  */
 struct RefinedGrid3d : public dg::Grid3d
@@ -537,6 +540,7 @@ RefinedGrid2d::RefinedGrid2d( const dg::RefinedGrid3d& g) :
  * @brief A refined cartesian grid
  *
  * @ingroup basicgrids
+ * @deprecated
  * @tparam container
  */
 template<class container>
@@ -569,6 +573,7 @@ struct CartesianRefinedGrid2d : public dg::RefinedGrid2d
 
 namespace create{
 
+///@deprecated
 cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const dg::RefinedGrid2d& g_fine)
 {
     dg::Grid2d g = g_fine.associated();
@@ -577,6 +582,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const dg::Refine
     return dg::create::interpolation( x,y, g);
 
 }
+///@deprecated
 cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const dg::RefinedGrid2d& g_fine)
 {
     cusp::coo_matrix<int, double, cusp::host_memory> temp = interpolation( g_fine), A;
@@ -584,6 +590,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const dg::Refin
     return A;
 }
 
+///@deprecated
 cusp::coo_matrix<int, double, cusp::host_memory> projection( const dg::RefinedGrid2d& g_fine)
 {
     //form the adjoint
@@ -609,6 +616,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> projection( const dg::RefinedGr
     return A;
 }
 
+///@deprecated
 cusp::coo_matrix<int, double, cusp::host_memory> smoothing( const dg::RefinedGrid2d& g)
 {
     cusp::coo_matrix<int, double, cusp::host_memory> A = interpolation(g);
@@ -619,6 +627,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> smoothing( const dg::RefinedGri
     return C; 
 }
 
+///@deprecated
 cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const dg::RefinedGrid3d& g_fine)
 {
     dg::Grid3d g = g_fine.associated();
@@ -628,6 +637,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const dg::Refine
     return dg::create::interpolation( x,y,z, g);
 }
 
+///@deprecated
 cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const dg::RefinedGrid3d& g_fine)
 {
     cusp::coo_matrix<int, double, cusp::host_memory> temp = interpolation( g_fine), A;
@@ -635,6 +645,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const dg::Refin
     return A;
 }
 
+///@deprecated
 cusp::coo_matrix<int, double, cusp::host_memory> projection( const dg::RefinedGrid3d& g_fine)
 {
     //form the adjoint
@@ -660,6 +671,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> projection( const dg::RefinedGr
     return A;
 }
 
+///@deprecated
 cusp::coo_matrix<int, double, cusp::host_memory> smoothing( const dg::RefinedGrid3d& g)
 {
     cusp::coo_matrix<int, double, cusp::host_memory> A = interpolation(g);
-- 
GitLab


From 0d337db63c4c07e16cd1ea74fdbb3c4c26247cb3 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sun, 23 Jul 2017 21:21:43 +0200
Subject: [PATCH 072/453] make curvilinear use clone of generators

---
 inc/dg/backend/grid.h            | 23 ++++++++---
 inc/dg/backend/mpi_grid.h        | 16 ++++---
 inc/geometries/curvilinear.h     | 71 +++++++++++++++++++++++---------
 inc/geometries/magnetic_field.h  |  4 +-
 inc/geometries/mpi_curvilinear.h | 65 +++++++++++++++++++++--------
 5 files changed, 129 insertions(+), 50 deletions(-)

diff --git a/inc/dg/backend/grid.h b/inc/dg/backend/grid.h
index 7edae7b6a..bee863200 100644
--- a/inc/dg/backend/grid.h
+++ b/inc/dg/backend/grid.h
@@ -389,9 +389,15 @@ struct Grid2d
     }
   protected:
     virtual ~Grid2d(){}
-    Grid2d(const Grid2d& src){}
-    Grid2d& operator=(const Grid2d& src){}
-    ///@copydoc set()
+    Grid2d(const Grid2d& src):dlt_(src.dlt_){*this = src;}
+    Grid2d& operator=(const Grid2d& src){
+        x0_=src.x0_, x1_=src.x1_,y0_=src.y0_,y1_=src.y1_;
+        lx_=src.lx_,ly_=src.ly_;
+        n_=src.n_,Nx_=src.Nx_,Ny_=src.Ny_;
+        hx_=src.hx_,hy_=src.hy_;
+        bcx_=src.bcx_,bcy_=src.bcy_;
+        dlt_=src.dlt_;
+    }
     virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny) {
         n_ = new_n, Nx_ = new_Nx, Ny_ = new_Ny;
         hx_ = lx_/(double)Nx_, hy_ = ly_/(double)Ny_;
@@ -703,8 +709,15 @@ struct Grid3d
     }
   protected:
     virtual ~Grid3d(){}
-    Grid3d(const Grid3d& src){}
-    Grid3d& operator=(const Grid3d& src){}
+    Grid3d(const Grid3d& src):dlt_(src.dlt_){*this = src;}
+    Grid3d& operator=(const Grid3d& src){ //use default in C++11
+        x0_=src.x0_, x1_=src.x1_,y0_=src.y0_,y1_=src.y1_,z0_=src.z0_,z1_=src.z1_;
+        lx_=src.lx_,ly_=src.ly_,lz_=src.lz_;
+        n_=src.n_,Nx_=src.Nx_,Ny_=src.Ny_,Nz_=src.Nz_;
+        hx_=src.hx_,hy_=src.hy_,hz_=src.hz_;
+        bcx_=src.bcx_,bcy_=src.bcy_,bcz_=src.bcz_;
+        dlt_=src.dlt_;
+    }
     virtual void do_set(unsigned new_n, unsigned new_Ny, unsigned new_Nz)
     {
         n_ = new_n, Nx_ = new_Nx, Ny_ = new_Ny, Nz_ = new_Nz;
diff --git a/inc/dg/backend/mpi_grid.h b/inc/dg/backend/mpi_grid.h
index 3d18295d5..1a03e7096 100644
--- a/inc/dg/backend/mpi_grid.h
+++ b/inc/dg/backend/mpi_grid.h
@@ -66,8 +66,8 @@ struct MPIGrid2d
     *           use the resize function instead, or set( g.n(), 2*g.global().Nx(), 2*g.global().Ny())
     */
     void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny) {
-        do_set( new_n,new_Nx,new_Ny);
         check_division( new_Nx, new_Ny, g.bcx(), g.bcy());
+        do_set( new_n,new_Nx,new_Ny);
     }
 
     /**
@@ -303,8 +303,10 @@ struct MPIGrid2d
     virtual const Grid2d& global() const {return g;}
     protected:
     virtual ~MPIGrid2d(){}
-    MPIGrid2d(const MPIGrid2d& src){}
-    MPIGrid2d& operator=(const MPIGrid2d& src){}
+    MPIGrid2d(const MPIGrid2d& src):g(src.g),comm(src.comm){}
+    MPIGrid2d& operator=(const MPIGrid2d& src){
+        g = src.g; comm = src.comm;
+    }
     virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny) {
         g.set(new_n,new_Nx,new_Ny);
     }
@@ -382,8 +384,8 @@ struct MPIGrid3d
      *           use the resize function instead, or set( g.n(), 2*g.global().Nx(), 2*g.global().Ny(), 2*g.global().Nz())
      */
     void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz) {
-        do_set(new_n,new_Nx,new_Ny,new_Nz);
         check_division( new_Nx,new_Ny,new_Nz,g.bcx(),g.bcy(),g.bcz());
+        do_set(new_n,new_Nx,new_Ny,new_Nz);
     }
 
 
@@ -639,8 +641,10 @@ struct MPIGrid3d
         g.set(new_n,new_Nx,new_Ny,new_Nz);
     }
     virtual ~MPIGrid3d(){}
-    MPIGrid3d(const MPIGrid3d& src){}
-    MPIGrid3d& operator=(const MPIGrid3d& src){}
+    MPIGrid3d(const MPIGrid3d& src):g(src.g),comm(src.comm){}
+    MPIGrid3d& operator=(const MPIGrid3d& src){
+        g = src.g; comm = src.comm;
+    }
     void init_X_boundaries( double global_x0, double global_x1)
     {
         g.init_X_boundaries(global_x0, global_x1);
diff --git a/inc/geometries/curvilinear.h b/inc/geometries/curvilinear.h
index 6e8b9cfda..796362352 100644
--- a/inc/geometries/curvilinear.h
+++ b/inc/geometries/curvilinear.h
@@ -40,16 +40,6 @@ struct CurvilinearGrid3d : public dg::Grid3d
         generator_ = generator;
         construct( n, Nx, Ny);
     }
-    /**
-    * @brief Reconstruct the grid coordinates
-    *
-    * @copydetails Grid3d::set()
-    * @attention the generator must still live when this function is called
-    */
-    void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny,unsigned new_Nz){
-        dg::Grid3d::set( new_n, new_Nx, new_Ny,new_Nz);
-        construct( new_n, new_Nx, new_Ny);
-    }
 
     perpendicular_grid perp_grid() const { return perpendicular_grid(*this);}
     const thrust::host_vector<double>& r()const{return r_;}
@@ -64,10 +54,32 @@ struct CurvilinearGrid3d : public dg::Grid3d
     const container& g_pp()const{return g_pp_;}
     const container& vol()const{return vol_;}
     const container& perpVol()const{return vol2d_;}
-    const geo::aGenerator * generator() const{return generator_;}
+    const geo::aGenerator & generator() const{return *generator_;}
+    bool isOrthonormal() const { return generator_->isOrthonormal();}
     bool isOrthogonal() const { return generator_->isOrthogonal();}
     bool isConformal() const { return generator_->isConformal();}
+    Curvilinear3d( const Curvilinear3d& src):Grid3d(src), g_xx_(src.g_xx_),g_xy_(src.g_xy_),g_yy_(src.g_yy_),g_pp_(src.g_pp_),vol_(src.vol_),vol2d_(src.vol2d_)
+    {
+        r_=src.r_,z_=src.z_,xr_=src.xr_,xz_=src.xz_,yr_=src.yr_,yz_=src.yz_;
+        generator_ = src.generator_->clone();
+    }
+    Curvilinear3d& operator=( const Curvilinear3d& src)
+    {
+        Grid3d::operator=(src);//call base class assignment
+        delete generator_;
+        g_xx_=src.g_xx_,g_xy_=src.g_xy_,g_yy_=src.g_yy_,g_pp_=src.g_pp_,vol_=src.vol_,vol2d_=src.vol2d_;
+        r_=src.r_,z_=src.z_,xr_=src.xr_,xz_=src.xz_,yr_=src.yr_,yz_=src.yz_;
+        generator_ = src.generator_->clone();
+        return *this;
+    }
+    ~Curvilinear3d(){
+        delete generator_;
+    }
     private:
+    virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny,unsigned new_Nz){
+        dg::Grid3d::do_set( new_n, new_Nx, new_Ny,new_Nz);
+        construct( new_n, new_Nx, new_Ny);
+    }
     void construct( unsigned n, unsigned Nx, unsigned Ny)
     {
         dg::Grid1d gY1d( 0, generator_->height(), n, Ny, dg::PER);
@@ -160,14 +172,6 @@ struct CurvilinearGrid2d : public dg::Grid2d
         thrust::copy( g.perpVol().begin(), g.perpVol().begin()+s, vol2d_.begin());
     }
 
-    void set(unsigned new_n, unsigned new_Nx, unsigned new_Ny)
-    {
-        dg::Grid2d::set( new_n, new_Nx, new_Ny);
-        CurvilinearGrid3d<container> g( generator_, new_n,new_Nx,new_Ny,1,bcx());
-        r_=g.r(), z_=g.z(), xr_=g.xr(), xz_=g.xz(), yr_=g.yr(), yz_=g.yz();
-        g_xx_=g.g_xx(), g_xy_=g.g_xy(), g_yy_=g.g_yy();
-        vol2d_=g.perpVol();
-    }
     const thrust::host_vector<double>& r()const{return r_;}
     const thrust::host_vector<double>& z()const{return z_;}
     const thrust::host_vector<double>& xr()const{return xr_;}
@@ -179,10 +183,37 @@ struct CurvilinearGrid2d : public dg::Grid2d
     const container& g_xy()const{return g_xy_;}
     const container& vol()const{return vol2d_;}
     const container& perpVol()const{return vol2d_;}
-    const geo::aGenerator * generator() const{return generator_;}
+    const geo::aGenerator& generator() const{return *generator_;}
+    bool isOrthonormal() const { return generator_->isOrthonormal();}
     bool isOrthogonal() const { return generator_->isOrthogonal();}
     bool isConformal() const { return generator_->isConformal();}
+
+    Curvilinear2d( const Curvilinear2d& src):Grid2d(src), g_xx_(src.g_xx_),g_xy_(src.g_xy_),g_yy_(src.g_yy_),vol2d_(src.vol2d_)
+    {
+        r_=src.r_,z_=src.z_,xr_=src.xr_,xz_=src.xz_,yr_=src.yr_,yz_=src.yz_;
+        generator_ = src.generator_->clone();
+    }
+    Curvilinear2d& operator=( const Curvilinear2d& src)
+    {
+        Grid2d::operator=(src); //call base class assignment
+        delete generator_;
+        g_xx_=src.g_xx_,g_xy_=src.g_xy_,g_yy_=src.g_yy_,vol2d_=src.vol2d_;
+        r_=src.r_,z_=src.z_,xr_=src.xr_,xz_=src.xz_,yr_=src.yr_,yz_=src.yz_;
+        generator_ = src.generator_->clone();
+        return *this;
+    }
+    ~Curvilinear2d(){
+        delete generator_;
+    }
     private:
+    virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny)
+    {
+        dg::Grid2d::do_set( new_n, new_Nx, new_Ny);
+        CurvilinearGrid3d<container> g( generator_, new_n,new_Nx,new_Ny,1,bcx());
+        r_=g.r(), z_=g.z(), xr_=g.xr(), xz_=g.xz(), yr_=g.yr(), yz_=g.yz();
+        g_xx_=g.g_xx(), g_xy_=g.g_xy(), g_yy_=g.g_yy();
+        vol2d_=g.perpVol();
+    }
     thrust::host_vector<double> r_, z_, xr_, xz_, yr_, yz_;
     container g_xx_, g_xy_, g_yy_, vol2d_;
     const geo::aGenerator* generator_;
diff --git a/inc/geometries/magnetic_field.h b/inc/geometries/magnetic_field.h
index 1aeb46aba..ee67aaf64 100644
--- a/inc/geometries/magnetic_field.h
+++ b/inc/geometries/magnetic_field.h
@@ -71,7 +71,7 @@ struct dg::geo::aCloneableBinaryFunctor : public dg::geo::aBinaryFunctor
     *
     * @return new copy of the functor
     */
-    virtual aBinaryFunctor* clone() const
+    virtual Derived* clone() const
     {
         return new Derived(static_cast<Derived const &>(*this));
     }
@@ -104,6 +104,7 @@ struct aTokamakMagneticField
     aTokamakMagneticField& operator=( const aTokamakMagneticField& mag)
     {
         aTokamakMagneticField temp(mag);
+        std::swap( temp.R0_, R0_);
         std::swap( temp.p_, p_);
         return *this;
     }
@@ -147,6 +148,7 @@ struct aTokamakMagneticField
         aBinaryFunctor* ipolR,
         aBinaryFunctor* ipolZ
         ):p_(10){ 
+            R0_(R0),
             p_[0] = psip,
             p_[1] = psipR,
             p_[2] = psipZ,
diff --git a/inc/geometries/mpi_curvilinear.h b/inc/geometries/mpi_curvilinear.h
index 5ae41ce45..ed760c403 100644
--- a/inc/geometries/mpi_curvilinear.h
+++ b/inc/geometries/mpi_curvilinear.h
@@ -30,18 +30,12 @@ struct CurvilinearMPIGrid3d : public dg::MPIGrid3d
     typedef dg::CurvilinearMPIGrid2d<LocalContainer> perpendicular_grid; //!< the two-dimensional grid
     typedef typename MPIContainer::container_type LocalContainer; //!< the local container type
 
-    CurvilinearMPIGrid3d( const geo::aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx, MPI_Comm comm): 
-        dg::MPIGrid3d( 0, generator->width(), 0., generator->height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, dg::PER, dg::PER, comm),
+    CurvilinearMPIGrid3d( const geo::aGenerator& generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx, MPI_Comm comm): 
+        dg::MPIGrid3d( 0, generator.width(), 0., generator.height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, dg::PER, dg::PER, comm),
         g( generator, n,Nx, Ny, local().Nz(), bcx)
     {
         divide_and_conquer();
     }
-    void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz)
-    {
-        dg::MPIGrid3d::set(new_n, new_Nx, new_Ny, new_Nz);
-        g.set( new_n, new_Nx, new_Ny, new_Nz);//construct new grid
-        divide_and_conquer();//distribute to processes
-    }
 
     perpendicular_grid perp_grid() const { return perpendicular_grid(*this);}
 
@@ -58,10 +52,30 @@ struct CurvilinearMPIGrid3d : public dg::MPIGrid3d
     const MPIContainer& vol()const{return vol_;}
     const MPIContainer& perpVol()const{return vol2d_;}
     const dg::CurvilinearGrid3d<LocalContainer>& global() const {return g;}
-    const geo::aGenerator* generator() const{return g.generator();}
+    const geo::aGenerator& generator() const{return g.generator();}
+    bool isOrthonormal() const { return g.isOrthonormal();}
     bool isOrthogonal() const { return g.isOrthogonal();}
     bool isConformal() const { return g.isConformal();}
+    MPICurvilinear3d( const MPICurvilinear3d& src):MPIGrid3d(src), g_xx_(src.g_xx_),g_xy_(src.g_xy_),g_yy_(src.g_yy_),g_pp_(src.g_pp_),vol_(src.vol_),vol2d_(src.vol2d_),g(src.g)
+    {
+        r_=src.r_,z_=src.z_,xr_=src.xr_,xz_=src.xz_,yr_=src.yr_,yz_=src.yz_;
+    }
+    MPICurvilinear3d& operator=( const MPICurvilinear3d& src)
+    {
+        MPIGrid3d::operator=(src);//call base class assignment
+        g_xx_=src.g_xx_,g_xy_=src.g_xy_,g_yy_=src.g_yy_,g_pp_=src.g_pp_,vol_=src.vol_,vol2d_=src.vol2d_;
+        r_=src.r_,z_=src.z_,xr_=src.xr_,xz_=src.xz_,yr_=src.yr_,yz_=src.yz_;
+        g = src.g;
+        return *this;
+    }
+    ~MPICurvilinear3d(){ }
     private:
+    virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz)
+    {
+        dg::MPIGrid3d::do_set(new_n, new_Nx, new_Ny, new_Nz);
+        g.set( new_n, new_Nx, new_Ny, new_Nz);//construct new grid
+        divide_and_conquer();//distribute to processes
+    }
     void divide_and_conquer( )
     {
         r_=dg::evaluate( dg::one, *this), z_=r_, xr_=r_, xz_=r_, yr_=r_, yz_=r_;
@@ -107,8 +121,8 @@ struct CurvilinearMPIGrid2d : public dg::MPIGrid2d
     typedef dg::CurvilinearCylindricalTag metric_category; 
     typedef typename MPIContainer::container_type LocalContainer; //!< the local container type
 
-    CurvilinearMPIGrid2d( const geo::aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx, MPI_Comm comm2d): 
-        dg::MPIGrid2d( 0, generator->width(), 0., generator->height(), n, Nx, Ny, bcx, dg::PER, comm2d),
+    CurvilinearMPIGrid2d( const geo::aGenerator& generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx, MPI_Comm comm2d): 
+        dg::MPIGrid2d( 0, generator.width(), 0., generator.height(), n, Nx, Ny, bcx, dg::PER, comm2d),
         g_( generator, n,Nx, Ny, bcx)
     {
         divide_and_conquer();
@@ -135,12 +149,6 @@ struct CurvilinearMPIGrid2d : public dg::MPIGrid2d
         thrust::copy( g.perpVol().data().begin(), g.perpVol().data().begin()+s, vol2d_.data().begin());
         
     }
-    void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny)
-    {
-        dg::MPIGrid3d::set(new_n, new_Nx, new_Ny);
-        g.set( new_n, new_Nx, new_Ny);//construct new grid
-        divide_and_conquer();//distribute to processes
-    }
 
     const dg::MPI_Vector<thrust::host_vector<double> >& r()const{return r_;}
     const dg::MPI_Vector<thrust::host_vector<double> >& z()const{return z_;}
@@ -154,10 +162,31 @@ struct CurvilinearMPIGrid2d : public dg::MPIGrid2d
     const MPIContainer& vol()const{return vol2d_;}
     const MPIContainer& perpVol()const{return vol2d_;}
     const dg::CurvilinearGrid2d<LocalContainer>& global() const {return g_;}
-    const geo::aGenerator* generator() const{return g.generator();}
+    const geo::aGenerator& generator() const{return g.generator();}
+    bool isOrthonormal() const { return g.isOrthonormal();}
     bool isOrthogonal() const { return g.isOrthogonal();}
     bool isConformal() const { return g.isConformal();}
+    MPICurvilinear2d( const MPICurvilinear2d& src):MPIGrid2d(src), g_xx_(src.g_xx_),g_xy_(src.g_xy_),g_yy_(src.g_yy_),vol2d_(src.vol2d_)
+    {
+        r_=src.r_,z_=src.z_,xr_=src.xr_,xz_=src.xz_,yr_=src.yr_,yz_=src.yz_;
+        g_ = src.g_;
+    }
+    MPICurvilinear2d& operator=( const MPICurvilinear2d& src)
+    {
+        MPIGrid2d::operator=(src); //call base class assignment
+        g_xx_=src.g_xx_,g_xy_=src.g_xy_,g_yy_=src.g_yy_,vol2d_=src.vol2d_;
+        r_=src.r_,z_=src.z_,xr_=src.xr_,xz_=src.xz_,yr_=src.yr_,yz_=src.yz_;
+        g_ = src.g_;
+        return *this;
+    }
+    ~MPICurvilinear2d(){ }
     private:
+    virtual void do set( unsigned new_n, unsigned new_Nx, unsigned new_Ny)
+    {
+        dg::MPIGrid3d::do_set(new_n, new_Nx, new_Ny);
+        g.set( new_n, new_Nx, new_Ny);//construct new grid
+        divide_and_conquer();//distribute to processes
+    }
     MPI_Comm get_reduced_comm( MPI_Comm src)
     {
         MPI_Comm planeComm;
-- 
GitLab


From dd5cbd5e06ace6b169ff00b40e2c7202804c876c Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sun, 23 Jul 2017 21:41:55 +0200
Subject: [PATCH 073/453] write identity generator in generator.h

---
 inc/geometries/generator.h | 55 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 55 insertions(+)

diff --git a/inc/geometries/generator.h b/inc/geometries/generator.h
index 268859476..c7807659c 100644
--- a/inc/geometries/generator.h
+++ b/inc/geometries/generator.h
@@ -72,5 +72,60 @@ struct aGridGenerator
    virtual ~aGridGenerator(){}
 };
 
+/**
+* @brief The identity coordinate transformation
+
+ @ingroup generators
+*/
+struct IdentityGenerator: public aGridGenerator
+{
+    virtual double width()  const{return lx_;} 
+    virtual double height() const{return ly_;}
+    virtual bool isOrthonormal() const{return true;}
+    virtual bool isOrthogonal() const{return true;}
+    virtual bool isConformal()const{return true;}
+    virtual IdentityGenerator* clone() const{return new IdentityGenerator(*this);}
+
+    /**
+    * @brief Define the 2d box in which to construct the coordinates
+    *
+    * @param x0 x-coordinate of lower left point
+    * @param x1 x-coordinate of upper right point
+    * @param y0 y-coordinate of lower left point
+    * @param y1 y-coordainte of upper right point
+    */
+    IdentityGenerator( double x0, double x1, double y0, double y1){
+        x0_ = x0; lx_ = (x1-x0);
+        y0_ = y0; ly_ = (y1-y0);
+    }
+
+    protected:
+    virtual void generate(
+         const thrust::host_vector<double>& zeta1d, 
+         const thrust::host_vector<double>& eta1d, 
+         thrust::host_vector<double>& x, 
+         thrust::host_vector<double>& y, 
+         thrust::host_vector<double>& zetaX, 
+         thrust::host_vector<double>& zetaY, 
+         thrust::host_vector<double>& etaX, 
+         thrust::host_vector<double>& etaY) const
+     {
+         for(unsigned i=0; i<eta1d.size();i++)
+             for(unsigned j=0; j<zeta1d.size();j++)
+             {
+                 x[i*zeta1d.size()+j] = x0_ + zeta1d[j];
+                 y[i*zeta1d.size()+j] = y0_ + eta1d[i];
+                 zetaX[i*zeta1d.size()+j] = 1;
+                 zetaY[i*zeta1d.size()+j] = 0;
+                 etaX[i*zeta1d.size()+j] = 0.;
+                 etaY[i*zeta1d.size()+j] = 1.;
+             }
+                 
+     }
+     private:
+     double x0_,y0_,lx_,ly_;
+    
+};
+
 }//namespace geo
 }//namespace dg
-- 
GitLab


From 784c5a55bb4083e249729d5d3bf200beaf683000 Mon Sep 17 00:00:00 2001
From: Matthias Home Office <Matthias.Wiesenberger@uibk.ac.at>
Date: Mon, 24 Jul 2017 15:43:10 +0200
Subject: [PATCH 074/453] unify cylindrical and curvilinear grids

---
 inc/dg/backend/evaluation.cuh                 |  6 +-
 inc/dg/geometry.h                             |  3 +-
 inc/{geometries => dg/geometry}/curvilinear.h | 34 ++++++---
 inc/dg/geometry/cylindrical.h                 | 58 --------------
 inc/{geometries => dg/geometry}/generator.h   | 10 ++-
 inc/dg/geometry/geometry_traits.h             | 49 +++++++-----
 .../geometry}/mpi_curvilinear.h               | 76 +++++++++++++++----
 inc/dg/geometry/mpi_grids.h                   | 62 +--------------
 inc/dg/geometry/refined_grid.h                |  2 +-
 inc/geometries/magnetic_field.h               | 10 +--
 10 files changed, 135 insertions(+), 175 deletions(-)
 rename inc/{geometries => dg/geometry}/curvilinear.h (86%)
 delete mode 100644 inc/dg/geometry/cylindrical.h
 rename inc/{geometries => dg/geometry}/generator.h (91%)
 rename inc/{geometries => dg/geometry}/mpi_curvilinear.h (75%)

diff --git a/inc/dg/backend/evaluation.cuh b/inc/dg/backend/evaluation.cuh
index d400ed6bb..4fc0a8dd7 100644
--- a/inc/dg/backend/evaluation.cuh
+++ b/inc/dg/backend/evaluation.cuh
@@ -54,8 +54,7 @@ thrust::host_vector<double> evaluate( double (f)(double), const Grid1d& g)
  * @param g The 2d grid on which to evaluate f
  *
  * @return  A dG Host Vector with values
- * @note Copies the binary Operator. This function is meant for small function objects, that
-            may be constructed during function call.
+ * @note if you don't like to copy f then just pass a (const) reference then the type should adapt
  */
 template< class BinaryOp>
 thrust::host_vector<double> evaluate( BinaryOp f, const Grid2d& g)
@@ -94,8 +93,7 @@ thrust::host_vector<double> evaluate( double(f)(double, double), const Grid2d& g
  * @param g The 3d grid on which to evaluate f
  *
  * @return  A dG Host Vector with values
- * @note Copies the ternary Operator. This function is meant for small function objects, that
-            may be constructed during function call.
+ * @note if you don't like to copy f then just pass a (const) reference then the type should adapt
  */
 template< class TernaryOp>
 thrust::host_vector<double> evaluate( TernaryOp f, const Grid3d& g)
diff --git a/inc/dg/geometry.h b/inc/dg/geometry.h
index a45e2944d..51d3db914 100644
--- a/inc/dg/geometry.h
+++ b/inc/dg/geometry.h
@@ -11,10 +11,11 @@
 #endif//MPI_VERSION
 #include "geometry/geometry_traits.h"
 #include "geometry/cartesian.h"
+#include "geometry/curvilinear.h"
 #include "geometry/cartesianX.h"
-#include "geometry/cylindrical.h"
 #ifdef MPI_VERSION
 #include "geometry/mpi_grids.h"
+#include "geometry/mpi_curvilinear.h"
 #endif//MPI_VERSION
 
 
diff --git a/inc/geometries/curvilinear.h b/inc/dg/geometry/curvilinear.h
similarity index 86%
rename from inc/geometries/curvilinear.h
rename to inc/dg/geometry/curvilinear.h
index 796362352..2063c37f4 100644
--- a/inc/geometries/curvilinear.h
+++ b/inc/dg/geometry/curvilinear.h
@@ -20,11 +20,18 @@ struct CurvilinearGrid2d;
  @tparam container models aContainer
  */
 template< class container>
-struct CurvilinearGrid3d : public dg::Grid3d
+struct CylindricalGrid3d : public dg::Grid3d
 {
-    typedef dg::CurvilinearCylindricalTag metric_category;
+    typedef dg::CurvilinearPerpTag metric_category;
     typedef CurvilinearGrid2d<container> perpendicular_grid;
 
+    CylindricalGrid3d( double R0, double R1, double Z0, double Z1, double phi0, double phi1, unsigned n, unsigned NR, unsigned NZ, unsigned Nphi, bc bcr = PER, bc bcz = PER, bc bcphi = PER): 
+        dg::Grid3d(0,R1-R0,0,Z1-Z0,phi0,phi1,n,NR,NZ,Nphi,bcR,bcZ,bcphi)
+        {
+            generator_ = new IdentityGenerator(R0,R1,Z0,Z1);
+            construct( n, NR, NZ);
+        }
+
     /*!@brief Constructor
     
      * @param generator must generate a grid
@@ -34,7 +41,7 @@ struct CurvilinearGrid3d : public dg::Grid3d
      @param Nz 
      @param bcx
      */
-    CurvilinearGrid3d( const geo::aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx=dg::DIR):
+    CylindricalGrid3d( const geo::aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx=dg::DIR):
         dg::Grid3d( 0, generator->width(), 0., generator->height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, dg::PER, dg::PER)
     { 
         generator_ = generator;
@@ -58,12 +65,12 @@ struct CurvilinearGrid3d : public dg::Grid3d
     bool isOrthonormal() const { return generator_->isOrthonormal();}
     bool isOrthogonal() const { return generator_->isOrthogonal();}
     bool isConformal() const { return generator_->isConformal();}
-    Curvilinear3d( const Curvilinear3d& src):Grid3d(src), g_xx_(src.g_xx_),g_xy_(src.g_xy_),g_yy_(src.g_yy_),g_pp_(src.g_pp_),vol_(src.vol_),vol2d_(src.vol2d_)
+    CylindricalGrid3d( const CylindricalGrid3d& src):Grid3d(src), g_xx_(src.g_xx_),g_xy_(src.g_xy_),g_yy_(src.g_yy_),g_pp_(src.g_pp_),vol_(src.vol_),vol2d_(src.vol2d_)
     {
         r_=src.r_,z_=src.z_,xr_=src.xr_,xz_=src.xz_,yr_=src.yr_,yz_=src.yz_;
         generator_ = src.generator_->clone();
     }
-    Curvilinear3d& operator=( const Curvilinear3d& src)
+    CylindricalGrid3d& operator=( const CylindricalGrid3d& src)
     {
         Grid3d::operator=(src);//call base class assignment
         delete generator_;
@@ -72,7 +79,7 @@ struct CurvilinearGrid3d : public dg::Grid3d
         generator_ = src.generator_->clone();
         return *this;
     }
-    ~Curvilinear3d(){
+    ~CylindricalGrid3d(){
         delete generator_;
     }
     private:
@@ -138,7 +145,14 @@ struct CurvilinearGrid3d : public dg::Grid3d
 template< class container>
 struct CurvilinearGrid2d : public dg::Grid2d
 {
-    typedef dg::CurvilinearCylindricalTag metric_category;
+    typedef dg::CurvilinearPerpTag metric_category;
+
+    CurvilinearGrid2d( double R0, double R1, double Z0, double Z1, unsigned n, unsigned NR, unsigned NZ, unsigned Nphi, bc bcR = PER, bc bcZ = PER): 
+        dg::Grid2d(0,R1-R0,0,Z1-Z0,n,NR,NZ,bcR,bcZ)
+        {
+            generator_ = new IdentityGenerator(R0,R1,Z0,Z1);
+            construct( n, NR, NZ);
+        }
     /*!@brief Constructor
     
      * @param generator must generate an orthogonal grid
@@ -150,14 +164,14 @@ struct CurvilinearGrid2d : public dg::Grid2d
     CurvilinearGrid2d( const geo::aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx=dg::DIR):
         dg::Grid2d( 0, generator->width(), 0., generator->height(), n, Nx, Ny, bcx, dg::PER)
     {
-        CurvilinearGrid3d<container> g( generator, n,Nx,Ny,1,bcx);
+        CylindricalGrid3d<container> g( generator, n,Nx,Ny,1,bcx);
         init_X_boundaries( g.x0(), g.x1());
         r_=g.r(), z_=g.z(), xr_=g.xr(), xz_=g.xz(), yr_=g.yr(), yz_=g.yz();
         g_xx_=g.g_xx(), g_xy_=g.g_xy(), g_yy_=g.g_yy();
         vol2d_=g.perpVol();
 
     }
-    CurvilinearGrid2d( const CurvilinearGrid3d<container>& g):
+    CurvilinearGrid2d( const CylindricalGrid3d<container>& g):
         dg::Grid2d( g.x0(), g.x1(), g.y0(), g.y1(), g.n(), g.Nx(), g.Ny(), g.bcx(), g.bcy())
     {
         generator_ = g.generator();
@@ -209,7 +223,7 @@ struct CurvilinearGrid2d : public dg::Grid2d
     virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny)
     {
         dg::Grid2d::do_set( new_n, new_Nx, new_Ny);
-        CurvilinearGrid3d<container> g( generator_, new_n,new_Nx,new_Ny,1,bcx());
+        CylindricalGrid3d<container> g( generator_, new_n,new_Nx,new_Ny,1,bcx());
         r_=g.r(), z_=g.z(), xr_=g.xr(), xz_=g.xz(), yr_=g.yr(), yz_=g.yz();
         g_xx_=g.g_xx(), g_xy_=g.g_xy(), g_yy_=g.g_yy();
         vol2d_=g.perpVol();
diff --git a/inc/dg/geometry/cylindrical.h b/inc/dg/geometry/cylindrical.h
deleted file mode 100644
index 52c86f026..000000000
--- a/inc/dg/geometry/cylindrical.h
+++ /dev/null
@@ -1,58 +0,0 @@
-#pragma once
-
-#include "../backend/functions.h"
-#include "../backend/grid.h"
-#include "../backend/evaluation.cuh"
-#include "geometry_traits.h"
-
-namespace dg
-{
-///@addtogroup basicgrids
-///@{
-
-/**
- * @brief three-dimensional Grid with Cartesian metric
- * 
- * @tparam container The container class for the volume element
- */
-template<class container>
-struct CylindricalGrid3d : public dg::Grid3d
-{
-    typedef OrthonormalCylindricalTag metric_category; 
-    typedef dg::CartesianGrid2d perpendicular_grid;
-    ///@copydoc Grid3d()
-    CylindricalGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = PER, bc bcy = PER, bc bcz = PER): 
-        dg::Grid3d(x0,x1,y0,y1,z0,z1,n,Nx,Ny,Nz,bcx,bcy,bcz),
-        R_(dg::evaluate( dg::cooX3d, *this)){}
-    /**
-     * @brief Construct from existing topology
-     *
-     * @param grid existing grid class
-     */
-    //is this constructor a good idea?? You could construct a Cylindrical Grid from any other Grid Type that derives from Grid3d
-    CylindricalGrid3d( const dg::Grid3d& grid):
-        dg::Grid3d(grid),
-        R_(dg::evaluate( dg::cooX3d, *this)){}
-
-    /**
-    * @brief Return the grid of the R-Z planes
-    * @return a Cartesian 2d grid of the R-Z plane
-    */
-    perpendicular_grid perp_grid() const { return dg::CartesianGrid2d( x0(), x1(), y0(), y1(), n(), Nx(), Ny(), bcx(), bcy());}
-    /**
-     * @brief The volume element R
-     * @return the volume element R
-     */
-    const container& vol()const {return R_;}
-    private:
-    virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz){
-        dg::Grid3d::set(new_n,new_Nx,new_Ny,new_Nz);
-        R_=dg::evaluate(dg::cooX3d, *this);
-    }
-    container R_;
-};
-
-///@}
-
-} //namespace dg
-
diff --git a/inc/geometries/generator.h b/inc/dg/geometry/generator.h
similarity index 91%
rename from inc/geometries/generator.h
rename to inc/dg/geometry/generator.h
index c7807659c..48b8e3708 100644
--- a/inc/geometries/generator.h
+++ b/inc/dg/geometry/generator.h
@@ -10,12 +10,13 @@ namespace geo
 A generator is there to construct coordinate transformations from physical coordinates
 \f$ x,y\f$ to the computational domain \f$\zeta, \eta\f$, which
 is a product space. 
+@note the origin of the computational space is assumed to be (0,0)
  @ingroup generators
 */
 struct aGridGenerator
 {
-    virtual double width()  const=0; //!<length in \f$ \zeta\f$ 
-    virtual double height() const=0; //!<length in \f$ \eta\f$
+    virtual double width()  const=0; //!<length in \f$ \zeta\f$ of the computational space
+    virtual double height() const=0; //!<length in \f$ \eta\f$ of the computational space
     virtual bool isOrthonormal() const{return false;} //!< true if coordinate system is orthonormal (false by default)
     virtual bool isOrthogonal() const{return false;} //!< true if coordinate system is orthogonal (false by default)
     virtual bool isConformal()const{return false;} //!< true if coordinate system is conformal (false by default)
@@ -73,8 +74,9 @@ struct aGridGenerator
 };
 
 /**
-* @brief The identity coordinate transformation
+* @brief The shifted identity coordinate transformation
 
+@note in fact it's not completely the identity because we assume that the origin is always (0,0) in the computational space
  @ingroup generators
 */
 struct IdentityGenerator: public aGridGenerator
@@ -87,7 +89,7 @@ struct IdentityGenerator: public aGridGenerator
     virtual IdentityGenerator* clone() const{return new IdentityGenerator(*this);}
 
     /**
-    * @brief Define the 2d box in which to construct the coordinates
+    * @brief Define the 2d box in the physical domain in which to construct the coordinates
     *
     * @param x0 x-coordinate of lower left point
     * @param x1 x-coordinate of upper right point
diff --git a/inc/dg/geometry/geometry_traits.h b/inc/dg/geometry/geometry_traits.h
index a19c2c962..1b27a1253 100644
--- a/inc/dg/geometry/geometry_traits.h
+++ b/inc/dg/geometry/geometry_traits.h
@@ -6,10 +6,9 @@ namespace dg
 {
 //categories
 //
-struct CurvilinearTag{};  //! 3d curvilinear
-struct CurvilinearCylindricalTag: public CurvilinearTag{}; //! perpVol, vol(), g_xx, g_xy, g_yy
-struct OrthonormalCylindricalTag:public CurvilinearCylindricalTag{}; //! vol(), cylindrical grid
-struct OrthonormalTag: public OrthonormalCylindricalTag{}; //! Cartesian grids
+struct CurvilinearTag{};  //! For completeness only (in fact we don't have a true 3d curvilinear grid right now)
+struct CurvilinearPerpTag: public CurvilinearTag{}; //! perpVol, vol(), g_xx, g_xy, g_yy (2x1 product grid, or 2d curvilinear)
+struct OrthonormalTag: public CurvilinearPerpTag{}; //! Cartesian grids  (3d or 2d)
 
 //memory_category and dimensionality Tags are already defined in grid.h
 
@@ -63,36 +62,44 @@ void doDivideVolume( container& inout, const Geometry& g, CurvilinearTag)
 };
 
 template <class container, class Geometry>
-void doMultiplyPerpVolume( container& inout, const Geometry& g, OrthonormalCylindricalTag)
+void doMultiplyPerpVolume( container& inout, const Geometry& g, OrthonormalTag)
 {
 };
 
 template <class container, class Geometry>
-void doMultiplyPerpVolume( container& inout, const Geometry& g, CurvilinearCylindricalTag)
+void doMultiplyPerpVolume( container& inout, const Geometry& g, CurvilinearPerpTag)
 {
-    dg::blas1::pointwiseDot( inout, g.perpVol(), inout);
+    if( !g.isOrthonormal())
+        dg::blas1::pointwiseDot( inout, g.perpVol(), inout);
 };
 
 template <class container, class Geometry>
-void doDividePerpVolume( container& inout, const Geometry& g, OrthonormalCylindricalTag)
+void doDividePerpVolume( container& inout, const Geometry& g, OrthonormalTag)
 {
 };
 
 template <class container, class Geometry>
-void doDividePerpVolume( container& inout, const Geometry& g, CurvilinearCylindricalTag)
+void doDividePerpVolume( container& inout, const Geometry& g, CurvilinearPerpTag)
 {
-    dg::blas1::pointwiseDivide( inout, g.perpVol(), inout);
+    if( !g.isOrthonormal())
+        dg::blas1::pointwiseDivide( inout, g.perpVol(), inout);
 };
 
 template <class container, class Geometry>
-void doRaisePerpIndex( container& in1, container& in2, container& out1, container& out2, const Geometry& g, OrthonormalCylindricalTag)
+void doRaisePerpIndex( container& in1, container& in2, container& out1, container& out2, const Geometry& g, OrthonormalTag)
 {
     in1.swap( out1);
     in2.swap( out2);
 };
 template <class container, class Geometry>
-void doRaisePerpIndex( container& in1, container& in2, container& out1, container& out2, const Geometry& g, CurvilinearCylindricalTag)
+void doRaisePerpIndex( container& in1, container& in2, container& out1, container& out2, const Geometry& g, CurvilinearPerpTag)
 {
+    if( g.isOrthonormal())
+    {
+        in1.swap( out1);
+        in2.swap( out2);
+        return;
+    }
     if( g.isOrthogonal())
     {
         dg::blas1::pointwiseDot( g.g_xx(), in1, out1); //gxx*v_x
@@ -106,13 +113,13 @@ void doRaisePerpIndex( container& in1, container& in2, container& out1, containe
 };
 
 template <class container, class Geometry>
-void doVolRaisePerpIndex( container& in1, container& in2, container& out1, container& out2, const Geometry& g, OrthonormalCylindricalTag)
+void doVolRaisePerpIndex( container& in1, container& in2, container& out1, container& out2, const Geometry& g, OrthonormalTag)
 {
     in1.swap( out1);
     in2.swap( out2);
 };
 template <class container, class Geometry>
-void doVolRaisePerpIndex( container& in1, container& in2, container& out1, container& out2, const Geometry& g, CurvilinearCylindricalTag)
+void doVolRaisePerpIndex( container& in1, container& in2, container& out1, container& out2, const Geometry& g, CurvilinearPerpTag)
 {
     if( g.isConformal())
     {
@@ -140,7 +147,7 @@ void doVolRaisePerpIndex( container& in1, container& in2, container& out1, conta
 template<class TernaryOp1, class TernaryOp2, class container, class Geometry> 
 void doPushForwardPerp( TernaryOp1 f1, TernaryOp2 f2, 
         container& vx, container& vy,
-        const Geometry& g, OrthonormalCylindricalTag)
+        const Geometry& g, OrthonormalTag)
 {
     typedef typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector host_vec;
     host_vec out1 = evaluate( f1, g);
@@ -152,7 +159,7 @@ void doPushForwardPerp( TernaryOp1 f1, TernaryOp2 f2,
 template<class TernaryOp1, class TernaryOp2, class container, class Geometry> 
 void doPushForwardPerp( TernaryOp1 f1, TernaryOp2 f2, 
         container& vx, container& vy,
-        const Geometry& g, CurvilinearCylindricalTag)
+        const Geometry& g, CurvilinearPerpTag)
 {
     typedef typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector host_vec;
     host_vec out1 = pullback( f1, g), temp1(out1);
@@ -168,7 +175,7 @@ void doPushForwardPerp( TernaryOp1 f1, TernaryOp2 f2,
 template<class FunctorRR, class FunctorRZ, class FunctorZZ, class container, class Geometry> 
 void doPushForwardPerp( FunctorRR chiRR, FunctorRZ chiRZ, FunctorZZ chiZZ,
         container& chixx, container& chixy, container& chiyy,
-        const Geometry& g, OrthonormalCylindricalTag)
+        const Geometry& g, OrthonormalTag)
 {
     typedef typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector host_vec;
     host_vec chixx_ = evaluate( chiRR, g);
@@ -182,7 +189,7 @@ void doPushForwardPerp( FunctorRR chiRR, FunctorRZ chiRZ, FunctorZZ chiZZ,
 template<class FunctorRR, class FunctorRZ, class FunctorZZ, class container, class Geometry> 
 void doPushForwardPerp( FunctorRR chiRR_, FunctorRZ chiRZ_, FunctorZZ chiZZ_,
         container& chixx, container& chixy, container& chiyy,
-        const Geometry& g, CurvilinearCylindricalTag)
+        const Geometry& g, CurvilinearPerpTag)
 {
     //compute the rhs
     typedef typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector host_vec;
@@ -286,7 +293,7 @@ thrust::host_vector<double> doPullback( BinaryOp f, const Geometry& g, Curviline
 }
 
 template< class TernaryOp, class Geometry>
-thrust::host_vector<double> doPullback( TernaryOp f, const Geometry& g, CurvilinearTag, ThreeDimensionalTag, SharedTag)
+thrust::host_vector<double> doPullback( TernaryOp f, const Geometry& g, CurvilinearPerpTag, ThreeDimensionalTag, SharedTag)
 {
     thrust::host_vector<double> vec( g.size());
     unsigned size2d = g.n()*g.n()*g.Nx()*g.Ny();
@@ -298,12 +305,12 @@ thrust::host_vector<double> doPullback( TernaryOp f, const Geometry& g, Curvilin
     return vec;
 }
 template< class BinaryOp, class Geometry>
-thrust::host_vector<double> doPullback( BinaryOp f, const Geometry& g, OrthonormalCylindricalTag, TwoDimensionalTag, SharedTag)
+thrust::host_vector<double> doPullback( BinaryOp f, const Geometry& g, OrthonormalTag, TwoDimensionalTag, SharedTag)
 {
     return evaluate( f, g);
 }
 template< class TernaryOp, class Geometry>
-thrust::host_vector<double> doPullback( TernaryOp f, const Geometry& g, OrthonormalCylindricalTag, ThreeDimensionalTag, SharedTag)
+thrust::host_vector<double> doPullback( TernaryOp f, const Geometry& g, OrthonormalTag, ThreeDimensionalTag, SharedTag)
 {
     return evaluate( f,g);
 }
diff --git a/inc/geometries/mpi_curvilinear.h b/inc/dg/geometry/mpi_curvilinear.h
similarity index 75%
rename from inc/geometries/mpi_curvilinear.h
rename to inc/dg/geometry/mpi_curvilinear.h
index ed760c403..c1477a2e2 100644
--- a/inc/geometries/mpi_curvilinear.h
+++ b/inc/dg/geometry/mpi_curvilinear.h
@@ -21,16 +21,40 @@ struct CurvilinearMPIGrid2d;
 ///@{
 
 /**
- * @tparam LocalContainer Vector class that holds metric coefficients
+ * This is s 2x1 product space grid
+ * @tparam MPIContainer Vector class that holds metric coefficients
  */
 template<class MPIContainer>
-struct CurvilinearMPIGrid3d : public dg::MPIGrid3d
+struct CylindricalMPIGrid3d : public dg::MPIGrid3d
 {
-    typedef dg::CurvilinearCylindricalTag metric_category; //!< metric tag
-    typedef dg::CurvilinearMPIGrid2d<LocalContainer> perpendicular_grid; //!< the two-dimensional grid
+    typedef dg::CurvilinearPerpTag metric_category; //!< metric tag
+    typedef dg::CylindricalMPIGrid2d<LocalContainer> perpendicular_grid; //!< the two-dimensional grid
     typedef typename MPIContainer::container_type LocalContainer; //!< the local container type
+    /**
+     * @copydoc Grid3d::Grid3d()
+     * @param comm a three-dimensional Cartesian communicator
+     * @note the paramateres given in the constructor are global parameters 
+     */
+    CylindricalMPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, MPI_Comm comm): 
+        dg::MPIGrid3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, comm),
+        g( new IdentityGenerator(x0,x1,y0,y1), n,Nx,Ny,local().Nz(), bcx,bcy,bcz)
+     {
+        divide_and_conquer();
+     }
+
+    /**
+     * @copydoc Grid3d::Grid3d()
+     * @param comm a three-dimensional Cartesian communicator
+     * @note the paramateres given in the constructor are global parameters 
+     */
+    CylindricalMPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz, MPI_Comm comm):
+        dg::MPIGrid3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, bcx, bcy, bcz, comm),
+        g( new IdentityGenerator(x0,x1,y0,y1), n,Nx,Ny,local().Nz(), bcx,bcy,bcz)
+     {
+        divide_and_conquer();
+     }
 
-    CurvilinearMPIGrid3d( const geo::aGenerator& generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx, MPI_Comm comm): 
+    CylindricalMPIGrid3d( const geo::aGenerator& generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx, MPI_Comm comm): 
         dg::MPIGrid3d( 0, generator.width(), 0., generator.height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, dg::PER, dg::PER, comm),
         g( generator, n,Nx, Ny, local().Nz(), bcx)
     {
@@ -51,16 +75,22 @@ struct CurvilinearMPIGrid3d : public dg::MPIGrid3d
     const MPIContainer& g_pp()const{return g_pp_;}
     const MPIContainer& vol()const{return vol_;}
     const MPIContainer& perpVol()const{return vol2d_;}
-    const dg::CurvilinearGrid3d<LocalContainer>& global() const {return g;}
+    /**
+     * @brief returns global non-MPI grid
+     *
+     * @attention the number of z-planes in this grid is the local number this process holds NOT the global one
+     * @return 
+     */
+    const dg::CylindricalGrid3d<LocalContainer>& global() const {return g;}
     const geo::aGenerator& generator() const{return g.generator();}
     bool isOrthonormal() const { return g.isOrthonormal();}
     bool isOrthogonal() const { return g.isOrthogonal();}
     bool isConformal() const { return g.isConformal();}
-    MPICurvilinear3d( const MPICurvilinear3d& src):MPIGrid3d(src), g_xx_(src.g_xx_),g_xy_(src.g_xy_),g_yy_(src.g_yy_),g_pp_(src.g_pp_),vol_(src.vol_),vol2d_(src.vol2d_),g(src.g)
+    MPICylindricalGrid3d( const MPICylindricalGrid3d& src):MPIGrid3d(src), g_xx_(src.g_xx_),g_xy_(src.g_xy_),g_yy_(src.g_yy_),g_pp_(src.g_pp_),vol_(src.vol_),vol2d_(src.vol2d_),g(src.g)
     {
         r_=src.r_,z_=src.z_,xr_=src.xr_,xz_=src.xz_,yr_=src.yr_,yz_=src.yz_;
     }
-    MPICurvilinear3d& operator=( const MPICurvilinear3d& src)
+    MPICylindricalGrid3d& operator=( const MPICylindricalGrid3d& src)
     {
         MPIGrid3d::operator=(src);//call base class assignment
         g_xx_=src.g_xx_,g_xy_=src.g_xy_,g_yy_=src.g_yy_,g_pp_=src.g_pp_,vol_=src.vol_,vol2d_=src.vol2d_;
@@ -68,7 +98,7 @@ struct CurvilinearMPIGrid3d : public dg::MPIGrid3d
         g = src.g;
         return *this;
     }
-    ~MPICurvilinear3d(){ }
+    ~MPICylindricalGrid3d(){ }
     private:
     virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz)
     {
@@ -109,17 +139,37 @@ struct CurvilinearMPIGrid3d : public dg::MPIGrid3d
     }
     dg::MPI_Vector<thrust::host_vector<double> > r_, z_, xr_, xz_, yr_, yz_; //3d vector
     MPIContainer g_xx_, g_xy_, g_yy_, g_pp_, vol_, vol2d_;
-    dg::CurvilinearGrid3d<LocalContainer> g;
+    dg::CylindricalGrid3d<LocalContainer> g;
 };
 
 /**
- * @tparam LocalContainer Vector class that holds metric coefficients
+ * @tparam MPIContainer Vector class that holds metric coefficients
  */
 template<class MPIContainer>
 struct CurvilinearMPIGrid2d : public dg::MPIGrid2d
 {
-    typedef dg::CurvilinearCylindricalTag metric_category; 
+    typedef dg::CurvilinearPerpTag metric_category; 
     typedef typename MPIContainer::container_type LocalContainer; //!< the local container type
+    /**
+     * @copydoc Grid2d::Grid2d()
+     * @param comm a two-dimensional Cartesian communicator
+     * @note the paramateres given in the constructor are global parameters 
+     */
+    CurvilinearMPIGrid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, MPI_Comm comm): dg::MPIGrid2d( x0, x1, y0, y1, n, Nx, Ny, comm),
+    g_(new IdentityGenerator(x0,x1,y0,y1),n,Nx,Ny){
+        divide_and_conquer();
+    
+    }
+
+    /**
+     * @copydoc Grid2d::Grid2d()
+     * @param comm a two-dimensional Cartesian communicator
+     * @note the paramateres given in the constructor are global parameters 
+     */
+    CurvilinearMPIGrid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx, bc bcy, MPI_Comm comm):dg::MPIGrid2d( x0, x1, y0, y1, n, Nx, Ny,bcx, bcy, comm),
+    g_(new IdentityGenerator(x0,x1,y0,y1),n,Nx,Ny){
+        divide_and_conquer();
+    }
 
     CurvilinearMPIGrid2d( const geo::aGenerator& generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx, MPI_Comm comm2d): 
         dg::MPIGrid2d( 0, generator.width(), 0., generator.height(), n, Nx, Ny, bcx, dg::PER, comm2d),
@@ -127,7 +177,7 @@ struct CurvilinearMPIGrid2d : public dg::MPIGrid2d
     {
         divide_and_conquer();
     }
-    CurvilinearMPIGrid2d( const CurvilinearMPIGrid3d<LocalContainer>& g):
+    CurvilinearMPIGrid2d( const CylindricalMPIGrid3d<LocalContainer>& g):
         dg::MPIGrid2d( g.global().x0(), g.global().x1(), g.global().y0(), g.global().y1(), g.global().n(), g.global().Nx(), g.global().Ny(), g.global().bcx(), g.global().bcy(), get_reduced_comm( g.communicator() )),
         r_(dg::evaluate( dg::one, *this)), z_(r_), xr_(r_), xz_(r_), yr_(r_), yz_(r_), 
         g_xx_(r_), g_xy_(g_xx_), g_yy_(g_xx_), vol2d_(g_xx_),
diff --git a/inc/dg/geometry/mpi_grids.h b/inc/dg/geometry/mpi_grids.h
index ead8d5ee6..58b9ae40e 100644
--- a/inc/dg/geometry/mpi_grids.h
+++ b/inc/dg/geometry/mpi_grids.h
@@ -2,7 +2,7 @@
 
 #include "geometry_traits.h"
 #include "../backend/mpi_grid.h"
-#include "cylindrical.h"
+#include "curvilinear_cylindrical.h"
 
 namespace dg
 {
@@ -59,60 +59,6 @@ struct CartesianMPIGrid3d : public dg::MPIGrid3d
 
 ///@}
 
-/**
- * @brief MPI version of Cylindrical grid
- *
- * @ingroup basicgrids
- * @tparam MPIContainer The MPI Vector container
- */
-template<class MPIContainer>
-struct CylindricalMPIGrid3d : public MPIGrid3d
-{
-    typedef typename MPIContainer::container_type LocalContainer; //!< the local container type
-    typedef OrthonormalCylindricalTag metric_category; 
-    typedef dg::CartesianMPIGrid2d perpendicular_grid;
-
-    /**
-     * @copydoc Grid3d::Grid3d()
-     * @param comm a three-dimensional Cartesian communicator
-     * @note the paramateres given in the constructor are global parameters 
-     */
-    CylindricalMPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, MPI_Comm comm): 
-        dg::MPIGrid3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, comm), 
-        R_( dg::evaluate( dg::cooX3d, *this)) { }
-
-    /**
-     * @copydoc Grid3d::Grid3d()
-     * @param comm a three-dimensional Cartesian communicator
-     * @note the paramateres given in the constructor are global parameters 
-     */
-    CylindricalMPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz, MPI_Comm comm):
-        dg::MPIGrid3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, bcx, bcy, bcz, comm),
-        R_( dg::evaluate( dg::cooX3d, *this))
-        {}
-
-    CylindricalMPIGrid3d( const MPIGrid3d& grid ):
-        MPIGrid3d( grid),
-        R_( dg::evaluate( dg::cooX3d, *this))
-    {}
-
-    const MPIContainer& vol() const { return R_;}
-    ///@copydoc CylindricalGrid3d::perp_grid()
-    perpendicular_grid perp_grid() const { 
-        MPI_Comm planeComm;
-        int remain_dims[] = {true,true,false}; //true true false
-        MPI_Cart_sub( communicator(), remain_dims, &planeComm);
-        return dg::CartesianMPIGrid2d( global().x0(), global().x1(), global().y0(), global().y1(), global().n(), global().Nx(), global().Ny(), global().bcx(), global().bcy(), planeComm);
-    }
-    private:
-    virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz){
-        MPIGrid3d::set(new_n,new_Ny,new_Nz);
-        R_=dg::evaluate(dg::cooX3d, *this);
-    }
-    MPIContainer R_;
-};
-///@}
-
 ///@cond
 /////////////////////////////////////////////////////MPI pullbacks/////////////////////////////////////////////////
 namespace detail{
@@ -138,7 +84,7 @@ MPI_Vector< thrust::host_vector<double> > doPullback( BinaryOp f, const Geometry
 }
 
 template< class TernaryOp, class Geometry>
-MPI_Vector< thrust::host_vector<double> > doPullback( TernaryOp f, const Geometry& g, CurvilinearTag, ThreeDimensionalTag, MPITag)
+MPI_Vector< thrust::host_vector<double> > doPullback( TernaryOp f, const Geometry& g, CurvilinearPerpTag, ThreeDimensionalTag, MPITag)
 {
     thrust::host_vector<double> vec( g.size());
     unsigned size2d = g.n()*g.n()*g.Nx()*g.Ny();
@@ -151,12 +97,12 @@ MPI_Vector< thrust::host_vector<double> > doPullback( TernaryOp f, const Geometr
     return v;
 }
 template< class BinaryOp, class Geometry>
-MPI_Vector< thrust::host_vector<double> > doPullback( BinaryOp f, const Geometry& g, OrthonormalCylindricalTag, TwoDimensionalTag, MPITag)
+MPI_Vector< thrust::host_vector<double> > doPullback( BinaryOp f, const Geometry& g, OrthonormalTag, TwoDimensionalTag, MPITag)
 {
     return evaluate( f, g);
 }
 template< class TernaryOp, class Geometry>
-MPI_Vector< thrust::host_vector<double> > doPullback( TernaryOp f, const Geometry& g, OrthonormalCylindricalTag, ThreeDimensionalTag, MPITag)
+MPI_Vector< thrust::host_vector<double> > doPullback( TernaryOp f, const Geometry& g, OrthonormalTag, ThreeDimensionalTag, MPITag)
 {
     return evaluate( f,g);
 }
diff --git a/inc/dg/geometry/refined_grid.h b/inc/dg/geometry/refined_grid.h
index 28b178d72..50a1d9ebe 100644
--- a/inc/dg/geometry/refined_grid.h
+++ b/inc/dg/geometry/refined_grid.h
@@ -546,7 +546,7 @@ RefinedGrid2d::RefinedGrid2d( const dg::RefinedGrid3d& g) :
 template<class container>
 struct CartesianRefinedGrid2d : public dg::RefinedGrid2d
 {
-    typedef CurvilinearCylindricalTag metric_category; 
+    typedef CurvilinearPerpTag metric_category; 
     CartesianRefinedGrid2d( unsigned multiple_x, unsigned multiple_y, double x0, double x1, double y0, double y1, unsigned n, unsigned n_old, unsigned Nx, unsigned Ny, bc bcx = PER, bc bcy = PER): dg::RefinedGrid2d(multiple_x, multiple_y,x0,x1,y0,y1,n,n_old,Nx,Ny,bcx,bcy), g_assoc_(x0,x1,y0,y1,n_old,Nx,Ny,bcx,bcy){ 
         dg::blas1::transfer( weightsX(), g_xx_);
         dg::blas1::transfer( weightsY(), g_yy_);
diff --git a/inc/geometries/magnetic_field.h b/inc/geometries/magnetic_field.h
index ee67aaf64..1b57e4fe2 100644
--- a/inc/geometries/magnetic_field.h
+++ b/inc/geometries/magnetic_field.h
@@ -131,11 +131,6 @@ struct aTokamakMagneticField
     /// \f$ \partial_Z I(\psi_p) \f$ 
     const aBinaryFunctor& ipolZ()const{return *p_[9];}
 
-    protected:
-    ~aTokamakMagneticField(){
-        for( unsigned i=0; i<p_.size(); i++)
-            delete p_[i];
-    }
     aTokamakMagneticField( double R0,
         aBinaryFunctor* psip,
         aBinaryFunctor* psipR,
@@ -160,6 +155,11 @@ struct aTokamakMagneticField
             p_[8] = ipolR,
             p_[9] = ipolZ,
         }
+    protected:
+    ~aTokamakMagneticField(){
+        for( unsigned i=0; i<p_.size(); i++)
+            delete p_[i];
+    }
     private:
     double R0_;
     std::vector<aBinaryFunctor*> p_;
-- 
GitLab


From 707925a453bef1bafb9825510ec1b44f35b44fbb Mon Sep 17 00:00:00 2001
From: Matthias Home Office <Matthias.Wiesenberger@uibk.ac.at>
Date: Mon, 24 Jul 2017 15:46:13 +0200
Subject: [PATCH 075/453] moved refined grid to geometry

---
 inc/{geometries => dg/geometry}/refined_curvilinear.h |  2 ++
 inc/geometries/README                                 | 10 +++-------
 2 files changed, 5 insertions(+), 7 deletions(-)
 rename inc/{geometries => dg/geometry}/refined_curvilinear.h (99%)

diff --git a/inc/geometries/refined_curvilinear.h b/inc/dg/geometry/refined_curvilinear.h
similarity index 99%
rename from inc/geometries/refined_curvilinear.h
rename to inc/dg/geometry/refined_curvilinear.h
index 062b7899b..20e4d6af3 100644
--- a/inc/geometries/refined_curvilinear.h
+++ b/inc/dg/geometry/refined_curvilinear.h
@@ -16,6 +16,7 @@ struct CurvilinearRefinedGrid2d;
 
 /**
  * @brief A curvilinear refined grid
+ * @deprecated
  */
 template< class container>
 struct CurvilinearRefinedGrid3d : public dg::RefinedGrid3d
@@ -102,6 +103,7 @@ struct CurvilinearRefinedGrid3d : public dg::RefinedGrid3d
 
 /**
  * @brief A curvilinear refined grid
+ * @deprecated
  */
 template< class container>
 struct CurvilinearRefinedGrid2d : public dg::RefinedGrid2d
diff --git a/inc/geometries/README b/inc/geometries/README
index 10720c2df..dd0f7bb49 100644
--- a/inc/geometries/README
+++ b/inc/geometries/README
@@ -1,21 +1,17 @@
 
 #grids
-    curvilinear.h
-    mpi_curvilinear.h
-    refined_curvilinear.h
-
     curvilinearX.h
     refined_curvilinearX.h
 
 #generators
-    generator.h
     simple_orthogonal.h
-    separatrix_orthogonal.h
     ribeiro.h
-    ribeiroX.h
     flux.h
     hector.h
 
+    separatrix_orthogonal.h
+    ribeiroX.h
+
     #utilities
         utilities.h
         utilitiesX.h
-- 
GitLab


From 3dc18795872149b355b79eeef8d1478d52247197 Mon Sep 17 00:00:00 2001
From: Matthias Home Office <Matthias.Wiesenberger@uibk.ac.at>
Date: Mon, 24 Jul 2017 16:02:34 +0200
Subject: [PATCH 076/453] update testfunctors and average

---
 inc/geometries/average.h      |  46 +++++------
 inc/geometries/testfunctors.h | 141 +++++++++++++++-------------------
 2 files changed, 83 insertions(+), 104 deletions(-)

diff --git a/inc/geometries/average.h b/inc/geometries/average.h
index 0724ec0c2..78e3f6a64 100644
--- a/inc/geometries/average.h
+++ b/inc/geometries/average.h
@@ -13,13 +13,11 @@ namespace geo
 /**
  * @brief Delta function for poloidal flux \f$ B_Z\f$
      \f[ |\nabla \psi_p|\delta(\psi_p(R,Z)-\psi_0) = \frac{\sqrt{ (\nabla \psi_p)^2}}{\sqrt{2\pi\varepsilon}} \exp\left(-\frac{(\psi_p(R,Z) - \psi_{0})^2}{2\varepsilon} \right)  \f]
-     @tparam MagneticField models aTokamakMagneticField
      @ingroup profiles
  */
-template<class MagneticField>
 struct DeltaFunction
 {
-    DeltaFunction(const MagneticField& c, double epsilon,double psivalue) :
+    DeltaFunction(const aTokamakMagneticField& c, double epsilon,double psivalue) :
         c_(c),
         epsilon_(epsilon),
         psivalue_(psivalue){
@@ -42,7 +40,7 @@ struct DeltaFunction
      */
     double operator()( double R, double Z) const
     {
-        double psip = c_.psip(R,Z), psipR = c_.psipR(R,Z), psipZ = c_.psipZ(R,Z);
+        double psip = c_.psip()(R,Z), psipR = c_.psipR()(R,Z), psipZ = c_.psipZ()(R,Z);
         return 1./sqrt(2.*M_PI*epsilon_)*
                exp(-( (psip-psivalue_)* (psip-psivalue_))/2./epsilon_)*sqrt(psipR*psipR +psipZ*psipZ);
     }
@@ -54,7 +52,7 @@ struct DeltaFunction
         return (*this)(R,Z);
     }
     private:
-    MagneticField c_;
+    aTokamakMagneticField c_;
     double epsilon_;
     double psivalue_;
 };
@@ -62,21 +60,19 @@ struct DeltaFunction
 /**
  * @brief Global safety factor
 \f[ \alpha(R,Z) = \frac{|B^\varphi|}{R|B^\eta|} = \frac{I_{pol}(R,Z)}{R|\nabla\psi_p|} \f]
-     @tparam MagneticField models aTokamakMagneticField
      @ingroup profiles
  */
-template<class MagneticField>
 struct Alpha
 {
-    Alpha( const MagneticField& c):c_(c){}
+    Alpha( const aTokamakMagneticField& c):c_(c){}
 
     /**
     * @brief \f[ \frac{ I_{pol}(R,Z)}{R \sqrt{\nabla\psi_p}} \f]
     */
     double operator()( double R, double Z) const
     {
-        double psipR = c_.psipR(R,Z), psipZ = c_.psipZ(R,Z);
-        return (1./R)*(c_.ipol(R,Z)/sqrt(psipR*psipR + psipZ*psipZ )) ;
+        double psipR = c_.psipR()(R,Z), psipZ = c_.psipZ()(R,Z);
+        return (1./R)*(c_.ipol()(R,Z)/sqrt(psipR*psipR + psipZ*psipZ )) ;
     }
     /**
      * @brief == operator()(R,Z)
@@ -86,7 +82,7 @@ struct Alpha
         return operator()(R,Z);
     }
     private:
-    MagneticField c_;
+    aTokamakMagneticField c_;
 };
 
 /**
@@ -94,12 +90,10 @@ struct Alpha
  \f[ \langle f\rangle(\psi_0) = \frac{1}{A} \int dV \delta(\psi_p(R,Z)-\psi_0) |\nabla\psi_p|f(R,Z) \f]
 
  with \f$ A = \int dV \delta(\psi_p(R,Z)-\psi_0)|\nabla\psi_p|\f$
- * @tparam MagneticField (models aTokamakMagneticField) This collective needs to contain at least the 2d functors, 
- * psip and its derivatives psipR and psipZ. 
  * @tparam container  The container class of the vector to average
  * @ingroup misc
  */
-template <class MagneticField, class container = thrust::host_vector<double> >
+template <class container = thrust::host_vector<double> >
 struct FluxSurfaceAverage
 {
      /**
@@ -108,15 +102,15 @@ struct FluxSurfaceAverage
      * @param c contains psip, psipR and psipZ
      * @param f container for global safety factor
      */
-    FluxSurfaceAverage(const dg::Grid2d& g2d, const MagneticField& c, const container& f) :
+    FluxSurfaceAverage(const dg::Grid2d& g2d, const aTokamakMagneticField& c, const container& f) :
     g2d_(g2d),
     f_(f),
-    deltaf_(geo::DeltaFunction<MagneticField>(c,0.0,0.0)),
+    deltaf_(geo::DeltaFunction(c,0.0,0.0)),
     w2d_ ( dg::create::weights( g2d_)),
     oneongrid_(dg::evaluate(dg::one,g2d_))              
     {
-        thrust::host_vector<double> psipRog2d  = dg::evaluate( c.psipR, g2d_);
-        thrust::host_vector<double> psipZog2d  = dg::evaluate( c.psipZ, g2d_);
+        thrust::host_vector<double> psipRog2d  = dg::evaluate( c.psipR(), g2d_);
+        thrust::host_vector<double> psipZog2d  = dg::evaluate( c.psipZ(), g2d_);
         double psipRmax = (double)thrust::reduce( psipRog2d.begin(), psipRog2d.end(),  0.,     thrust::maximum<double>()  );    
         //double psipRmin = (double)thrust::reduce( psipRog2d.begin(), psipRog2d.end(),  psipRmax,thrust::minimum<double>()  );
         double psipZmax = (double)thrust::reduce( psipZog2d.begin(), psipZog2d.end(), 0.,      thrust::maximum<double>()  );    
@@ -142,7 +136,7 @@ struct FluxSurfaceAverage
     private:
     dg::Grid2d g2d_;
     container f_;
-    geo::DeltaFunction<MagneticField> deltaf_;    
+    geo::DeltaFunction deltaf_;    
     const container w2d_;
     const container oneongrid_;
 };
@@ -152,12 +146,10 @@ struct FluxSurfaceAverage
 
 where \f$ \alpha\f$ is the dg::geo::Alpha functor.
  * @tparam container The container class to use aContainer
- * @tparam MagneticField (models aTokamakMagneticField) This collective needs to contain at least the 2d functors, 
- * psip and its derivatives psipR and psipZ. 
  * @ingroup misc
  *
  */
-template <class MagneticField, class container = thrust::host_vector<double> >
+template <class container = thrust::host_vector<double> >
 struct SafetyFactor
 {
      /**
@@ -166,15 +158,15 @@ struct SafetyFactor
      * @param c contains psip, psipR and psipZ
      * @param f container for global safety factor
      */
-    SafetyFactor(const dg::Grid2d& g2d, const MagneticField& c, const container& f) :
+    SafetyFactor(const dg::Grid2d& g2d, const aTokamakMagneticField& c, const container& f) :
     g2d_(g2d),
     f_(f), //why not directly use Alpha??
-    deltaf_(geo::DeltaFunction<MagneticField>(c,0.0,0.0)),
+    deltaf_(geo::DeltaFunction(c,0.0,0.0)),
     w2d_ ( dg::create::weights( g2d_)),
     oneongrid_(dg::evaluate(dg::one,g2d_))              
     {
-      thrust::host_vector<double> psipRog2d  = dg::evaluate( c.psipR, g2d_);
-      thrust::host_vector<double> psipZog2d  = dg::evaluate( c.psipZ, g2d_);
+      thrust::host_vector<double> psipRog2d  = dg::evaluate( c.psipR(), g2d_);
+      thrust::host_vector<double> psipZog2d  = dg::evaluate( c.psipZ(), g2d_);
       double psipRmax = (double)thrust::reduce( psipRog2d.begin(), psipRog2d.end(), 0.,     thrust::maximum<double>()  );    
       //double psipRmin = (double)thrust::reduce( psipRog2d.begin(), psipRog2d.end(),  psipRmax,thrust::minimum<double>()  );
       double psipZmax = (double)thrust::reduce( psipZog2d.begin(), psipZog2d.end(), 0.,      thrust::maximum<double>()  );    
@@ -199,7 +191,7 @@ struct SafetyFactor
     private:
     dg::Grid2d g2d_;
     container f_;
-    geo::DeltaFunction<MagneticField> deltaf_;    
+    geo::DeltaFunction deltaf_;    
     const container w2d_;
     const container oneongrid_;
 };
diff --git a/inc/geometries/testfunctors.h b/inc/geometries/testfunctors.h
index ef6d2db9e..16acca24e 100644
--- a/inc/geometries/testfunctors.h
+++ b/inc/geometries/testfunctors.h
@@ -5,7 +5,7 @@
 
 /*!@file
  *
- * MagneticField objects 
+ * aTokamakMagneticField objects 
  */
 namespace dg
 {
@@ -17,35 +17,32 @@ namespace geo
 ///@{
 
 
-template<class MagneticField>
 struct FuncNeu
 {
-    FuncNeu( const MagneticField& c):c_(c){}
-    double operator()(double R, double Z, double phi) const {return -c_.psip(R,Z)*cos(phi);
+    FuncNeu( const aTokamakMagneticField& c):c_(c){}
+    double operator()(double R, double Z, double phi) const {return -c_.psip()(R,Z)*cos(phi);
     }
     private:
-    MagneticField c_;
+    aTokamakMagneticField c_;
 };
 
-template<class MagneticField>
 struct DeriNeu
 {
-    DeriNeu( const MagneticField& c, double R0):c_(c), bhat_(c, R0){}
-    double operator()(double R, double Z, double phi) const {return c_.psip(R,Z)*bhat_(R,Z,phi)*sin(phi);
+    DeriNeu( const aTokamakMagneticField& c, double R0):c_(c), bhat_(c){}
+    double operator()(double R, double Z, double phi) const {return c_.psip()(R,Z)*bhat_(R,Z,phi)*sin(phi);
     }
     private:
-    MagneticField c_;
-    dg::geo::BHatP<MagneticField> bhat_;
+    aTokamakMagneticField c_;
+    dg::geo::BHatP<aTokamakMagneticField> bhat_;
 };
 
 //psi * cos(theta)
-template<class MagneticField>
 struct FuncDirPer
 {
-    FuncDirPer( const MagneticField& c, double R0, double psi_0, double psi_1, double k):
-        R_0_(R0), psi0_(psi_0), psi1_(psi_1), k_(k), c_(c) {}
+    FuncDirPer( const aTokamakMagneticField& c, double psi_0, double psi_1, double k):
+        R_0_(c.R0()), psi0_(psi_0), psi1_(psi_1), k_(k), c_(c) {}
     double operator()(double R, double Z) const {
-        double psip = c_.psip(R,Z);
+        double psip = c_.psip()(R,Z);
         double result = (psip-psi0_)*(psip-psi1_)*cos(k_*theta(R,Z));
         return 0.1*result;
     }
@@ -54,15 +51,15 @@ struct FuncDirPer
     }
     double dR( double R, double Z)const
     {
-        double psip = c_.psip(R,Z), psipR = c_.psipR(R,Z), theta_ = k_*theta(R,Z);
+        double psip = c_.psip()(R,Z), psipR = c_.psipR()(R,Z), theta_ = k_*theta(R,Z);
         double result = (2.*psip*psipR - (psi0_+psi1_)*psipR)*cos(theta_) 
             - (psip-psi0_)*(psip-psi1_)*sin(theta_)*k_*thetaR(R,Z);
         return 0.1*result;
     }
     double dRR( double R, double Z)const
     {
-        double psip = c_.psip(R,Z), psipR = c_.psipR(R,Z), theta_=k_*theta(R,Z), thetaR_=k_*thetaR(R,Z);
-        double psipRR = c_.psipRR(R,Z);
+        double psip = c_.psip()(R,Z), psipR = c_.psipR()(R,Z), theta_=k_*theta(R,Z), thetaR_=k_*thetaR(R,Z);
+        double psipRR = c_.psipRR()(R,Z);
         double result = (2.*(psipR*psipR + psip*psipRR) - (psi0_+psi1_)*psipRR)*cos(theta_)
             - 2.*(2.*psip*psipR-(psi0_+psi1_)*psipR)*sin(theta_)*thetaR_
             - (psip-psi0_)*(psip-psi1_)*(k_*thetaRR(R,Z)*sin(theta_)+cos(theta_)*thetaR_*thetaR_);
@@ -71,15 +68,15 @@ struct FuncDirPer
     }
     double dZ( double R, double Z)const
     {
-        double psip = c_.psip(R,Z), psipZ = c_.psipZ(R,Z), theta_=k_*theta(R,Z);
+        double psip = c_.psip()(R,Z), psipZ = c_.psipZ()(R,Z), theta_=k_*theta(R,Z);
         double result = (2*psip*psipZ - (psi0_+psi1_)*psipZ)*cos(theta_) 
             - (psip-psi0_)*(psip-psi1_)*sin(theta_)*k_*thetaZ(R,Z);
         return 0.1*result;
     }
     double dZZ( double R, double Z)const
     {
-        double psip = c_.psip(R,Z), psipZ = c_.psipZ(R,Z), theta_=k_*theta(R,Z), thetaZ_=k_*thetaZ(R,Z);
-        double psipZZ = c_.psipZZ(R,Z);
+        double psip = c_.psip()(R,Z), psipZ = c_.psipZ()(R,Z), theta_=k_*theta(R,Z), thetaZ_=k_*thetaZ(R,Z);
+        double psipZZ = c_.psipZZ()(R,Z);
         double result = (2.*(psipZ*psipZ + psip*psipZZ) - (psi0_+psi1_)*psipZZ)*cos(theta_)
             - 2.*(2.*psip*psipZ-(psi0_+psi1_)*psipZ)*sin(theta_)*thetaZ_
             - (psip-psi0_)*(psip-psi1_)*(k_*thetaZZ(R,Z)*sin(theta_) + cos(theta_)*thetaZ_*thetaZ_ );
@@ -108,14 +105,13 @@ struct FuncDirPer
     double thetaZZ( double R, double Z) const { return -thetaRR(R,Z);}
     double R_0_;
     double psi0_, psi1_, k_;
-    const MagneticField c_;
+    const aTokamakMagneticField c_;
 };
 
 //takes the magnetic field as chi
-template<class MagneticField>
 struct EllipticDirPerM
 {
-    EllipticDirPerM( const MagneticField& c, double R0, double psi_0, double psi_1, double k): func_(c, R0, psi_0, psi_1, k), bmod_(c, R0), br_(c, R0), bz_(c, R0) {}
+    EllipticDirPerM( const aTokamakMagneticField& c, double psi_0, double psi_1, double k): func_(c, psi_0, psi_1, k), bmod_(c), br_(c), bz_(c) {}
     double operator()(double R, double Z, double phi) const {
         return this->operator()(R,Z);}
     double operator()(double R, double Z) const {
@@ -124,17 +120,16 @@ struct EllipticDirPerM
 
     }
     private:
-    FuncDirPer<MagneticField> func_;
-    dg::geo::Bmodule<MagneticField> bmod_;
-    dg::geo::BR<MagneticField> br_;
-    dg::geo::BZ<MagneticField> bz_;
+    FuncDirPer func_;
+    dg::geo::Bmodule bmod_;
+    dg::geo::BR br_;
+    dg::geo::BZ bz_;
 };
 
 //Blob function
-template<class MagneticField>
 struct FuncDirNeu
 {
-    FuncDirNeu( const MagneticField& c, double psi_0, double psi_1, double R_blob, double Z_blob, double sigma_blob, double amp_blob):
+    FuncDirNeu( const aTokamakMagneticField& c, double psi_0, double psi_1, double R_blob, double Z_blob, double sigma_blob, double amp_blob):
         psi0_(psi_0), psi1_(psi_1), 
         cauchy_(R_blob, Z_blob, sigma_blob, sigma_blob, amp_blob){}
 
@@ -184,10 +179,9 @@ struct FuncDirNeu
 
 
 //takes the magnetic field multiplied by (1+0.5sin(theta)) as chi
-template<class MagneticField>
 struct BmodTheta
 {
-    BmodTheta( const MagneticField& c, double R0): R_0_(R0), bmod_(c, R0){}
+    BmodTheta( const aTokamakMagneticField& c): R_0_(c.R0()), bmod_(c){}
     double operator()(double R,double Z, double phi) const{
         return this->operator()(R,Z);}
     double operator()(double R,double Z) const{
@@ -202,15 +196,14 @@ struct BmodTheta
             return 2.*M_PI-acos( dR/sqrt( dR*dR + Z*Z));
     }
     double R_0_;
-    dg::geo::Bmodule<MagneticField> bmod_;
+    dg::geo::Bmodule bmod_;
 
 };
 
 //take BmodTheta as chi
-template<class MagneticField>
 struct EllipticDirNeuM
 {
-    EllipticDirNeuM( const MagneticField& c, double R0, double psi_0, double psi_1, double R_blob, double Z_blob, double sigma_blob, double amp_blob): R_0_(R0), 
+    EllipticDirNeuM( const aTokamakMagneticField& c, double psi_0, double psi_1, double R_blob, double Z_blob, double sigma_blob, double amp_blob): R_0_(c.R0()), 
     func_(c, psi_0, psi_1, R_blob, Z_blob, sigma_blob,amp_blob), bmod_(c, R0), br_(c, R0), bz_(c, R0) {}
     double operator()(double R, double Z) const {
         double bmod = bmod_(R,Z), br = br_(R,Z), bz = bz_(R,Z), theta_ = theta(R,Z);
@@ -240,59 +233,57 @@ struct EllipticDirNeuM
         return dR/(dR*dR+Z*Z);
     }
     double R_0_;
-    FuncDirNeu<MagneticField> func_;
-    dg::geo::Bmodule<MagneticField> bmod_;
-    dg::geo::BR<MagneticField> br_;
-    dg::geo::BZ<MagneticField> bz_;
+    FuncDirNeu func_;
+    dg::geo::Bmodule bmod_;
+    dg::geo::BR br_;
+    dg::geo::BZ bz_;
 };
 
 //the psi surfaces
-template<class MagneticField>
 struct FuncXDirNeu
 {
-    FuncXDirNeu( const MagneticField& c, double psi_0, double psi_1):
+    FuncXDirNeu( const aTokamakMagneticField& c, double psi_0, double psi_1):
         c_(c), psi0_(psi_0), psi1_(psi_1){}
 
     double operator()(double R, double Z, double phi) const {
         return this->operator()(R,Z);}
     double operator()(double R, double Z) const {
-        double psip = c_.psip(R,Z);
+        double psip = c_.psip()(R,Z);
         return (psip-psi0_)*(psip-psi1_);
     }
     double dR( double R, double Z)const
     {
-        double psip = c_.psip(R,Z), psipR = c_.psipR(R,Z);
+        double psip = c_.psip()(R,Z), psipR = c_.psipR()(R,Z);
         return (2.*psip-psi0_-psi1_)*psipR;
     }
     double dRR( double R, double Z)const
     {
-        double psip = c_.psip(R,Z), psipR = c_.psipR(R,Z);
-        double psipRR = c_.psipRR(R,Z);
+        double psip = c_.psip()(R,Z), psipR = c_.psipR()(R,Z);
+        double psipRR = c_.psipRR()(R,Z);
         return (2.*(psipR*psipR + psip*psipRR) - (psi0_+psi1_)*psipRR);
             
     }
     double dZ( double R, double Z)const
     {
-        double psip = c_.psip(R,Z), psipZ = c_.psipZ(R,Z);
+        double psip = c_.psip()(R,Z), psipZ = c_.psipZ()(R,Z);
         return (2*psip-psi0_-psi1_)*psipZ;
     }
     double dZZ( double R, double Z)const
     {
-        double psip = c_.psip(R,Z), psipZ = c_.psipZ(R,Z);
-        double psipZZ = c_.psipZZ(R,Z);
+        double psip = c_.psip()(R,Z), psipZ = c_.psipZ()(R,Z);
+        double psipZZ = c_.psipZZ()(R,Z);
         return (2.*(psipZ*psipZ + psip*psipZZ) - (psi0_+psi1_)*psipZZ);
     }
     private:
-    MagneticField c_;
+    aTokamakMagneticField c_;
     double psi0_, psi1_;
 };
 
 //take Bmod as chi
-template<class MagneticField>
 struct EllipticXDirNeuM
 {
-    EllipticXDirNeuM( const MagneticField& c, double R0, double psi_0, double psi_1): R_0_(R0), 
-    func_(c, psi_0, psi_1), bmod_(c, R0), br_(c, R0), bz_(c, R0) {}
+    EllipticXDirNeuM( const aTokamakMagneticField& c, double psi_0, double psi_1): R_0_(c.R0()), 
+    func_(c, psi_0, psi_1), bmod_(c), br_(c), bz_(c) {}
     double operator()(double R, double Z) const {
         double bmod = bmod_(R,Z), br = br_(R,Z), bz = bz_(R,Z);
         double chi = 1e4+bmod; //bmod can be zero for a Taylor state(!)
@@ -308,17 +299,16 @@ struct EllipticXDirNeuM
     }
     private:
     double R_0_;
-    FuncXDirNeu<MagneticField> func_;
-    dg::geo::Bmodule<MagneticField> bmod_;
-    dg::geo::BR<MagneticField> br_;
-    dg::geo::BZ<MagneticField> bz_;
+    FuncXDirNeu func_;
+    dg::geo::Bmodule bmod_;
+    dg::geo::BR br_;
+    dg::geo::BZ bz_;
 };
 
 //take Blob and chi=1
-template<class MagneticField>
 struct EllipticBlobDirNeuM
 {
-    EllipticBlobDirNeuM( const MagneticField& c, double psi_0, double psi_1, double R_blob, double Z_blob, double sigma_blob, double amp_blob): 
+    EllipticBlobDirNeuM( const aTokamakMagneticField& c, double psi_0, double psi_1, double R_blob, double Z_blob, double sigma_blob, double amp_blob): 
     func_(c, psi_0, psi_1, R_blob, Z_blob, sigma_blob, amp_blob){}
     double operator()(double R, double Z) const {
         return -( func_.dRR(R,Z) + func_.dZZ(R,Z) );
@@ -328,32 +318,30 @@ struct EllipticBlobDirNeuM
     }
     private:
     double R_0_;
-    FuncDirNeu<MagneticField> func_;
+    FuncDirNeu func_;
 };
 
-template<class MagneticField>
 struct EllipticDirSimpleM
 {
-    EllipticDirSimpleM( const MagneticField& c, double psi_0, double psi_1, double R_blob, double Z_blob, double sigma_blob, double amp_blob): func_(c, psi_0, psi_1, R_blob, Z_blob, sigma_blob, amp_blob) {}
+    EllipticDirSimpleM( const aTokamakMagneticField& c, double psi_0, double psi_1, double R_blob, double Z_blob, double sigma_blob, double amp_blob): func_(c, psi_0, psi_1, R_blob, Z_blob, sigma_blob, amp_blob) {}
     double operator()(double R, double Z, double phi) const {
         return -(( 1./R*func_.dR(R,Z) + func_.dRR(R,Z) + func_.dZZ(R,Z) ));
 
     }
     private:
-    FuncDirNeu<MagneticField> func_;
+    FuncDirNeu func_;
 };
 
 /**
  * @brief testfunction to test the parallel derivative 
       \f[ f(R,Z,\varphi) = -\frac{\cos(\varphi)}{R\hat b_\varphi} \f]
  */ 
-template<class MagneticField>
 struct TestFunction
 {
-    TestFunction( const MagneticField& c, double R0) :  
-        bhatR_(c, R0),
-        bhatZ_(c, R0),
-        bhatP_(c, R0) {}
+    TestFunction( const aTokamakMagneticField& c) :  
+        bhatR_(c),
+        bhatZ_(c),
+        bhatP_(c) {}
     /**
      * @brief \f[ f(R,Z,\varphi) = -\frac{\cos(\varphi)}{R\hat b_\varphi} \f]
      */ 
@@ -375,22 +363,21 @@ struct TestFunction
 
     }
     private:
-    dg::geo::BHatR<MagneticField> bhatR_;
-    dg::geo::BHatZ<MagneticField> bhatZ_;
-    dg::geo::BHatP<MagneticField> bhatP_;
+    dg::geo::BHatR bhatR_;
+    dg::geo::BHatZ bhatZ_;
+    dg::geo::BHatP bhatP_;
 };
 
 /**
  * @brief analyitcal solution of the parallel derivative of the testfunction
  *  \f[ \nabla_\parallel(R,Z,\varphi) f = \frac{\sin(\varphi)}{R}\f]
  */ 
-template<class MagneticField>
 struct DeriTestFunction
 {
-    DeriTestFunction( const MagneticField& c, double R0) :
-        bhatR_(c, R0),
-        bhatZ_(c, R0),
-        bhatP_(c, R0) {}
+    DeriTestFunction( const aTokamakMagneticField& c) :
+        bhatR_(c),
+        bhatZ_(c),
+        bhatP_(c) {}
 /**
  * @brief \f[ \nabla_\parallel f = \frac{\sin(\varphi)}{R}\f]
  */ 
@@ -415,9 +402,9 @@ struct DeriTestFunction
 
     }
     private:
-    dg::geo::BHatR<MagneticField> bhatR_;
-    dg::geo::BHatZ<MagneticField> bhatZ_;
-    dg::geo::BHatP<MagneticField> bhatP_;
+    dg::geo::BHatR bhatR_;
+    dg::geo::BHatZ bhatZ_;
+    dg::geo::BHatP bhatP_;
 };
 
 ///@} 
-- 
GitLab


From 386c3be772583e3c214fdec2375b0a9fa04fa296 Mon Sep 17 00:00:00 2001
From: Matthias Home Office <Matthias.Wiesenberger@uibk.ac.at>
Date: Mon, 24 Jul 2017 16:51:01 +0200
Subject: [PATCH 077/453] average and toroidal small changes

---
 inc/geometries/average.h  | 1 +
 inc/geometries/toroidal.h | 4 +---
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/inc/geometries/average.h b/inc/geometries/average.h
index 78e3f6a64..794a238b6 100644
--- a/inc/geometries/average.h
+++ b/inc/geometries/average.h
@@ -1,6 +1,7 @@
 #pragma once
 
 #include <thrust/host_vector.h>
+#include "magnetic_field.h"
 
 /*!@file
  *
diff --git a/inc/geometries/toroidal.h b/inc/geometries/toroidal.h
index ddffc5998..cb5019b71 100644
--- a/inc/geometries/toroidal.h
+++ b/inc/geometries/toroidal.h
@@ -1,9 +1,7 @@
 #pragma once
 
-#include "functors.h"
 #include "magnetic_field.h"
 
-
 namespace dg{
 namespace geo{
 namespace toroidal{
@@ -16,7 +14,7 @@ struct Constant:public aCloneableBinaryOperator<Constant>
     double c_;
 };
 /**
- * @brief Models a slab toroidal field (models aTokamakMagneticField)
+ * @brief Models a slab toroidal field 
  *
  * \f$ B=\frac{R_0}{R}\f$, \f$ \psi_p = 1\f$ and \f$ I = 1\f$.
  @note The solovev field can also be made to model a todoidal slab field
-- 
GitLab


From 40f98cfd8cc0c2c96aa5ed8e7fe536d2fcbab112 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Mon, 24 Jul 2017 22:21:33 +0200
Subject: [PATCH 078/453] added fluxfunctions.h header

---
 inc/geometries/fluxfunctions.h  | 114 ++++++++++++++++++++++++++++++++
 inc/geometries/hector.h         |   2 +-
 inc/geometries/magnetic_field.h | 114 +++++++-------------------------
 3 files changed, 139 insertions(+), 91 deletions(-)
 create mode 100644 inc/geometries/fluxfunctions.h

diff --git a/inc/geometries/fluxfunctions.h b/inc/geometries/fluxfunctions.h
new file mode 100644
index 000000000..f9a865041
--- /dev/null
+++ b/inc/geometries/fluxfunctions.h
@@ -0,0 +1,114 @@
+#pragma once
+
+namespace dg
+{
+namespace geo
+{
+/**
+* @brief This functor represents functions written in cylindrical coordinates
+        that are independent of the angle phi
+  @ingroup fluxfunctions
+*/
+struct aBinaryFunctor
+{
+    /**
+    * @brief The function value
+    *
+    * @param R radius (cylindrical coordinate)
+    * @param Z height (cylindrical coordinate)
+    *
+    * @return f(R,Z)
+    */
+    virtual double operator()(double R, double Z) const=0;
+    /**
+    * @brief Redirects to the 2D version
+    *
+    * @param R radius (cylindrical coordinate)
+    * @param Z height (cylindrical coordinate)
+    * @param phi angle (cylindrical coordinate)
+    *
+    * @return f(R,Z)
+    */
+    double operator()(double R, double Z, double phi)
+    {
+        return this->operator()(R,Z);
+    }
+    /**
+    * @brief abstract copy of a binary functor
+    *
+    * @return a functor on the heap
+    */
+    virtual aBinaryFunctor* clone()const=0;
+    protected:
+    virtual ~aBinaryFunctor(){}
+    /**
+    * @brief We do not allow object slicing so the copy is protected
+    */
+    aBinaryFunctor(const aBinaryFunctor&){}
+    /**
+    * @brief We do not allow object slicing so the assignment is protected
+    */
+    aBinaryFunctor& operator=(const aBinaryFunctor&){return *this}
+};
+
+/**
+* @brief Implementation helper for the clone pattern
+
+    https://katyscode.wordpress.com/2013/08/22/c-polymorphic-cloning-and-the-crtp-curiously-recurring-template-pattern/
+  @ingroup fluxfunctions
+*/
+template<class Derived>
+struct dg::geo::aCloneableBinaryFunctor : public dg::geo::aBinaryFunctor
+{
+    /**
+    * @brief Returns a copy of the functor dynamically allocated on the heap
+    *
+    * @return new copy of the functor
+    */
+    virtual Derived* clone() const
+    {
+        return new Derived(static_cast<Derived const &>(*this));
+    }
+};
+
+
+
+struct aBinaryFunctorBundle
+{
+    aBinaryFunctorBundle( 
+        aBinaryFunctor* psip,
+        aBinaryFunctor* psipR,
+        aBinaryFunctor* psipZ,
+        aBinaryFunctor* psipRR,
+        aBinaryFunctor* psipRZ,
+        aBinaryFunctor* psipZZ,
+        ):p_(6){ 
+            R0_(R0),
+            p_[0] = psip,
+            p_[1] = psipR,
+            p_[2] = psipZ,
+            p_[3] = psipRR,
+            p_[4] = psipRZ,
+            p_[5] = psipZZ,
+        }
+    aBinaryFunctorBundle( const aBinaryFunctorBundle& b)
+    {
+        //deep copy
+        for( unsigned i=0; i<p_.size(); i++)
+            p_[i] = b.p_[i]->clone();
+    }
+    aBinaryFunctorBundle& operator=( const aBinaryFunctorBundle& b)
+    {
+        aBinaryFunctorBundle temp(b);
+        std::swap( temp.p_, p_);
+        return *this;
+    }
+    ~aBinaryFunctorBundle(){
+        for( unsigned i=0; i<p_.size(); i++)
+            delete p_[i];
+    }
+    private:
+    std::vector<aBinaryFunctor*> p_;
+};
+}//namespace geo
+}//namespace dg
diff --git a/inc/geometries/hector.h b/inc/geometries/hector.h
index f8c2ebe23..ba9dcdd55 100644
--- a/inc/geometries/hector.h
+++ b/inc/geometries/hector.h
@@ -3,10 +3,10 @@
 #include <vector>
 #include "dg/backend/grid.h"
 #include "dg/backend/interpolation.cuh"
+#include "dg/geometry/curvilinear.h"
 #include "dg/elliptic.h"
 #include "dg/cg.h"
 #include "flux.h"
-#include "curvilinear.h"
 #include "adaption.h"
 #include "generator.h"
 
diff --git a/inc/geometries/magnetic_field.h b/inc/geometries/magnetic_field.h
index 1b57e4fe2..024418944 100644
--- a/inc/geometries/magnetic_field.h
+++ b/inc/geometries/magnetic_field.h
@@ -10,72 +10,6 @@ namespace dg
 namespace geo
 {
 
-/**
-* @brief This functor represents functions written in cylindrical coordinates
-        that are independent of the angle phi
-  @ingroup fluxfunctions
-*/
-struct aBinaryFunctor
-{
-    /**
-    * @brief The function value
-    *
-    * @param R radius (cylindrical coordinate)
-    * @param Z height (cylindrical coordinate)
-    *
-    * @return f(R,Z)
-    */
-    virtual double operator()(double R, double Z) const=0;
-    /**
-    * @brief Redirects to the 2D version
-    *
-    * @param R radius (cylindrical coordinate)
-    * @param Z height (cylindrical coordinate)
-    * @param phi angle (cylindrical coordinate)
-    *
-    * @return f(R,Z)
-    */
-    double operator()(double R, double Z, double phi)
-    {
-        return this->operator()(R,Z);
-    }
-    /**
-    * @brief abstract copy of a binary functor
-    *
-    * @return a functor on the heap
-    */
-    virtual aBinaryFunctor* clone()const=0;
-    protected:
-    virtual ~aBinaryFunctor(){}
-    /**
-    * @brief We do not allow object slicing so the copy is protected
-    */
-    aBinaryFunctor(const aBinaryFunctor&){}
-    /**
-    * @brief We do not allow object slicing so the assignment is protected
-    */
-    aBinaryFunctor& operator=(const aBinaryFunctor&){return *this}
-};
-
-/**
-* @brief Implementation helper for the clone pattern
-
-    https://katyscode.wordpress.com/2013/08/22/c-polymorphic-cloning-and-the-crtp-curiously-recurring-template-pattern/
-  @ingroup fluxfunctions
-*/
-template<class Derived>
-struct dg::geo::aCloneableBinaryFunctor : public dg::geo::aBinaryFunctor
-{
-    /**
-    * @brief Returns a copy of the functor dynamically allocated on the heap
-    *
-    * @return new copy of the functor
-    */
-    virtual Derived* clone() const
-    {
-        return new Derived(static_cast<Derived const &>(*this));
-    }
-};
 
 ///@addtogroup magnetic
 ///@{
@@ -95,6 +29,30 @@ struct dg::geo::aCloneableBinaryFunctor : public dg::geo::aBinaryFunctor
 */
 struct aTokamakMagneticField
 {
+    aTokamakMagneticField( double R0,
+        aBinaryFunctor* psip,
+        aBinaryFunctor* psipR,
+        aBinaryFunctor* psipZ,
+        aBinaryFunctor* psipRR,
+        aBinaryFunctor* psipRZ,
+        aBinaryFunctor* psipZZ,
+        aBinaryFunctor* laplacePsip,
+        aBinaryFunctor* ipol,
+        aBinaryFunctor* ipolR,
+        aBinaryFunctor* ipolZ
+        ):p_(10){ 
+            R0_(R0),
+            p_[0] = psip,
+            p_[1] = psipR,
+            p_[2] = psipZ,
+            p_[3] = psipRR,
+            p_[4] = psipRZ,
+            p_[5] = psipZZ,
+            p_[6] = laplacePsip,
+            p_[7] = psip,
+            p_[8] = ipolR,
+            p_[9] = ipolZ,
+        }
     aTokamakMagneticField( const aTokamakMagneticField& mag)
     {
         R0_ = mag.R0_;
@@ -131,30 +89,6 @@ struct aTokamakMagneticField
     /// \f$ \partial_Z I(\psi_p) \f$ 
     const aBinaryFunctor& ipolZ()const{return *p_[9];}
 
-    aTokamakMagneticField( double R0,
-        aBinaryFunctor* psip,
-        aBinaryFunctor* psipR,
-        aBinaryFunctor* psipZ,
-        aBinaryFunctor* psipRR,
-        aBinaryFunctor* psipRZ,
-        aBinaryFunctor* psipZZ,
-        aBinaryFunctor* laplacePsip,
-        aBinaryFunctor* ipol,
-        aBinaryFunctor* ipolR,
-        aBinaryFunctor* ipolZ
-        ):p_(10){ 
-            R0_(R0),
-            p_[0] = psip,
-            p_[1] = psipR,
-            p_[2] = psipZ,
-            p_[3] = psipRR,
-            p_[4] = psipRZ,
-            p_[5] = psipZZ,
-            p_[6] = laplacePsip,
-            p_[7] = psip,
-            p_[8] = ipolR,
-            p_[9] = ipolZ,
-        }
     protected:
     ~aTokamakMagneticField(){
         for( unsigned i=0; i<p_.size(); i++)
-- 
GitLab


From 89a99ddcbabd38b5d7592f6460fe0445fad0e196 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Mon, 24 Jul 2017 23:06:23 +0200
Subject: [PATCH 079/453] created functor containers for functions and
 derivatives

---
 inc/geometries/fluxfunctions.h  | 130 ++++++++++++++++++++++++++------
 inc/geometries/hector.h         |   1 +
 inc/geometries/magnetic_field.h |  57 ++++----------
 3 files changed, 123 insertions(+), 65 deletions(-)

diff --git a/inc/geometries/fluxfunctions.h b/inc/geometries/fluxfunctions.h
index f9a865041..9c0c94194 100644
--- a/inc/geometries/fluxfunctions.h
+++ b/inc/geometries/fluxfunctions.h
@@ -4,10 +4,12 @@ namespace dg
 {
 namespace geo
 {
+///@addtogroup fluxfunctions
+///@{
+
 /**
 * @brief This functor represents functions written in cylindrical coordinates
         that are independent of the angle phi
-  @ingroup fluxfunctions
 */
 struct aBinaryFunctor
 {
@@ -55,7 +57,6 @@ struct aBinaryFunctor
 * @brief Implementation helper for the clone pattern
 
     https://katyscode.wordpress.com/2013/08/22/c-polymorphic-cloning-and-the-crtp-curiously-recurring-template-pattern/
-  @ingroup fluxfunctions
 */
 template<class Derived>
 struct dg::geo::aCloneableBinaryFunctor : public dg::geo::aBinaryFunctor
@@ -71,26 +72,12 @@ struct dg::geo::aCloneableBinaryFunctor : public dg::geo::aBinaryFunctor
     }
 };
 
-
-
+/**
+* @brief Manage and deep copy binary Functors on the heap
+*/
 struct aBinaryFunctorBundle
 {
-    aBinaryFunctorBundle( 
-        aBinaryFunctor* psip,
-        aBinaryFunctor* psipR,
-        aBinaryFunctor* psipZ,
-        aBinaryFunctor* psipRR,
-        aBinaryFunctor* psipRZ,
-        aBinaryFunctor* psipZZ,
-        ):p_(6){ 
-            R0_(R0),
-            p_[0] = psip,
-            p_[1] = psipR,
-            p_[2] = psipZ,
-            p_[3] = psipRR,
-            p_[4] = psipRZ,
-            p_[5] = psipZZ,
-        }
+    protected:
     aBinaryFunctorBundle( const aBinaryFunctorBundle& b)
     {
         //deep copy
@@ -107,8 +94,109 @@ struct aBinaryFunctorBundle
         for( unsigned i=0; i<p_.size(); i++)
             delete p_[i];
     }
-    private:
     std::vector<aBinaryFunctor*> p_;
 };
+
+/**
+* @brief This struct bundles a function and its first derivatives
+*/
+struct BinaryFunctorsLvl1 : public aBinaryFunctorBundle
+{
+    /**
+    * @brief Take ownership of newly allocated functors
+    *
+    * @param f f 
+    * @param fx partial f / partial x
+    * @param fy partial f / partial y
+    */
+    BinaryFunctorsLvl1( aBinaryFunctor* f, aBinaryFunctor* fx, aBinaryFunctor* fy): p_(3)
+    {
+        p_[0] = f;
+        p_[1] = fx;
+        p_[2] = fy;
+    }
+    ///f 
+    const aBinaryFunctor& f()const{return *p_[0];}
+    /// partial f / partial x
+    const aBinaryFunctor& fx()const{return *p_[1];}
+    /// partial f / partial y
+    const aBinaryFunctor& fy()const{return *p_[2];}
+};
+/**
+* @brief This struct bundles a function and its first and second derivatives
+*/
+struct BinaryFunctorsLvl2 : public aBinaryFunctorBundle
+{
+    /**
+    * @brief Take ownership of newly allocated functors
+    *
+    * @param f f 
+    * @param fx partial f / partial x
+    * @param fy partial f / partial y
+    * @param fxx partial2 f / partial x2
+    * @param fxy partial2 f / partial x /partial y
+    * @param fyy partial2 f / partial y2
+    */
+    BinaryFunctorsLvl2( aBinaryFunctor* f, aBinaryFunctor* fx, aBinaryFunctor* fy,
+    aBinaryFunctor* fxx, aBinaryFunctor* fxy, aBinaryFunctor* fyy): p_(6)
+    {
+        p_[0] = f;
+        p_[1] = fx;
+        p_[2] = fy;
+        p_[3] = fxx;
+        p_[4] = fxy;
+        p_[5] = fyy;
+    }
+    /// f
+    const aBinaryFunctor& f()const{return *p_[0];}
+    /// partial f /partial x
+    const aBinaryFunctor& fx()const{return *p_[1];}
+    /// partial f /partial y
+    const aBinaryFunctor& fy()const{return *p_[2];}
+    /// partial^2f/partial x^2
+    const aBinaryFunctor& fxx()const{return *p_[3];}
+    /// partial^2 f / partial x partial y
+    const aBinaryFunctor& fxy()const{return *p_[4];}
+    /// partial^2f/partial y^2
+    const aBinaryFunctor& fyy()const{return *p_[5];}
+};
+/**
+* @brief This struct bundles a symmetric tensor and its divergence
+*/
+struct BinarySymmTensorLvl1 : public aBinaryFunctorBundle
+{
+    /**
+    * @brief Take ownership of newly allocated functors
+    *
+    * let's assume the tensor is called chi
+    * @param chi_xx contravariant xx component
+    * @param chi_xy contravariant xy component
+    * @param chi_yy contravariant yy component
+     * @param divChiX \f$ \partial_x \chi^{xx} + \partial_y\chi^{yx}\f$ is the x-component of the divergence of the tensor \f$ \chi\f$
+     * @param divChiY \f$ \partial_x \chi^{xy} + \partial_y\chi^{yy}\f$ is the y-component of the divergence of the tensor \f$ \chi \f$
+    */
+    BinarySymmTensorLvl1( aBinaryFunctor* chi_xx, aBinaryFunctor* chi_xy, aBinaryFunctor* chi_yy,
+    aBinaryFunctor* divChiX, aBinaryFunctor* divChiY): p_(5)
+    {
+        p_[0] = chi_xx;
+        p_[1] = chi_xy;
+        p_[2] = chi_yy;
+        p_[3] = divChiX;
+        p_[4] = divChiY;
+    }
+    ///xx component
+    const aBinaryFunctor& xx()const{return *p_[0];}
+    ///xy component
+    const aBinaryFunctor& xy()const{return *p_[1];}
+    ///yy component
+    const aBinaryFunctor& yy()const{return *p_[2];}
+    ///x component of the divergence 
+    const aBinaryFunctor& divX()const{return *p_[3];}
+    ///y component of the divergence 
+    const aBinaryFunctor& divY()const{return *p_[4];}
+};
+
+
+///@}
 }//namespace geo
 }//namespace dg
diff --git a/inc/geometries/hector.h b/inc/geometries/hector.h
index ba9dcdd55..71faa11b2 100644
--- a/inc/geometries/hector.h
+++ b/inc/geometries/hector.h
@@ -5,6 +5,7 @@
 #include "dg/backend/interpolation.cuh"
 #include "dg/geometry/curvilinear.h"
 #include "dg/elliptic.h"
+#include "fluxfunctions.h"
 #include "dg/cg.h"
 #include "flux.h"
 #include "adaption.h"
diff --git a/inc/geometries/magnetic_field.h b/inc/geometries/magnetic_field.h
index 024418944..78a9ebbbd 100644
--- a/inc/geometries/magnetic_field.h
+++ b/inc/geometries/magnetic_field.h
@@ -1,5 +1,6 @@
 #pragma once
 
+#include "fluxfunctions.h"
 
 /*!@file
  *
@@ -36,67 +37,35 @@ struct aTokamakMagneticField
         aBinaryFunctor* psipRR,
         aBinaryFunctor* psipRZ,
         aBinaryFunctor* psipZZ,
-        aBinaryFunctor* laplacePsip,
         aBinaryFunctor* ipol,
         aBinaryFunctor* ipolR,
         aBinaryFunctor* ipolZ
-        ):p_(10){ 
-            R0_(R0),
-            p_[0] = psip,
-            p_[1] = psipR,
-            p_[2] = psipZ,
-            p_[3] = psipRR,
-            p_[4] = psipRZ,
-            p_[5] = psipZZ,
-            p_[6] = laplacePsip,
-            p_[7] = psip,
-            p_[8] = ipolR,
-            p_[9] = ipolZ,
-        }
-    aTokamakMagneticField( const aTokamakMagneticField& mag)
-    {
-        R0_ = mag.R0_;
-        for( unsigned i=0; i<p_.size(); i++)
-            p_[i] = mag.p_[i]->clone();
-    }
-    aTokamakMagneticField& operator=( const aTokamakMagneticField& mag)
-    {
-        aTokamakMagneticField temp(mag);
-        std::swap( temp.R0_, R0_);
-        std::swap( temp.p_, p_);
-        return *this;
-    }
+        ): R0_(R0), psip_(psip,psipR,psipZ,psipRR,psipRZ,psipZZ),ipol_(ipol,ipolR,ipolZ){ }
     /// \f$ R_0 \f$ 
     double R0()const {return R0_;}
     /// \f$ \psi_p(R,Z)\f$, where R, Z are cylindrical coordinates
-    const aBinaryFunctor& psip()const{return *p_[0];}
+    const aBinaryFunctor& psip()const{return psip_.f();}
     /// \f$ \partial_R \psi_p(R,Z)\f$, where R, Z are cylindrical coordinates
-    const aBinaryFunctor& psipR()const{return *p_[1];}
+    const aBinaryFunctor& psipR()const{return psip_.fx();}
     /// \f$ \partial_Z \psi_p(R,Z)\f$, where R, Z are cylindrical coordinates
-    const aBinaryFunctor& psipZ()const{return *p_[2];}
+    const aBinaryFunctor& psipZ()const{return psip_.fy();}
     /// \f$ \partial_R\partial_R \psi_p(R,Z)\f$, where R, Z are cylindrical coordinates
-    const aBinaryFunctor& psipRR()const{return *p_[3];}
+    const aBinaryFunctor& psipRR()const{return psip_.fxx();}
     /// \f$ \partial_R\partial_Z \psi_p(R,Z)\f$, where R, Z are cylindrical coordinates
-    const aBinaryFunctor& psipRZ()const{return *p_[4];}
+    const aBinaryFunctor& psipRZ()const{return psip_.fxy();}
     /// \f$ \partial_Z\partial_Z \psi_p(R,Z)\f$, where R, Z are cylindrical coordinates
-    const aBinaryFunctor& psipZZ()const{return *p_[5];}
-    /// 2d Laplacian of \f$ \psi_p\f$ 
-    const aBinaryFunctor& laplacePsip()const{return *p_[6];}
+    const aBinaryFunctor& psipZZ()const{return psip_.fyy();}
     /// \f$ I(\psi_p) \f$ the current
-    const aBinaryFunctor& ipol()const{return *p_[7];}
+    const aBinaryFunctor& ipol()const{return ipol_.f();}
     /// \f$ \partial_R I(\psi_p) \f$ 
-    const aBinaryFunctor& ipolR()const{return *p_[8];}
+    const aBinaryFunctor& ipolR()const{return ipol_.fx();}
     /// \f$ \partial_Z I(\psi_p) \f$ 
-    const aBinaryFunctor& ipolZ()const{return *p_[9];}
+    const aBinaryFunctor& ipolZ()const{return ipol_.fy();}
 
-    protected:
-    ~aTokamakMagneticField(){
-        for( unsigned i=0; i<p_.size(); i++)
-            delete p_[i];
-    }
     private:
     double R0_;
-    std::vector<aBinaryFunctor*> p_;
+    BinaryFunctorsLvl2 psip_;
+    BinaryFunctorsLvl1 ipol_;
 };
 
 
-- 
GitLab


From f6e2a68774c1a7c640b24f4e0eefa286c2a6babf Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Tue, 25 Jul 2017 10:45:40 +0200
Subject: [PATCH 080/453] make grid class copyable again

---
 inc/dg/backend/grid.h          |  4 ++--
 inc/dg/backend/mpi_grid.h      | 10 +++++-----
 inc/geometries/fluxfunctions.h |  4 +++-
 3 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/inc/dg/backend/grid.h b/inc/dg/backend/grid.h
index bee863200..2f34e6c22 100644
--- a/inc/dg/backend/grid.h
+++ b/inc/dg/backend/grid.h
@@ -387,7 +387,6 @@ struct Grid2d
         if( (x>=x0_ && x <= x1_) && (y>=y0_ && y <= y1_)) return true; 
         return false;
     }
-  protected:
     virtual ~Grid2d(){}
     Grid2d(const Grid2d& src):dlt_(src.dlt_){*this = src;}
     Grid2d& operator=(const Grid2d& src){
@@ -398,6 +397,7 @@ struct Grid2d
         bcx_=src.bcx_,bcy_=src.bcy_;
         dlt_=src.dlt_;
     }
+    protected:
     virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny) {
         n_ = new_n, Nx_ = new_Nx, Ny_ = new_Ny;
         hx_ = lx_/(double)Nx_, hy_ = ly_/(double)Ny_;
@@ -707,7 +707,6 @@ struct Grid3d
             return true; 
         return false;
     }
-  protected:
     virtual ~Grid3d(){}
     Grid3d(const Grid3d& src):dlt_(src.dlt_){*this = src;}
     Grid3d& operator=(const Grid3d& src){ //use default in C++11
@@ -718,6 +717,7 @@ struct Grid3d
         bcx_=src.bcx_,bcy_=src.bcy_,bcz_=src.bcz_;
         dlt_=src.dlt_;
     }
+    protected:
     virtual void do_set(unsigned new_n, unsigned new_Ny, unsigned new_Nz)
     {
         n_ = new_n, Nx_ = new_Nx, Ny_ = new_Ny, Nz_ = new_Nz;
diff --git a/inc/dg/backend/mpi_grid.h b/inc/dg/backend/mpi_grid.h
index 1a03e7096..655494d7e 100644
--- a/inc/dg/backend/mpi_grid.h
+++ b/inc/dg/backend/mpi_grid.h
@@ -301,12 +301,12 @@ struct MPIGrid2d
      * @return non-MPI Grid object
      */
     virtual const Grid2d& global() const {return g;}
-    protected:
     virtual ~MPIGrid2d(){}
     MPIGrid2d(const MPIGrid2d& src):g(src.g),comm(src.comm){}
     MPIGrid2d& operator=(const MPIGrid2d& src){
         g = src.g; comm = src.comm;
     }
+    protected:
     virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny) {
         g.set(new_n,new_Nx,new_Ny);
     }
@@ -636,15 +636,15 @@ struct MPIGrid3d
      *@copydoc MPIGrid2d::global()const
      */
     virtual const Grid3d& global() const {return g;}
-    protected:
-    virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz) {
-        g.set(new_n,new_Nx,new_Ny,new_Nz);
-    }
     virtual ~MPIGrid3d(){}
     MPIGrid3d(const MPIGrid3d& src):g(src.g),comm(src.comm){}
     MPIGrid3d& operator=(const MPIGrid3d& src){
         g = src.g; comm = src.comm;
     }
+    protected:
+    virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz) {
+        g.set(new_n,new_Nx,new_Ny,new_Nz);
+    }
     void init_X_boundaries( double global_x0, double global_x1)
     {
         g.init_X_boundaries(global_x0, global_x1);
diff --git a/inc/geometries/fluxfunctions.h b/inc/geometries/fluxfunctions.h
index 9c0c94194..2bb474982 100644
--- a/inc/geometries/fluxfunctions.h
+++ b/inc/geometries/fluxfunctions.h
@@ -41,8 +41,8 @@ struct aBinaryFunctor
     * @return a functor on the heap
     */
     virtual aBinaryFunctor* clone()const=0;
-    protected:
     virtual ~aBinaryFunctor(){}
+    protected:
     /**
     * @brief We do not allow object slicing so the copy is protected
     */
@@ -78,6 +78,8 @@ struct dg::geo::aCloneableBinaryFunctor : public dg::geo::aBinaryFunctor
 struct aBinaryFunctorBundle
 {
     protected:
+    /// constructor is deleted
+    aBinaryFunctorBundle();
     aBinaryFunctorBundle( const aBinaryFunctorBundle& b)
     {
         //deep copy
-- 
GitLab


From 2ddb1efe761aa9d0d44071f08ae412c0d6e05361 Mon Sep 17 00:00:00 2001
From: Matthias Home Office <Matthias.Wiesenberger@uibk.ac.at>
Date: Tue, 25 Jul 2017 14:28:07 +0200
Subject: [PATCH 081/453] added fluxfunctions

---
 inc/geometries/flux.h          |  1 +
 inc/geometries/fluxfunctions.h | 29 +++++---------
 inc/geometries/utilities.h     | 72 ++++++++++++++--------------------
 3 files changed, 40 insertions(+), 62 deletions(-)

diff --git a/inc/geometries/flux.h b/inc/geometries/flux.h
index a323dc0ed..583f3d40c 100644
--- a/inc/geometries/flux.h
+++ b/inc/geometries/flux.h
@@ -9,6 +9,7 @@
 #include "dg/runge_kutta.h"
 #include "dg/nullstelle.h"
 #include "dg/geometry.h"
+#include "fluxfunctions.h"
 #include "ribeiro.h"
 #include "generator.h"
 
diff --git a/inc/geometries/fluxfunctions.h b/inc/geometries/fluxfunctions.h
index 2bb474982..74ec92dda 100644
--- a/inc/geometries/fluxfunctions.h
+++ b/inc/geometries/fluxfunctions.h
@@ -120,14 +120,14 @@ struct BinaryFunctorsLvl1 : public aBinaryFunctorBundle
     ///f 
     const aBinaryFunctor& f()const{return *p_[0];}
     /// partial f / partial x
-    const aBinaryFunctor& fx()const{return *p_[1];}
+    const aBinaryFunctor& dfx()const{return *p_[1];}
     /// partial f / partial y
-    const aBinaryFunctor& fy()const{return *p_[2];}
+    const aBinaryFunctor& dfy()const{return *p_[2];}
 };
 /**
 * @brief This struct bundles a function and its first and second derivatives
 */
-struct BinaryFunctorsLvl2 : public aBinaryFunctorBundle
+struct BinaryFunctorsLvl2 : public BinaryFunctorLvl1
 {
     /**
     * @brief Take ownership of newly allocated functors
@@ -140,27 +140,18 @@ struct BinaryFunctorsLvl2 : public aBinaryFunctorBundle
     * @param fyy partial2 f / partial y2
     */
     BinaryFunctorsLvl2( aBinaryFunctor* f, aBinaryFunctor* fx, aBinaryFunctor* fy,
-    aBinaryFunctor* fxx, aBinaryFunctor* fxy, aBinaryFunctor* fyy): p_(6)
+    aBinaryFunctor* fxx, aBinaryFunctor* fxy, aBinaryFunctor* fyy): BinaryFunctorLvl1(f,fx,fy) 
     {
-        p_[0] = f;
-        p_[1] = fx;
-        p_[2] = fy;
-        p_[3] = fxx;
-        p_[4] = fxy;
-        p_[5] = fyy;
+        p_.append( fxx);
+        p_.append( fxy);
+        p_.append( fyy);
     }
-    /// f
-    const aBinaryFunctor& f()const{return *p_[0];}
-    /// partial f /partial x
-    const aBinaryFunctor& fx()const{return *p_[1];}
-    /// partial f /partial y
-    const aBinaryFunctor& fy()const{return *p_[2];}
     /// partial^2f/partial x^2
-    const aBinaryFunctor& fxx()const{return *p_[3];}
+    const aBinaryFunctor& dfxx()const{return *p_[3];}
     /// partial^2 f / partial x partial y
-    const aBinaryFunctor& fxy()const{return *p_[4];}
+    const aBinaryFunctor& dfxy()const{return *p_[4];}
     /// partial^2f/partial y^2
-    const aBinaryFunctor& fyy()const{return *p_[5];}
+    const aBinaryFunctor& dfyy()const{return *p_[5];}
 };
 /**
 * @brief This struct bundles a symmetric tensor and its divergence
diff --git a/inc/geometries/utilities.h b/inc/geometries/utilities.h
index 2492fd699..8b8fa1632 100644
--- a/inc/geometries/utilities.h
+++ b/inc/geometries/utilities.h
@@ -1,4 +1,5 @@
 #pragma once
+#include "fluxfunctions.h"
 
 ///@cond
 namespace dg
@@ -15,14 +16,13 @@ namespace flux{
  * \f[  d Z/d \theta =   B^Z/B^\theta \f],
  * \f[  d y/d \theta =   B^y/B^\theta\f]
  */ 
-template< class PsiR, class PsiZ, class Ipol>
 struct FieldRZYT
 {
-    FieldRZYT( PsiR psiR, PsiZ psiZ, Ipol ipol, double R0, double Z0): R_0_(R0), psipR_(psiR), psipZ_(psiZ),ipol_(ipol){}
+    FieldRZYT( const BinaryFunctorsLvl2& psip, const BinaryFunctorsLvl1& ipol, double R0, double Z0): R_0_(R0), Z_0_(Z0), psip_(psip), ipol_(ipol){}
     void operator()( const dg::HVec& y, dg::HVec& yp) const
     {
-        double psipR = psipR_(y[0], y[1]), psipZ = psipZ_(y[0],y[1]);
-        double ipol=ipol_(y[0], y[1]);
+        double psipR = psip_.dfx()(y[0], y[1]), psipZ = psip_.dfy()(y[0],y[1]);
+        double ipol=ipol_.f()(y[0], y[1]);
         yp[0] =  psipZ;//fieldR
         yp[1] = -psipR;//fieldZ
         yp[2] =ipol/y[0];
@@ -34,19 +34,17 @@ struct FieldRZYT
     }
   private:
     double R_0_, Z_0_;
-    PsiR psipR_;
-    PsiZ psipZ_;
-    Ipol ipol_;
+    BinaryFunctorsLvl2 psip_;
+    BinaryFunctorsLvl1 ipol_;
 };
 
-template< class PsiR, class PsiZ, class Ipol>
 struct FieldRZYZ
 {
-    FieldRZYZ( PsiR psiR, PsiZ psiZ, Ipol ipol): psipR_(psiR), psipZ_(psiZ), ipol_(ipol){}
+    FieldRZYZ( const BinaryFunctorsLvl2& psip, const BinaryFunctorsLvl1& ipol):psip_(psip), ipol_(ipol) {}
     void operator()( const dg::HVec& y, dg::HVec& yp) const
     {
-        double psipR = psipR_(y[0], y[1]), psipZ = psipZ_(y[0],y[1]);
-        double ipol=ipol_(y[0], y[1]);
+        double psipR = psip_.dfx()(y[0], y[1]), psipZ = psip_.dfy()(y[0],y[1]);
+        double ipol=ipol_.f()(y[0], y[1]);
         yp[0] =  psipZ;//fieldR
         yp[1] = -psipR;//fieldZ
         yp[2] =   ipol/y[0]; //fieldYbar
@@ -55,9 +53,8 @@ struct FieldRZYZ
         yp[1] =  1.;
     }
   private:
-    PsiR psipR_;
-    PsiZ psipZ_;
-    Ipol ipol_;
+    BinaryFunctorsLvl2 psip_;
+    BinaryFunctorsLvl1 ipol_;
 };
 
 /**
@@ -65,24 +62,22 @@ struct FieldRZYZ
  * \f[  d R/d y =   B^R/B^y \f], 
  * \f[  d Z/d y =   B^Z/B^y \f],
  */ 
-template< class PsiR, class PsiZ, class Ipol>
 struct FieldRZY
 {
-    FieldRZY( PsiR psiR, PsiZ psiZ, Ipol ipol): f_(1.), psipR_(psiR), psipZ_(psiZ),ipol_(ipol){}
+    FieldRZY( const BinaryFunctorsLvl2& psip, const BinaryFunctorsLvl1& ipol): f_(1.), psip_(psip), ipol_(ipol){}
     void set_f(double f){ f_ = f;}
     void operator()( const dg::HVec& y, dg::HVec& yp) const
     {
-        double psipR = psipR_(y[0], y[1]), psipZ = psipZ_(y[0],y[1]);
-        double ipol=ipol_(y[0], y[1]);
+        double psipR = psip_.dfx()(y[0], y[1]), psipZ = psip_.dfy()(y[0],y[1]);
+        double ipol=ipol_.f()(y[0], y[1]);
         double fnorm = y[0]/ipol/f_;       
         yp[0] =  (psipZ)*fnorm;
         yp[1] = -(psipR)*fnorm;
     }
   private:
     double f_;
-    PsiR psipR_;
-    PsiZ psipZ_;
-    Ipol ipol_;
+    BinaryFunctorsLvl2 psip_;
+    BinaryFunctorsLvl1 ipol_;
 };
 
 /**
@@ -96,34 +91,32 @@ struct FieldRZY
     -\frac{\partial^2 \psi_p}{\partial R \partial Z} y_Z\right]+ 
     \frac{\partial \psi_p}{\partial Z} \left(\frac{1}{I(\psi_p)} \frac{\partial I(\psi_p)}{\partial \psi_p} -\frac{1}{q(\psi_p)} \frac{\partial q(\psi_p)}{\partial \psi_p}\right)\f],
  */ 
-template<class PsiR, class PsiZ, class PsiRR, class PsiRZ, class PsiZZ, class Ipol, class IpolR, class IpolZ>
 struct FieldRZYRYZY
 {
-    FieldRZYRYZY( PsiR psiR, PsiZ psiZ, PsiRR psiRR, PsiRZ psiRZ, PsiZZ psiZZ, Ipol ipol, IpolR ipolR, IpolZ ipolZ): 
-        psipR_(psiR), psipZ_(psiZ), psipRR_(psiRR), psipRZ_(psiRZ), psipZZ_(psiZZ), ipol_(ipol), ipolR_(ipolR), ipolZ_(ipolZ){ f_ = f_prime_ = 1.;}
+    FieldRZYRYZY(const BinaryFunctorsLvl2& psip, const BinaryFunctorsLvl1& ipol): f_(1.), f_prime_(1), psip_(psip), ipol_(ipol){}
     void set_f( double new_f){ f_ = new_f;}
     void set_fp( double new_fp){ f_prime_ = new_fp;}
     void initialize( double R0, double Z0, double& yR, double& yZ)
     {
-        double psipR = psipR_(R0, Z0), psipZ = psipZ_(R0,Z0);
+        double psipR = psip_.dfx()(R0, Z0), psipZ = psip_.dfy()(R0,Z0);
         double psip2 = (psipR*psipR+ psipZ*psipZ);
-        double fnorm =R0/ipol_(R0,Z0)/f_; //=Rq/I
+        double fnorm =R0/ipol_.f()(R0,Z0)/f_; //=Rq/I
         yR = -psipZ_(R0, Z0)/psip2/fnorm;
         yZ = +psipR_(R0, Z0)/psip2/fnorm;
     }
     void derive( double R0, double Z0, double& xR, double& xZ)
     {
-        xR = +f_*psipR_(R0, Z0);
-        xZ = +f_*psipZ_(R0, Z0);
+        xR = +f_*psip_.dfx()(R0, Z0);
+        xZ = +f_*psip_.dfy()(R0, Z0);
     }
     
     void operator()( const dg::HVec& y, dg::HVec& yp) const
     {
-        double psipR = psipR_(y[0], y[1]), psipZ = psipZ_(y[0],y[1]);
-        double psipRR = psipRR_(y[0], y[1]), psipRZ = psipRZ_(y[0],y[1]), psipZZ = psipZZ_(y[0],y[1]);
-        double ipol=ipol_(y[0], y[1]);
-        double ipolR=ipolR_(y[0], y[1]);
-        double ipolZ=ipolZ_(y[0], y[1]);
+        double psipR = psip_.dfx()(y[0], y[1]), psipZ = psip_.dfy()(y[0],y[1]);
+        double psipRR = psip_.dfxx()(y[0], y[1]), psipRZ = psip_.dfxy()(y[0],y[1]), psipZZ = psip_.dfyy()(y[0],y[1]);
+        double ipol=ipol_.f()(y[0], y[1]);
+        double ipolR=ipol_.dfx()(y[0], y[1]);
+        double ipolZ=ipol_.dfy()(y[0], y[1]);
         double fnorm =y[0]/ipol/f_; //=R/(I/q)
 
         yp[0] = -(psipZ)*fnorm;
@@ -134,23 +127,16 @@ struct FieldRZYRYZY
     }
   private:
     double f_, f_prime_;
-    PsiR psipR_;
-    PsiZ psipZ_;
-    PsiRR psipRR_;
-    PsiRZ psipRZ_;
-    PsiZZ psipZZ_;
-    Ipol ipol_;
-    IpolR ipolR_;
-    IpolZ ipolZ_;
+    BinaryFunctorsLvl2 psip_;
+    BinaryFunctorsLvl1 ipol_;
 };
 
 }//namespace flux
 namespace ribeiro{
 
-template< class PsiR, class PsiZ>
 struct FieldRZYT
 {
-    FieldRZYT( PsiR psiR, PsiZ psiZ, double R0, double Z0): R_0_(R0), Z_0_(Z0), psipR_(psiR), psipZ_(psiZ){}
+    FieldRZYT( const BinaryFunctorsLvl2, double R0, double Z0): R_0_(R0), Z_0_(Z0), psipR_(psiR), psipZ_(psiZ){}
     void operator()( const dg::HVec& y, dg::HVec& yp) const
     {
         double psipR = psipR_(y[0], y[1]), psipZ = psipZ_(y[0],y[1]);
-- 
GitLab


From bd51d9ce72a07b70e612ff8cd65619a771df28f5 Mon Sep 17 00:00:00 2001
From: Matthias Home Office <Matthias.Wiesenberger@uibk.ac.at>
Date: Tue, 25 Jul 2017 17:42:39 +0200
Subject: [PATCH 082/453] work on backend for generators

---
 inc/geometries/adaption.h          | 202 ++++++++++-------------------
 inc/geometries/flux.h              |  95 ++++----------
 inc/geometries/fluxfunctions.h     |  63 +++++----
 inc/geometries/guenther.h          |  16 ---
 inc/geometries/magnetic_field.h    |   4 +
 inc/geometries/ribeiro.h           |  63 ++++-----
 inc/geometries/simple_orthogonal.h |  66 ++++------
 inc/geometries/solovev.h           |  35 -----
 inc/geometries/taylor.h            |  14 --
 inc/geometries/utilities.h         | 152 +++++++++-------------
 10 files changed, 233 insertions(+), 477 deletions(-)

diff --git a/inc/geometries/adaption.h b/inc/geometries/adaption.h
index 2990c04c8..9638b8d95 100644
--- a/inc/geometries/adaption.h
+++ b/inc/geometries/adaption.h
@@ -1,5 +1,6 @@
 #pragma once
 
+#include "fluxfunctions.h"
 
 /*!@file 
  *
@@ -14,62 +15,41 @@ namespace geo
 ///@cond
 namespace detail{
 
-template<class PsiX, class PsiY, class LaplacePsi, class Chi, class ChiX, class ChiY>
-struct LaplaceAdaptPsi
+struct LaplaceAdaptPsi: public aCloneableBinaryFunctor<LaplaceAdaptPsi>
 {
-    LaplaceAdaptPsi( const PsiX& psiX, const PsiY& psiY, 
-    const LaplacePsi& laplacePsi,
-    const Chi& chi, const ChiX& chiX, const ChiY& chiY):
-        psiX_(psiX), psiY_(psiY), laplacePsi_(laplacePsi),
-        chi_(chi), chiX_(chiX), chiY_(chiY){}
+    LaplaceAdaptPsi( const BinaryFunctorsLvl2& psi, const BinaryFunctorsLvl1& chi) : psi_(psi), chi_(chi){}
     double operator()(double x, double y)
     {
-        return psiX_(x,y)*chiX_(x,y)+psiY_(x,y)*chiY_(x,y)+chi_(x,y)*laplacePsi_(x,y);
+        return  psi_.dfx()(x,y)*chi_.dfx()(x,y) +
+                psi_.dfy()(x,y)*chi_.dfy()(x,y) +
+                chi_.f()(x,y)*( psi_.dfxx()(x,y) + psi_.dfyy()(x,y));
     }
     private:
-    PsiX psiX_;
-    PsiY psiY_;
-    LaplacePsi laplacePsi_;
-    Chi chi_;
-    ChiX chiX_;
-    ChiY chiY_;
+    BinaryFunctorsLvl2 psi_;
+    BinaryFunctorsLvl1 chi_;
 };
 
-template<class PsiX, class PsiY, class PsiXX, class PsiXY, class PsiYY, class ChiXX, class ChiXY, class ChiYY, class DivChiX, class DivChiY>
-struct LaplaceChiPsi
+struct LaplaceChiPsi: public aCloneableBinaryFunctor<LaplaceChiPsi>
 {
-    LaplaceChiPsi( const PsiX& psiX, const PsiY& psiY, const PsiXX& psiXX,
-    const PsiXY& psiXY, const PsiYY& psiYY, const ChiXX& chiXX, const ChiXY& chiXY,
-    const ChiYY& chiYY, const DivChiX& divChiX, const DivChiY& divChiY):
-        psiX_(psiX), psiY_(psiY), psiXX_(psiXX), psiXY_(psiXY), psiYY_(psiYY),
-        chiXX_(chiXX), chiXY_(chiXY), chiYY_(chiYY), divChiX_(divChiX), divChiY_(divChiY){}
+    LaplaceChiPsi( const BinaryFunctorsLvl2& psi, const BinarySymmTensorLvl1& chi)
+        psi_(psi), chi_(chi){}
     double operator()(double x, double y)
     {
-        return psiXX_(x,y)*chiXX_(x,y)+2.*psiXY_(x,y)*chiXY_(x,y)+psiYY_(x,y)*chiYY_(x,y)
-            + divChiX_(x,y)*psiX_(x,y) + divChiY_(x,y)*psiY_(x,y);
+        return psi_.dfxx()(x,y)*chi_.xx()(x,y)+2.*psi_.dfxy()(x,y)*chi_.xy()(x,y)+psi_.dfyy()(x,y)*chi_.yy()(x,y)
+            + chi_.divX()(x,y)*psi_.dfx()(x,y) + chi_.divY()(x,y)*psi_.dfy()(x,y);
     }
 
     private:
-    PsiX psiX_;
-    PsiY psiY_;
-    PsiXX psiXX_;
-    PsiXY psiXY_;
-    PsiYY psiYY_;
-    ChiXX chiXX_;
-    ChiXY chiXY_;
-    ChiYY chiYY_; 
-    DivChiX divChiX_;
-    DivChiY divChiY_;
+    BinaryFunctorsLvl2 psi_;
+    BinarySymmTensorLvl1 chi_;
 };
 
-template <class PsiXX, class PsiYY>
-struct LaplacePsi
+struct LaplacePsi: public aCloneableBinaryFunctor<LaplacePsi>
 {
-    LaplacePsi( const PsiXX& psiXX, const PsiYY& psiYY):psiXX_(psiXX), psiYY_(psiYY){}
-    double operator()(double x, double y){return psiXX_(x,y)+psiYY_(x,y);}
+    LaplacePsi( const BinaryFunctorsLvl2& psi): psi_(psi){}
+    double operator()(double x, double y){return psi_.dfxx()(x,y)+psi_.dfyy()(x,y);}
     private:
-        PsiXX psiXX_;
-        PsiYY psiYY_;
+    BinaryFunctorsLvl2 psi_;
 };
 
 }//namespace detail
@@ -81,96 +61,75 @@ struct LaplacePsi
 /**
  * @brief  A weight function for the Hector algorithm
  *\f[ |\nabla\psi|^{-1} = (\psi_x^2 + \psi_y^2)^{-1/2} \f]
- @tparam PsiX models aBinaryOperator
- @tparam PsiY models aBinaryOperator
  */
-template<class PsiX, class PsiY>
-struct NablaPsiInv
+struct NablaPsiInv: public aCloneableBinaryFunctor<NablaPsiInv>
 {
     /**
-     * @brief Construct with derivatives
+     * @brief Construct with function container
      *
-     @param psiX \f$ \psi_x\f$ derivative in x
-     @param psiY \f$ \psi_y\f$ derivative in y
+     * @param psi \f$ \psi(x,y)\f$ and its first derivatives
      */
-    NablaPsiInv( const PsiX& psiX, const PsiY& psiY): psiX_(psiX), psiY_(psiY){}
+    NablaPsiInv( const BinaryFunctorsLvl1& psi): psi_(psi){}
     /**
      * @brief  A weight function for the Hector algorithm
      * \f[ |\nabla\psi|^{-1} = (\psi_x^2 + \psi_y^2)^{-1/2} \f]
      */
     double operator()(double x, double y)
     {
-        double psiX = psiX_(x,y), psiY = psiY_(x,y);
+        double psiX = psi_.dfx()(x,y), psiY = psi_.dfy()(x,y);
         return 1./sqrt(psiX*psiX+psiY*psiY);
     }
     private:
-    PsiX psiX_;
-    PsiY psiY_;
+    BinaryFunctorsLvl1 psi_;
 };
 
 /**
  * @brief Derivative of the weight function
  *\f[\partial_x|\nabla\psi|^{-1} \f]
  */
-template<class PsiX, class PsiY, class PsiXX, class PsiXY, class PsiYY>
-struct NablaPsiInvX
+struct NablaPsiInvX: public aCloneableBinaryFunctor<NablaPsiInvX>
 {
-    NablaPsiInvX( const PsiX& psiX, const PsiY& psiY, const PsiXX& psiXX, const PsiXY& psiXY, const PsiYY& psiYY):
-    psiX_(psiX), psiY_(psiY), psiXX_(psiXX), psiXY_(psiXY), psiYY_(psiYY)
-    {}
+    NablaPsiInvX( const BinaryFunctorsLvl2& psi):psi_(psi) {}
     double operator()(double x, double y)
     {
-        double psiX = psiX_(x,y), psiY = psiY_(x,y);
-        double psiXX = psiXX_(x,y), psiXY = psiXY_(x,y);
+        double psiX = psi_.dfx()(x,y), psiY = psi_.dfy()(x,y);
+        double psiXX = psi_.dfxx()(x,y), psiXY = psi_.dfxy()(x,y);
         double psip = sqrt( psiX*psiX+psiY*psiY);
         return -(psiX*psiXX+psiY*psiXY)/psip/psip/psip;
     }
     
     private:
-    PsiX psiX_;
-    PsiY psiY_;
-    PsiXX psiXX_;
-    PsiXY psiXY_;
-    PsiYY psiYY_;
+    BinaryFunctorsLvl2 psi_;
 };
 
 /**
  * @brief Derivative of the weight function
  *\f[ \partial_y|\nabla\psi|^{-1} \f]
  */
-template<class PsiX, class PsiY, class PsiXX, class PsiXY, class PsiYY>
-struct NablaPsiInvY
+struct NablaPsiInvY: public aCloneableBinaryFunctor<NablaPsiInvY>
 {
-    NablaPsiInvY( const PsiX& psiX, const PsiY& psiY, const PsiXX& psiXX, const PsiXY& psiXY, const PsiYY& psiYY):
-    psiX_(psiX), psiY_(psiY), psiXX_(psiXX), psiXY_(psiXY), psiYY_(psiYY)
-    {}
+    NablaPsiInvY( const BinaryFunctorsLvl2& psi):psi_(psi) {}
     double operator()(double x, double y)
     {
-        double psiX = psiX_(x,y), psiY = psiY_(x,y);
-        double psiYY = psiYY_(x,y), psiXY = psiXY_(x,y);
+        double psiX = psi_.dfx()(x,y), psiY = psi_.dfy()(x,y);
+        double psiYY = psi_.dfyy()(x,y), psiXY = psi_.dfxy()(x,y);
         double psip = sqrt( psiX*psiX+psiY*psiY);
         return -(psiX*psiXY+psiY*psiYY)/psip/psip/psip;
     }
     
     private:
-    PsiX psiX_;
-    PsiY psiY_;
-    PsiXX psiXX_;
-    PsiXY psiXY_;
-    PsiYY psiYY_;
+    BinaryFunctorsLvl2 psi_;
 };
 
 /**
  * @brief A container class that contains all NablaPsiInv functors
  */
-template<class PsiX, class PsiY, class PsiXX, class PsiXY, class PsiYY>
-struct NablaPsiInvCollective
+BinaryFunctorsLvl1 make_NablaPsiInvCollective( const BinaryFunctorsLvl2& psi)
 {
-    NablaPsiInvCollective( const PsiX& psiX, const PsiY& psiY, const PsiXX& psiXX, const PsiXY& psiXY, const PsiYY& psiYY):nablaPsiInv(psiX, psiY), nablaPsiInvX(psiX, psiY, psiXX, psiXY, psiYY), nablaPsiInvY( psiX, psiY, psiXX, psiXY, psiYY){}
-    NablaPsiInv<PsiX, PsiY> nablaPsiInv; //!< inverse of nabla psi
-    NablaPsiInvX<PsiX, PsiY, PsiXX, PsiXY, PsiYY> nablaPsiInvX; //!< derivative of nabla psi
-    NablaPsiInvY<PsiX, PsiY, PsiXX, PsiXY, PsiYY> nablaPsiInvY; //!< derivative of nabla psi
-};
+    BinaryFunctorsLvl1 temp( new NablaPsiInv(psi), new NablaPsiInvX(psi), new NablaPsiInvY( psi));
+    return temp;
+}
+
 
 
 /**
@@ -180,13 +139,12 @@ struct NablaPsiInvCollective
  * with
  * \f[ \det \chi = (\varepsilon+(\nabla\psi)^2)(\varepsilon+k^2(\nabla\psi)^2)\f]
  */
-template<class PsiX, class PsiY>
-struct Liseikin_XX
+struct Liseikin_XX: public aCloneableBinaryFunctor<Liseikin_XX>
 {
-    Liseikin_XX(PsiX psiX, PsiY psiY, double k, double eps):k_(k), eps_(eps), psiX_(psiX), psiY_(psiY){}
+    Liseikin_XX(BinaryFunctorsLvl1& psi, double k, double eps):k_(k), eps_(eps), psi_(psi){}
     double operator()(double x, double y)
     {
-        double psiX = psiX_(x,y), psiY = psiY_(x,y), k2 = k_*k_;
+        double psiX = psi_.dfx()(x,y), psiY = psi_.dfy()(x,y), k2 = k_*k_;
         double psip2 = psiX*psiX+psiY*psiY;
         double sqrtG = sqrt((eps_+psip2)*(eps_+k2*psip2));
         return (psiY*psiY+k2*psiX*psiX + eps_)/sqrtG;
@@ -194,8 +152,7 @@ struct Liseikin_XX
 
     private:
     double k_, eps_;
-    PsiX psiX_;
-    PsiY psiY_;
+    BinaryFunctorsLvl1 psi_;
 };
 
 /**
@@ -205,13 +162,12 @@ struct Liseikin_XX
  * with
  * \f[ \det \chi = (\varepsilon+(\nabla\psi)^2)(\varepsilon+k^2(\nabla\psi)^2)\f]
  */
-template<class PsiX, class PsiY>
-struct Liseikin_XY
+struct Liseikin_XY: public aCloneableBinaryFunctor<Liseikin_XY>
 {
-    Liseikin_XY(PsiX psiX, PsiY psiY, double k, double eps):k_(k), eps_(eps), psiX_(psiX), psiY_(psiY){}
+    Liseikin_XY(BinaryFunctorsLvl1& psi, double k, double eps):k_(k), eps_(eps), psi_(psi){}
     double operator()(double x, double y)
     {
-        double psiX = psiX_(x,y), psiY = psiY_(x,y), k2 = k_*k_;
+        double psiX = psi_.dfx()(x,y), psiY = psi_.dfy()(x,y), k2 = k_*k_;
         double psip2 = psiX*psiX+psiY*psiY;
         double sqrtG = sqrt((eps_+psip2)*(eps_+k2*psip2));
         return (-psiX*psiY+k2*psiX*psiY)/sqrtG;
@@ -230,13 +186,12 @@ struct Liseikin_XY
  * with
  * \f[ \det \chi = (\varepsilon+(\nabla\psi)^2)(\varepsilon+k^2(\nabla\psi)^2)\f]
  */
-template<class PsiX, class PsiY>
-struct Liseikin_YY
+struct Liseikin_YY: public aCloneableBinaryFunctor<Liseikin_YY>
 {
-    Liseikin_YY(PsiX psiX, PsiY psiY, double k, double eps):k_(k), eps_(eps), psiX_(psiX), psiY_(psiY){}
+    Liseikin_YY(BinaryFunctorsLvl1& psi, double k, double eps):k_(k), eps_(eps), psi_(psi){}
     double operator()(double x, double y)
     {
-        double psiX = psiX_(x,y), psiY = psiY_(x,y), k2 = k_*k_;
+        double psiX = psi_.dfx()(x,y), psiY = psi_.dfy()(x,y), k2 = k_*k_;
         double psip2 = psiX*psiX+psiY*psiY;
         double sqrtG = sqrt((eps_+psip2)*(eps_+k2*psip2));
         return (eps_+psiX*psiX+k2*psiY*psiY)/sqrtG;
@@ -244,22 +199,20 @@ struct Liseikin_YY
 
     private:
     double k_, eps_;
-    PsiX psiX_;
-    PsiY psiY_;
+    BinaryFunctorsLvl1 psi_;
 };
 
 /**
  * @brief The x-component of the divergence of the Liseikin monitor metric
  * \f[ \partial_x \chi^{xx} + \partial_y\chi^{yx}\f]
  */
-template<class PsiX, class PsiY, class PsiXX, class PsiXY, class PsiYY>
-struct DivLiseikinX
+struct DivLiseikinX: public aCloneableBinaryFunctor<DivLiseikinX>
 {
-    DivLiseikinX(PsiX psiX, PsiY psiY, PsiXX psiXX, PsiXY psiXY, PsiYY psiYY, double k, double eps):k_(k), eps_(eps), psiX_(psiX), psiY_(psiY), psiXX_(psiXX), psiXY_(psiXY), psiYY_(psiYY){}
+    DivLiseikinX(BinaryFunctorsLvl2& psi, double k, double eps):k_(k), eps_(eps), psi_(psi){}
     double operator()(double x, double y)
     {
-        double psiX = psiX_(x,y), psiY = psiY_(x,y), k2 = k_*k_;
-        double psiXX = psiXX_(x,y), psiXY = psiXY_(x,y), psiYY=psiYY_(x,y);
+        double psiX = psi_.dfx()(x,y), psiY = psi_.dfy()(x,y), k2 = k_*k_;
+        double psiXX = psi_.dfxx()(x,y), psiXY = psi_.dfxy()(x,y), psiYY=psi_.dfyy()(x,y);
         double psiY2 = psiY*psiY, psiY3=psiY*psiY2, psiY4=psiY2*psiY2, psiY5=psiY4*psiY;
         double psiX2 = psiX*psiX, psiX4=psiX2*psiX2;
         double psip2 = psiX*psiX+psiY*psiY;
@@ -273,25 +226,20 @@ struct DivLiseikinX
 
     private:
     double k_, eps_;
-    PsiX psiX_;
-    PsiY psiY_;
-    PsiXX psiXX_;
-    PsiXY psiXY_;
-    PsiYY psiYY_;
+    BinaryFunctorsLvl2 psi_;
 };
 
 /**
  * @brief The y-component of the divergence of the Liseikin monitor metric
  * \f[ \partial_x \chi^{xy} + \partial_y\chi^{yy}\f]
  */
-template<class PsiX, class PsiY, class PsiXX, class PsiXY, class PsiYY>
-struct DivLiseikinY
+struct DivLiseikinY : public aCloneableBinaryFunctor<DivLiseikinY>
 {
-    DivLiseikinY(PsiX psiX, PsiY psiY, PsiXX psiXX, PsiXY psiXY, PsiYY psiYY, double k, double eps):k_(k), eps_(eps), psiX_(psiX), psiY_(psiY), psiXX_(psiXX), psiXY_(psiXY), psiYY_(psiYY){}
+    DivLiseikinY(BinaryFunctorsLvl2& psi, double k, double eps):k_(k), eps_(eps), psi_(psi){}
     double operator()(double x, double y)
     {
-        double psiX = psiX_(x,y), psiY = psiY_(x,y), k2 = k_*k_;
-        double psiXX = psiXX_(x,y), psiXY = psiXY_(x,y), psiYY=psiYY_(x,y);
+        double psiX = psi_.dfx()(x,y), psiY = psi_.dfy()(x,y), k2 = k_*k_;
+        double psiXX = psi_.dfxx()(x,y), psiXY = psi_.dfxy()(x,y), psiYY=psi_.dfyy()(x,y);
         double psiX2 = psiX*psiX, psiX3=psiX*psiX2, psiX4=psiX2*psiX2, psiX5=psiX4*psiX;
         double psiY2 = psiY*psiY, psiY4 = psiY2*psiY2;
         double psip2 = psiX*psiX+psiY*psiY;
@@ -300,37 +248,19 @@ struct DivLiseikinY
                 k2*psiX4*psiY*(2.*psiYY-psiXX)+psiY*(eps_+k2*psiY2)
                 *(eps_*psiYY+(eps_+psiY2)*psiXX)+k2*psiX5*psiXY+
                 psiX3*(-(eps_+2.*k2*psiY2)*psiXY+eps_*(1.+k2)*psiXY) +
-                psiX*(-(eps_+2.*psiY2)*(eps_+k2*psiY2)*psiXY + (eps_*eps_-k2*psiY4)*psiXY))/sqrtG/sqrtG/sqrtG;
+                psiX*(-(eps_+2.*psiY2)*(eps_+k2*psiY2)*psiXY + (eps_*eps_-k2*psiY4)*psiXY)/sqrtG/sqrtG/sqrtG;
     }
 
     private:
     double k_, eps_;
-    PsiX psiX_;
-    PsiY psiY_;
-    PsiXX psiXX_;
-    PsiXY psiXY_;
-    PsiYY psiYY_;
+    BinaryFunctorsLvl2 psi_;
 };
 
-/**
- * @brief A structure that contains all functors of the Liseikin monitor metric
- */
-template<class PsiX, class PsiY, class PsiXX, class PsiXY, class PsiYY>
-struct LiseikinCollective
+BinarySymmTensorLvl1 make_LiseikinCollective( const BinaryFunctorsLvl2& psi)
 {
-    LiseikinCollective( const PsiX& psiX, const PsiY& psiY, const PsiXX& psiXX, const PsiXY& psiXY, const PsiYY& psiYY, double k, double eps):
-        chi_XX(psiX, psiY, k, eps), 
-        chi_XY(psiX, psiY, k, eps), 
-        chi_YY(psiX, psiY, k, eps), 
-        divChiX(psiX, psiY, psiXX, psiXY, psiYY, k, eps), 
-        divChiY(psiX, psiY, psiXX, psiXY, psiYY, k, eps){}
-    Liseikin_XX<PsiX, PsiY> chi_XX; //!< a metric tensor component
-    Liseikin_XY<PsiX, PsiY> chi_XY; //!< a metric tensor component
-    Liseikin_YY<PsiX, PsiY> chi_YY; //!< a metric tensor component
-    DivLiseikinX<PsiX, PsiY, PsiXX, PsiXY, PsiYY> divChiX; //!< divergence of metric
-    DivLiseikinY<PsiX, PsiY, PsiXX, PsiXY, PsiYY> divChiY; //!< divergence of metric
-};
-
+    BinarySymmTensorLvl1 temp( new Liseikin_XX(psi), new Liseikin_XY(psi), new Liseikin_YY(psi), new DivLiseikinX(psi), new DivLiseikinY(psi));
+    return temp;
+}
 ///@}
 
 }//namespace geo
diff --git a/inc/geometries/flux.h b/inc/geometries/flux.h
index 583f3d40c..37d7ccd6e 100644
--- a/inc/geometries/flux.h
+++ b/inc/geometries/flux.h
@@ -27,16 +27,15 @@ namespace detail
 
 //This leightweights struct and its methods finds the initial R and Z values and the coresponding f(\psi) as 
 //good as it can, i.e. until machine precision is reached
-template< class Psi, class PsiX, class PsiY, class Ipol>
 struct Fpsi
 {
     
     //firstline = 0 -> conformal, firstline = 1 -> equalarc
-    Fpsi( Psi psi, PsiX psiX, PsiY psiY, Ipol ipol, double x0, double y0): 
-        psip_(psi), fieldRZYT_(psiX, psiY, ipol, x0, y0), fieldRZtau_(psiX, psiY)
+    Fpsi( const BinaryFunctorsLvl1& psip, const BinaryFunctorsLvl1& ipol, double x0, double y0): 
+        psip_(psip), fieldRZYT_(psip, ipol, x0, y0), fieldRZtau_(psip)
     {
         X_init = x0, Y_init = y0;
-        while( fabs( psiX(X_init, Y_init)) <= 1e-10 && fabs( psiY( X_init, Y_init)) <= 1e-10)
+        while( fabs( psip.dfx()(X_init, Y_init)) <= 1e-10 && fabs( psip.dfy()( X_init, Y_init)) <= 1e-10)
             X_init +=  1.; 
     }
     //finds the starting points for the integration in y direction
@@ -50,7 +49,7 @@ struct Fpsi
         while( (eps < eps_old || eps > 1e-7) && eps > 1e-14)
         {
             eps_old = eps; end2d_old = end2d;
-            N*=2; dg::stepperRK17( fieldRZtau_, begin2d, end2d, psip_(X_init, Y_init), psi, N);
+            N*=2; dg::stepperRK17( fieldRZtau_, begin2d, end2d, psip_.f()(X_init, Y_init), psi, N);
             eps = sqrt( (end2d[0]-end2d_old[0])*(end2d[0]-end2d_old[0]) + (end2d[1]-end2d_old[1])*(end2d[1]-end2d_old[1]));
         }
         X_init = R_0 = end2d_old[0], Y_init = Z_0 = end2d_old[1];
@@ -113,9 +112,9 @@ struct Fpsi
 
     private:
     double X_init, Y_init;
-    Psi psip_;
-    dg::geo::flux::FieldRZYT<PsiX, PsiY, Ipol> fieldRZYT_;
-    dg::geo::FieldRZtau<PsiX, PsiY> fieldRZtau_;
+    BinaryFunctorsLvl1 psip_;
+    dg::geo::flux::FieldRZYT fieldRZYT_;
+    dg::geo::FieldRZtau fieldRZtau_;
 
 };
 
@@ -126,31 +125,14 @@ struct Fpsi
 /**
  * @brief A symmetry flux generator
  * @ingroup generators
- * @tparam Psi All the template parameters must model aBinaryOperator i.e. the bracket operator() must be callable with two arguments and return a double. 
- * @tparam PsiX models aBinaryOperator 
- * @tparam PsiY models aBinaryOperator 
- * @tparam PsiXX models aBinaryOperator 
- * @tparam PsiXY models aBinaryOperator 
- * @tparam PsiYY models aBinaryOperator 
- * @tparam Ipol models aBinaryOperator 
- * @tparam IpolX models aBinaryOperator 
- * @tparam IpolY models aBinaryOperator 
  */
-template< class Psi, class PsiX, class PsiY, class PsiXX, class PsiXY, class PsiYY, class Ipol, class IpolX, class IpolY>
 struct FluxGenerator : public aGridGenerator
 {
     /**
      * @brief Construct a symmetry flux grid generator
      *
-     * @param psi \f$ \psi(x,y)\f$ the flux function in Cartesian coordinates (x,y)
-     @param psiX \f$ \psi_x\f$ its derivative in x
-     @param psiY \f$ \psi_y\f$ its derivative in y
-     @param psiXX \f$ \psi_{xx}\f$ second derivative
-     @param psiXY \f$ \psi_{xy}\f$ second derivative
-     @param psiYY \f$ \psi_{yy}\f$ second derivative
-     * @param ipol \f$ I(x,y)\f$ the current function in Cartesian coordinates (x,y)
-     * @param ipolX \f$ I_x(x,y)\f$ its derivative in x
-     * @param ipolY \f$ I_y(x,y)\f$ its derivative in x
+     * @param psi \f$ \psi(x,y)\f$ the flux function and its derivatives in Cartesian coordinates (x,y)
+     * @param ipol \f$ I(x,y)\f$ the current function and its derivatives in Cartesian coordinates (x,y)
      * @param psi_0 first boundary 
      * @param psi_1 second boundary
      * @param x0 a point in the inside of the ring bounded by psi0 (shouldn't be the O-point)
@@ -158,19 +140,19 @@ struct FluxGenerator : public aGridGenerator
      * @param mode This parameter indicates the adaption type used to create the grid: 0 is no adaption, 1 is an equalarc adaption
      * @note If mode == 1 then this class does the same as the RibeiroFluxGenerator
      */
-    FluxGenerator( Psi psi, PsiX psiX, PsiY psiY, PsiXX psiXX, PsiXY psiXY, PsiYY psiYY, Ipol ipol, IpolX ipolX, IpolY ipolY, double psi_0, double psi_1, double x0, double y0, int mode=0):
-        psi_(psi), psiX_(psiX), psiY_(psiY), psiXX_(psiXX), psiXY_(psiXY), psiYY_(psiYY), ipol_(ipol), ipolR_(ipolX), ipolZ_(ipolY), mode_(mode)
+    FluxGenerator( const BinaryFunctorsLvl2& psi, const BinaryFunctorsLvl1 ipol, double psi_0, double psi_1, double x0, double y0, int mode=0):
+        psi_(psi), ipol_(ipol), mode_(mode)
     {
         psi0_ = psi_0, psi1_ = psi_1;
         assert( psi_1 != psi_0);
         if( mode==0)
         {
-            flux::detail::Fpsi<Psi, PsiX, PsiY, Ipol> fpsi(psi, psiX, psiY, ipol, x0, y0);
+            flux::detail::Fpsi fpsi(psi, ipol, x0, y0);
             f0_ = fabs( fpsi.construct_f( psi_0, x0_, y0_));
         }
         else
         {
-            ribeiro::detail::Fpsi<Psi, PsiX, PsiY> fpsi(psi, psiX, psiY, x0, y0, mode);
+            ribeiro::detail::Fpsi fpsi(psi, x0, y0, mode);
             f0_ = fabs( fpsi.construct_f( psi_0, x0_, y0_));
         }
         if( psi_1 < psi_0) f0_*=-1;
@@ -213,10 +195,10 @@ struct FluxGenerator : public aGridGenerator
             psi_x[i] = zeta1d[i]/f0_ +psi0_;
 
         //std::cout << "In grid function:\n";
-        flux::detail::Fpsi<Psi, PsiX, PsiY, Ipol> fpsi(psi_, psiX_, psiY_, ipol_, x0_, y0_);
-        dg::geo::flux::FieldRZYRYZY<PsiX, PsiY, PsiXX, PsiXY, PsiYY, Ipol, IpolX, IpolY> fieldRZYRYZY(psiX_, psiY_, psiXX_, psiXY_, psiYY_, ipol_, ipolR_, ipolZ_);
-        ribeiro::detail::Fpsi<Psi, PsiX, PsiY> fpsiRibeiro(psi_, psiX_, psiY_, x0_, y0_, mode_);
-        dg::geo::equalarc::FieldRZYRYZY<PsiX, PsiY, PsiXX, PsiXY, PsiYY> fieldRZYRYZYequalarc(psiX_, psiY_, psiXX_, psiXY_, psiYY_);
+        flux::detail::Fpsi fpsi(psi_, ipol_, x0_, y0_);
+        dg::geo::flux::FieldRZYRYZY fieldRZYRYZY(psi_, ipol_);
+        ribeiro::detail::Fpsi fpsiRibeiro(psi_, x0_, y0_, mode_);
+        dg::geo::equalarc::FieldRZYRYZY fieldRZYRYZYequalarc(psi_);
         thrust::host_vector<double> fx_;
         fx_.resize( zeta1d.size());
         thrust::host_vector<double> f_p(fx_);
@@ -236,15 +218,8 @@ struct FluxGenerator : public aGridGenerator
             }
         }
     }
-    Psi psi_;
-    PsiX psiX_;
-    PsiY psiY_;
-    PsiXX psiXX_;
-    PsiXY psiXY_;
-    PsiYY psiYY_;
-    Ipol ipol_;
-    IpolX ipolR_;
-    IpolY ipolZ_;
+    BinaryFunctorsLvl2 psi_;
+    BinaryFunctorsLvl1 ipol_;
     double f0_, lx_, x0_, y0_, psi0_, psi1_;
     int mode_;
 };
@@ -252,12 +227,6 @@ struct FluxGenerator : public aGridGenerator
 /**
  * @brief Same as the Ribeiro class just but uses psi as a flux label directly
  * @ingroup generators
- * @tparam Psi All the template parameters must model aBinaryOperator i.e. the bracket operator() must be callable with two arguments and return a double. 
-     * @tparam PsiX models aBinaryOperator 
-     * @tparam PsiY models aBinaryOperator 
-     * @tparam PsiXX models aBinaryOperator 
-     * @tparam PsiXY models aBinaryOperator 
-     * @tparam PsiYY models aBinaryOperator 
  */
 template< class Psi, class PsiX, class PsiY, class PsiXX, class PsiXY, class PsiYY>
 struct RibeiroFluxGenerator : public aGridGenerator
@@ -265,24 +234,19 @@ struct RibeiroFluxGenerator : public aGridGenerator
     /**
      * @brief Construct a flux aligned grid generator
      *
-     * @param psi \f$ \psi(x,y)\f$ the flux function in Cartesian coordinates (x,y)
-     @param psiX \f$ \psi_x\f$ its derivative in x
-     @param psiY \f$ \psi_y\f$ its derivative in y
-     @param psiXX \f$ \psi_{xx}\f$ second derivative
-     @param psiXY \f$ \psi_{xy}\f$ second derivative
-     @param psiYY \f$ \psi_{yy}\f$ second derivative
+     * @param psi \f$ \psi(x,y)\f$ the flux function and its derivatives in Cartesian coordinates (x,y)
      * @param psi_0 first boundary 
      * @param psi_1 second boundary
      * @param x0 a point in the inside of the ring bounded by psi0 (shouldn't be the O-point)
      * @param y0 a point in the inside of the ring bounded by psi0 (shouldn't be the O-point)
      * @param mode This parameter indicates the adaption type used to create the grid: 0 is no adaption, 1 is an equalarc adaption
      */
-    RibeiroFluxGenerator( Psi psi, PsiX psiX, PsiY psiY, PsiXX psiXX, PsiXY psiXY, PsiYY psiYY, double psi_0, double psi_1, double x0, double y0, int mode=0):
-        psi_(psi), psiX_(psiX), psiY_(psiY), psiXX_(psiXX), psiXY_(psiXY), psiYY_(psiYY), mode_(mode)
+    RibeiroFluxGenerator( const BinaryFunctorsLvl2& psi, double psi_0, double psi_1, double x0, double y0, int mode=0):
+        psi_(psi), mode_(mode)
     {
         psi0_ = psi_0, psi1_ = psi_1;
         assert( psi_1 != psi_0);
-        ribeiro::detail::Fpsi<Psi, PsiX, PsiY> fpsi(psi, psiX, psiY, x0, y0, mode);
+        ribeiro::detail::Fpsi fpsi(psi, x0, y0, mode);
         f0_ = fabs( fpsi.construct_f( psi_0, x0_, y0_));
         if( psi_1 < psi_0) f0_*=-1;
         lx_ =  f0_*(psi_1-psi_0);
@@ -322,9 +286,9 @@ struct RibeiroFluxGenerator : public aGridGenerator
         for( unsigned i=0; i<psi_x.size(); i++)
             psi_x[i] = zeta1d[i]/f0_ +psi0_;
 
-        ribeiro::detail::Fpsi<Psi, PsiX, PsiY> fpsi(psi_, psiX_, psiY_, x0_, y0_, mode_);
-        dg::geo::ribeiro::FieldRZYRYZY<PsiX, PsiY, PsiXX, PsiXY, PsiYY> fieldRZYRYZYribeiro(psiX_, psiY_, psiXX_, psiXY_, psiYY_);
-        dg::geo::equalarc::FieldRZYRYZY<PsiX, PsiY, PsiXX, PsiXY, PsiYY> fieldRZYRYZYequalarc(psiX_, psiY_, psiXX_, psiXY_, psiYY_);
+        ribeiro::detail::Fpsi fpsi(psi_, x0_, y0_, mode_);
+        dg::geo::ribeiro::FieldRZYRYZY fieldRZYRYZYribeiro(psi_);
+        dg::geo::equalarc::FieldRZYRYZY fieldRZYRYZYequalarc(psi_);
         thrust::host_vector<double> fx_;
         fx_.resize( zeta1d.size());
         thrust::host_vector<double> f_p(fx_);
@@ -344,12 +308,7 @@ struct RibeiroFluxGenerator : public aGridGenerator
             }
         }
     }
-    Psi psi_;
-    PsiX psiX_;
-    PsiY psiY_;
-    PsiXX psiXX_;
-    PsiXY psiXY_;
-    PsiYY psiYY_;
+    BinaryFunctorsLvl2 psip_;
     double f0_, lx_, x0_, y0_, psi0_, psi1_;
     int mode_;
 };
diff --git a/inc/geometries/fluxfunctions.h b/inc/geometries/fluxfunctions.h
index 74ec92dda..f23531dfd 100644
--- a/inc/geometries/fluxfunctions.h
+++ b/inc/geometries/fluxfunctions.h
@@ -82,6 +82,7 @@ struct aBinaryFunctorBundle
     aBinaryFunctorBundle();
     aBinaryFunctorBundle( const aBinaryFunctorBundle& b)
     {
+        p_.resize( b.p_.size());
         //deep copy
         for( unsigned i=0; i<p_.size(); i++)
             p_[i] = b.p_[i]->clone();
@@ -107,9 +108,9 @@ struct BinaryFunctorsLvl1 : public aBinaryFunctorBundle
     /**
     * @brief Take ownership of newly allocated functors
     *
-    * @param f f 
-    * @param fx partial f / partial x
-    * @param fy partial f / partial y
+    * @param f \f$ f(x,y)\f$ the function in some coordinates (x,y)
+    * @param fx \f$ \partial f / \partial x \f$ its derivative in the first coordinate
+    * @param fy \f$ \partial f / \partial y \f$ its derivative in the second coordinate
     */
     BinaryFunctorsLvl1( aBinaryFunctor* f, aBinaryFunctor* fx, aBinaryFunctor* fy): p_(3)
     {
@@ -117,40 +118,36 @@ struct BinaryFunctorsLvl1 : public aBinaryFunctorBundle
         p_[1] = fx;
         p_[2] = fy;
     }
-    ///f 
+    /// \f$ f \f$
     const aBinaryFunctor& f()const{return *p_[0];}
-    /// partial f / partial x
+    /// \f$ \partial f / \partial x \f$ 
     const aBinaryFunctor& dfx()const{return *p_[1];}
-    /// partial f / partial y
+    /// \f$ \partial f / \partial y\f$
     const aBinaryFunctor& dfy()const{return *p_[2];}
 };
 /**
 * @brief This struct bundles a function and its first and second derivatives
 */
-struct BinaryFunctorsLvl2 : public BinaryFunctorLvl1
+struct BinaryFunctorsLvl2 : public BinaryFunctorsLvl1
 {
     /**
-    * @brief Take ownership of newly allocated functors
-    *
-    * @param f f 
-    * @param fx partial f / partial x
-    * @param fy partial f / partial y
-    * @param fxx partial2 f / partial x2
-    * @param fxy partial2 f / partial x /partial y
-    * @param fyy partial2 f / partial y2
+    * @copydoc BinaryFunctorsLvl1
+    * @param fxx \f$ \partial^2 f / \partial x^2\f$ second derivative in first coordinate
+    * @param fxy \f$ \partial^2 f / \partial x \partial y\f$ second mixed derivative 
+    * @param fyy \f$ \partial^2 f / \partial y^2\f$ second derivative in second coordinate
     */
     BinaryFunctorsLvl2( aBinaryFunctor* f, aBinaryFunctor* fx, aBinaryFunctor* fy,
-    aBinaryFunctor* fxx, aBinaryFunctor* fxy, aBinaryFunctor* fyy): BinaryFunctorLvl1(f,fx,fy) 
+    aBinaryFunctor* fxx, aBinaryFunctor* fxy, aBinaryFunctor* fyy): BinaryFunctorsLvl1(f,fx,fy) 
     {
-        p_.append( fxx);
-        p_.append( fxy);
-        p_.append( fyy);
+        p_.push_back( fxx);
+        p_.push_back( fxy);
+        p_.push_back( fyy);
     }
-    /// partial^2f/partial x^2
+    /// \f$ \partial^2f/\partial x^2\f$
     const aBinaryFunctor& dfxx()const{return *p_[3];}
-    /// partial^2 f / partial x partial y
+    /// \f$ \partial^2 f / \partial x \partial y\f$
     const aBinaryFunctor& dfxy()const{return *p_[4];}
-    /// partial^2f/partial y^2
+    /// \f$ \partial^2f/\partial y^2\f$
     const aBinaryFunctor& dfyy()const{return *p_[5];}
 };
 /**
@@ -159,12 +156,12 @@ struct BinaryFunctorsLvl2 : public BinaryFunctorLvl1
 struct BinarySymmTensorLvl1 : public aBinaryFunctorBundle
 {
     /**
-    * @brief Take ownership of newly allocated functors
-    *
-    * let's assume the tensor is called chi
-    * @param chi_xx contravariant xx component
-    * @param chi_xy contravariant xy component
-    * @param chi_yy contravariant yy component
+     * @brief Take ownership of newly allocated functors
+     *
+     * let's assume the tensor is called \f$ \chi \f$ (chi)
+     * @param chi_xx contravariant xx component \f$ \chi^{xx}\f$ 
+     * @param chi_xy contravariant xy component \f$ \chi^{xy}\f$ 
+     * @param chi_yy contravariant yy component \f$ \chi^{yy}\f$ 
      * @param divChiX \f$ \partial_x \chi^{xx} + \partial_y\chi^{yx}\f$ is the x-component of the divergence of the tensor \f$ \chi\f$
      * @param divChiY \f$ \partial_x \chi^{xy} + \partial_y\chi^{yy}\f$ is the y-component of the divergence of the tensor \f$ \chi \f$
     */
@@ -177,15 +174,15 @@ struct BinarySymmTensorLvl1 : public aBinaryFunctorBundle
         p_[3] = divChiX;
         p_[4] = divChiY;
     }
-    ///xx component
+    ///xy component \f$ \chi^{xx}\f$ 
     const aBinaryFunctor& xx()const{return *p_[0];}
-    ///xy component
+    ///xy component \f$ \chi^{xy}\f$ 
     const aBinaryFunctor& xy()const{return *p_[1];}
-    ///yy component
+    ///yy component \f$ \chi^{yy}\f$ 
     const aBinaryFunctor& yy()const{return *p_[2];}
-    ///x component of the divergence 
+     /// \f$ \partial_x \chi^{xx} + \partial_y\chi^{yx}\f$ is the x-component of the divergence of the tensor \f$ \chi\f$
     const aBinaryFunctor& divX()const{return *p_[3];}
-    ///y component of the divergence 
+     /// \f$ \partial_x \chi^{xy} + \partial_y\chi^{yy}\f$ is the y-component of the divergence of the tensor \f$ \chi \f$
     const aBinaryFunctor& divY()const{return *p_[4];}
 };
 
diff --git a/inc/geometries/guenther.h b/inc/geometries/guenther.h
index cb5573124..5a79eb38c 100644
--- a/inc/geometries/guenther.h
+++ b/inc/geometries/guenther.h
@@ -108,21 +108,6 @@ struct PsipRZ : public aCloneableBinaryFunctor<PsipRZ>
     double R_0;
 };
 
-/**
- * @brief \f[-\pi^2\cos(\pi(R-R_0)/2)\cos(\pi Z/2)/2\f]
- */
-struct LaplacePsip : public aCloneableBinaryFunctor<LaplacePsip>
-{
-    LaplacePsip( double R_0 ): psipRR_(R_0), psipZZ_(R_0){}
-    double operator()(double R, double Z) const
-    {    
-        return psipRR_(R,Z) + psipZZ_(R,Z);
-    }
-  private:
-    PsipRR psipRR_;
-    PsipZZ psipZZ_;
-};
-
 /**
  * @brief \f[ I_0\f]
  */
@@ -162,7 +147,6 @@ struct MagneticField : public dg::geo::aTokamakMagneticField
         new PsipRR(gp), 
         new PsipRZ(gp), 
         new PsipZZ(gp), 
-        new LaplacePsip(gp), 
         new Ipol(gp), 
         new IpolR(gp), 
         new IpolZ(gp)){}
diff --git a/inc/geometries/magnetic_field.h b/inc/geometries/magnetic_field.h
index 78a9ebbbd..59988080f 100644
--- a/inc/geometries/magnetic_field.h
+++ b/inc/geometries/magnetic_field.h
@@ -30,6 +30,7 @@ namespace geo
 */
 struct aTokamakMagneticField
 {
+    //maybe construct using make functions instead of deriving from it?
     aTokamakMagneticField( double R0,
         aBinaryFunctor* psip,
         aBinaryFunctor* psipR,
@@ -62,6 +63,9 @@ struct aTokamakMagneticField
     /// \f$ \partial_Z I(\psi_p) \f$ 
     const aBinaryFunctor& ipolZ()const{return ipol_.fy();}
 
+    const BinaryFunctorsLvl2& get_psip() const{return psip_;}
+    const BinaryFunctorsLvl1& get_ipol() const{return ipol_;}
+
     private:
     double R0_;
     BinaryFunctorsLvl2 psip_;
diff --git a/inc/geometries/ribeiro.h b/inc/geometries/ribeiro.h
index 1873321db..91ef81b07 100644
--- a/inc/geometries/ribeiro.h
+++ b/inc/geometries/ribeiro.h
@@ -26,14 +26,13 @@ namespace detail
 
 //This leightweights struct and its methods finds the initial R and Z values and the coresponding f(\psi) as 
 //good as it can, i.e. until machine precision is reached
-template< class Psi, class PsiX, class PsiY>
 struct Fpsi
 {
-    Fpsi( Psi psi, PsiX psiX, PsiY psiY, double x0, double y0, int mode): 
-        psip_(psi), fieldRZYTribeiro_(psiX, psiY, x0, y0),fieldRZYTequalarc_(psiX, psiY, x0, y0), fieldRZtau_(psiX, psiY), mode_(mode)
+    Fpsi( const BinaryFuncotsLvl1& psi_, double x0, double y0, int mode): 
+        psip_(psi), fieldRZYTribeiro_(psi,x0, y0),fieldRZYTequalarc_(psi, x0, y0), fieldRZtau_(psi), mode_(mode)
     {
         R_init = x0; Z_init = y0;
-        while( fabs( psiX(R_init, Z_init)) <= 1e-10 && fabs( psiY( R_init, Z_init)) <= 1e-10)
+        while( fabs( psi.dfx()(R_init, Z_init)) <= 1e-10 && fabs( psi.dfy()( R_init, Z_init)) <= 1e-10)
             R_init = x0 + 1.; Z_init = y0;
     }
     //finds the starting points for the integration in y direction
@@ -48,7 +47,7 @@ struct Fpsi
         while( (eps < eps_old || eps > 1e-7) && eps > 1e-14)
         {
             eps_old = eps; end2d_old = end2d;
-            N*=2; dg::stepperRK17( fieldRZtau_, begin2d, end2d, psip_(R_init, Z_init), psi, N);
+            N*=2; dg::stepperRK17( fieldRZtau_, begin2d, end2d, psip_.f()(R_init, Z_init), psi, N);
             eps = sqrt( (end2d[0]-end2d_old[0])*(end2d[0]-end2d_old[0]) + (end2d[1]-end2d_old[1])*(end2d[1]-end2d_old[1]));
         }
         R_init = R_0 = end2d_old[0], Z_init = Z_0 = end2d_old[1];
@@ -150,19 +149,18 @@ struct Fpsi
 
     private:
     double R_init, Z_init;
-    Psi psip_;
-    dg::geo::ribeiro::FieldRZYT<PsiX, PsiY> fieldRZYTribeiro_;
-    dg::geo::equalarc::FieldRZYT<PsiX, PsiY> fieldRZYTequalarc_;
-    dg::geo::FieldRZtau<PsiX, PsiY> fieldRZtau_;
+    BinaryFunctorsLvl1 psip_;
+    dg::geo::ribeiro::FieldRZYT fieldRZYTribeiro_;
+    dg::geo::equalarc::FieldRZYT fieldRZYTequalarc_;
+    dg::geo::FieldRZtau fieldRZtau_;
     int mode_;
 };
 
 //This struct computes -2pi/f with a fixed number of steps for all psi
-template<class Psi, class PsiR, class PsiZ>
 struct FieldFinv
 {
-    FieldFinv( Psi psi, PsiR psiR, PsiZ psiZ, double x0, double y0, unsigned N_steps, int mode):
-        fpsi_(psi, psiR, psiZ, x0, y0, mode), fieldRZYTribeiro_(psiR, psiZ, x0, y0), fieldRZYTequalarc_(psiR, psiZ, x0, y0), N_steps(N_steps), mode_(mode) { }
+    FieldFinv( const BinaryFunctorsLvl1& psi, double x0, double y0, unsigned N_steps, int mode):
+        fpsi_(psi, x0, y0, mode), fieldRZYTribeiro_(psi, x0, y0), fieldRZYTequalarc_(psi, x0, y0), N_steps(N_steps), mode_(mode) { }
     void operator()(const thrust::host_vector<double>& psi, thrust::host_vector<double>& fpsiM) 
     { 
         thrust::host_vector<double> begin( 3, 0), end(begin), end_old(begin);
@@ -173,9 +171,9 @@ struct FieldFinv
         //std::cout <<"fpsiMinverse is "<<fpsiM[0]<<" "<<-1./fpsi_(psi[0])<<" "<<eps<<"\n";
     }
     private:
-    Fpsi<Psi, PsiR, PsiZ> fpsi_;
-    dg::geo::ribeiro::FieldRZYT<PsiR, PsiZ> fieldRZYTribeiro_;
-    dg::geo::equalarc::FieldRZYT<PsiR, PsiZ> fieldRZYTequalarc_;
+    Fpsi fpsi_;
+    dg::geo::ribeiro::FieldRZYT fieldRZYTribeiro_;
+    dg::geo::equalarc::FieldRZYT fieldRZYTequalarc_;
     unsigned N_steps;
     int mode_;
 };
@@ -186,37 +184,25 @@ struct FieldFinv
 /**
  * @brief A two-dimensional grid based on "almost-conformal" coordinates by %Ribeiro and Scott 2010 (models aGenerator)
  * @ingroup generators
- * @tparam Psi All the template parameters must model aBinaryOperator
-     * @tparam PsiX models aBinaryOperator 
-     * @tparam PsiY models aBinaryOperator 
-     * @tparam PsiXX models aBinaryOperator 
-     * @tparam PsiXY models aBinaryOperator 
-     * @tparam PsiYY models aBinaryOperator 
  */
-template< class Psi, class PsiX, class PsiY, class PsiXX, class PsiXY, class PsiYY>
 struct Ribeiro : public aGenerator
 {
     /**
      * @brief Construct a near-conformal grid generator
      *
      * @param psi psi is the flux function in Cartesian coordinates (x,y), psiX is its derivative in x, psiY the derivative in y, psiXX the second derivative in x, etc.
-     * @param psi \f$ \psi(x,y)\f$ the flux function in Cartesian coordinates (x,y)
-     @param psiX \f$ \psi_x\f$ its derivative in x
-     @param psiY \f$ \psi_y\f$ its derivative in y
-     @param psiXX \f$ \psi_{xx}\f$ second derivative
-     @param psiXY \f$ \psi_{xy}\f$ second derivative
-     @param psiYY \f$ \psi_{yy}\f$ second derivative
+     * @param psi \f$ \psi(x,y)\f$ the flux function and its derivatives in Cartesian coordinates (x,y)
      * @param psi_0 first boundary 
      * @param psi_1 second boundary
      * @param x0 a point in the inside of the ring bounded by psi0 (shouldn't be the O-point)
      * @param y0 a point in the inside of the ring bounded by psi0 (shouldn't be the O-point)
      * @param mode This parameter indicates the adaption type used to create the grid: 0 is no adaption, 1 is an equalarc adaption
      */
-    Ribeiro( Psi psi, PsiX psiX, PsiY psiY, PsiXX psiXX, PsiXY psiXY, PsiYY psiYY, double psi_0, double psi_1, double x0, double y0, int mode = 0):
-        psi_(psi), psiX_(psiX), psiY_(psiY), psiXX_(psiXX), psiXY_(psiXY), psiYY_(psiYY), mode_(mode)
+    Ribeiro( const BinaryFunctorsLvl2& psi, double psi_0, double psi_1, double x0, double y0, int mode = 0):
+        psi_(psi), mode_(mode)
     {
         assert( psi_1 != psi_0);
-        ribeiro::detail::Fpsi<Psi, PsiX, PsiY> fpsi(psi, psiX, psiY, x0, y0, mode);
+        ribeiro::detail::Fpsi fpsi(psi, x0, y0, mode);
         lx_ = fabs(fpsi.find_x1( psi_0, psi_1));
         x0_=x0, y0_=y0, psi0_=psi_0, psi1_=psi_1;
         //std::cout << "lx_ = "<<lx_<<"\n";
@@ -250,14 +236,14 @@ struct Ribeiro : public aGenerator
          thrust::host_vector<double>& etaY) const
     {
         //compute psi(x) for a grid on x and call construct_rzy for all psi
-        ribeiro::detail::FieldFinv<Psi, PsiX, PsiY> fpsiMinv_(psi_, psiX_, psiY_, x0_,y0_, 500, mode_);
+        ribeiro::detail::FieldFinv fpsiMinv_(psi_, psiX_, psiY_, x0_,y0_, 500, mode_);
         thrust::host_vector<double> psi_x, fx_;
         dg::geo::detail::construct_psi_values( fpsiMinv_, psi0_, psi1_, 0., zeta1d, lx_, psi_x, fx_);
 
         //std::cout << "In grid function:\n";
-        ribeiro::detail::Fpsi<Psi, PsiX, PsiY> fpsi(psi_, psiX_, psiY_, x0_, y0_, mode_);
-        dg::geo::ribeiro::FieldRZYRYZY<PsiX, PsiY, PsiXX, PsiXY, PsiYY> fieldRZYRYZYribeiro(psiX_, psiY_, psiXX_, psiXY_, psiYY_);
-        dg::geo::equalarc::FieldRZYRYZY<PsiX, PsiY, PsiXX, PsiXY, PsiYY> fieldRZYRYZYequalarc(psiX_, psiY_, psiXX_, psiXY_, psiYY_);
+        ribeiro::detail::Fpsi fpsi(psi_, x0_, y0_, mode_);
+        dg::geo::ribeiro::FieldRZYRYZY fieldRZYRYZYribeiro(psi_);
+        dg::geo::equalarc::FieldRZYRYZY fieldRZYRYZYequalarc(psi_);
         thrust::host_vector<double> f_p(fx_);
         unsigned Nx = zeta1d.size(), Ny = eta1d.size();
         for( unsigned i=0; i<zeta1d.size(); i++)
@@ -275,12 +261,7 @@ struct Ribeiro : public aGenerator
             }
         }
     }
-    Psi psi_;
-    PsiX psiX_;
-    PsiY psiY_;
-    PsiXX psiXX_;
-    PsiXY psiXY_;
-    PsiYY psiYY_;
+    BinaryFunctorsLvl2 psi_;
     double lx_, x0_, y0_, psi0_, psi1_;
     int mode_; //0 = ribeiro, 1 = equalarc
 };
diff --git a/inc/geometries/simple_orthogonal.h b/inc/geometries/simple_orthogonal.h
index dae70342f..651e64937 100644
--- a/inc/geometries/simple_orthogonal.h
+++ b/inc/geometries/simple_orthogonal.h
@@ -26,16 +26,15 @@ namespace detail
 
 //This leightweights struct and its methods finds the initial R and Z values and the coresponding f(\psi) as 
 //good as it can, i.e. until machine precision is reached
-template< class Psi, class PsiX, class PsiY>
 struct Fpsi
 {
     
     //firstline = 0 -> conformal, firstline = 1 -> equalarc
-    Fpsi( Psi psi, PsiX psiX, PsiY psiY, double x0, double y0, int firstline): 
-        psip_(psi), fieldRZYTconf_(psiX, psiY, x0, y0),fieldRZYTequl_(psiX, psiY, x0, y0), fieldRZtau_(psiX, psiY)
+    Fpsi( const BinaryFunctorsLvl1& psi, double x0, double y0, int firstline): 
+        psip_(psi), fieldRZYTconf_(psi, x0, y0),fieldRZYTequl_(psi, x0, y0), fieldRZtau_(psi)
     {
         X_init = x0, Y_init = y0;
-        while( fabs( psiX(X_init, Y_init)) <= 1e-10 && fabs( psiY( X_init, Y_init)) <= 1e-10)
+        while( fabs( psi.dfx()(X_init, Y_init)) <= 1e-10 && fabs( psi.dfy()( X_init, Y_init)) <= 1e-10)
             X_init +=  1.; 
         firstline_ = firstline;
     }
@@ -50,7 +49,7 @@ struct Fpsi
         while( (eps < eps_old || eps > 1e-7) && eps > 1e-14)
         {
             eps_old = eps; end2d_old = end2d;
-            N*=2; dg::stepperRK17( fieldRZtau_, begin2d, end2d, psip_(X_init, Y_init), psi, N);
+            N*=2; dg::stepperRK17( fieldRZtau_, begin2d, end2d, psip_.f()(X_init, Y_init), psi, N);
             eps = sqrt( (end2d[0]-end2d_old[0])*(end2d[0]-end2d_old[0]) + (end2d[1]-end2d_old[1])*(end2d[1]-end2d_old[1]));
         }
         X_init = R_0 = end2d_old[0], Y_init = Z_0 = end2d_old[1];
@@ -88,17 +87,16 @@ struct Fpsi
     private:
     int firstline_;
     double X_init, Y_init;
-    Psi psip_;
-    dg::geo::ribeiro::FieldRZYT<PsiX, PsiY> fieldRZYTconf_;
-    dg::geo::equalarc::FieldRZYT<PsiX, PsiY> fieldRZYTequl_;
-    dg::geo::FieldRZtau<PsiX, PsiY> fieldRZtau_;
+    BinaryFunctorsLvl1 psip_;
+    dg::geo::ribeiro::FieldRZYT fieldRZYTconf_;
+    dg::geo::equalarc::FieldRZYT fieldRZYTequl_;
+    dg::geo::FieldRZtau fieldRZtau_;
 
 };
 
 //compute the vector of r and z - values that form one psi surface
 //assumes y_0 = 0
-template <class PsiX, class PsiY>
-void compute_rzy( PsiX psiX, PsiY psiY, const thrust::host_vector<double>& y_vec,
+void compute_rzy( const BinaryFunctorsLvl1& psi, const thrust::host_vector<double>& y_vec,
         thrust::host_vector<double>& r, 
         thrust::host_vector<double>& z, 
         double R_0, double Z_0, double f_psi, int mode ) 
@@ -110,8 +108,8 @@ void compute_rzy( PsiX psiX, PsiY psiY, const thrust::host_vector<double>& y_vec
     thrust::host_vector<double> begin( 2, 0), end(begin), temp(begin);
     begin[0] = R_0, begin[1] = Z_0;
     //std::cout <<f_psi<<" "<<" "<< begin[0] << " "<<begin[1]<<"\t";
-    dg::geo::ribeiro::FieldRZY<PsiX, PsiY> fieldRZYconf(psiX, psiY);
-    dg::geo::equalarc::FieldRZY<PsiX, PsiY> fieldRZYequi(psiX, psiY);
+    dg::geo::ribeiro::FieldRZY fieldRZYconf(psi);
+    dg::geo::equalarc::FieldRZY fieldRZYequi(psi);
     fieldRZYconf.set_f(f_psi);
     fieldRZYequi.set_f(f_psi);
     unsigned steps = 1;
@@ -151,14 +149,11 @@ void compute_rzy( PsiX psiX, PsiY psiY, const thrust::host_vector<double>& y_vec
 
 //This struct computes -2pi/f with a fixed number of steps for all psi
 //and provides the Nemov algorithm for orthogonal grid
-//template< class PsiX, class PsiY, class PsiXX, class PsiXY, class PsiYY, class LaplacePsiX, class LaplacePsiY>
-template< class PsiX, class PsiY, class LaplacePsi>
 struct Nemov
 {
-    Nemov( PsiX psiX, PsiY psiY, LaplacePsi laplacePsi, double f0, int mode):
+    Nemov( const BinaryFunctorsLvl2 psi, double f0, int mode):
         f0_(f0), mode_(mode),
-        psipR_(psiX), psipZ_(psiY),
-        laplacePsip_( laplacePsi)
+        psip_(psi), 
             { }
     void initialize( 
         const thrust::host_vector<double>& r_init, //1d intial values
@@ -175,8 +170,8 @@ struct Nemov
                 h_init[i] = f0_;
             if(mode_ == 1)
             {
-                double psipR = psipR_(r_init[i], z_init[i]), 
-                       psipZ = psipZ_(r_init[i], z_init[i]);
+                double psipR = psip_.dfx()(r_init[i], z_init[i]), 
+                       psipZ = psip_.dfy()(r_init[i], z_init[i]);
                 double psip2 = (psipR*psipR+psipZ*psipZ);
                 h_init[i]  = f0_/sqrt(psip2); //equalarc
             }
@@ -194,12 +189,12 @@ struct Nemov
         double psipR, psipZ, psip2;
         for( unsigned i=0; i<size; i++)
         {
-            psipR = psipR_(y[0][i], y[1][i]), psipZ = psipZ_(y[0][i], y[1][i]);
+            psipR = psip_.dfx()(y[0][i], y[1][i]), psipZ = psip_.dfy()(y[0][i], y[1][i]);
             //psipRR = psipRR_(y[0][i], y[1][i]), psipRZ = psipRZ_(y[0][i], y[1][i]), psipZZ = psipZZ_(y[0][i], y[1][i]);
             psip2 = f0_*(psipR*psipR+psipZ*psipZ);
             yp[0][i] = psipR/psip2;
             yp[1][i] = psipZ/psip2;
-            yp[2][i] = y[2][i]*( -laplacePsip_(y[0][i], y[1][i]) )/psip2;
+            yp[2][i] = y[2][i]*( - psip_.dfxx()(y[0][i], y[1][i]) - psip_.dfyy()(y[0][i], y[1][i]) )/psip2;
             //yp[3][i] = ( -(2.*psipRR+psipZZ)*y[3][i] - psipRZ*y[4][i] - laplacePsipR_(y[0][i], y[1][i])*y[2][i])/psip2;
             //yp[4][i] = ( -psipRZ*y[3][i] - (2.*psipZZ+psipRR)*y[4][i] - laplacePsipZ_(y[0][i], y[1][i])*y[2][i])/psip2;
         }
@@ -207,9 +202,7 @@ struct Nemov
     private:
     double f0_;
     int mode_;
-    PsiX psipR_;
-    PsiY psipZ_;
-    LaplacePsi laplacePsip_;
+    BinaryFunctorsLvl2 psip_;
 };
 
 template<class Nemov>
@@ -282,30 +275,25 @@ void construct_rz( Nemov nemov,
  *
  * Psi is the radial coordinate and you can choose various discretizations of the first line
  * @ingroup generators
- * @tparam Psi All the template parameters must model a Binary-operator i.e. the bracket operator() must be callable with two arguments and return a double. 
  */
-template< class Psi, class PsiX, class PsiY, class LaplacePsi>
 struct SimpleOrthogonal : public aGridGenerator
 {
     /**
      * @brief Construct a simple orthogonal grid 
      *
-     * @param psi \f$\psi(x,y)\f$ is the flux function in Cartesian coordinates (x,y)
-     * @param psiX \f$ \psi_x\f$ is its derivative in x
-     * @param psiY \f$ \psi_y\f$ ...
-     * @param laplacePsi \f$ \Delta\psi\f$ 
+     * @param psi \f$\psi(x,y)\f$ is the flux function and its derivatives in Cartesian coordinates (x,y)
      * @param psi_0 first boundary 
      * @param psi_1 second boundary
      * @param x0 a point in the inside of the ring bounded by psi0 (shouldn't be the O-point)
      * @param y0 a point in the inside of the ring bounded by psi0 (shouldn't be the O-point)
      * @param firstline This parameter indicates the adaption type used to create the orthogonal grid: 0 is no adaption, 1 is an equalarc adaption
      */
-    SimpleOrthogonal( Psi psi, PsiX psiX, PsiY psiY, LaplacePsi laplacePsi, double psi_0, double psi_1, double x0, double y0, int firstline =0):
-        psiX_(psiX), psiY_(psiY), laplacePsi_(laplacePsi)
+    SimpleOrthogonal(const BinaryFunctorsLvl2& psi, double psi_0, double psi_1, double x0, double y0, int firstline =0):
+        psi_(psi)
     {
         assert( psi_1 != psi_0);
         firstline_ = firstline;
-        orthogonal::detail::Fpsi<Psi, PsiX, PsiY> fpsi(psi, psiX, psiY, x0, y0, firstline);
+        orthogonal::detail::Fpsi fpsi(psi, x0, y0, firstline);
         f0_ = fabs( fpsi.construct_f( psi_0, R0_, Z0_));
         if( psi_1 < psi_0) f0_*=-1;
         lz_ =  f0_*(psi_1-psi_0);
@@ -353,23 +341,21 @@ struct SimpleOrthogonal : public aGridGenerator
     {
         thrust::host_vector<double> r_init, z_init;
         orthogonal::detail::compute_rzy( psiX_, psiY_, eta1d, r_init, z_init, R0_, Z0_, f0_, firstline_);
-        orthogonal::detail::Nemov<PsiX, PsiY, LaplacePsi> nemov(psiX_, psiY_, laplacePsi_, f0_, firstline_);
+        orthogonal::detail::Nemov nemov(psi_, f0_, firstline_);
         thrust::host_vector<double> h;
         orthogonal::detail::construct_rz(nemov, 0., zeta1d, r_init, z_init, x, y, h);
         unsigned size = x.size();
         for( unsigned idx=0; idx<size; idx++)
         {
-            double psipR = psiX_(x[idx], y[idx]);
-            double psipZ = psiY_(x[idx], y[idx]);
+            double psipR = psi_.dfx()(x[idx], y[idx]);
+            double psipZ = psi_.dfy()(x[idx], y[idx]);
             zetaX[idx] = f0_*psipR;
             zetaY[idx] = f0_*psipZ;
             etaX[idx] = -h[idx]*psipZ;
             etaY[idx] = +h[idx]*psipR;
         }
     }
-    PsiX psiX_;
-    PsiY psiY_;
-    LaplacePsi laplacePsi_;
+    BinaryFunctorsLvl2 psi_;
     double f0_, lz_, R0_, Z0_;
     int firstline_;
 };
diff --git a/inc/geometries/solovev.h b/inc/geometries/solovev.h
index 81802d169..c5f741284 100644
--- a/inc/geometries/solovev.h
+++ b/inc/geometries/solovev.h
@@ -391,27 +391,6 @@ struct PsipRZ: public aCloneableBinaryFunctor<PsipRZ>
     std::vector<double> c_;
 };
 
-
-/**
- * @brief  \f[\frac{\partial^2  \hat{\psi}_p }{ \partial \hat{R}^2 } + \frac{\partial^2  \hat{\psi}_p }{ \partial \hat{Z}^2 } \f] 
- */
-struct LaplacePsip: public aCloneableBinaryFunctor<LaplacePsip>
-{
-    LaplacePsip( GeomParameters gp ): psipRR_(gp), psipZZ_(gp){}
-    /**
-     * @brief  \f[\frac{\partial^2  \hat{\psi}_p }{ \partial \hat{R}^2 } + \frac{\partial^2  \hat{\psi}_p }{ \partial \hat{Z}^2 } \f] 
-     */
-    double operator()(double R, double Z) const
-    {    
-        return psipRR_(R,Z) + psipZZ_(R,Z);
-    }
-  private:
-    PsipRR psipRR_;
-    PsipZZ psipZZ_;
-};
-
-
-
 /**
  * @brief \f[\hat{I}\f] 
  */ 
@@ -473,7 +452,6 @@ struct MagneticField : public dg::geo::aTokamakMagneticField
         new PsipRR(gp), 
         new PsipRZ(gp), 
         new PsipZZ(gp), 
-        new LaplacePsip(gp), 
         new Ipol(gp), 
         new IpolR(gp), 
         new IpolZ(gp)){}
@@ -657,19 +635,6 @@ struct PsipRZ: public aCloneableBinaryFunctor<PsipRZ>
     dg::Cauchy cauchy_;
 };
 
-struct LaplacePsip: public aCloneableBinaryFunctor<LaplacePsip>
-{
-    LaplacePsip( GeomParameters gp ): psipRR_(gp), psipZZ_(gp){}
-    double operator()(double R, double Z) const
-    {    
-        return psipRR_(R,Z) + psipZZ_(R,Z);
-    }
-  private:
-    solovev::mod::PsipRR psipRR_;
-    solovev::mod::PsipZZ psipZZ_;
-};
-
-
 } //namespace mod
 ///@endcond
 
diff --git a/inc/geometries/taylor.h b/inc/geometries/taylor.h
index 579adaf8d..9f34e6e0a 100644
--- a/inc/geometries/taylor.h
+++ b/inc/geometries/taylor.h
@@ -264,19 +264,6 @@ struct PsipRZ: public aCloneableBinaryFunctor<PsipRZ>
     std::vector<double> c_;
 };
 
-struct LaplacePsip: public aCloneableBinaryFunctor<LaplacePsip>
-{
-    LaplacePsip( solovev::GeomParameters gp ): psipRR_(gp), psipZZ_(gp){}
-    double operator()(double R, double Z) const
-    {    
-        return psipRR_(R,Z) + psipZZ_(R,Z);
-    }
-  private:
-    PsipRR psipRR_;
-    PsipZZ psipZZ_;
-};
-
-
 /**
  * @brief \f[\hat{I} = c_{12}\psi\f] 
  */ 
@@ -336,7 +323,6 @@ struct MagneticField : public dg::geo::aTokamakMagneticField
         new PsipRR(gp), 
         new PsipRZ(gp), 
         new PsipZZ(gp), 
-        new LaplacePsip(gp), 
         new Ipol(gp), 
         new IpolR(gp), 
         new IpolZ(gp)){}
diff --git a/inc/geometries/utilities.h b/inc/geometries/utilities.h
index 8b8fa1632..9bcab190b 100644
--- a/inc/geometries/utilities.h
+++ b/inc/geometries/utilities.h
@@ -18,7 +18,7 @@ namespace flux{
  */ 
 struct FieldRZYT
 {
-    FieldRZYT( const BinaryFunctorsLvl2& psip, const BinaryFunctorsLvl1& ipol, double R0, double Z0): R_0_(R0), Z_0_(Z0), psip_(psip), ipol_(ipol){}
+    FieldRZYT( const BinaryFunctorsLvl1& psip, const BinaryFunctorsLvl1& ipol, double R0, double Z0): R_0_(R0), Z_0_(Z0), psip_(psip), ipol_(ipol){}
     void operator()( const dg::HVec& y, dg::HVec& yp) const
     {
         double psipR = psip_.dfx()(y[0], y[1]), psipZ = psip_.dfy()(y[0],y[1]);
@@ -34,13 +34,12 @@ struct FieldRZYT
     }
   private:
     double R_0_, Z_0_;
-    BinaryFunctorsLvl2 psip_;
-    BinaryFunctorsLvl1 ipol_;
+    BinaryFunctorsLvl1 psip_, ipol_;
 };
 
 struct FieldRZYZ
 {
-    FieldRZYZ( const BinaryFunctorsLvl2& psip, const BinaryFunctorsLvl1& ipol):psip_(psip), ipol_(ipol) {}
+    FieldRZYZ( const BinaryFunctorsLvl1& psip, const BinaryFunctorsLvl1& ipol):psip_(psip), ipol_(ipol) {}
     void operator()( const dg::HVec& y, dg::HVec& yp) const
     {
         double psipR = psip_.dfx()(y[0], y[1]), psipZ = psip_.dfy()(y[0],y[1]);
@@ -53,8 +52,7 @@ struct FieldRZYZ
         yp[1] =  1.;
     }
   private:
-    BinaryFunctorsLvl2 psip_;
-    BinaryFunctorsLvl1 ipol_;
+    BinaryFunctorsLvl1 psip_, ipol_;
 };
 
 /**
@@ -76,8 +74,7 @@ struct FieldRZY
     }
   private:
     double f_;
-    BinaryFunctorsLvl2 psip_;
-    BinaryFunctorsLvl1 ipol_;
+    BinaryFunctorsLvl1 psip_,ipol_;
 };
 
 /**
@@ -101,8 +98,8 @@ struct FieldRZYRYZY
         double psipR = psip_.dfx()(R0, Z0), psipZ = psip_.dfy()(R0,Z0);
         double psip2 = (psipR*psipR+ psipZ*psipZ);
         double fnorm =R0/ipol_.f()(R0,Z0)/f_; //=Rq/I
-        yR = -psipZ_(R0, Z0)/psip2/fnorm;
-        yZ = +psipR_(R0, Z0)/psip2/fnorm;
+        yR = -psipZ/psip2/fnorm;
+        yZ = +psipR/psip2/fnorm;
     }
     void derive( double R0, double Z0, double& xR, double& xZ)
     {
@@ -136,10 +133,10 @@ namespace ribeiro{
 
 struct FieldRZYT
 {
-    FieldRZYT( const BinaryFunctorsLvl2, double R0, double Z0): R_0_(R0), Z_0_(Z0), psipR_(psiR), psipZ_(psiZ){}
+    FieldRZYT( const BinaryFunctorsLvl1& psip, double R0, double Z0): R_0_(R0), Z_0_(Z0), psip_(psip)
     void operator()( const dg::HVec& y, dg::HVec& yp) const
     {
-        double psipR = psipR_(y[0], y[1]), psipZ = psipZ_(y[0],y[1]);
+        double psipR = psip_.dfx()(y[0], y[1]), psipZ = psip_.dfy()(y[0],y[1]);
         double psip2 = psipR*psipR+psipZ*psipZ;
         yp[0] = -psipZ;//fieldR
         yp[1] = +psipR;//fieldZ
@@ -155,17 +152,15 @@ struct FieldRZYT
     }
   private:
     double R_0_, Z_0_;
-    PsiR psipR_;
-    PsiZ psipZ_;
+    BinaryFunctorsLvl1 psip_;
 };
 
-template< class PsiR, class PsiZ>
 struct FieldRZYZ
 {
-    FieldRZYZ( PsiR psiR, PsiZ psiZ): psipR_(psiR), psipZ_(psiZ){}
+    FieldRZYZ( const BinaryFunctorsLvl1& psip): psip_(psip){}
     void operator()( const dg::HVec& y, dg::HVec& yp) const
     {
-        double psipR = psipR_(y[0], y[1]), psipZ = psipZ_(y[0],y[1]);
+        double psipR = psip_.dfx()(y[0], y[1]), psipZ = psip_.dfy()(y[0],y[1]);
         double psip2 = psipR*psipR+psipZ*psipZ;
         yp[0] = -psipZ;//fieldR
         yp[1] =  psipR;//fieldZ
@@ -178,18 +173,16 @@ struct FieldRZYZ
         yp[1] =  1.;
     }
   private:
-    PsiR psipR_;
-    PsiZ psipZ_;
+    BinaryFunctorsLvl1 psip_;
 };
 
-template <class PsiR, class PsiZ>
 struct FieldRZY
 {
-    FieldRZY( PsiR psiR, PsiZ psiZ): f_(1.), psipR_(psiR), psipZ_(psiZ){}
+    FieldRZY( const BinaryFunctorsLvl1& psip): f_(1.),psip_(psip){}
     void set_f(double f){ f_ = f;}
     void operator()( const dg::HVec& y, dg::HVec& yp) const
     {
-        double psipR = psipR_(y[0], y[1]), psipZ = psipZ_(y[0],y[1]);
+        double psipR = psip_.dfx()(y[0], y[1]), psipZ = psip_.dfy()(y[0],y[1]);
         double psip2 = psipR*psipR+psipZ*psipZ;
         //yp[0] = +psipZ/f_;//volume 
         //yp[1] = -psipR/f_;//volume 
@@ -202,33 +195,30 @@ struct FieldRZY
     }
   private:
     double f_;
-    PsiR psipR_;
-    PsiZ psipZ_;
+    BinaryFunctorsLvl1 psip_;
 };
 
 
-template<class PsiR, class PsiZ, class PsiRR, class PsiRZ, class PsiZZ>
 struct FieldRZYRYZY
 {
-    FieldRZYRYZY( PsiR psiR, PsiZ psiZ, PsiRR psiRR, PsiRZ psiRZ, PsiZZ psiZZ): 
-        psipR_(psiR), psipZ_(psiZ), psipRR_(psiRR), psipRZ_(psiRZ), psipZZ_(psiZZ){ f_ = f_prime_ = 1.;}
+    FieldRZYRYZY( const BinaryFunctorsLvl2& psip):psip_(psip){ f_ = f_prime_ = 1.;}
     void set_f( double new_f){ f_ = new_f;}
     void set_fp( double new_fp){ f_prime_ = new_fp;}
     void initialize( double R0, double Z0, double& yR, double& yZ)
     {
-        yR = -f_*psipZ_(R0, Z0);
-        yZ = +f_*psipR_(R0, Z0);
+        yR = -f_*psip_.dfy()(R0, Z0);
+        yZ = +f_*psip_.dfx()(R0, Z0);
     }
     void derive( double R0, double Z0, double& xR, double& xZ)
     {
-        xR = +f_*psipR_(R0, Z0);
-        xZ = +f_*psipZ_(R0, Z0);
+        xR = +f_*psip_.dfx()(R0, Z0);
+        xZ = +f_*psip_.dfy()(R0, Z0);
     }
 
     void operator()( const dg::HVec& y, dg::HVec& yp) const
     {
-        double psipR = psipR_(y[0], y[1]), psipZ = psipZ_(y[0],y[1]);
-        double psipRR = psipRR_(y[0], y[1]), psipRZ = psipRZ_(y[0],y[1]), psipZZ = psipZZ_(y[0],y[1]);
+        double psipR = psip_.dfx()(y[0], y[1]), psipZ = psip_.dfy()(y[0],y[1]);
+        double psipRR = psip_.dfxx()(y[0], y[1]), psipRZ = psip_.dfxy()(y[0],y[1]), psipZZ = psip_.dfyy()(y[0],y[1]);
         double psip2 = (psipR*psipR+ psipZ*psipZ);
 
         yp[0] =  -psipZ/f_/psip2;
@@ -240,23 +230,18 @@ struct FieldRZYRYZY
     }
   private:
     double f_, f_prime_;
-    PsiR psipR_;
-    PsiZ psipZ_;
-    PsiRR psipRR_;
-    PsiRZ psipRZ_;
-    PsiZZ psipZZ_;
+    BinaryFunctorsLvl2 psip_;
 };
 }//namespace ribeiro
 namespace equalarc{
 
 
-template< class PsiR, class PsiZ>
 struct FieldRZYT
 {
-    FieldRZYT( PsiR psiR, PsiZ psiZ, double R0, double Z0): R_0_(R0), Z_0_(Z0), psipR_(psiR), psipZ_(psiZ){}
+    FieldRZYT( const BinaryFunctorsLvl1& psip, double R0, double Z0): R_0_(R0), Z_0_(Z0), psip_(psip){}
     void operator()( const dg::HVec& y, dg::HVec& yp) const
     {
-        double psipR = psipR_(y[0], y[1]), psipZ = psipZ_(y[0],y[1]);
+        double psipR = psip_.dfx()(y[0], y[1]), psipZ = psip_.dfy()(y[0],y[1]);
         double psip2 = psipR*psipR+psipZ*psipZ;
         yp[0] = -psipZ;//fieldR
         yp[1] = +psipR;//fieldZ
@@ -272,17 +257,15 @@ struct FieldRZYT
     }
   private:
     double R_0_, Z_0_;
-    PsiR psipR_;
-    PsiZ psipZ_;
+    BinaryFunctorsLvl1 psip_;
 };
 
-template< class PsiR, class PsiZ>
 struct FieldRZYZ
 {
-    FieldRZYZ( PsiR psiR, PsiZ psiZ): psipR_(psiR), psipZ_(psiZ){}
+    FieldRZYZ( const BinaryFunctorsLvl1& psip): psip_(psip){}
     void operator()( const dg::HVec& y, dg::HVec& yp) const
     {
-        double psipR = psipR_(y[0], y[1]), psipZ = psipZ_(y[0],y[1]);
+        double psipR = psip_.dfx()(y[0], y[1]), psipZ = psip_.dfy()(y[0],y[1]);
         double psip2 = psipR*psipR+psipZ*psipZ;
         yp[0] = -psipZ;//fieldR
         yp[1] = +psipR;//fieldZ
@@ -295,18 +278,16 @@ struct FieldRZYZ
         yp[1] =  1.;
     }
   private:
-    PsiR psipR_;
-    PsiZ psipZ_;
+    BinaryFunctorsLvl1 psip_;
 };
 
-template <class PsiR, class PsiZ>
 struct FieldRZY
 {
-    FieldRZY( PsiR psiR, PsiZ psiZ): f_(1.), psipR_(psiR), psipZ_(psiZ){}
+    FieldRZY( const BinaryFunctorsLvl1& psip):f_(1.), psip_(psip){}
     void set_f(double f){ f_ = f;}
     void operator()( const dg::HVec& y, dg::HVec& yp) const
     {
-        double psipR = psipR_(y[0], y[1]), psipZ = psipZ_(y[0],y[1]);
+        double psipR = psip_.dfx()(y[0], y[1]), psipZ = psip_.dfy()(y[0],y[1]);
         double psip2 = psipR*psipR+psipZ*psipZ;
         //yp[0] = +psipZ/f_;//volume 
         //yp[1] = -psipR/f_;//volume 
@@ -319,35 +300,32 @@ struct FieldRZY
     }
   private:
     double f_;
-    PsiR psipR_;
-    PsiZ psipZ_;
+    BinaryFunctorsLvl1 psip_;
 };
 
 
-template<class PsiR, class PsiZ, class PsiRR, class PsiRZ, class PsiZZ>
 struct FieldRZYRYZY
 {
-    FieldRZYRYZY( PsiR psiR, PsiZ psiZ, PsiRR psiRR, PsiRZ psiRZ, PsiZZ psiZZ): 
-        psipR_(psiR), psipZ_(psiZ), psipRR_(psiRR), psipRZ_(psiRZ), psipZZ_(psiZZ){ f_ = f_prime_ = 1.;}
+    FieldRZYRYZY( const BinaryFunctorsLvl2& psip):  psip_(psip){ f_ = f_prime_ = 1.;}
     void set_f( double new_f){ f_ = new_f;}
     void set_fp( double new_fp){ f_prime_ = new_fp;}
     void initialize( double R0, double Z0, double& yR, double& yZ)
     {
-        double psipR = psipR_(R0, Z0), psipZ = psipZ_(R0,Z0);
+        double psipR = psip_.dfx()(R0, Z0), psipZ = psip_.dfy()(R0,Z0);
         double psip2 = (psipR*psipR+ psipZ*psipZ);
-        yR = -f_*psipZ_(R0, Z0)/sqrt(psip2);
-        yZ = +f_*psipR_(R0, Z0)/sqrt(psip2);
+        yR = -f_*psipZ/sqrt(psip2);
+        yZ = +f_*psipR/sqrt(psip2);
     }
     void derive( double R0, double Z0, double& xR, double& xZ)
     {
-        xR = +f_*psipR_(R0, Z0);
-        xZ = +f_*psipZ_(R0, Z0);
+        xR = +f_*psip_.dfx()(R0, Z0);
+        xZ = +f_*psip_.dfy()(R0, Z0);
     }
 
     void operator()( const dg::HVec& y, dg::HVec& yp) const
     {
-        double psipR = psipR_(y[0], y[1]), psipZ = psipZ_(y[0],y[1]);
-        double psipRR = psipRR_(y[0], y[1]), psipRZ = psipRZ_(y[0],y[1]), psipZZ = psipZZ_(y[0],y[1]);
+        double psipR = psip_.dfx()(y[0], y[1]), psipZ = psip_.dfy()(y[0],y[1]);
+        double psipRR = psip_.dfxx()(y[0], y[1]), psipRZ = psip_.dfxy()(y[0],y[1]), psipZZ = psip_.dfyy()(y[0],y[1]);
         double psip2 = (psipR*psipR+ psipZ*psipZ);
 
         yp[0] =  -psipZ/f_/sqrt(psip2);
@@ -359,41 +337,34 @@ struct FieldRZYRYZY
     }
   private:
     double f_, f_prime_;
-    PsiR psipR_;
-    PsiZ psipZ_;
-    PsiRR psipRR_;
-    PsiRZ psipRZ_;
-    PsiZZ psipZZ_;
+    BinaryFunctorsLvl2 psip_;
 };
 
 }//namespace equalarc
 
-template < class PsiR, class PsiZ>
 struct FieldRZtau
 {
-    FieldRZtau( PsiR psiR, PsiZ psiZ): psipR_(psiR), psipZ_(psiZ){}
+    FieldRZtau(const BinaryFunctorsLvl1& psip): psip_(psip){}
     void operator()( const dg::HVec& y, dg::HVec& yp) const
     {
-        double psipR = psipR_(y[0], y[1]), psipZ = psipZ_(y[0],y[1]);
+        double psipR = psip_.dfx()(y[0], y[1]), psipZ = psip_.dfy()(y[0],y[1]);
         double psi2 = psipR*psipR+ psipZ*psipZ;
         yp[0] =  psipR/psi2;
         yp[1] =  psipZ/psi2;
     }
   private:
-    PsiR psipR_;
-    PsiZ psipZ_;
+    BinaryFunctorsLvl1 psip_;
 };
 
-template<class PsiX, class PsiY, class PsiXX, class PsiXY, class PsiYY>
 struct HessianRZtau
 {
-    HessianRZtau( PsiX psiX, PsiY psiY, PsiXX psiXX, PsiXY psiXY, PsiYY psiYY): norm_(false), quad_(1), psipR_(psiX), psipZ_(psiY), psipRR_(psiXX), psipRZ_(psiXY), psipZZ_(psiYY){}
+    HessianRZtau( const BinaryFunctorsLvl2): norm_(false), quad_(1), psip_(psip){}
     // if true goes into positive Z - direction and X else
     void set_quadrant( int quadrant) {quad_ = quadrant;}
     void set_norm( bool normed) {norm_ = normed;}
     void operator()( const dg::HVec& y, dg::HVec& yp) const
     {
-        double psipRZ = psipRZ_(y[0], y[1]);
+        double psipRZ = psip_.dfxy(y[0], y[1]);
         if( psipRZ == 0)
         {
             if(      quad_ == 0) { yp[0] = 1; yp[1] = 0; }
@@ -403,7 +374,7 @@ struct HessianRZtau
         }
         else
         {
-            double psipRR = psipRR_(y[0], y[1]), psipZZ = psipZZ_(y[0],y[1]);
+            double psipRR = psip_.dfxx(y[0], y[1]), psipZZ = psip_.dfyy(y[0],y[1]);
             double T = psipRR + psipZZ; 
             double D = psipZZ*psipRR - psipRZ*psipRZ;
             double L1 = 0.5*T+sqrt( 0.25*T*T-D); // > 0
@@ -415,7 +386,7 @@ struct HessianRZtau
         }
         if( norm_) 
         {
-            double vgradpsi = yp[0]*psipR_(y[0],y[1]) + yp[1]*psipZ_(y[0],y[1]);
+            double vgradpsi = yp[0]*psip_.dfx()(y[0],y[1]) + yp[1]*psip_.dfy()(y[0],y[1]);
             yp[0] /= vgradpsi, yp[1] /= vgradpsi;
         }
         else
@@ -427,9 +398,9 @@ struct HessianRZtau
     }
     void newton_iteration( const dg::HVec&y, dg::HVec& yp)
     {
-        double psipRZ = psipRZ_(y[0], y[1]);
-        double psipRR = psipRR_(y[0], y[1]), psipZZ = psipZZ_(y[0],y[1]);
-        double psipR = psipR_(y[0], y[1]), psipZ = psipZ_(y[0], y[1]);
+        double psipRZ = psip_.dfxy()(y[0], y[1]);
+        double psipRR = psip_.dfxx()(y[0], y[1]), psipZZ = psip_.dfyy()(y[0],y[1]);
+        double psipR = psip_.dfx()(y[0], y[1]), psipZ = psip_.dfy()(y[0], y[1]);
         double Dinv = 1./(psipZZ*psipRR - psipRZ*psipRZ);
         yp[0] = y[0] - Dinv*(psipZZ*psipR - psipRZ*psipZ);
         yp[1] = y[1] - Dinv*(-psipRZ*psipR + psipRR*psipZ);
@@ -437,22 +408,17 @@ struct HessianRZtau
   private:
     bool norm_;
     int quad_;
-    PsiX  psipR_;
-    PsiY  psipZ_;
-    PsiXX psipRR_;
-    PsiXY psipRZ_;
-    PsiYY psipZZ_;
+    BinaryFunctorsLvl2 psip_;
 };
 
-template<class Psi, class PsiX, class PsiY>
 struct MinimalCurve
 {
-    MinimalCurve(Psi psi, PsiX psiX, PsiY psiY): norm_(false), 
-        psip_(psi), psipR_(psiX), psipZ_(psiY){}
+    MinimalCurve(const BinaryFunctorsLvl1& psip): norm_(false), 
+        psip_(psip){}
     void set_norm( bool normed) {norm_ = normed;}
     void operator()( const dg::HVec& y, dg::HVec& yp) const
     {
-        double psipR = psipR_(y[0], y[1]), psipZ = psipZ_(y[0], y[1]);
+        double psipR = psip_.dfx()(y[0], y[1]), psipZ = psip_.dfy()(y[0], y[1]);
         yp[0] = y[2];
         yp[1] = y[3];
         //double psipRZ = psipRZ_(y[0], y[1]), psipR = psipR_(y[0], y[1]), psipZ = psipZ_(y[0], y[1]), psipRR=psipRR_(y[0], y[1]), psipZZ=psipZZ_(y[0], y[1]); 
@@ -460,7 +426,7 @@ struct MinimalCurve
         //double grad2 = psipR*psipR+psipZ*psipZ;
         //yp[2] = D2/(1.+grad2) * psipR ;
         //yp[3] = D2/(1.+grad2) * psipZ ;
-        if( psip_(y[0], y[1]) < 0)
+        if( psip_.f()(y[0], y[1]) < 0)
         {
             yp[2] = -10.*psipR;
             yp[3] = -10.*psipZ;
@@ -479,9 +445,7 @@ struct MinimalCurve
     }
   private:
     bool norm_;
-    Psi  psip_;
-    PsiX psipR_;
-    PsiY psipZ_;
+    BinaryFunctorsLvl1 psip_;
 };
 ////////////////////////////////////////////////////////////////////////////////
 
-- 
GitLab


From 8d7bf0f44d37995d4377b75175fbb8e217c941b3 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Tue, 25 Jul 2017 22:38:35 +0200
Subject: [PATCH 083/453] reworked Hector with BinaryFunctors

---
 inc/dg/backend/grid.h          |   8 +-
 inc/dg/backend/mpi_grid.h      |  12 +--
 inc/geometries/flux.h          |   1 -
 inc/geometries/fluxfunctions.h |   7 ++
 inc/geometries/hector.h        | 153 +++++++++------------------------
 inc/geometries/toroidal.h      |   7 --
 6 files changed, 58 insertions(+), 130 deletions(-)

diff --git a/inc/dg/backend/grid.h b/inc/dg/backend/grid.h
index 2f34e6c22..b16e9fd5b 100644
--- a/inc/dg/backend/grid.h
+++ b/inc/dg/backend/grid.h
@@ -289,13 +289,13 @@ struct Grid2d
     Grid2d local_grid() const {return *this;}
 
     /**
-    * @brief Resize the number of cells relative to the old ones
+    * @brief Multiply the number of cells with a given factor
     *
     * With this function you can resize the grid ignorantly of its current size
     * @param fx new number of cells is fx*Nx()
     * @param fy new number of cells is fy*Ny()
     */
-    void resize( double fx, double fy){
+    void multiplyCellNumber( double fx, double fy){
         set(n_, floor(fx*(double)Nx_+0.5), floor(fy*(double)Ny_+0.5));
     }
     /**
@@ -478,8 +478,8 @@ struct Grid3d
         lx_ = (x1_-x0_), ly_ = (y1_-y0_), lz_ = (z1_-z0_);
         hx_ = lx_/(double)Nx_, hy_ = ly_/(double)Ny_, hz_ = lz_/(double)Nz_;
     }
-    ///@copydoc Grid2d::resize()
-    void resize( double fx, double fy){
+    ///@copydoc Grid2d::multiplyCellNumber()
+    void multiplyCellNumber( double fx, double fy){
         set(n_, floor(fx*(double)Nx_+0.5), floor(fy*(double)Ny_+0.5), Nz());
     }
     /**
diff --git a/inc/dg/backend/mpi_grid.h b/inc/dg/backend/mpi_grid.h
index 655494d7e..db17ff120 100644
--- a/inc/dg/backend/mpi_grid.h
+++ b/inc/dg/backend/mpi_grid.h
@@ -51,19 +51,19 @@ struct MPIGrid2d
     }
 
     /**
-    * @brief Resize the number of cells relative to the old ones
+    * @brief Multiply the number of cells with a given factor
     *
     * With this function you can resize the grid ignorantly of its current size
     * @param fx new global number of cells is fx*global().Nx()
     * @param fy new global number of cells is fy*global().Ny()
     */
-    void resize( double fx, double fy){
+    void multiplyCellNumber( double fx, double fy){
         set(n_, floor(fx*(double)global.Nx()+0.5), floor(fy*(double)global.Ny()+0.5));
     }
     /**
     * @copydoc Grid2d::set(unsigned,unsigned,unsigned)
     * @attention these are global parameters, i.e. set( g.n(), 2*g.Nx(), 2*g.Ny()) is NOT(!) what you want
-    *           use the resize function instead, or set( g.n(), 2*g.global().Nx(), 2*g.global().Ny())
+    *           use the multiplyCellNumber function instead, or set( g.n(), 2*g.global().Nx(), 2*g.global().Ny())
     */
     void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny) {
         check_division( new_Nx, new_Ny, g.bcx(), g.bcy());
@@ -374,14 +374,14 @@ struct MPIGrid3d
     {
         check_division( Nx, Ny, Nz, bcx, bcy, bcz);
     }
-    ///@copydoc MPIGrid2d::resize()
-    void resize( double fx, double fy){
+    ///@copydoc MPIGrid2d::multiplyCellNumber()
+    void multiplyCellNumber( double fx, double fy){
         set(n_, floor(fx*(double)global.Nx()+0.5), floor(fy*(double)global.Ny()+0.5), global.Nz());
     }
     /**
      * @copydoc Grid3d::set(unsigned,unsigned,unsigned,unsigned)
      * @attention these are global parameters, i.e. set( g.n(), 2*g.Nx(), 2*g.Ny(), 2*g.Nz()) is NOT(!) what you want
-     *           use the resize function instead, or set( g.n(), 2*g.global().Nx(), 2*g.global().Ny(), 2*g.global().Nz())
+     *           use the multiply function instead, or set( g.n(), 2*g.global().Nx(), 2*g.global().Ny(), 2*g.global().Nz())
      */
     void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz) {
         check_division( new_Nx,new_Ny,new_Nz,g.bcx(),g.bcy(),g.bcz());
diff --git a/inc/geometries/flux.h b/inc/geometries/flux.h
index 37d7ccd6e..d221a8cc0 100644
--- a/inc/geometries/flux.h
+++ b/inc/geometries/flux.h
@@ -228,7 +228,6 @@ struct FluxGenerator : public aGridGenerator
  * @brief Same as the Ribeiro class just but uses psi as a flux label directly
  * @ingroup generators
  */
-template< class Psi, class PsiX, class PsiY, class PsiXX, class PsiXY, class PsiYY>
 struct RibeiroFluxGenerator : public aGridGenerator
 {
     /**
diff --git a/inc/geometries/fluxfunctions.h b/inc/geometries/fluxfunctions.h
index f23531dfd..ae9d773d9 100644
--- a/inc/geometries/fluxfunctions.h
+++ b/inc/geometries/fluxfunctions.h
@@ -186,6 +186,13 @@ struct BinarySymmTensorLvl1 : public aBinaryFunctorBundle
     const aBinaryFunctor& divY()const{return *p_[4];}
 };
 
+struct Constant:public aCloneableBinaryOperator<Constant> 
+{ 
+    Constant(double c):c_(c){}
+    double operator()(double R,double Z)const{return c_;}
+    private:
+    double c_;
+};
 
 ///@}
 }//namespace geo
diff --git a/inc/geometries/hector.h b/inc/geometries/hector.h
index 71faa11b2..f24a183a7 100644
--- a/inc/geometries/hector.h
+++ b/inc/geometries/hector.h
@@ -211,18 +211,7 @@ struct Hector : public aGenerator
     /**
      * @brief Construct a conformal grid 
      *
-     * @tparam Psi models aBinaryOperator 
-     * @tparam PsiX models aBinaryOperator 
-     * @tparam PsiY models aBinaryOperator 
-     * @tparam PsiXX models aBinaryOperator 
-     * @tparam PsiXY models aBinaryOperator 
-     * @tparam PsiYY models aBinaryOperator 
-     * @param psi \f$ \psi(x,y)\f$ the flux function in Cartesian coordinates (x,y)
-     @param psiX \f$ \psi_x\f$ its derivative in x
-     @param psiY \f$ \psi_y\f$ its derivative in y
-     @param psiXX \f$ \psi_{xx}\f$ second derivative
-     @param psiXY \f$ \psi_{xy}\f$ second derivative
-     @param psiYY \f$ \psi_{yy}\f$ second derivative
+     * @param psi \f$ \psi(x,y)\f$ the flux function and its derivatives in Cartesian coordinates (x,y)
      * @param psi0 first boundary 
      * @param psi1 second boundary
      * @param X0 a point in the inside of the ring bounded by psi0 (shouldn't be the O-point)
@@ -233,14 +222,12 @@ struct Hector : public aGenerator
      * @param eps_u the accuracy of u
      * @param verbose If true convergence details are printed to std::cout
      */
-    template< class Psi, class PsiX, class PsiY, class PsiXX ,class PsiXY, class PsiYY >
-    Hector( Psi psi, PsiX psiX, PsiY psiY, PsiXX psiXX, PsiXY psiXY, PsiYY psiYY, double psi0, double psi1, double X0, double Y0, unsigned n = 13, unsigned Nx = 2, unsigned Ny = 10, double eps_u = 1e-10, bool verbose=false) : 
-        generator_( new dg::geo::RibeiroFluxGenerator<Psi,PsiX,PsiY,PsiXX, PsiXY, PsiYY>(psi, psiX, psiY, psiXX, psiXY, psiYY, psi0, psi1, X0, Y0,1)),
-        g2d_(generator_, n, Nx, Ny, dg::DIR)
+    Hector( const BinaryFunctorsLvl2& psi, double psi0, double psi1, double X0, double Y0, unsigned n = 13, unsigned Nx = 2, unsigned Ny = 10, double eps_u = 1e-10, bool verbose=false) : 
+        g2d_(new dg::geo::RibeiroFluxGenerator(psi, psi0, psi1, X0, Y0,1), n, Nx, Ny, dg::DIR)
     {
         //first construct u_
-        container u = construct_grid_and_u( psi, psiX, psiY, psiXX, psiXY, psiYY, dg::ONE(), dg::geo::detail::LaplacePsi<PsiXX, PsiYY>(psiXX, psiYY), psi0, psi1, X0, Y0, n, Nx, Ny, eps_u , verbose);
-        construct( u, psi0, psi1, dg::ONE(), dg::ZERO(), dg::ONE() );
+        container u = construct_grid_and_u( dg::geo::Constant(1), dg::geo::detail::LaplacePsi(psi), psi0, psi1, X0, Y0, n, Nx, Ny, eps_u , verbose);
+        construct( u, psi0, psi1, dg::geo::Constant(1.), dg::geo::Constant(0.), dg::geo::Constant(1.) );
         conformal_=orthogonal_=true;
         ////we actually don't need u_ but it makes a good testcase 
         //container psi__;
@@ -254,24 +241,8 @@ struct Hector : public aGenerator
     /**
      * @brief Construct an orthogonal grid with adaption
      *
-     * @tparam Psi  models aBinaryOperator 
-     * @tparam PsiX models aBinaryOperator 
-     * @tparam PsiY models aBinaryOperator 
-     * @tparam PsiXX models aBinaryOperator 
-     * @tparam PsiXY models aBinaryOperator 
-     * @tparam PsiYY models aBinaryOperator 
-     * @tparam Chi  models aBinaryOperator 
-     * @tparam ChiX models aBinaryOperator 
-     * @tparam ChiY models aBinaryOperator 
-     * @param psi \f$ \psi(x,y)\f$ the flux function in Cartesian coordinates (x,y)
-     @param psiX \f$ \psi_x\f$ its derivative in x
-     @param psiY \f$ \psi_y\f$ its derivative in y
-     @param psiXX \f$ \psi_{xx}\f$ second derivative
-     @param psiXY \f$ \psi_{xy}\f$ second derivative
-     @param psiYY \f$ \psi_{yy}\f$ second derivative
-     * @param chi \f$ \chi(x,y)\f$  is the adaption function in Cartesian coordinates (x,y)
-     @param chiX \f$ \chi_x\f$ its derivative in x 
-     @param chiY \f$ \chi_y\f$ its derivative in y 
+     * @param psi \f$ \psi(x,y)\f$ the flux function and its derivatives in Cartesian coordinates (x,y)
+     * @param chi \f$ \chi(x,y)\f$  the adaption function and its derivatives in Cartesian coordinates (x,y)
      * @param psi0 first boundary 
      * @param psi1 second boundary
      * @param X0 a point in the inside of the ring bounded by psi0 (shouldn't be the O-point)
@@ -282,15 +253,13 @@ struct Hector : public aGenerator
      * @param eps_u the accuracy of u
      * @param verbose If true convergence details are printed to std::cout
      */
-    template< class Psi, class PsiX, class PsiY, class PsiXX, class PsiXY, class PsiYY, class Chi, class ChiX, class ChiY>
-    Hector( Psi psi, PsiX psiX, PsiY psiY, PsiXX psiXX, PsiXY psiXY, PsiYY psiYY, Chi chi, ChiX chiX, ChiY chiY, double psi0, double psi1, double X0, double Y0, unsigned n = 13, unsigned Nx = 2, unsigned Ny = 10, double eps_u = 1e-10, bool verbose=false) : 
-        generator_( new dg::geo::RibeiroFluxGenerator<Psi,PsiX,PsiY,PsiXX, PsiXY, PsiYY>(psi, psiX, psiY, psiXX, psiXY, psiYY, psi0, psi1, X0, Y0,1)),
-        g2d_(generator_, n, Nx, Ny, dg::DIR)
+    Hector( const BinaryFunctorsLvl2& psi, const BinaryFunctorsLvl1& chi, double psi0, double psi1, double X0, double Y0, unsigned n = 13, unsigned Nx = 2, unsigned Ny = 10, double eps_u = 1e-10, bool verbose=false) : 
+        g2d_(new dg::geo::RibeiroFluxGenerator(psi, psi0, psi1, X0, Y0,1), n, Nx, Ny, dg::DIR)
     {
-        dg::geo::detail::LaplaceAdaptPsi<PsiX, PsiY, dg::geo::detail::LaplacePsi<PsiXX, PsiYY>, Chi, ChiX, ChiY> lapAdaPsi( psiX, psiY, dg::geo::detail::LaplacePsi<PsiXX, PsiYY>(psiXX, psiYY), chi, chiX, chiY);
+        dg::geo::detail::LaplaceAdaptPsi lapAdaPsi( psi, chi);
         //first construct u_
-        container u = construct_grid_and_u( psi, psiX, psiY, psiXX, psiXY, psiYY, chi, lapAdaPsi, psi0, psi1, X0, Y0, n, Nx, Ny, eps_u , verbose);
-        construct( u, psi0, psi1, chi, dg::ZERO(), chi );
+        container u = construct_grid_and_u( chi.f(), lapAdaPsi, psi0, psi1, X0, Y0, n, Nx, Ny, eps_u , verbose);
+        construct( u, psi0, psi1, chi.f(),dg::geo::Constant(0), chi.f() );
         orthogonal_=true;
         conformal_=false;
         ////we actually don't need u_ but it makes a good testcase 
@@ -305,26 +274,8 @@ struct Hector : public aGenerator
     /**
      * @brief Construct a curvilinear grid with monitor metric
      *
-     * @tparam Psi   models aBinaryOperator 
-     * @tparam PsiX  models aBinaryOperator 
-     * @tparam PsiY  models aBinaryOperator 
-     * @tparam PsiXX models aBinaryOperator 
-     * @tparam PsiXY models aBinaryOperator 
-     * @tparam PsiYY models aBinaryOperator 
-     * @tparam Chi   models aBinaryOperator 
-     * @tparam ChiX  models aBinaryOperator 
-     * @tparam ChiY  models aBinaryOperator 
-     * @param psi \f$ \psi(x,y)\f$ the flux function in Cartesian coordinates (x,y)
-     @param psiX \f$ \psi_x\f$ its derivative in x
-     @param psiY \f$ \psi_y\f$ its derivative in y
-     @param psiXX \f$ \psi_{xx}\f$ second derivative
-     @param psiXY \f$ \psi_{xy}\f$ second derivative
-     @param psiYY \f$ \psi_{yy}\f$ second derivative
-      @param chi_XX \f$  \chi^{xx}(x,y)\f$  is the contravariant xx-component of the adaption tensor in Cartesian coordinates (x,y)
-      @param chi_XY \f$  \chi^{xy}(x,y)\f$  is the contravariant xy-component of the adaption tensor in Cartesian coordinates (x,y)
-      @param chi_YY \f$  \chi^{yy}(x,y)\f$  is the contravariant yy-component of the adaption tensor in Cartesian coordinates (x,y)
-     * @param divChiX \f$ \partial_x \chi^{xx} + \partial_y\chi^{yx}\f$ is the x-component of the divergence of the tensor \f$ \chi\f$
-     * @param divChiY \f$ \partial_x \chi^{xy} + \partial_y\chi^{yy}\f$ is the y-component of the divergence of the tensor \f$ \chi \f$
+     * @param psi the flux function \f$ \psi(x,y)\f$ and its derivatives in Cartesian coordinates (x,y)
+      @param chi the symmetric adaption tensor \f$\chi(x,y)\f$ and its divergence in Cartesian coordinates (x,y)
      * @param psi0 first boundary 
      * @param psi1 second boundary
      * @param X0 a point in the inside of the ring bounded by psi0 (shouldn't be the O-point)
@@ -335,25 +286,14 @@ struct Hector : public aGenerator
      * @param eps_u the accuracy of u
      * @param verbose If true convergence details are printed to std::cout
      */
-    template< class Psi, class PsiX, class PsiY, class PsiXX, class PsiXY, class PsiYY, class Chi_XX, class Chi_XY, class Chi_YY, class DivChiX, class DivChiY>
-    Hector( Psi psi, PsiX psiX, PsiY psiY, 
-            PsiXX psiXX, PsiXY psiXY, PsiYY psiYY,  
-            Chi_XX chi_XX, Chi_XY chi_XY, Chi_YY chi_YY, 
-            DivChiX divChiX, DivChiY divChiY,
+    Hector( const BinaryFunctorsLvl2& psi,const BinarySymmTensorLvl1& chi,
             double psi0, double psi1, double X0, double Y0, unsigned n = 13, unsigned Nx = 2, unsigned Ny = 10, double eps_u = 1e-10, bool verbose=false) : 
-        generator_( new dg::geo::RibeiroFluxGenerator<Psi,PsiX,PsiY,PsiXX, PsiXY, PsiYY>(psi, psiX, psiY, psiXX, psiXY, psiYY, psi0, psi1, X0, Y0,1)),
-        g2d_(generator_, n, Nx, Ny, dg::DIR)
+        g2d_(new dg::geo::RibeiroFluxGenerator(psi, psi0, psi1, X0, Y0,1), n, Nx, Ny, dg::DIR)
     {
-        dg::geo::detail::LaplaceChiPsi<PsiX, PsiY, PsiXX, PsiXY, PsiYY, Chi_XX, Chi_XY, Chi_YY, DivChiX, DivChiY>
-            lapChiPsi( psiX, psiY, psiXX, psiXY, psiYY, 
-                chi_XX, chi_XY, chi_YY, divChiX, divChiY);
         //first construct u_
-        container u = construct_grid_and_u( 
-                psi, psiX, psiY, 
-                psiXX, psiXY, psiYY,
-                chi_XX, chi_XY, chi_YY, lapChiPsi, 
+        container u = construct_grid_and_u( psi, chi, 
                 psi0, psi1, X0, Y0, n, Nx, Ny, eps_u , verbose);
-        construct( u, psi0, psi1, chi_XX, chi_XY, chi_YY);
+        construct( u, psi0, psi1, chi.xx(), chi.xy(), chi.yy());
         orthogonal_=conformal_=false;
         ////we actually don't need u_ but it makes a good testcase 
         //container psi__;
@@ -398,7 +338,6 @@ struct Hector : public aGenerator
      * @return  orthogonal zeta, eta grid
      */
     const dg::CurvilinearGrid2d<container>& internal_grid() const {return g2d_;}
-    virtual ~Hector() { delete generator_;}
     virtual Hector* clone() const{return new Hector(*this);}
     private:
     virtual void generate( const thrust::host_vector<double>& u1d, 
@@ -437,11 +376,7 @@ struct Hector : public aGenerator
         //std::cout << "Error in u is "<<eps<<std::endl;
     }
 
-    //make Hector non-copyable
-    Hector( const Hector& src);
-    Hector& operator=( const Hector& src);
-    template< class Psi, class PsiX, class PsiY, class PsiXX, class PsiXY, class PsiYY, class Chi, class LaplaceChiPsi>
-    container construct_grid_and_u( Psi psi, PsiX psiX, PsiY psiY, PsiXX psiXX, PsiXY psiXY, PsiYY psiYY, Chi chi, LaplaceChiPsi lapCP, double psi0, double psi1, double X0, double Y0, unsigned n, unsigned Nx, unsigned Ny, double eps_u , bool verbose) 
+    container construct_grid_and_u( const aBinaryFunctor& chi, const aBinaryFunctor& lapChiPsi, double psi0, double psi1, double X0, double Y0, unsigned n, unsigned Nx, unsigned Ny, double eps_u , bool verbose) 
     {
         //first find u( \zeta, \eta)
         double eps = 1e10, eps_old = 2e10;
@@ -451,22 +386,21 @@ struct Hector : public aGenerator
         ellipticD_old.set_chi( adapt);
 
         container u_old = dg::evaluate( dg::zero, g2d_old), u(u_old);
-        container lapu = dg::pullback( lapCP, g2d_old);
+        container lapu = dg::pullback( lapChiPsi, g2d_old);
         dg::Invert<container > invert_old( u_old, n*n*Nx*Ny, eps_u);
         unsigned number = invert_old( ellipticD_old, u_old, lapu);
         while( (eps < eps_old||eps > 1e-7) && eps > eps_u)
         {
             eps = eps_old;
-            Nx*=2, Ny*=2;
-            dg::CurvilinearGrid2d<container> g2d(generator_, n, Nx, Ny, dg::DIR);
+            g2d_.multiplyCellNumber(2,2);
             if(verbose) std::cout << "Nx "<<Nx<<" Ny "<<Ny<<std::flush;
-            dg::Elliptic<dg::CurvilinearGrid2d<container>, Matrix, container> ellipticD( g2d, dg::DIR, dg::PER, dg::not_normed, dg::centered);
-            adapt = dg::pullback(chi, g2d);
+            dg::Elliptic<dg::CurvilinearGrid2d<container>, Matrix, container> ellipticD( g2d_, dg::DIR, dg::PER, dg::not_normed, dg::centered);
+            adapt = dg::pullback(chi, g2d_);
             ellipticD.set_chi( adapt);
-            lapu = dg::pullback( lapCP, g2d);
-            const container vol2d = dg::create::weights( g2d);
-            const IMatrix Q = dg::create::interpolation( g2d, g2d_old);
-            u = dg::evaluate( dg::zero, g2d);
+            lapu = dg::pullback( lapChiPsi, g2d_);
+            const container vol2d = dg::create::weights( g2d_);
+            const IMatrix Q = dg::create::interpolation( g2d_, g2d_old);
+            u = dg::evaluate( dg::zero, g2d_);
             container u_diff( u);
             dg::blas2::gemv( Q, u_old, u_diff);
 
@@ -477,38 +411,36 @@ struct Hector : public aGenerator
             if(verbose) std::cout <<" iter "<<number<<" error "<<eps<<"\n";
             g2d_old = g2d;
             u_old = u;
-            g2d_ = g2d;
             number++;//get rid of warning
         }
         return u;
     }
 
-    template< class Psi, class PsiX, class PsiY, class PsiXX, class PsiXY, class PsiYY, class Chi_XX, class Chi_XY, class Chi_YY, class LaplaceChiPsi>
-    container construct_grid_and_u( Psi psi, PsiX psiX, PsiY psiY, PsiXX psiXX, PsiXY psiXY, PsiYY psiYY, 
-            Chi_XX chi_XX, Chi_XY chi_XY, Chi_YY chi_YY, LaplaceChiPsi lapCP, double psi0, double psi1, double X0, double Y0, unsigned n, unsigned Nx, unsigned Ny, double eps_u, bool verbose ) 
+    container construct_grid_and_u( const BinaryFunctorsLvl2& psi, 
+            const BinarySymmTensorLvl1& chi, double psi0, double psi1, double X0, double Y0, unsigned n, unsigned Nx, unsigned Ny, double eps_u, bool verbose ) 
     {
+        dg::geo::detail::LaplaceChiPsi lapChiPsi( psi, chi);
         //first find u( \zeta, \eta)
         double eps = 1e10, eps_old = 2e10;
         dg::CurvilinearGrid2d<container> g2d_old = g2d_;
         dg::TensorElliptic<dg::CurvilinearGrid2d<container>, Matrix, container> ellipticD_old( g2d_old, dg::DIR, dg::PER, dg::not_normed, dg::centered);
-        ellipticD_old.set( chi_XX, chi_XY, chi_YY);
+        ellipticD_old.set( chi.xx(), chi.xy(), chi.yy());
 
         container u_old = dg::evaluate( dg::zero, g2d_old), u(u_old);
-        container lapu = dg::pullback( lapCP, g2d_old);
+        container lapu = dg::pullback( lapChiPsi, g2d_old);
         dg::Invert<container > invert_old( u_old, n*n*Nx*Ny, eps_u);
         unsigned number = invert_old( ellipticD_old, u_old, lapu);
         while( (eps < eps_old||eps > 1e-7) && eps > eps_u)
         {
             eps = eps_old;
-            Nx*=2, Ny*=2;
-            dg::CurvilinearGrid2d<container> g2d(generator_, n, Nx, Ny, dg::DIR);
+            g2d_.multiplyCellNumber(2,2);
             if(verbose)std::cout << "Nx "<<Nx<<" Ny "<<Ny<<std::flush;
-            dg::TensorElliptic<dg::CurvilinearGrid2d<container>, Matrix, container> ellipticD( g2d, dg::DIR, dg::PER, dg::not_normed, dg::centered);
-            ellipticD.set( chi_XX, chi_XY, chi_YY );
-            lapu = dg::pullback( lapCP, g2d);
-            const container vol2d = dg::create::weights( g2d);
-            const IMatrix Q = dg::create::interpolation( g2d, g2d_old);
-            u = dg::evaluate( dg::zero, g2d);
+            dg::TensorElliptic<dg::CurvilinearGrid2d<container>, Matrix, container> ellipticD( g2d_, dg::DIR, dg::PER, dg::not_normed, dg::centered);
+            ellipticD.set( chi.xx(), chi.xy(), chi.yy() );
+            lapu = dg::pullback( lapChiPsi, g2d_);
+            const container vol2d = dg::create::weights( g2d_);
+            const IMatrix Q = dg::create::interpolation( g2d_, g2d_old);
+            u = dg::evaluate( dg::zero, g2d_);
             container u_diff( u);
             dg::blas2::gemv( Q, u_old, u_diff);
 
@@ -517,16 +449,14 @@ struct Hector : public aGenerator
             dg::blas1::axpby( 1. ,u, -1., u_diff);
             eps = sqrt( dg::blas2::dot( u_diff, vol2d, u_diff) / dg::blas2::dot( u, vol2d, u) );
             if(verbose) std::cout <<" iter "<<number<<" error "<<eps<<"\n";
-            g2d_old = g2d;
+            g2d_old = g2d_;
             u_old = u;
-            g2d_ = g2d;
             number++;//get rid of warning
         }
         return u;
     }
 
-    template< class Chi_XX, class Chi_XY, class Chi_YY>
-    void construct(const container& u, double psi0, double psi1, Chi_XX chi_XX, Chi_XY chi_XY, Chi_YY chi_YY)
+    void construct(const container& u, double psi0, double psi1, const aBinaryFunctor& chi_XX, const aBinaryFunctor& chi_XY, const aBinaryFunctor& chi_YY)
     {
         //now compute u_zeta and u_eta 
         Matrix dzeta = dg::create::dx( g2d_, dg::DIR);
@@ -593,7 +523,6 @@ struct Hector : public aGenerator
     double c0_, lu_;
     thrust::host_vector<double> u_, ux_, uy_, vx_, vy_;
     thrust::host_vector<double> etaV_, zetaU_, etaU_;
-    aGenerator* generator_;
     dg::CurvilinearGrid2d<container> g2d_;
 
 };
diff --git a/inc/geometries/toroidal.h b/inc/geometries/toroidal.h
index cb5019b71..5f105e666 100644
--- a/inc/geometries/toroidal.h
+++ b/inc/geometries/toroidal.h
@@ -6,13 +6,6 @@ namespace dg{
 namespace geo{
 namespace toroidal{
 
-struct Constant:public aCloneableBinaryOperator<Constant> 
-{ 
-    Constant(double c):c_(c){}
-    double operator()(double R,double Z)const{return c_;}
-    private:
-    double c_;
-};
 /**
  * @brief Models a slab toroidal field 
  *
-- 
GitLab


From c1eec1cf008af355ef69e10b5815a2c6622e13a0 Mon Sep 17 00:00:00 2001
From: Matthias Home Office <Matthias.Wiesenberger@uibk.ac.at>
Date: Wed, 26 Jul 2017 13:48:56 +0200
Subject: [PATCH 084/453] added adaption function and class for aBinaryFunctor

---
 inc/geometries/fluxfunctions.h | 27 ++++++++++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/inc/geometries/fluxfunctions.h b/inc/geometries/fluxfunctions.h
index ae9d773d9..32b0de398 100644
--- a/inc/geometries/fluxfunctions.h
+++ b/inc/geometries/fluxfunctions.h
@@ -54,7 +54,7 @@ struct aBinaryFunctor
 };
 
 /**
-* @brief Implementation helper for the clone pattern
+* @brief Intermediate implementation helper class for the clone pattern with CRTP
 
     https://katyscode.wordpress.com/2013/08/22/c-polymorphic-cloning-and-the-crtp-curiously-recurring-template-pattern/
 */
@@ -71,6 +71,31 @@ struct dg::geo::aCloneableBinaryFunctor : public dg::geo::aBinaryFunctor
         return new Derived(static_cast<Derived const &>(*this));
     }
 };
+/**
+ * @brief With this adapater class you can make any Functor cloneable
+ *
+ * @tparam BinaryFunctor must overload the operator() like
+ * double operator()(double,double)const;
+ */
+template<class BinaryFunctor>
+struct Adapter : public dg::geo::aCloneableBinaryFunctor<Adapter>
+{
+    Adapter( const BinaryFunctor& f):f_(f){}
+    double operator()(double x, double y)const{return f_(x,y);}
+    private:
+    BinaryFunctor f_;
+};
+/**
+ * @brief Use this function when you want to convert any Functor to aBinaryFunctor
+ *
+ * @tparam BinaryFunctor must overload the operator() like
+ * double operator()(double,double)const;
+ * @param f const reference to a functor class
+ * @return a newly allocated instance of aBinaryFunctor on the heap
+ * @note the preferred way is to derive your Functor from aCloneableBinaryFunctor but if you can't or don't want to for whatever reason then use this to make one
+ */
+temmplate<class BinaryFunctor>
+aBinaryFunctor* make_aBinaryFunctor(const BinaryFunctor& f){return new Adapter<BinaryFunctor>(f);}
 
 /**
 * @brief Manage and deep copy binary Functors on the heap
-- 
GitLab


From 092a87261aa0e48f881ade607697e7fd8057bd20 Mon Sep 17 00:00:00 2001
From: Matthias Home Office <Matthias.Wiesenberger@uibk.ac.at>
Date: Wed, 26 Jul 2017 15:23:34 +0200
Subject: [PATCH 085/453] made inplace raiseIndex function

---
 inc/dg/elliptic.h                 |  6 +++---
 inc/dg/geometry.h                 | 36 ++++++++++++++++++-------------
 inc/dg/geometry/geometry_traits.h | 24 ++++++++++-----------
 3 files changed, 36 insertions(+), 30 deletions(-)

diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index 98a22bb56..81054cbe9 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -147,10 +147,10 @@ class Elliptic
     void symv( const Vector& x, Vector& y) 
     {
         //compute gradient
-        dg::blas2::gemv( rightx, x, tempx); //R_x*f 
-        dg::blas2::gemv( righty, x, tempy); //R_y*f
+        dg::blas2::gemv( rightx, x, gradx); //R_x*f 
+        dg::blas2::gemv( righty, x, y); //R_y*f
 
-        dg::geo::volRaisePerpIndex( tempx, tempy, gradx, y, g_);
+        dg::geo::volRaisePerpIndex( gradx, y, gradx, y, g_);
 
         //multiply with chi 
         dg::blas1::pointwiseDot( xchi, gradx, gradx); //Chi*R_x*x 
diff --git a/inc/dg/geometry.h b/inc/dg/geometry.h
index 51d3db914..2241c4b06 100644
--- a/inc/dg/geometry.h
+++ b/inc/dg/geometry.h
@@ -82,20 +82,23 @@ void divideVolume( container& inout, const Geometry& g)
 }
 
 /**
- * @brief Raises the index of a covariant vector in 2d
+ * @brief Raises the index of a covariant vector with the help of the projection tensor in 2d
  *
- * Computes \f$ v^i = g^{ij}v_j\f$ for \f$ i,j\in \{1,2\}\f$ in the two dimensions of a 2x1 product space
+ * The projection tensor is defined as \f[ h^{ij} := g^{ij} - b^ib^ĵ \f]
+ * where \f$ b^i\f$ are the contravariant components of a unit vector.
+ * Here, we assume that the projection tensor equals the perpendicular metric tensor:
+ * Compute \f$ v^i = g^{ij}v_j\f$ for \f$ i,j\in \{1,2\}\f$ in the two dimensions of a 2x1 product space
  * @tparam container the container class
  * @tparam Geometry the geometry class
- * @param covX (input) covariant first component (may get destroyed!!)
- * @param covY (input) covariant second component (may get destroyed!!)
- * @param contraX (output) contravariant first component
- * @param contraY (output) contravariant second component
+ * @param covX (input) covariant first component 
+ * @param covY (input) covariant second component
+ * @param contraX (output) contravariant first component (can be the same as covX)
+ * @param contraY (output) contravariant second component (can be the same as covY)
  * @param g The geometry object
- * @note covX, covY, contraX and contraY may not be the same
+ * @note if contraX==covX and/or contraY==covY the transformation is done in-place
  */
 template<class container, class Geometry>
-void raisePerpIndex( container& covX, container& covY, container& contraX, container& contraY, const Geometry& g)
+void raisePerpIndex( const container& covX, const container& covY, container& contraX, container& contraY, const Geometry& g)
 {
     assert( &covX != &contraX);
     assert( &covY != &contraY);
@@ -104,18 +107,21 @@ void raisePerpIndex( container& covX, container& covY, container& contraX, conta
 
 }
 /**
- * @brief Raises the index of a covariant vector in 2d and multiplies the perpendicular volume
+ * @brief Raises the index of a covariant vector in 2d with the help of the projection tensor in 2d and multiplies the perpendicular volume
  *
- * Computes \f$ v^i = \sqrt{g/g_{zz}} g^{ij}v_j\f$ for \f$ i,j\in \{1,2\}\f$ in the two dimensions of a 2x1 product space. This special 
+ * The projection tensor is defined as \f[ h^{ij} := g^{ij} - b^ib^ĵ \f]
+ * where \f$ b^i\f$ are the contravariant components of a unit vector.
+ * Here, we assume that the projection tensor equals the perpendicular metric tensor:
+ * Compute \f$ v^i = \sqrt{g/g_{zz}} g^{ij}v_j\f$ for \f$ i,j\in \{1,2\}\f$ in the two dimensions of a 2x1 product space. This special 
  * form occurs in the discretization of elliptic operators which is why it get's a special function.
  * @tparam container the container class
  * @tparam Geometry the geometry class
- * @param covX (input) covariant first component (may get destroyed!!)
- * @param covY (input) covariant second component (may get destroyed!!)
- * @param contraX (output) contravariant first component
- * @param contraY (output) contravariant second component
+ * @param covX (input) covariant first component 
+ * @param covY (input) covariant second component
+ * @param contraX (output) contravariant first component (can be the same as covX)
+ * @param contraY (output) contravariant second component (can be the same as covY)
  * @param g The geometry object
- * @note covX, covY, contraX and contraY may not be the same
+ * @note if contraX==covX and/or contraY==covY the transformation is done in-place
  */
 template<class container, class Geometry>
 void volRaisePerpIndex( container& covX, container& covY, container& contraX, container& contraY, const Geometry& g)
diff --git a/inc/dg/geometry/geometry_traits.h b/inc/dg/geometry/geometry_traits.h
index 1b27a1253..25505d7e3 100644
--- a/inc/dg/geometry/geometry_traits.h
+++ b/inc/dg/geometry/geometry_traits.h
@@ -86,18 +86,18 @@ void doDividePerpVolume( container& inout, const Geometry& g, CurvilinearPerpTag
 };
 
 template <class container, class Geometry>
-void doRaisePerpIndex( container& in1, container& in2, container& out1, container& out2, const Geometry& g, OrthonormalTag)
+void doRaisePerpIndex( const container& in1, const container& in2, container& out1, container& out2, const Geometry& g, OrthonormalTag)
 {
-    in1.swap( out1);
-    in2.swap( out2);
+    out1=in1;//the container type should check for self-assignment
+    out2=in2;//the container type should check for self-assignment
 };
 template <class container, class Geometry>
-void doRaisePerpIndex( container& in1, container& in2, container& out1, container& out2, const Geometry& g, CurvilinearPerpTag)
+void doRaisePerpIndex( const container& in1, const container& in2, container& out1, container& out2, const Geometry& g, CurvilinearPerpTag)
 {
     if( g.isOrthonormal())
     {
-        in1.swap( out1);
-        in2.swap( out2);
+        out1=in1;//the container type should check for self-assignment
+        out2=in2;//the container type should check for self-assignment
         return;
     }
     if( g.isOrthogonal())
@@ -113,18 +113,18 @@ void doRaisePerpIndex( container& in1, container& in2, container& out1, containe
 };
 
 template <class container, class Geometry>
-void doVolRaisePerpIndex( container& in1, container& in2, container& out1, container& out2, const Geometry& g, OrthonormalTag)
+void doVolRaisePerpIndex( const container& in1, const container& in2, container& out1, container& out2, const Geometry& g, OrthonormalTag)
 {
-    in1.swap( out1);
-    in2.swap( out2);
+    out1=in1;//the container type should check for self-assignment
+    out2=in2;//the container type should check for self-assignment
 };
 template <class container, class Geometry>
-void doVolRaisePerpIndex( container& in1, container& in2, container& out1, container& out2, const Geometry& g, CurvilinearPerpTag)
+void doVolRaisePerpIndex( const container& in1, const container& in2, container& out1, container& out2, const Geometry& g, CurvilinearPerpTag)
 {
     if( g.isConformal())
     {
-        in1.swap( out1);
-        in2.swap( out2);
+        out1=in1;//the container type should check for self-assignment
+        out2=in2;//the container type should check for self-assignment
         return;
     }
     if( g.isOrthogonal())
-- 
GitLab


From b7616f88262a1d7f3589bdf10ae8c20f4722aa89 Mon Sep 17 00:00:00 2001
From: Matthias Home Office <Matthias.Wiesenberger@uibk.ac.at>
Date: Wed, 26 Jul 2017 16:38:17 +0200
Subject: [PATCH 086/453] some notes on projection tensor and check
 self-assignment in curvilinear grids

---
 inc/dg/geometry.h               |  6 ++---
 inc/dg/geometry/curvilinear.h   | 43 ++++++++++++++++++---------------
 inc/geometries/magnetic_field.h |  8 +++---
 3 files changed, 31 insertions(+), 26 deletions(-)

diff --git a/inc/dg/geometry.h b/inc/dg/geometry.h
index 2241c4b06..3e96fb72b 100644
--- a/inc/dg/geometry.h
+++ b/inc/dg/geometry.h
@@ -86,7 +86,7 @@ void divideVolume( container& inout, const Geometry& g)
  *
  * The projection tensor is defined as \f[ h^{ij} := g^{ij} - b^ib^ĵ \f]
  * where \f$ b^i\f$ are the contravariant components of a unit vector.
- * Here, we assume that the projection tensor equals the perpendicular metric tensor:
+ * Here, we assume that the metric forms a 2x1 product space and b only has one component \f$ b^3b^3 = g_{33}^{-1} = g^{33}\f$:
  * Compute \f$ v^i = g^{ij}v_j\f$ for \f$ i,j\in \{1,2\}\f$ in the two dimensions of a 2x1 product space
  * @tparam container the container class
  * @tparam Geometry the geometry class
@@ -111,8 +111,8 @@ void raisePerpIndex( const container& covX, const container& covY, container& co
  *
  * The projection tensor is defined as \f[ h^{ij} := g^{ij} - b^ib^ĵ \f]
  * where \f$ b^i\f$ are the contravariant components of a unit vector.
- * Here, we assume that the projection tensor equals the perpendicular metric tensor:
- * Compute \f$ v^i = \sqrt{g/g_{zz}} g^{ij}v_j\f$ for \f$ i,j\in \{1,2\}\f$ in the two dimensions of a 2x1 product space. This special 
+ * Here, we assume that the metric forms a 2x1 product space and b only has one component \f$ b^3b^3 = g_{33}^{-1} = g^{33}\f$:
+ * Compute \f$ v^i = \sqrt{g/g_{33}} g^{ij}v_j\f$ for \f$ i,j\in \{1,2\}\f$ in the two dimensions of a 2x1 product space. This special 
  * form occurs in the discretization of elliptic operators which is why it get's a special function.
  * @tparam container the container class
  * @tparam Geometry the geometry class
diff --git a/inc/dg/geometry/curvilinear.h b/inc/dg/geometry/curvilinear.h
index 2063c37f4..70347736e 100644
--- a/inc/dg/geometry/curvilinear.h
+++ b/inc/dg/geometry/curvilinear.h
@@ -73,13 +73,16 @@ struct CylindricalGrid3d : public dg::Grid3d
     CylindricalGrid3d& operator=( const CylindricalGrid3d& src)
     {
         Grid3d::operator=(src);//call base class assignment
-        delete generator_;
-        g_xx_=src.g_xx_,g_xy_=src.g_xy_,g_yy_=src.g_yy_,g_pp_=src.g_pp_,vol_=src.vol_,vol2d_=src.vol2d_;
-        r_=src.r_,z_=src.z_,xr_=src.xr_,xz_=src.xz_,yr_=src.yr_,yz_=src.yz_;
-        generator_ = src.generator_->clone();
+        if( &src!=this)
+        {
+            delete generator_;
+            g_xx_=src.g_xx_,g_xy_=src.g_xy_,g_yy_=src.g_yy_,g_pp_=src.g_pp_,vol_=src.vol_,vol2d_=src.vol2d_;
+            r_=src.r_,z_=src.z_,xr_=src.xr_,xz_=src.xz_,yr_=src.yr_,yz_=src.yz_;
+            generator_ = src.generator_->clone();
+        }
         return *this;
     }
-    ~CylindricalGrid3d(){
+    virtual ~CylindricalGrid3d(){
         delete generator_;
     }
     private:
@@ -136,7 +139,7 @@ struct CylindricalGrid3d : public dg::Grid3d
     }
     thrust::host_vector<double> r_, z_, xr_, xz_, yr_, yz_;
     container g_xx_, g_xy_, g_yy_, g_pp_, vol_, vol2d_;
-    const geo::aGenerator* generator_;
+    geo::aGenerator* generator_;
 };
 
 /**
@@ -164,12 +167,7 @@ struct CurvilinearGrid2d : public dg::Grid2d
     CurvilinearGrid2d( const geo::aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx=dg::DIR):
         dg::Grid2d( 0, generator->width(), 0., generator->height(), n, Nx, Ny, bcx, dg::PER)
     {
-        CylindricalGrid3d<container> g( generator, n,Nx,Ny,1,bcx);
-        init_X_boundaries( g.x0(), g.x1());
-        r_=g.r(), z_=g.z(), xr_=g.xr(), xz_=g.xz(), yr_=g.yr(), yz_=g.yz();
-        g_xx_=g.g_xx(), g_xy_=g.g_xy(), g_yy_=g.g_yy();
-        vol2d_=g.perpVol();
-
+        construct( n,Nx,Ny);
     }
     CurvilinearGrid2d( const CylindricalGrid3d<container>& g):
         dg::Grid2d( g.x0(), g.x1(), g.y0(), g.y1(), g.n(), g.Nx(), g.Ny(), g.bcx(), g.bcy())
@@ -210,27 +208,34 @@ struct CurvilinearGrid2d : public dg::Grid2d
     Curvilinear2d& operator=( const Curvilinear2d& src)
     {
         Grid2d::operator=(src); //call base class assignment
-        delete generator_;
-        g_xx_=src.g_xx_,g_xy_=src.g_xy_,g_yy_=src.g_yy_,vol2d_=src.vol2d_;
-        r_=src.r_,z_=src.z_,xr_=src.xr_,xz_=src.xz_,yr_=src.yr_,yz_=src.yz_;
-        generator_ = src.generator_->clone();
+        if( &src!=this)
+        {
+            delete generator_;
+            g_xx_=src.g_xx_,g_xy_=src.g_xy_,g_yy_=src.g_yy_,vol2d_=src.vol2d_;
+            r_=src.r_,z_=src.z_,xr_=src.xr_,xz_=src.xz_,yr_=src.yr_,yz_=src.yz_;
+            generator_ = src.generator_->clone();
+        }
         return *this;
     }
-    ~Curvilinear2d(){
+    virtual ~Curvilinear2d(){
         delete generator_;
     }
     private:
     virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny)
     {
         dg::Grid2d::do_set( new_n, new_Nx, new_Ny);
-        CylindricalGrid3d<container> g( generator_, new_n,new_Nx,new_Ny,1,bcx());
+        construct( new_n, new_Nx, new_Ny);
+    }
+    void construct( unsigned n, unsigned Nx, unsigned Ny)
+    {
+        CylindricalGrid3d<container> g( generator_, n,Nx,Ny,1,bcx());
         r_=g.r(), z_=g.z(), xr_=g.xr(), xz_=g.xz(), yr_=g.yr(), yz_=g.yz();
         g_xx_=g.g_xx(), g_xy_=g.g_xy(), g_yy_=g.g_yy();
         vol2d_=g.perpVol();
     }
     thrust::host_vector<double> r_, z_, xr_, xz_, yr_, yz_;
     container g_xx_, g_xy_, g_yy_, vol2d_;
-    const geo::aGenerator* generator_;
+    geo::aGenerator* generator_;
 };
 
 ///@}
diff --git a/inc/geometries/magnetic_field.h b/inc/geometries/magnetic_field.h
index 59988080f..66b1e7107 100644
--- a/inc/geometries/magnetic_field.h
+++ b/inc/geometries/magnetic_field.h
@@ -23,10 +23,10 @@ namespace geo
  \f]
  where \f$ R_0\f$ is a normalization constant, \f$ I\f$ the current 
  and \f$ \psi_p\f$ the poloidal flux function.
- 
- This class holds and controls instances of aBinaryFunctor dynamically 
- on the heap.
- @note an instance of this class cannot be constructed nor deleted directly but it can be copied and assigned
+ @note We implicitly also assume the toroidal field line approximation, i.e. all curvature
+ and other perpendicular terms created with this field will assume that the perpendicular 
+ direction lies within the
+ R-Z planes of a cylindrical grid (The plane \f$ \perp \hat e_\varphi\f$ )
 */
 struct aTokamakMagneticField
 {
-- 
GitLab


From 2aae46def0424f5a8977587d4c872c279e4bdd18 Mon Sep 17 00:00:00 2001
From: Matthias Home Office <Matthias.Wiesenberger@uibk.ac.at>
Date: Wed, 26 Jul 2017 17:34:51 +0200
Subject: [PATCH 087/453] made r and z 2d and added 1d phi vector in
 curvilinear grids

---
 inc/dg/backend/grid.h             |  2 ++
 inc/dg/backend/mpi_grid.h         |  2 ++
 inc/dg/geometry/curvilinear.h     | 38 +++++++++++++++--------
 inc/dg/geometry/geometry_traits.h |  4 +--
 inc/dg/geometry/mpi_curvilinear.h | 51 +++++++++++++++++++------------
 inc/dg/geometry/mpi_grids.h       |  6 ++--
 6 files changed, 63 insertions(+), 40 deletions(-)

diff --git a/inc/dg/backend/grid.h b/inc/dg/backend/grid.h
index b16e9fd5b..6609dd416 100644
--- a/inc/dg/backend/grid.h
+++ b/inc/dg/backend/grid.h
@@ -307,6 +307,7 @@ struct Grid2d
     */
     void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny) {
         assert( new_n> 0 && new_Nx > 0  && new_Ny > 0);
+        if( new_n == n_ && new_Nx == Nx_ && new_Ny == Ny_) return;
         do_set(new_n,new_Nx,new_Ny);
     }
 
@@ -492,6 +493,7 @@ struct Grid3d
     */
     void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz) {
         assert( new_n>0); assert( new_Nx > 0  && new_Ny > 0); assert( new_Nz > 0);
+        if( new_n == n_ && new_Nx == Nx_ && new_Ny == Ny_ && new_Nz == Nz_) return;
         do_set(new_n, new_Nx,new_Ny,new_Nz);
     }
 
diff --git a/inc/dg/backend/mpi_grid.h b/inc/dg/backend/mpi_grid.h
index db17ff120..dd3a0e213 100644
--- a/inc/dg/backend/mpi_grid.h
+++ b/inc/dg/backend/mpi_grid.h
@@ -67,6 +67,7 @@ struct MPIGrid2d
     */
     void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny) {
         check_division( new_Nx, new_Ny, g.bcx(), g.bcy());
+        if( new_n == g.n() && new_Nx == g.Nx() && new_Ny == g.Ny()) return;
         do_set( new_n,new_Nx,new_Ny);
     }
 
@@ -385,6 +386,7 @@ struct MPIGrid3d
      */
     void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz) {
         check_division( new_Nx,new_Ny,new_Nz,g.bcx(),g.bcy(),g.bcz());
+        if( new_n == g.n() && new_Nx == g.Nx() && new_Ny == g.Ny() && new_Nz == g.Nz()) return;
         do_set(new_n,new_Nx,new_Ny,new_Nz);
     }
 
diff --git a/inc/dg/geometry/curvilinear.h b/inc/dg/geometry/curvilinear.h
index 70347736e..f36bc20fd 100644
--- a/inc/dg/geometry/curvilinear.h
+++ b/inc/dg/geometry/curvilinear.h
@@ -29,31 +29,41 @@ struct CylindricalGrid3d : public dg::Grid3d
         dg::Grid3d(0,R1-R0,0,Z1-Z0,phi0,phi1,n,NR,NZ,Nphi,bcR,bcZ,bcphi)
         {
             generator_ = new IdentityGenerator(R0,R1,Z0,Z1);
-            construct( n, NR, NZ);
+            construct( n, NR, NZ,Nphi);
         }
 
     /*!@brief Constructor
     
+     * the coordinates of the computational space are called x,y,z
      * @param generator must generate a grid
-     * @param n 
-     * @param Nx
-     @param Ny
-     @param Nz 
-     @param bcx
+     * @param n number of %Gaussian nodes in x and y
+     * @param Nx number of cells in x
+     @param Ny number of cells in y 
+     @param Nz  number of cells z
+     @param bcx boundary condition in x
+     @note the boundary conditions for y and z are set periodic
      */
     CylindricalGrid3d( const geo::aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx=dg::DIR):
         dg::Grid3d( 0, generator->width(), 0., generator->height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, dg::PER, dg::PER)
     { 
         generator_ = generator;
-        construct( n, Nx, Ny);
+        construct( n, Nx, Ny,Nz);
     }
 
     perpendicular_grid perp_grid() const { return perpendicular_grid(*this);}
+    /// a 2d vector containing R(x,y)
     const thrust::host_vector<double>& r()const{return r_;}
+    /// a 2d vector containing Z(x,y)
     const thrust::host_vector<double>& z()const{return z_;}
+    /// a 1d vector containing the phi values
+    const thrust::host_vector<double>& phi()const{return phi_;}
+    /// a 3d vector containing dx/dR
     const thrust::host_vector<double>& xr()const{return xr_;}
+    /// a 3d vector containing dy/dR
     const thrust::host_vector<double>& yr()const{return yr_;}
+    /// a 3d vector containing dx/dZ
     const thrust::host_vector<double>& xz()const{return xz_;}
+    /// a 3d vector containing dy/dZ
     const thrust::host_vector<double>& yz()const{return yz_;}
     const container& g_xx()const{return g_xx_;}
     const container& g_yy()const{return g_yy_;}
@@ -67,7 +77,7 @@ struct CylindricalGrid3d : public dg::Grid3d
     bool isConformal() const { return generator_->isConformal();}
     CylindricalGrid3d( const CylindricalGrid3d& src):Grid3d(src), g_xx_(src.g_xx_),g_xy_(src.g_xy_),g_yy_(src.g_yy_),g_pp_(src.g_pp_),vol_(src.vol_),vol2d_(src.vol2d_)
     {
-        r_=src.r_,z_=src.z_,xr_=src.xr_,xz_=src.xz_,yr_=src.yr_,yz_=src.yz_;
+        r_=src.r_,z_=src.z_,phi_=src.phi_,xr_=src.xr_,xz_=src.xz_,yr_=src.yr_,yz_=src.yz_;
         generator_ = src.generator_->clone();
     }
     CylindricalGrid3d& operator=( const CylindricalGrid3d& src)
@@ -77,7 +87,7 @@ struct CylindricalGrid3d : public dg::Grid3d
         {
             delete generator_;
             g_xx_=src.g_xx_,g_xy_=src.g_xy_,g_yy_=src.g_yy_,g_pp_=src.g_pp_,vol_=src.vol_,vol2d_=src.vol2d_;
-            r_=src.r_,z_=src.z_,xr_=src.xr_,xz_=src.xz_,yr_=src.yr_,yz_=src.yz_;
+            r_=src.r_,z_=src.z_,phi_=src.phi_,xr_=src.xr_,xz_=src.xz_,yr_=src.yr_,yz_=src.yz_;
             generator_ = src.generator_->clone();
         }
         return *this;
@@ -88,12 +98,14 @@ struct CylindricalGrid3d : public dg::Grid3d
     private:
     virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny,unsigned new_Nz){
         dg::Grid3d::do_set( new_n, new_Nx, new_Ny,new_Nz);
-        construct( new_n, new_Nx, new_Ny);
+        construct( new_n, new_Nx, new_Ny,new_Nz);
     }
-    void construct( unsigned n, unsigned Nx, unsigned Ny)
+    void construct( unsigned n, unsigned Nx, unsigned Ny,unsigned Nz)
     {
         dg::Grid1d gY1d( 0, generator_->height(), n, Ny, dg::PER);
         dg::Grid1d gX1d( 0., generator_->width(), n, Nx);
+        dg::Grid1d gphi( z0(), z1(), 1, Nz);
+        phi_ = dg::create::abscissas( gphi);
         thrust::host_vector<double> x_vec = dg::evaluate( dg::cooX1d, gX1d);
         thrust::host_vector<double> y_vec = dg::evaluate( dg::cooX1d, gY1d);
         (*generator_)( x_vec, y_vec, r_, z_, xr_, xz_, yr_, yz_);
@@ -105,7 +117,7 @@ struct CylindricalGrid3d : public dg::Grid3d
     {
         //lift to 3D grid
         unsigned size = this->size();
-        r_.resize( size), z_.resize(size), xr_.resize(size), yr_.resize( size), xz_.resize( size), yz_.resize(size);
+        xr_.resize(size), yr_.resize( size), xz_.resize( size), yz_.resize(size);
         unsigned Nx = this->n()*this->Nx(), Ny = this->n()*this->Ny();
         for( unsigned k=1; k<this->Nz(); k++)
             for( unsigned i=0; i<Nx*Ny; i++)
@@ -137,7 +149,7 @@ struct CylindricalGrid3d : public dg::Grid3d
         dg::blas1::pointwiseDivide( tempxx, r_, tempxx); //1/R^2
         g_pp_=tempxx;
     }
-    thrust::host_vector<double> r_, z_, xr_, xz_, yr_, yz_;
+    thrust::host_vector<double> r_, z_, phi_, xr_, xz_, yr_, yz_;
     container g_xx_, g_xy_, g_yy_, g_pp_, vol_, vol2d_;
     geo::aGenerator* generator_;
 };
diff --git a/inc/dg/geometry/geometry_traits.h b/inc/dg/geometry/geometry_traits.h
index 25505d7e3..469d6c251 100644
--- a/inc/dg/geometry/geometry_traits.h
+++ b/inc/dg/geometry/geometry_traits.h
@@ -297,11 +297,9 @@ thrust::host_vector<double> doPullback( TernaryOp f, const Geometry& g, Curvilin
 {
     thrust::host_vector<double> vec( g.size());
     unsigned size2d = g.n()*g.n()*g.Nx()*g.Ny();
-    Grid1d gz( g.z0(), g.z1(), 1, g.Nz());
-    thrust::host_vector<double> absz = create::abscissas( gz);
     for( unsigned k=0; k<g.Nz(); k++)
         for( unsigned i=0; i<size2d; i++)
-            vec[k*size2d+i] = f( g.r()[k*size2d+i], g.z()[k*size2d+i], absz[k]);
+            vec[k*size2d+i] = f( g.r()[i], g.z()[i], g.phi()[k]);
     return vec;
 }
 template< class BinaryOp, class Geometry>
diff --git a/inc/dg/geometry/mpi_curvilinear.h b/inc/dg/geometry/mpi_curvilinear.h
index c1477a2e2..b6d16c360 100644
--- a/inc/dg/geometry/mpi_curvilinear.h
+++ b/inc/dg/geometry/mpi_curvilinear.h
@@ -63,8 +63,9 @@ struct CylindricalMPIGrid3d : public dg::MPIGrid3d
 
     perpendicular_grid perp_grid() const { return perpendicular_grid(*this);}
 
-    const dg::MPI_Vector<thrust::host_vector<double> >& r()const{return r_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& z()const{return z_;}
+    const thrust::host_vector<double>& r()const{return r_;}
+    const thrust::host_vector<double>& z()const{return z_;}
+    const thrust::host_vector<double>& phi()const{return phi_;}
     const dg::MPI_Vector<thrust::host_vector<double> >& xr()const{return xr_;}
     const dg::MPI_Vector<thrust::host_vector<double> >& yr()const{return yr_;}
     const dg::MPI_Vector<thrust::host_vector<double> >& xz()const{return xz_;}
@@ -88,13 +89,13 @@ struct CylindricalMPIGrid3d : public dg::MPIGrid3d
     bool isConformal() const { return g.isConformal();}
     MPICylindricalGrid3d( const MPICylindricalGrid3d& src):MPIGrid3d(src), g_xx_(src.g_xx_),g_xy_(src.g_xy_),g_yy_(src.g_yy_),g_pp_(src.g_pp_),vol_(src.vol_),vol2d_(src.vol2d_),g(src.g)
     {
-        r_=src.r_,z_=src.z_,xr_=src.xr_,xz_=src.xz_,yr_=src.yr_,yz_=src.yz_;
+        r_=src.r_,z_=src.z_,phi_=src.phi_,xr_=src.xr_,xz_=src.xz_,yr_=src.yr_,yz_=src.yz_;
     }
     MPICylindricalGrid3d& operator=( const MPICylindricalGrid3d& src)
     {
         MPIGrid3d::operator=(src);//call base class assignment
         g_xx_=src.g_xx_,g_xy_=src.g_xy_,g_yy_=src.g_yy_,g_pp_=src.g_pp_,vol_=src.vol_,vol2d_=src.vol2d_;
-        r_=src.r_,z_=src.z_,xr_=src.xr_,xz_=src.xz_,yr_=src.yr_,yz_=src.yz_;
+        r_=src.r_,z_=src.z_,phi_=src.phi_,xr_=src.xr_,xz_=src.xz_,yr_=src.yr_,yz_=src.yz_;
         g = src.g;
         return *this;
     }
@@ -108,8 +109,11 @@ struct CylindricalMPIGrid3d : public dg::MPIGrid3d
     }
     void divide_and_conquer( )
     {
-        r_=dg::evaluate( dg::one, *this), z_=r_, xr_=r_, xz_=r_, yr_=r_, yz_=r_;
-        dg::blas1::transfer( r_, g_xx_);
+        r_.resize( this->n()*this->n()*this->Nx()*this->Ny());
+        z_ = r_;
+        phi_.resize( this->Nz());
+        xr_=dg::evaluate( dg::one, *this), xz_=xr_, yr_=xr_, yz_=xr_;
+        dg::blas1::transfer( xr_, g_xx_);
         g_xy_=g_xx_, g_yy_=g_xx_, g_pp_=g_xx_, vol_=g_xx_, vol2d_=g_xx_;
         //divide and conquer
         int dims[3], periods[3], coords[3];
@@ -121,10 +125,13 @@ struct CylindricalMPIGrid3d : public dg::MPIGrid3d
                     //for( unsigned px=0; px<dims[0]; px++)
                         for( unsigned j=0; j<this->n()*this->Nx(); j++)
                         {
+                            unsigned idx2D1 = i*this->n()*this->Nx() + j;
+                            unsigned idx2D2 = ((coords[1]*this->n()*this->Ny()+i)*dims[0] + coords[0])*this->n()*this->Nx() + j;
+                            r_[idx2D1] = g.r()[idx2D2];
+                            z_[idx2D1] = g.z()[idx2D2];
+                            phi_[s] = g.phi()[coords[2]*this->Nz()+s];
                             unsigned idx1 = (s*this->n()*this->Ny()+i)*this->n()*this->Nx() + j;
                             unsigned idx2 = (((s*dims[1]+coords[1])*this->n()*this->Ny()+i)*dims[0] + coords[0])*this->n()*this->Nx() + j;
-                            r_.data()[idx1] = g.r()[idx2];
-                            z_.data()[idx1] = g.z()[idx2];
                             xr_.data()[idx1] = g.xr()[idx2];
                             xz_.data()[idx1] = g.xz()[idx2];
                             yr_.data()[idx1] = g.yr()[idx2];
@@ -137,7 +144,8 @@ struct CylindricalMPIGrid3d : public dg::MPIGrid3d
                             vol2d_.data()[idx1] = g.perpVol()[idx2];
                         }
     }
-    dg::MPI_Vector<thrust::host_vector<double> > r_, z_, xr_, xz_, yr_, yz_; //3d vector
+    thrust::host_vector<double> r_,z_,phi_;
+    dg::MPI_Vector<thrust::host_vector<double> > xr_, xz_, yr_, yz_; //3d vector
     MPIContainer g_xx_, g_xy_, g_yy_, g_pp_, vol_, vol2d_;
     dg::CylindricalGrid3d<LocalContainer> g;
 };
@@ -179,15 +187,15 @@ struct CurvilinearMPIGrid2d : public dg::MPIGrid2d
     }
     CurvilinearMPIGrid2d( const CylindricalMPIGrid3d<LocalContainer>& g):
         dg::MPIGrid2d( g.global().x0(), g.global().x1(), g.global().y0(), g.global().y1(), g.global().n(), g.global().Nx(), g.global().Ny(), g.global().bcx(), g.global().bcy(), get_reduced_comm( g.communicator() )),
-        r_(dg::evaluate( dg::one, *this)), z_(r_), xr_(r_), xz_(r_), yr_(r_), yz_(r_), 
-        g_xx_(r_), g_xy_(g_xx_), g_yy_(g_xx_), vol2d_(g_xx_),
+        xr_(dg::evaluate( dg::one, *this)), xz_(xr_), yr_(xr_), yz_(xr_), 
+        g_xx_(xr_), g_xy_(g_xx_), g_yy_(g_xx_), vol2d_(g_xx_),
         g_( g.global())
     {
+        r_=g.r(); 
+        z_=g.z(); 
         unsigned s = this->size();
         for( unsigned i=0; i<s; i++)
         {
-            r_.data()[i]=g.r().data()[i]; 
-            z_.data()[i]=g.z().data()[i]; 
             xr_.data()[i]=g.xr().data()[i]; 
             xz_.data()[i]=g.xz().data()[i]; 
             yr_.data()[i]=g.yr().data()[i]; 
@@ -200,8 +208,8 @@ struct CurvilinearMPIGrid2d : public dg::MPIGrid2d
         
     }
 
-    const dg::MPI_Vector<thrust::host_vector<double> >& r()const{return r_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& z()const{return z_;}
+    const thrust::host_vector<double>& r()const{return r_;}
+    const thrust::host_vector<double>& z()const{return z_;}
     const dg::MPI_Vector<thrust::host_vector<double> >& xr()const{return xr_;}
     const dg::MPI_Vector<thrust::host_vector<double> >& yr()const{return yr_;}
     const dg::MPI_Vector<thrust::host_vector<double> >& xz()const{return xz_;}
@@ -246,8 +254,10 @@ struct CurvilinearMPIGrid2d : public dg::MPIGrid2d
     }
     void divide_and_conquer()
     {
-        r_=dg::evaluate( dg::one, *this), z_=r_, xr_=r_, xz_=r_, yr_=r_, yz_=r_;
-        dg::blas1::transfer( r_, g_xx_);
+        r_.resize( this->n()*this->n()*this->Nx()*this->Ny());
+        z_=r_;
+        xr_=dg::evaluate( dg::one, *this), xz_=xr_, yr_=xr_, yz_=xr_;
+        dg::blas1::transfer( xr_, g_xx_);
         g_xy_=g_xx_, g_yy_=g_xx_, vol2d_=g_xx_;
         //divide and conquer
         int dims[2], periods[2], coords[2];
@@ -260,8 +270,8 @@ struct CurvilinearMPIGrid2d : public dg::MPIGrid2d
                         {
                             unsigned idx1 = i*this->n()*this->Nx() + j;
                             unsigned idx2 = ((coords[1]*this->n()*this->Ny()+i)*dims[0] + coords[0])*this->n()*this->Nx() + j;
-                            r_.data()[idx1] = g_.r()[idx2];
-                            z_.data()[idx1] = g_.z()[idx2];
+                            r_[idx1] = g_.r()[idx2];
+                            z_[idx1] = g_.z()[idx2];
                             xr_.data()[idx1] = g_.xr()[idx2];
                             xz_.data()[idx1] = g_.xz()[idx2];
                             yr_.data()[idx1] = g_.yr()[idx2];
@@ -273,7 +283,8 @@ struct CurvilinearMPIGrid2d : public dg::MPIGrid2d
                         }
     }
 
-    dg::MPI_Vector<thrust::host_vector<double> > r_, z_, xr_, xz_, yr_, yz_; //2d vector
+    thrust::host_vector<double> r_,z_;
+    dg::MPI_Vector<thrust::host_vector<double> > xr_, xz_, yr_, yz_; //2d vector
     dg::MPIContainer g_xx_, g_xy_, g_yy_, vol2d_;
     dg::CurvilinearGrid2d<LocalContainer> g_;
 };
diff --git a/inc/dg/geometry/mpi_grids.h b/inc/dg/geometry/mpi_grids.h
index 58b9ae40e..5b08f36f0 100644
--- a/inc/dg/geometry/mpi_grids.h
+++ b/inc/dg/geometry/mpi_grids.h
@@ -78,7 +78,7 @@ MPI_Vector< thrust::host_vector<double> > doPullback( BinaryOp f, const Geometry
 {
     thrust::host_vector<double> vec( g.size());
     for( unsigned i=0; i<g.size(); i++)
-        vec[i] = f( g.r().data()[i], g.z().data()[i]);
+        vec[i] = f( g.r()[i], g.z()[i]);
     MPI_Vector<thrust::host_vector<double> > v( vec, g.communicator());
     return v;
 }
@@ -88,11 +88,9 @@ MPI_Vector< thrust::host_vector<double> > doPullback( TernaryOp f, const Geometr
 {
     thrust::host_vector<double> vec( g.size());
     unsigned size2d = g.n()*g.n()*g.Nx()*g.Ny();
-    Grid1d gz( g.z0(), g.z1(), 1, g.Nz());
-    thrust::host_vector<double> absz = create::abscissas( gz);
     for( unsigned k=0; k<g.Nz(); k++)
         for( unsigned i=0; i<size2d; i++)
-            vec[k*size2d+i] = f( g.r().data()[k*size2d+i], g.z().data()[k*size2d+i], absz[k]);
+            vec[k*size2d+i] = f( g.r()[i], g.z()[i], g.phi()[k]);
     MPI_Vector<thrust::host_vector<double> > v( vec, g.communicator());
     return v;
 }
-- 
GitLab


From ec962710a2e31bec35f71c5ea0df67a169cb09fb Mon Sep 17 00:00:00 2001
From: Matthias Home Office <Matthias.Wiesenberger@uibk.ac.at>
Date: Wed, 26 Jul 2017 17:47:57 +0200
Subject: [PATCH 088/453] debug curvilinear

---
 inc/dg/geometry/curvilinear.h | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/inc/dg/geometry/curvilinear.h b/inc/dg/geometry/curvilinear.h
index f36bc20fd..3b9a5157f 100644
--- a/inc/dg/geometry/curvilinear.h
+++ b/inc/dg/geometry/curvilinear.h
@@ -122,8 +122,6 @@ struct CylindricalGrid3d : public dg::Grid3d
         for( unsigned k=1; k<this->Nz(); k++)
             for( unsigned i=0; i<Nx*Ny; i++)
             {
-                r_[k*Nx*Ny+i] = r_[(k-1)*Nx*Ny+i];
-                z_[k*Nx*Ny+i] = z_[(k-1)*Nx*Ny+i];
                 xr_[k*Nx*Ny+i] = xr_[(k-1)*Nx*Ny+i];
                 xz_[k*Nx*Ny+i] = xz_[(k-1)*Nx*Ny+i];
                 yr_[k*Nx*Ny+i] = yr_[(k-1)*Nx*Ny+i];
@@ -133,13 +131,16 @@ struct CylindricalGrid3d : public dg::Grid3d
     //compute metric elements from xr, xz, yr, yz, r and z
     void construct_metric( )
     {
-        thrust::host_vector<double> tempxx( r_), tempxy(r_), tempyy(r_), tempvol(r_);
-        for( unsigned i = 0; i<this->size(); i++)
+        thrust::host_vector<double> tempxx( xr_), tempxy(xr_), tempyy(xr_), tempvol(xr_), tempvol2d(xr_);
+        unsigned size2d = this->n()*this->n()*this->Nx()*this->Ny();
+        for( unsigned i = 0; i<this->Nz(); i++)
+        for( unsigned j = 0; j<size2d; j++)
         {
-            tempxx[i] = (xr_[i]*xr_[i]+xz_[i]*xz_[i]);
-            tempxy[i] = (yr_[i]*xr_[i]+yz_[i]*xz_[i]);
-            tempyy[i] = (yr_[i]*yr_[i]+yz_[i]*yz_[i]);
-            tempvol[i] = r_[i]/sqrt( tempxx[i]*tempyy[i] -tempxy[i]*tempxy[i] );
+            unsigned idx = i*size2d+j;
+            tempxx[idx] = (xr_[idx]*xr_[idx]+xz_[idx]*xz_[idx]);
+            tempxy[idx] = (yr_[idx]*xr_[idx]+yz_[idx]*xz_[idx]);
+            tempyy[idx] = (yr_[idx]*yr_[idx]+yz_[idx]*yz_[idx]);
+            tempvol[idx] = r_[j]/sqrt( tempxx[idx]*tempyy[idx] -tempxy[idx]*tempxy[idx] );
         }
         g_xx_=tempxx, g_xy_=tempxy, g_yy_=tempyy, vol_=tempvol;
         dg::blas1::pointwiseDivide( tempvol, r_, tempvol);
-- 
GitLab


From 9ea0645a137b86d86a8185649ae5722d8fe68167 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 27 Jul 2017 00:41:18 +0200
Subject: [PATCH 089/453] introduced handle class for cloneable objects and
 used in Grids and Functor containers

---
 inc/dg/backend/mpi_grid.h         |   7 +-
 inc/dg/geometry/curvilinear.h     | 134 ++++++++++-----------------
 inc/dg/geometry/generator.h       |  40 +++++++++
 inc/dg/geometry/mpi_curvilinear.h | 145 +++++++++++++-----------------
 inc/geometries/fluxfunctions.h    |  87 +++++++-----------
 5 files changed, 185 insertions(+), 228 deletions(-)

diff --git a/inc/dg/backend/mpi_grid.h b/inc/dg/backend/mpi_grid.h
index dd3a0e213..1d814a792 100644
--- a/inc/dg/backend/mpi_grid.h
+++ b/inc/dg/backend/mpi_grid.h
@@ -298,10 +298,11 @@ struct MPIGrid2d
     /**
      * @brief Return the global non-MPI grid 
      *
-     * The global grid contains the global boundaries and cell numbers. This is the grid that we would have to use in a non-MPI implementation.
+     * The global grid contains the global boundaries and cell numbers. 
+     * This is the grid that we would have to use in a non-MPI implementation.
      * @return non-MPI Grid object
      */
-    virtual const Grid2d& global() const {return g;}
+    const Grid2d& global() const {return g;}
     virtual ~MPIGrid2d(){}
     MPIGrid2d(const MPIGrid2d& src):g(src.g),comm(src.comm){}
     MPIGrid2d& operator=(const MPIGrid2d& src){
@@ -637,7 +638,7 @@ struct MPIGrid3d
     /**
      *@copydoc MPIGrid2d::global()const
      */
-    virtual const Grid3d& global() const {return g;}
+    const Grid3d& global() const {return g;}
     virtual ~MPIGrid3d(){}
     MPIGrid3d(const MPIGrid3d& src):g(src.g),comm(src.comm){}
     MPIGrid3d& operator=(const MPIGrid3d& src){
diff --git a/inc/dg/geometry/curvilinear.h b/inc/dg/geometry/curvilinear.h
index 3b9a5157f..cda264945 100644
--- a/inc/dg/geometry/curvilinear.h
+++ b/inc/dg/geometry/curvilinear.h
@@ -28,7 +28,7 @@ struct CylindricalGrid3d : public dg::Grid3d
     CylindricalGrid3d( double R0, double R1, double Z0, double Z1, double phi0, double phi1, unsigned n, unsigned NR, unsigned NZ, unsigned Nphi, bc bcr = PER, bc bcz = PER, bc bcphi = PER): 
         dg::Grid3d(0,R1-R0,0,Z1-Z0,phi0,phi1,n,NR,NZ,Nphi,bcR,bcZ,bcphi)
         {
-            generator_ = new IdentityGenerator(R0,R1,Z0,Z1);
+            handle_.set( new IdentityGenerator(R0,R1,Z0,Z1));
             construct( n, NR, NZ,Nphi);
         }
 
@@ -43,10 +43,10 @@ struct CylindricalGrid3d : public dg::Grid3d
      @param bcx boundary condition in x
      @note the boundary conditions for y and z are set periodic
      */
-    CylindricalGrid3d( const geo::aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx=dg::DIR):
-        dg::Grid3d( 0, generator->width(), 0., generator->height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, dg::PER, dg::PER)
+    CylindricalGrid3d( const geo::aGenerator& generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx=dg::DIR):
+        dg::Grid3d( 0, generator.width(), 0., generator.height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, dg::PER, dg::PER)
     { 
-        generator_ = generator;
+        handle_ = generator;
         construct( n, Nx, Ny,Nz);
     }
 
@@ -71,30 +71,10 @@ struct CylindricalGrid3d : public dg::Grid3d
     const container& g_pp()const{return g_pp_;}
     const container& vol()const{return vol_;}
     const container& perpVol()const{return vol2d_;}
-    const geo::aGenerator & generator() const{return *generator_;}
-    bool isOrthonormal() const { return generator_->isOrthonormal();}
-    bool isOrthogonal() const { return generator_->isOrthogonal();}
-    bool isConformal() const { return generator_->isConformal();}
-    CylindricalGrid3d( const CylindricalGrid3d& src):Grid3d(src), g_xx_(src.g_xx_),g_xy_(src.g_xy_),g_yy_(src.g_yy_),g_pp_(src.g_pp_),vol_(src.vol_),vol2d_(src.vol2d_)
-    {
-        r_=src.r_,z_=src.z_,phi_=src.phi_,xr_=src.xr_,xz_=src.xz_,yr_=src.yr_,yz_=src.yz_;
-        generator_ = src.generator_->clone();
-    }
-    CylindricalGrid3d& operator=( const CylindricalGrid3d& src)
-    {
-        Grid3d::operator=(src);//call base class assignment
-        if( &src!=this)
-        {
-            delete generator_;
-            g_xx_=src.g_xx_,g_xy_=src.g_xy_,g_yy_=src.g_yy_,g_pp_=src.g_pp_,vol_=src.vol_,vol2d_=src.vol2d_;
-            r_=src.r_,z_=src.z_,phi_=src.phi_,xr_=src.xr_,xz_=src.xz_,yr_=src.yr_,yz_=src.yz_;
-            generator_ = src.generator_->clone();
-        }
-        return *this;
-    }
-    virtual ~CylindricalGrid3d(){
-        delete generator_;
-    }
+    const geo::aGenerator & generator() const{return handle_.get()}
+    bool isOrthonormal() const { return handle_.get().isOrthonormal();}
+    bool isOrthogonal() const { return handle_.get().isOrthogonal();}
+    bool isConformal() const { return handle_.get().isConformal();}
     private:
     virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny,unsigned new_Nz){
         dg::Grid3d::do_set( new_n, new_Nx, new_Ny,new_Nz);
@@ -102,36 +82,31 @@ struct CylindricalGrid3d : public dg::Grid3d
     }
     void construct( unsigned n, unsigned Nx, unsigned Ny,unsigned Nz)
     {
-        dg::Grid1d gY1d( 0, generator_->height(), n, Ny, dg::PER);
-        dg::Grid1d gX1d( 0., generator_->width(), n, Nx);
+        dg::Grid1d gY1d( 0., y0(), n, Ny, dg::PER);
+        dg::Grid1d gX1d( 0., x0(), n, Nx);
         dg::Grid1d gphi( z0(), z1(), 1, Nz);
         phi_ = dg::create::abscissas( gphi);
         thrust::host_vector<double> x_vec = dg::evaluate( dg::cooX1d, gX1d);
         thrust::host_vector<double> y_vec = dg::evaluate( dg::cooX1d, gY1d);
-        (*generator_)( x_vec, y_vec, r_, z_, xr_, xz_, yr_, yz_);
-        init_X_boundaries( 0., generator_->width());
-        lift3d( ); //lift to 3D grid
-        construct_metric();
-    }
-    void lift3d( )
-    {
+        handle_.get()( x_vec, y_vec, r_, z_, xr_, xz_, yr_, yz_);
         //lift to 3D grid
         unsigned size = this->size();
         xr_.resize(size), yr_.resize( size), xz_.resize( size), yz_.resize(size);
-        unsigned Nx = this->n()*this->Nx(), Ny = this->n()*this->Ny();
+        unsigned size2d = this->n()*this->n()*this->Nx()*this->Ny();
         for( unsigned k=1; k<this->Nz(); k++)
-            for( unsigned i=0; i<Nx*Ny; i++)
+            for( unsigned i=0; i<size2d; i++)
             {
-                xr_[k*Nx*Ny+i] = xr_[(k-1)*Nx*Ny+i];
-                xz_[k*Nx*Ny+i] = xz_[(k-1)*Nx*Ny+i];
-                yr_[k*Nx*Ny+i] = yr_[(k-1)*Nx*Ny+i];
-                yz_[k*Nx*Ny+i] = yz_[(k-1)*Nx*Ny+i];
+                xr_[k*size2d+i] = xr_[(k-1)*size2d+i];
+                xz_[k*size2d+i] = xz_[(k-1)*size2d+i];
+                yr_[k*size2d+i] = yr_[(k-1)*size2d+i];
+                yz_[k*size2d+i] = yz_[(k-1)*size2d+i];
             }
+        construct_metric();
     }
     //compute metric elements from xr, xz, yr, yz, r and z
     void construct_metric( )
     {
-        thrust::host_vector<double> tempxx( xr_), tempxy(xr_), tempyy(xr_), tempvol(xr_), tempvol2d(xr_);
+        thrust::host_vector<double> tempxx( size()), tempxy(size()), tempyy(size()), tempvol(size()), tempvol2d(size()), temppp(size());
         unsigned size2d = this->n()*this->n()*this->Nx()*this->Ny();
         for( unsigned i = 0; i<this->Nz(); i++)
         for( unsigned j = 0; j<size2d; j++)
@@ -140,19 +115,22 @@ struct CylindricalGrid3d : public dg::Grid3d
             tempxx[idx] = (xr_[idx]*xr_[idx]+xz_[idx]*xz_[idx]);
             tempxy[idx] = (yr_[idx]*xr_[idx]+yz_[idx]*xz_[idx]);
             tempyy[idx] = (yr_[idx]*yr_[idx]+yz_[idx]*yz_[idx]);
-            tempvol[idx] = r_[j]/sqrt( tempxx[idx]*tempyy[idx] -tempxy[idx]*tempxy[idx] );
+            tempvol2d[idx] = 1./sqrt( tempxx[idx]*tempyy[idx] -tempxy[idx]*tempxy[idx] );
+            tempvol[idx] = r_[j]*tempvol2d[idx];
+            temppp[idx] = 1./r_[j]/r_[j];
         }
-        g_xx_=tempxx, g_xy_=tempxy, g_yy_=tempyy, vol_=tempvol;
-        dg::blas1::pointwiseDivide( tempvol, r_, tempvol);
-        vol2d_ = tempvol;
-        thrust::host_vector<double> ones = dg::evaluate( dg::one, *this);
-        dg::blas1::pointwiseDivide( ones, r_, tempxx);
-        dg::blas1::pointwiseDivide( tempxx, r_, tempxx); //1/R^2
-        g_pp_=tempxx;
+        //now transfer to device
+        dg::blas1::transfer(tempxx,     g_xx_);
+        dg::blas1::transfer(tempxy,     g_xy_);
+        dg::blas1::transfer(tempyy,     g_yy_);
+        dg::blas1::transfer(tempvol,    vol_);
+        dg::blas1::transfer(tempvol2d,  vol2d_);
+        dg::blas1::transfer(temppp,     g_pp_);
     }
-    thrust::host_vector<double> r_, z_, phi_, xr_, xz_, yr_, yz_;
+    thrust::host_vector<double> r_, z_, phi_;  //2d and 1d vectors
+    thrust::host_vector<double> xr_, xz_, yr_, yz_;
     container g_xx_, g_xy_, g_yy_, g_pp_, vol_, vol2d_;
-    geo::aGenerator* generator_;
+    dg::Handle<geo::aGenerator> handle_;
 };
 
 /**
@@ -164,33 +142,34 @@ struct CurvilinearGrid2d : public dg::Grid2d
     typedef dg::CurvilinearPerpTag metric_category;
 
     CurvilinearGrid2d( double R0, double R1, double Z0, double Z1, unsigned n, unsigned NR, unsigned NZ, unsigned Nphi, bc bcR = PER, bc bcZ = PER): 
-        dg::Grid2d(0,R1-R0,0,Z1-Z0,n,NR,NZ,bcR,bcZ)
+        dg::Grid2d(0,R1-R0,0,Z1-Z0,n,NR,NZ,bcR,bcZ), handle_(new IdentityGenerator(R0,R1,Z0,Z1))
         {
-            generator_ = new IdentityGenerator(R0,R1,Z0,Z1);
             construct( n, NR, NZ);
         }
     /*!@brief Constructor
     
-     * @param generator must generate an orthogonal grid
+     * @param generator must generate an orthogonal grid (class takes ownership of the pointer)
      * @param n number of polynomial coefficients
      * @param Nx number of cells in first coordinate
      @param Ny number of cells in second coordinate
      @param bcx boundary condition in first coordinate
      */
-    CurvilinearGrid2d( const geo::aGenerator* generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx=dg::DIR):
-        dg::Grid2d( 0, generator->width(), 0., generator->height(), n, Nx, Ny, bcx, dg::PER)
+    CurvilinearGrid2d( const geo::aGenerator& generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx=dg::DIR):
+        dg::Grid2d( 0, generator.width(), 0., generator.height(), n, Nx, Ny, bcx, dg::PER), handle_(generator)
     {
         construct( n,Nx,Ny);
     }
     CurvilinearGrid2d( const CylindricalGrid3d<container>& g):
-        dg::Grid2d( g.x0(), g.x1(), g.y0(), g.y1(), g.n(), g.Nx(), g.Ny(), g.bcx(), g.bcy())
+        dg::Grid2d( g.x0(), g.x1(), g.y0(), g.y1(), g.n(), g.Nx(), g.Ny(), g.bcx(), g.bcy()), handle_(g.generator())
     {
-        generator_ = g.generator();
+        r_ = g.r();
+        z_ = g.z();
         unsigned s = this->size();
-        r_.resize( s), z_.resize(s), xr_.resize(s), xz_.resize(s), yr_.resize(s), yz_.resize(s);
+        xr_.resize(s), xz_.resize(s), yr_.resize(s), yz_.resize(s);
         g_xx_.resize( s), g_xy_.resize(s), g_yy_.resize(s), vol2d_.resize(s);
-        for( unsigned i=0; i<s; i++)
-        { r_[i]=g.r()[i], z_[i]=g.z()[i], xr_[i]=g.xr()[i], xz_[i]=g.xz()[i], yr_[i]=g.yr()[i], yz_[i]=g.yz()[i];}
+        for( unsigned i=0; i<s; i++) { 
+            xr_[i]=g.xr()[i], xz_[i]=g.xz()[i], yr_[i]=g.yr()[i], yz_[i]=g.yz()[i];
+        }
         thrust::copy( g.g_xx().begin(), g.g_xx().begin()+s, g_xx_.begin());
         thrust::copy( g.g_xy().begin(), g.g_xy().begin()+s, g_xy_.begin());
         thrust::copy( g.g_yy().begin(), g.g_yy().begin()+s, g_yy_.begin());
@@ -208,31 +187,10 @@ struct CurvilinearGrid2d : public dg::Grid2d
     const container& g_xy()const{return g_xy_;}
     const container& vol()const{return vol2d_;}
     const container& perpVol()const{return vol2d_;}
-    const geo::aGenerator& generator() const{return *generator_;}
+    const geo::aGenerator& generator() const{return handle_.get();}
     bool isOrthonormal() const { return generator_->isOrthonormal();}
     bool isOrthogonal() const { return generator_->isOrthogonal();}
     bool isConformal() const { return generator_->isConformal();}
-
-    Curvilinear2d( const Curvilinear2d& src):Grid2d(src), g_xx_(src.g_xx_),g_xy_(src.g_xy_),g_yy_(src.g_yy_),vol2d_(src.vol2d_)
-    {
-        r_=src.r_,z_=src.z_,xr_=src.xr_,xz_=src.xz_,yr_=src.yr_,yz_=src.yz_;
-        generator_ = src.generator_->clone();
-    }
-    Curvilinear2d& operator=( const Curvilinear2d& src)
-    {
-        Grid2d::operator=(src); //call base class assignment
-        if( &src!=this)
-        {
-            delete generator_;
-            g_xx_=src.g_xx_,g_xy_=src.g_xy_,g_yy_=src.g_yy_,vol2d_=src.vol2d_;
-            r_=src.r_,z_=src.z_,xr_=src.xr_,xz_=src.xz_,yr_=src.yr_,yz_=src.yz_;
-            generator_ = src.generator_->clone();
-        }
-        return *this;
-    }
-    virtual ~Curvilinear2d(){
-        delete generator_;
-    }
     private:
     virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny)
     {
@@ -241,14 +199,14 @@ struct CurvilinearGrid2d : public dg::Grid2d
     }
     void construct( unsigned n, unsigned Nx, unsigned Ny)
     {
-        CylindricalGrid3d<container> g( generator_, n,Nx,Ny,1,bcx());
+        CylindricalGrid3d<container> g( handle_, n,Nx,Ny,1,bcx());
         r_=g.r(), z_=g.z(), xr_=g.xr(), xz_=g.xz(), yr_=g.yr(), yz_=g.yz();
         g_xx_=g.g_xx(), g_xy_=g.g_xy(), g_yy_=g.g_yy();
         vol2d_=g.perpVol();
     }
     thrust::host_vector<double> r_, z_, xr_, xz_, yr_, yz_;
     container g_xx_, g_xy_, g_yy_, vol2d_;
-    geo::aGenerator* generator_;
+    dg::Hanle<geo::aGenerator> handle_;
 };
 
 ///@}
diff --git a/inc/dg/geometry/generator.h b/inc/dg/geometry/generator.h
index 48b8e3708..b2ffd0510 100644
--- a/inc/dg/geometry/generator.h
+++ b/inc/dg/geometry/generator.h
@@ -130,4 +130,44 @@ struct IdentityGenerator: public aGridGenerator
 };
 
 }//namespace geo
+///@cond
+
+//there is probably a better class in boost...
+///Helper class to avoid rule of three in grid classes
+///Actually it's a very very basic smart pointer that implements 
+///a deep copy using the clone() method 
+template<class cloneable>
+struct Handle
+{
+    Handle():ptr_(0){}
+    /// take ownership of the pointer
+    Handle( cloneable* ptr): ptr_(ptr){}
+    /// clone given value
+    Handle( const cloneable& src): ptr_(src.clone()){}
+    Handle( const Handle& src):ptr_(0) {
+        if(ptr_!=0) ptr_ = src.ptr->clone(); //deep copy
+    }
+    Handle& operator=( Handle src) {
+        this->swap( src );
+        return *this;
+    }
+    ~Handle(){
+        if(ptr_!=0) delete ptr_;
+    }
+    const cloneable& get()const {return *ptr_;}
+    void set( cloneable* ptr){ 
+        Handle tmp(ptr);
+        *this=tmp;
+    }
+    void set( const cloneable& src){ 
+        Handle tmp(src);
+        *this=tmp;
+    }
+    void swap( Handle& src){
+        std::swap(ptr_,src.ptr_);
+    }
+    private:
+    cloneable* ptr_;
+};
+///@endcond
 }//namespace dg
diff --git a/inc/dg/geometry/mpi_curvilinear.h b/inc/dg/geometry/mpi_curvilinear.h
index b6d16c360..aa2328996 100644
--- a/inc/dg/geometry/mpi_curvilinear.h
+++ b/inc/dg/geometry/mpi_curvilinear.h
@@ -36,7 +36,7 @@ struct CylindricalMPIGrid3d : public dg::MPIGrid3d
      * @note the paramateres given in the constructor are global parameters 
      */
     CylindricalMPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, MPI_Comm comm): 
-        dg::MPIGrid3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, comm),
+        dg::MPIGrid3d( 0, x1-x0, 0, y1-y0, z0, z1, n, Nx, Ny, Nz, comm),
         g( new IdentityGenerator(x0,x1,y0,y1), n,Nx,Ny,local().Nz(), bcx,bcy,bcz)
      {
         divide_and_conquer();
@@ -49,16 +49,18 @@ struct CylindricalMPIGrid3d : public dg::MPIGrid3d
      */
     CylindricalMPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz, MPI_Comm comm):
         dg::MPIGrid3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, bcx, bcy, bcz, comm),
-        g( new IdentityGenerator(x0,x1,y0,y1), n,Nx,Ny,local().Nz(), bcx,bcy,bcz)
+        handle_( new IdentityGenerator(x0,x1,y0,y1))
      {
-        divide_and_conquer();
+        CylindricalGrid3d<LocalContainer> g(generator,n,Nx,Ny,this->Nz());
+        divide_and_conquer(g);
      }
 
     CylindricalMPIGrid3d( const geo::aGenerator& generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx, MPI_Comm comm): 
         dg::MPIGrid3d( 0, generator.width(), 0., generator.height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, dg::PER, dg::PER, comm),
-        g( generator, n,Nx, Ny, local().Nz(), bcx)
+        handle_( generator)
     {
-        divide_and_conquer();
+        CylindricalGrid3d<LocalContainer> g(generator,n,Nx,Ny,this->Nz());
+        divide_and_conquer(g);
     }
 
     perpendicular_grid perp_grid() const { return perpendicular_grid(*this);}
@@ -76,38 +78,18 @@ struct CylindricalMPIGrid3d : public dg::MPIGrid3d
     const MPIContainer& g_pp()const{return g_pp_;}
     const MPIContainer& vol()const{return vol_;}
     const MPIContainer& perpVol()const{return vol2d_;}
-    /**
-     * @brief returns global non-MPI grid
-     *
-     * @attention the number of z-planes in this grid is the local number this process holds NOT the global one
-     * @return 
-     */
-    const dg::CylindricalGrid3d<LocalContainer>& global() const {return g;}
     const geo::aGenerator& generator() const{return g.generator();}
     bool isOrthonormal() const { return g.isOrthonormal();}
     bool isOrthogonal() const { return g.isOrthogonal();}
     bool isConformal() const { return g.isConformal();}
-    MPICylindricalGrid3d( const MPICylindricalGrid3d& src):MPIGrid3d(src), g_xx_(src.g_xx_),g_xy_(src.g_xy_),g_yy_(src.g_yy_),g_pp_(src.g_pp_),vol_(src.vol_),vol2d_(src.vol2d_),g(src.g)
-    {
-        r_=src.r_,z_=src.z_,phi_=src.phi_,xr_=src.xr_,xz_=src.xz_,yr_=src.yr_,yz_=src.yz_;
-    }
-    MPICylindricalGrid3d& operator=( const MPICylindricalGrid3d& src)
-    {
-        MPIGrid3d::operator=(src);//call base class assignment
-        g_xx_=src.g_xx_,g_xy_=src.g_xy_,g_yy_=src.g_yy_,g_pp_=src.g_pp_,vol_=src.vol_,vol2d_=src.vol2d_;
-        r_=src.r_,z_=src.z_,phi_=src.phi_,xr_=src.xr_,xz_=src.xz_,yr_=src.yr_,yz_=src.yz_;
-        g = src.g;
-        return *this;
-    }
-    ~MPICylindricalGrid3d(){ }
     private:
     virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz)
     {
         dg::MPIGrid3d::do_set(new_n, new_Nx, new_Ny, new_Nz);
-        g.set( new_n, new_Nx, new_Ny, new_Nz);//construct new grid
-        divide_and_conquer();//distribute to processes
+        dg::CylindricalGrid3d<thrust::host_vector> g( handle_.get(), new_n, new_Nx,new_Ny,Nz());
+        divide_and_conquer(g);//distribute to processes
     }
-    void divide_and_conquer( )
+    void divide_and_conquer( const dg::CylindricalGrid3d<thrust::host_vector<double> & g )
     {
         r_.resize( this->n()*this->n()*this->Nx()*this->Ny());
         z_ = r_;
@@ -115,6 +97,7 @@ struct CylindricalMPIGrid3d : public dg::MPIGrid3d
         xr_=dg::evaluate( dg::one, *this), xz_=xr_, yr_=xr_, yz_=xr_;
         dg::blas1::transfer( xr_, g_xx_);
         g_xy_=g_xx_, g_yy_=g_xx_, g_pp_=g_xx_, vol_=g_xx_, vol2d_=g_xx_;
+        thrust::host_vector<double> tempxx(size()), tempxy(size()),tempyy(size()),temppp(size()),tempvol2d(size()),tempvol(size());
         //divide and conquer
         int dims[3], periods[3], coords[3];
         MPI_Cart_get( comm, 3, dims, periods, coords);
@@ -136,18 +119,25 @@ struct CylindricalMPIGrid3d : public dg::MPIGrid3d
                             xz_.data()[idx1] = g.xz()[idx2];
                             yr_.data()[idx1] = g.yr()[idx2];
                             yz_.data()[idx1] = g.yz()[idx2];
-                            g_xx_.data()[idx1] = g.g_xx()[idx2];
-                            g_xy_.data()[idx1] = g.g_xy()[idx2];
-                            g_yy_.data()[idx1] = g.g_yy()[idx2];
-                            g_pp_.data()[idx1] = g.g_pp()[idx2];
-                            vol_.data()[idx1] = g.vol()[idx2];
-                            vol2d_.data()[idx1] = g.perpVol()[idx2];
+
+                            tempxx[idx1] = g.g_xx()[idx2];
+                            tempxy[idx1] = g.g_xy()[idx2];
+                            tempyy[idx1] = g.g_yy()[idx2];
+                            temppp[idx1] = g.g_pp()[idx2];
+                            tempvol[idx1] = g.vol()[idx2];
+                            tempvol2d[idx1] = g.perpVol()[idx2];
                         }
+        dg::blas1::transfer( tempxx, g_xx_.data());
+        dg::blas1::transfer( tempxy, g_xy_.data());
+        dg::blas1::transfer( tempyy, g_yy_.data());
+        dg::blas1::transfer( temppp, g_pp_.data());
+        dg::blas1::transfer( tempvol, vol_.data());
+        dg::blas1::transfer( tempvol2d, vol2d_.data());
     }
     thrust::host_vector<double> r_,z_,phi_;
     dg::MPI_Vector<thrust::host_vector<double> > xr_, xz_, yr_, yz_; //3d vector
     MPIContainer g_xx_, g_xy_, g_yy_, g_pp_, vol_, vol2d_;
-    dg::CylindricalGrid3d<LocalContainer> g;
+    Handle<dg::geo::aGenerator> handle_;
 };
 
 /**
@@ -174,22 +164,23 @@ struct CurvilinearMPIGrid2d : public dg::MPIGrid2d
      * @param comm a two-dimensional Cartesian communicator
      * @note the paramateres given in the constructor are global parameters 
      */
-    CurvilinearMPIGrid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx, bc bcy, MPI_Comm comm):dg::MPIGrid2d( x0, x1, y0, y1, n, Nx, Ny,bcx, bcy, comm),
-    g_(new IdentityGenerator(x0,x1,y0,y1),n,Nx,Ny){
-        divide_and_conquer();
+    CurvilinearMPIGrid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx, bc bcy, MPI_Comm comm):dg::MPIGrid2d( 0, x1-x0, 0, y1-y0, n, Nx, Ny,bcx, bcy, comm),
+    handle_(new IdentityGenerator(x0,x1,y0,y1)){
+        dg::CurvilinearGrid2d<thrust::host_vector<double> > g(generator, n, Nx, Ny);
+        divide_and_conquer(g);
     }
 
     CurvilinearMPIGrid2d( const geo::aGenerator& generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx, MPI_Comm comm2d): 
-        dg::MPIGrid2d( 0, generator.width(), 0., generator.height(), n, Nx, Ny, bcx, dg::PER, comm2d),
-        g_( generator, n,Nx, Ny, bcx)
+        dg::MPIGrid2d( 0, generator.width(), 0., generator.height(), n, Nx, Ny, bcx, dg::PER, comm2d),handle_(generator)
     {
-        divide_and_conquer();
+        dg::CurvilinearGrid2d<thrust::host_vector<double> > g(generator, n, Nx, Ny);
+        divide_and_conquer(g);
     }
     CurvilinearMPIGrid2d( const CylindricalMPIGrid3d<LocalContainer>& g):
         dg::MPIGrid2d( g.global().x0(), g.global().x1(), g.global().y0(), g.global().y1(), g.global().n(), g.global().Nx(), g.global().Ny(), g.global().bcx(), g.global().bcy(), get_reduced_comm( g.communicator() )),
         xr_(dg::evaluate( dg::one, *this)), xz_(xr_), yr_(xr_), yz_(xr_), 
         g_xx_(xr_), g_xy_(g_xx_), g_yy_(g_xx_), vol2d_(g_xx_),
-        g_( g.global())
+        handle_(g.generator())
     {
         r_=g.r(); 
         z_=g.z(); 
@@ -205,7 +196,6 @@ struct CurvilinearMPIGrid2d : public dg::MPIGrid2d
         thrust::copy( g.g_xy().data().begin(), g.g_xy().data().begin()+s, g_xy_.data().begin());
         thrust::copy( g.g_yy().data().begin(), g.g_yy().data().begin()+s, g_yy_.data().begin());
         thrust::copy( g.perpVol().data().begin(), g.perpVol().data().begin()+s, vol2d_.data().begin());
-        
     }
 
     const thrust::host_vector<double>& r()const{return r_;}
@@ -219,31 +209,16 @@ struct CurvilinearMPIGrid2d : public dg::MPIGrid2d
     const MPIContainer& g_xy()const{return g_xy_;}
     const MPIContainer& vol()const{return vol2d_;}
     const MPIContainer& perpVol()const{return vol2d_;}
-    const dg::CurvilinearGrid2d<LocalContainer>& global() const {return g_;}
     const geo::aGenerator& generator() const{return g.generator();}
     bool isOrthonormal() const { return g.isOrthonormal();}
     bool isOrthogonal() const { return g.isOrthogonal();}
     bool isConformal() const { return g.isConformal();}
-    MPICurvilinear2d( const MPICurvilinear2d& src):MPIGrid2d(src), g_xx_(src.g_xx_),g_xy_(src.g_xy_),g_yy_(src.g_yy_),vol2d_(src.vol2d_)
-    {
-        r_=src.r_,z_=src.z_,xr_=src.xr_,xz_=src.xz_,yr_=src.yr_,yz_=src.yz_;
-        g_ = src.g_;
-    }
-    MPICurvilinear2d& operator=( const MPICurvilinear2d& src)
-    {
-        MPIGrid2d::operator=(src); //call base class assignment
-        g_xx_=src.g_xx_,g_xy_=src.g_xy_,g_yy_=src.g_yy_,vol2d_=src.vol2d_;
-        r_=src.r_,z_=src.z_,xr_=src.xr_,xz_=src.xz_,yr_=src.yr_,yz_=src.yz_;
-        g_ = src.g_;
-        return *this;
-    }
-    ~MPICurvilinear2d(){ }
     private:
-    virtual void do set( unsigned new_n, unsigned new_Nx, unsigned new_Ny)
+    virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny)
     {
         dg::MPIGrid3d::do_set(new_n, new_Nx, new_Ny);
-        g.set( new_n, new_Nx, new_Ny);//construct new grid
-        divide_and_conquer();//distribute to processes
+        dg::CurvilinearGrid2d<thrust::host_vector<double> > g( handle_.get(), new_n, new_Nx, new_Ny);
+        divide_and_conquer(g);//distribute to processes
     }
     MPI_Comm get_reduced_comm( MPI_Comm src)
     {
@@ -252,41 +227,45 @@ struct CurvilinearMPIGrid2d : public dg::MPIGrid2d
         MPI_Cart_sub( src, remain_dims, &planeComm);
         return planeComm;
     }
-    void divide_and_conquer()
+    void divide_and_conquer(const dg::CurvilinearGrid2d<thrust::host_vector<double> >& g_)
     {
-        r_.resize( this->n()*this->n()*this->Nx()*this->Ny());
+        r_.resize( size());
         z_=r_;
         xr_=dg::evaluate( dg::one, *this), xz_=xr_, yr_=xr_, yz_=xr_;
+        thrust::host_vector<double> tempxx(size()), tempxy(size()),tempyy(size()), tempvol(size());
         dg::blas1::transfer( xr_, g_xx_);
         g_xy_=g_xx_, g_yy_=g_xx_, vol2d_=g_xx_;
         //divide and conquer
         int dims[2], periods[2], coords[2];
         MPI_Cart_get( communicator(), 2, dims, periods, coords);
-        init_X_boundaries( g_.x0(), g_.x1());
-            //for( unsigned py=0; py<dims[1]; py++)
-                for( unsigned i=0; i<this->n()*this->Ny(); i++)
-                    //for( unsigned px=0; px<dims[0]; px++)
-                        for( unsigned j=0; j<this->n()*this->Nx(); j++)
-                        {
-                            unsigned idx1 = i*this->n()*this->Nx() + j;
-                            unsigned idx2 = ((coords[1]*this->n()*this->Ny()+i)*dims[0] + coords[0])*this->n()*this->Nx() + j;
-                            r_[idx1] = g_.r()[idx2];
-                            z_[idx1] = g_.z()[idx2];
-                            xr_.data()[idx1] = g_.xr()[idx2];
-                            xz_.data()[idx1] = g_.xz()[idx2];
-                            yr_.data()[idx1] = g_.yr()[idx2];
-                            yz_.data()[idx1] = g_.yz()[idx2];
-                            g_xx_.data()[idx1] = g_.g_xx()[idx2];
-                            g_xy_.data()[idx1] = g_.g_xy()[idx2];
-                            g_yy_.data()[idx1] = g_.g_yy()[idx2];
-                            vol2d_.data()[idx1] = g_.perpVol()[idx2];
-                        }
+        //for( unsigned py=0; py<dims[1]; py++)
+            for( unsigned i=0; i<this->n()*this->Ny(); i++)
+                //for( unsigned px=0; px<dims[0]; px++)
+                    for( unsigned j=0; j<this->n()*this->Nx(); j++)
+                    {
+                        unsigned idx1 = i*this->n()*this->Nx() + j;
+                        unsigned idx2 = ((coords[1]*this->n()*this->Ny()+i)*dims[0] + coords[0])*this->n()*this->Nx() + j;
+                        r_[idx1] = g_.r()[idx2];
+                        z_[idx1] = g_.z()[idx2];
+                        xr_.data()[idx1] = g_.xr()[idx2];
+                        xz_.data()[idx1] = g_.xz()[idx2];
+                        yr_.data()[idx1] = g_.yr()[idx2];
+                        yz_.data()[idx1] = g_.yz()[idx2];
+                        tempxx[idx1] = g_.g_xx()[idx2];
+                        tempxy[idx1] = g_.g_xy()[idx2];
+                        tempyy[idx1] = g_.g_yy()[idx2];
+                        tempvol2d[idx1] = g_.perpVol()[idx2];
+                    }
+        dg::blas1::transfer( tempxx, g_xx_.data());
+        dg::blas1::transfer( tempxy, g_xy_.data());
+        dg::blas1::transfer( tempyy, g_yy_.data());
+        dg::blas1::transfer( tempvol, vol2d_.data());
     }
 
     thrust::host_vector<double> r_,z_;
     dg::MPI_Vector<thrust::host_vector<double> > xr_, xz_, yr_, yz_; //2d vector
     dg::MPIContainer g_xx_, g_xy_, g_yy_, vol2d_;
-    dg::CurvilinearGrid2d<LocalContainer> g_;
+    dg::Handle<dg::geo::aGenerator> handle_;
 };
 ///@}
 }//namespace dg
diff --git a/inc/geometries/fluxfunctions.h b/inc/geometries/fluxfunctions.h
index 32b0de398..5957e6e4a 100644
--- a/inc/geometries/fluxfunctions.h
+++ b/inc/geometries/fluxfunctions.h
@@ -97,38 +97,10 @@ struct Adapter : public dg::geo::aCloneableBinaryFunctor<Adapter>
 temmplate<class BinaryFunctor>
 aBinaryFunctor* make_aBinaryFunctor(const BinaryFunctor& f){return new Adapter<BinaryFunctor>(f);}
 
-/**
-* @brief Manage and deep copy binary Functors on the heap
-*/
-struct aBinaryFunctorBundle
-{
-    protected:
-    /// constructor is deleted
-    aBinaryFunctorBundle();
-    aBinaryFunctorBundle( const aBinaryFunctorBundle& b)
-    {
-        p_.resize( b.p_.size());
-        //deep copy
-        for( unsigned i=0; i<p_.size(); i++)
-            p_[i] = b.p_[i]->clone();
-    }
-    aBinaryFunctorBundle& operator=( const aBinaryFunctorBundle& b)
-    {
-        aBinaryFunctorBundle temp(b);
-        std::swap( temp.p_, p_);
-        return *this;
-    }
-    ~aBinaryFunctorBundle(){
-        for( unsigned i=0; i<p_.size(); i++)
-            delete p_[i];
-    }
-    std::vector<aBinaryFunctor*> p_;
-};
-
 /**
 * @brief This struct bundles a function and its first derivatives
 */
-struct BinaryFunctorsLvl1 : public aBinaryFunctorBundle
+struct BinaryFunctorsLvl1 
 {
     /**
     * @brief Take ownership of newly allocated functors
@@ -137,18 +109,20 @@ struct BinaryFunctorsLvl1 : public aBinaryFunctorBundle
     * @param fx \f$ \partial f / \partial x \f$ its derivative in the first coordinate
     * @param fy \f$ \partial f / \partial y \f$ its derivative in the second coordinate
     */
-    BinaryFunctorsLvl1( aBinaryFunctor* f, aBinaryFunctor* fx, aBinaryFunctor* fy): p_(3)
+    BinaryFunctorsLvl1( aBinaryFunctor* f, aBinaryFunctor* fx, aBinaryFunctor* fy): 
     {
-        p_[0] = f;
-        p_[1] = fx;
-        p_[2] = fy;
+        p_[0].set(f);
+        p_[1].set(fx);
+        p_[2].set(fy);
     }
     /// \f$ f \f$
-    const aBinaryFunctor& f()const{return *p_[0];}
+    const aBinaryFunctor& f()const{return p_[0].get();}
     /// \f$ \partial f / \partial x \f$ 
-    const aBinaryFunctor& dfx()const{return *p_[1];}
+    const aBinaryFunctor& dfx()const{return p_[1].get();}
     /// \f$ \partial f / \partial y\f$
-    const aBinaryFunctor& dfy()const{return *p_[2];}
+    const aBinaryFunctor& dfy()const{return p_[2].get();}
+    private:
+    Handle<aBinaryFunctor> p_[3];
 };
 /**
 * @brief This struct bundles a function and its first and second derivatives
@@ -164,21 +138,24 @@ struct BinaryFunctorsLvl2 : public BinaryFunctorsLvl1
     BinaryFunctorsLvl2( aBinaryFunctor* f, aBinaryFunctor* fx, aBinaryFunctor* fy,
     aBinaryFunctor* fxx, aBinaryFunctor* fxy, aBinaryFunctor* fyy): BinaryFunctorsLvl1(f,fx,fy) 
     {
-        p_.push_back( fxx);
-        p_.push_back( fxy);
-        p_.push_back( fyy);
+        p_[0].set( fxx);
+        p_[1].set( fxy);
+        p_[2].set( fyy);
     }
     /// \f$ \partial^2f/\partial x^2\f$
-    const aBinaryFunctor& dfxx()const{return *p_[3];}
+    const aBinaryFunctor& dfxx()const{return p_[0].get();}
     /// \f$ \partial^2 f / \partial x \partial y\f$
-    const aBinaryFunctor& dfxy()const{return *p_[4];}
+    const aBinaryFunctor& dfxy()const{return p_[1].get();}
     /// \f$ \partial^2f/\partial y^2\f$
-    const aBinaryFunctor& dfyy()const{return *p_[5];}
+    const aBinaryFunctor& dfyy()const{return p_[2].get();}
+    private:
+    Handle<aBinaryFunctor> p_[3];
 };
+
 /**
 * @brief This struct bundles a symmetric tensor and its divergence
 */
-struct BinarySymmTensorLvl1 : public aBinaryFunctorBundle
+struct BinarySymmTensorLvl1
 {
     /**
      * @brief Take ownership of newly allocated functors
@@ -191,24 +168,26 @@ struct BinarySymmTensorLvl1 : public aBinaryFunctorBundle
      * @param divChiY \f$ \partial_x \chi^{xy} + \partial_y\chi^{yy}\f$ is the y-component of the divergence of the tensor \f$ \chi \f$
     */
     BinarySymmTensorLvl1( aBinaryFunctor* chi_xx, aBinaryFunctor* chi_xy, aBinaryFunctor* chi_yy,
-    aBinaryFunctor* divChiX, aBinaryFunctor* divChiY): p_(5)
+    aBinaryFunctor* divChiX, aBinaryFunctor* divChiY)
     {
-        p_[0] = chi_xx;
-        p_[1] = chi_xy;
-        p_[2] = chi_yy;
-        p_[3] = divChiX;
-        p_[4] = divChiY;
+        p_[0].set( chi_xx);
+        p_[1].set( chi_xy);
+        p_[2].set( chi_yy);
+        p_[3].set( divChiX);
+        p_[4].set( divChiY);
     }
     ///xy component \f$ \chi^{xx}\f$ 
-    const aBinaryFunctor& xx()const{return *p_[0];}
+    const aBinaryFunctor& xx()const{return p_[0].get();}
     ///xy component \f$ \chi^{xy}\f$ 
-    const aBinaryFunctor& xy()const{return *p_[1];}
+    const aBinaryFunctor& xy()const{return p_[1].get();}
     ///yy component \f$ \chi^{yy}\f$ 
-    const aBinaryFunctor& yy()const{return *p_[2];}
+    const aBinaryFunctor& yy()const{return p_[2].get();}
      /// \f$ \partial_x \chi^{xx} + \partial_y\chi^{yx}\f$ is the x-component of the divergence of the tensor \f$ \chi\f$
-    const aBinaryFunctor& divX()const{return *p_[3];}
+    const aBinaryFunctor& divX()const{return p_[3].get();}
      /// \f$ \partial_x \chi^{xy} + \partial_y\chi^{yy}\f$ is the y-component of the divergence of the tensor \f$ \chi \f$
-    const aBinaryFunctor& divY()const{return *p_[4];}
+    const aBinaryFunctor& divY()const{return p_[4].get();}
+    private:
+    Handle<aBinaryFunctor> p_[5];
 };
 
 struct Constant:public aCloneableBinaryOperator<Constant> 
-- 
GitLab


From c6971b1ce1eabf5961aaa0ebf80288e735a693ad Mon Sep 17 00:00:00 2001
From: Matthias Home Office <Matthias.Wiesenberger@uibk.ac.at>
Date: Fri, 28 Jul 2017 14:58:25 +0200
Subject: [PATCH 090/453] introduced Vector function container

---
 inc/dg/geometry.h                 | 28 +++++++++++++++-------------
 inc/dg/geometry/curvilinear.h     |  4 ++++
 inc/dg/geometry/generator.h       |  2 +-
 inc/dg/geometry/geometry_traits.h | 19 +++++++++++++++++++
 inc/geometries/ds.h               |  6 +++---
 inc/geometries/fieldaligned.h     | 18 +++++++-----------
 inc/geometries/fluxfunctions.h    | 29 +++++++++++++++++++++++------
 7 files changed, 72 insertions(+), 34 deletions(-)

diff --git a/inc/dg/geometry.h b/inc/dg/geometry.h
index 3e96fb72b..3b4c00909 100644
--- a/inc/dg/geometry.h
+++ b/inc/dg/geometry.h
@@ -25,19 +25,14 @@
  */
 
 /*
-Comment: there is a little design flaw in basing the geometry routines also on template programming. 
-For example it is a flaw that we have to recompile when we want to use an Orthogonal instead
-of a Curvilinear grid. Also, the performance gain from using template should be minimal.
-At the same time we cannot get rid of the distinction between Grids and MPIGrids. 
-There is also nothing wrong with recompiling there 
-as this is always accompanied with changing the hardware the program runs on. 
-Now, the question is how severe this flaw actually affects execution in 
-real life, i.e. do users want to change between grids during a parameter scan?
-It might not be that severe especially with the generator mechanism to 
-construct grids. 
-If ever a problem: A possible solution could be to make the geometry functions virtual 
-in the Grid and the MPI_Grid each (we still need both). Then objects would have to
-keep pointers to the grid-base-class (Grid* or MPIGrid*) and work through these.
+Comment: we want to use template functions for the geometry routines
+because we fundamentally have to distinguish between Grid and MPIGrid. 
+A change in hardware (from shared to distributed memory) always 
+necessitates a recompilation. 
+We try to avoid recompilation when we change the grid 
+at least for the cases in which you don't have to change the model anyways.
+The point is that these functions are not really performance critical 
+so we shouldn't make the distinctions too fine grained.
 */
 namespace dg{
 
@@ -194,6 +189,13 @@ void pushForwardPerp(
     pushForwardPerp<double(double, double, double), double(double, double, double), Geometry>( fR, fZ, out1, out2, g); 
 }
 ///@endcond
+template<class Functor1, class Functor2, class Functor3 class container, class Geometry> 
+void pushForward( Functor1 vR, Functor2 vZ, Functor3 vPhi,
+        container& vx, container& vy, container& vz,
+        const Geometry& g)
+{
+    dg::geo::detail::doPushForward( vR, vZ,vPhi,vx, vy,vz, g, typename GeometryTraits<Geometry>::metric_category() ); 
+}
 
 /**
  * @brief Push forward a symmetric 2d tensor from cylindrical or Cartesian to a new coordinate system
diff --git a/inc/dg/geometry/curvilinear.h b/inc/dg/geometry/curvilinear.h
index cda264945..568d0f670 100644
--- a/inc/dg/geometry/curvilinear.h
+++ b/inc/dg/geometry/curvilinear.h
@@ -15,6 +15,10 @@ template< class container>
 struct CurvilinearGrid2d; 
 ///@endcond
 
+//when we make a 3d grid with eta and phi swapped the metric structure and the transformation changes 
+//In practise it can only be orthogonal due to the projection matrix in the elliptic operator
+
+
 /**
  * @brief A three-dimensional grid based on curvilinear coordinates
  @tparam container models aContainer
diff --git a/inc/dg/geometry/generator.h b/inc/dg/geometry/generator.h
index b2ffd0510..0720ae1cb 100644
--- a/inc/dg/geometry/generator.h
+++ b/inc/dg/geometry/generator.h
@@ -31,7 +31,7 @@ struct aGridGenerator
     * @param zetaY (output) the list of \f$ N_\eta N_\zeta\f$ elements \f$ \partial\zeta/\partial y (\zeta_i, \eta_j)\f$ 
     * @param etaX (output) the list of \f$ N_\eta N_\zeta\f$ elements \f$ \partial\eta/\partial x (\zeta_i, \eta_j)\f$ 
     * @param etaY (output) the list of \f$ N_\eta N_\zeta\f$ elements \f$ \partial\eta/\partial y (\zeta_i, \eta_j)\f$ 
-    * @note the \f$ \zeta\f$ coordinate is contiguous in memory
+    * @note the first (\f$ \zeta\f$) coordinate shall be constructed contiguously in memory, i.e. the resuling lists are \f$ x_0=x(\zeta_0, \eta_0),\ x_1=x(\zeta_1, \eta_0)\ x_2=x(\zeta_2, \eta_0)\dots x_{NM-1}=x(\zeta_{N-1} \eta_{M-1})\f$
     * @note All the resulting vectors are write-only and get properly resized
     */
     void operator()( 
diff --git a/inc/dg/geometry/geometry_traits.h b/inc/dg/geometry/geometry_traits.h
index 469d6c251..65c9d4182 100644
--- a/inc/dg/geometry/geometry_traits.h
+++ b/inc/dg/geometry/geometry_traits.h
@@ -172,6 +172,25 @@ void doPushForwardPerp( TernaryOp1 f1, TernaryOp2 f2,
     dg::blas1::transfer( out2, vy);
 }
 
+template<class TernaryOp1, class TernaryOp2, class TernaryOp3 class container, class Geometry> 
+void doPushForward( TernaryOp1 f1, TernaryOp2 f2, TernaryOp3,
+        container& vx, container& vy, container& vz,
+        const Geometry& g, CurvilinearPerpTag)
+{
+    typedef typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector host_vec;
+    doPushForwardPerp(f1,f2,vx,vy,CurvilinearPerpTag());
+    host_vec out3 = pullback( f3, g), temp3(out3);
+    dg::blas1::pointwiseDot( g.zP(), temp3, out3);
+    dg::blas1::transfer( out3, vz);
+}
+template<class TernaryOp1, class TernaryOp2, class TernaryOp3 class container, class Geometry> 
+void doPushForward( TernaryOp1 f1, TernaryOp2 f2, TernaryOp3,
+        container& vx, container& vy, container& vz,
+        const Geometry& g, OrthonormalTag)
+{
+    doPushForwardPerp( f1,f2,f3,vx,vy,vz,OrthonormalTag());
+}
+
 template<class FunctorRR, class FunctorRZ, class FunctorZZ, class container, class Geometry> 
 void doPushForwardPerp( FunctorRR chiRR, FunctorRZ chiRZ, FunctorZZ chiZZ,
         container& chixx, container& chixy, container& chiyy,
diff --git a/inc/geometries/ds.h b/inc/geometries/ds.h
index e0a506ba2..7e353c700 100644
--- a/inc/geometries/ds.h
+++ b/inc/geometries/ds.h
@@ -43,7 +43,7 @@ struct DS
     @param jumpX determines if a jump matrix is added in X-direction
     */
     template<class MagneticField, class Geometry>
-    DS(const MagneticField& field, Geometry, dg::norm no=dg::normed, dg::direction dir = dg::centered, bool jumpX = true, unsigned mx=1, unsigned my=1);
+    DS(const MagneticField& field, Geometry, dg::norm no=dg::normed, dg::direction dir = dg::centered, bool dependsOnX = true, bool dependsOnY=true, unsigned mx=1, unsigned my=1);
 
     /**
     * @brief Apply the forward derivative on a 3d vector
@@ -215,8 +215,8 @@ struct DS
 ////////////////////////////////////DEFINITIONS////////////////////////////////////////
 
 template<class I, class M, class container>
-template <class MagneticField, class Geometry>
-DS< I, M,container>::DS(MagneticField mag, Geometry grid, dg::norm no, dg::direction dir, bool jumpX, unsigned mx, unsigned my):
+template <class Geometry>
+DS< I, M,container>::DS(const BinaryVectorLvl1& vector, Geometry grid, dg::norm no, dg::direction dir, bool jumpX, bool jumpY, unsigned mx, unsigned my):
         jumpX( dg::create::jumpX( grid)),
         jumpY( dg::create::jumpY( grid)),
         tempP( dg::evaluate( dg::zero, grid())), temp0( tempP), tempM( tempP), 
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index 8e255503a..9448ffa10 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -97,24 +97,20 @@ struct Field
 template< class GeometryPerp>
 struct DSField
 {
-    template<class MagneticField>
-    DSField( const MagneticField& c, const GeometryPerp& g)
+    DSField( const BinaryVectorLvl0& v, const GeometryPerp& g)
     {
-        InvB<MagneticField> invB(c);
-        FieldR<MagneticField> fieldR(c);
-        FieldZ<MagneticField> fieldZ(c);
+        thrust::host_vector<double> vx, vy, vz;
         thrust::host_vector<double> b_zeta, b_eta;
         dg::geo::pushForwardPerp( fieldR, fieldZ, b_zeta, b_eta, g);
         FieldP<MagneticField> fieldP(c);
         thrust::host_vector<double> b_phi = dg::pullback( fieldP, g);
-        Bmodule<MagneticField> bmod( c);
         thrust::host_vector<double> b_mod = dg::pullback( bmod, g);
         dg::blas1::pointwiseDivide( b_zeta, b_phi, b_zeta);
         dg::blas1::pointwiseDivide( b_eta,  b_phi, b_eta);
         dg::blas1::pointwiseDivide( b_mod,  b_phi, b_mod);
-        dzetadphi_ = dg::forward_transform( b_zeta, g );
-        detadphi_  = dg::forward_transform( b_eta, g );
-        dsdphi_    = dg::forward_transform( b_mod, g );
+        dxdz_ = dg::forward_transform( b_zeta, g );
+        dypz_ = dg::forward_transform( b_eta, g );
+        dsdz_ = dg::forward_transform( b_mod, g );
     }
 
     void operator()(thrust::host_vector<double> y, thrust::host_vector<double>& yp)
@@ -331,8 +327,8 @@ struct FieldAligned
         by the bcz variable from the grid and can be changed by the set_boundaries function. 
         If there is no limiter the boundary condition is periodic.
     */
-    template <class MagneticField, class Geometry, class Limiter>
-    FieldAligned(MagnetricField field, Geometry grid, unsigned multiplyX, unsigned multiplyY, double eps = 1e-4, Limiter limit = DefaultLimiter(), dg::bc globalbcz = dg::DIR, double deltaPhi = -1);
+    template <class Geometry, class Limiter>
+    FieldAligned(const BinaryVectorLvl0& vec, Geometry grid, unsigned multiplyX, unsigned multiplyY, double eps = 1e-4, Limiter limit = DefaultLimiter(), dg::bc globalbcz = dg::DIR, bool dependsOnX=true, bool dependsOnY=true, double deltaPhi = -1);
 
     /**
     * @brief Set boundary conditions in the limiter region
diff --git a/inc/geometries/fluxfunctions.h b/inc/geometries/fluxfunctions.h
index 5957e6e4a..87d618d3c 100644
--- a/inc/geometries/fluxfunctions.h
+++ b/inc/geometries/fluxfunctions.h
@@ -78,9 +78,9 @@ struct dg::geo::aCloneableBinaryFunctor : public dg::geo::aBinaryFunctor
  * double operator()(double,double)const;
  */
 template<class BinaryFunctor>
-struct Adapter : public dg::geo::aCloneableBinaryFunctor<Adapter>
+struct BinaryFunctorAdapter : public dg::geo::aCloneableBinaryFunctor<Adapter>
 {
-    Adapter( const BinaryFunctor& f):f_(f){}
+    BinaryFunctorAdapter( const BinaryFunctor& f):f_(f){}
     double operator()(double x, double y)const{return f_(x,y);}
     private:
     BinaryFunctor f_;
@@ -95,7 +95,7 @@ struct Adapter : public dg::geo::aCloneableBinaryFunctor<Adapter>
  * @note the preferred way is to derive your Functor from aCloneableBinaryFunctor but if you can't or don't want to for whatever reason then use this to make one
  */
 temmplate<class BinaryFunctor>
-aBinaryFunctor* make_aBinaryFunctor(const BinaryFunctor& f){return new Adapter<BinaryFunctor>(f);}
+aBinaryFunctor* make_aBinaryFunctor(const BinaryFunctor& f){return new BinaryFunctorAdapter<BinaryFunctor>(f);}
 
 /**
 * @brief This struct bundles a function and its first derivatives
@@ -152,9 +152,7 @@ struct BinaryFunctorsLvl2 : public BinaryFunctorsLvl1
     Handle<aBinaryFunctor> p_[3];
 };
 
-/**
-* @brief This struct bundles a symmetric tensor and its divergence
-*/
+/// A symmetric 2d tensor field and its divergence
 struct BinarySymmTensorLvl1
 {
     /**
@@ -190,6 +188,25 @@ struct BinarySymmTensorLvl1
     Handle<aBinaryFunctor> p_[5];
 };
 
+/// A vector field with three components that depend only on the first two coordinates
+struct BinaryVectorLvl0
+{
+    BinaryVectorLvl0( aBinaryFunctor* v_x, aBinaryFunctor* v_y, aBinaryFunctor* v_z)
+    {
+        p_[0].set(v_x);
+        p_[1].set(v_y);
+        p_[2].set(v_z);
+    }
+    /// x-component of the vector
+    const aBinaryFunctor& x()const{return p_[0];}
+    /// y-component of the vector
+    const aBinaryFunctor& y()const{return p_[1];}
+    /// z-component of the vector
+    const aBinaryFunctor& z()const{return p_[2];}
+    private:
+    Handle<aBinaryFunctor> p_[3];
+};
+
 struct Constant:public aCloneableBinaryOperator<Constant> 
 { 
     Constant(double c):c_(c){}
-- 
GitLab


From 72b8050134b750be0da4d5333ba75b023bed33ed Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 27 Jul 2017 15:51:49 +0200
Subject: [PATCH 091/453] made doPullback default pullback for curvilinear
 grids and redirect 2d perpVolume function to Volume function

---
 inc/dg/geometry.h                 |  6 +++--
 inc/dg/geometry/curvilinear.h     | 29 +++++++++++++---------
 inc/dg/geometry/generator.h       |  6 ++---
 inc/dg/geometry/geometry_traits.h | 41 ++++++++++++++++++++++---------
 inc/dg/geometry/mpi_curvilinear.h | 29 +++++++++++++---------
 inc/dg/geometry/mpi_grids.h       | 10 ++------
 6 files changed, 74 insertions(+), 47 deletions(-)

diff --git a/inc/dg/geometry.h b/inc/dg/geometry.h
index 3b4c00909..bb4daac70 100644
--- a/inc/dg/geometry.h
+++ b/inc/dg/geometry.h
@@ -136,11 +136,12 @@ void volRaisePerpIndex( container& covX, container& covY, container& contraX, co
  * @tparam Geometry the geometry class
  * @param inout input (contains result on output)
  * @param g The geometry object
+ * @note if g is two-dimensional this function will redirect to multiplyVolume()
  */
 template<class container, class Geometry>
 void multiplyPerpVolume( container& inout, const Geometry& g)
 {
-    dg::geo::detail::doMultiplyPerpVolume( inout, g, typename dg::GeometryTraits<Geometry>::metric_category());
+    dg::geo::detail::doMultiplyPerpVolume( inout, g, typename GeometryTraits<Geometry>::dimensionality());
 }
 
 /**
@@ -151,11 +152,12 @@ void multiplyPerpVolume( container& inout, const Geometry& g)
  * @tparam Geometry the geometry class
  * @param inout input (contains result on output)
  * @param g The geometry object
+ * @note if g is two-dimensional this function will redirect to divideVolume()
  */
 template<class container, class Geometry>
 void dividePerpVolume( container& inout, const Geometry& g)
 {
-    dg::geo::detail::doDividePerpVolume( inout, g, typename dg::GeometryTraits<Geometry>::metric_category());
+    dg::geo::detail::doDividePerpVolume( inout, g, typename GeometryTraits<Geometry>::dimensionality());
 }
 
 /**
diff --git a/inc/dg/geometry/curvilinear.h b/inc/dg/geometry/curvilinear.h
index 568d0f670..082c5cee9 100644
--- a/inc/dg/geometry/curvilinear.h
+++ b/inc/dg/geometry/curvilinear.h
@@ -32,7 +32,7 @@ struct CylindricalGrid3d : public dg::Grid3d
     CylindricalGrid3d( double R0, double R1, double Z0, double Z1, double phi0, double phi1, unsigned n, unsigned NR, unsigned NZ, unsigned Nphi, bc bcr = PER, bc bcz = PER, bc bcphi = PER): 
         dg::Grid3d(0,R1-R0,0,Z1-Z0,phi0,phi1,n,NR,NZ,Nphi,bcR,bcZ,bcphi)
         {
-            handle_.set( new IdentityGenerator(R0,R1,Z0,Z1));
+            handle_.set( new ShiftedIdentityGenerator(R0,R1,Z0,Z1));
             construct( n, NR, NZ,Nphi);
         }
 
@@ -55,12 +55,20 @@ struct CylindricalGrid3d : public dg::Grid3d
     }
 
     perpendicular_grid perp_grid() const { return perpendicular_grid(*this);}
-    /// a 2d vector containing R(x,y)
-    const thrust::host_vector<double>& r()const{return r_;}
-    /// a 2d vector containing Z(x,y)
-    const thrust::host_vector<double>& z()const{return z_;}
-    /// a 1d vector containing the phi values
-    const thrust::host_vector<double>& phi()const{return phi_;}
+
+    /**
+    * @brief Function to do the pullback
+    * @note Don't call! Use the pullback() function
+    */
+    template<class TernaryOp>
+    thrust::host_vector<double> doPullback( TernaryOp f)const {
+        thrust::host_vector<double> vec( size());
+        unsigned size2d = n()*n()*Nx()*Ny();
+        for( unsigned k=0; k<Nz(); k++)
+            for( unsigned i=0; i<size2d; i++)
+                vec[k*size2d+i] = f( r_[i], z_[i], phi_[k]);
+        return vec;
+    }
     /// a 3d vector containing dx/dR
     const thrust::host_vector<double>& xr()const{return xr_;}
     /// a 3d vector containing dy/dR
@@ -146,7 +154,7 @@ struct CurvilinearGrid2d : public dg::Grid2d
     typedef dg::CurvilinearPerpTag metric_category;
 
     CurvilinearGrid2d( double R0, double R1, double Z0, double Z1, unsigned n, unsigned NR, unsigned NZ, unsigned Nphi, bc bcR = PER, bc bcZ = PER): 
-        dg::Grid2d(0,R1-R0,0,Z1-Z0,n,NR,NZ,bcR,bcZ), handle_(new IdentityGenerator(R0,R1,Z0,Z1))
+        dg::Grid2d(0,R1-R0,0,Z1-Z0,n,NR,NZ,bcR,bcZ), handle_(new ShiftedIdentityGenerator(R0,R1,Z0,Z1))
         {
             construct( n, NR, NZ);
         }
@@ -155,8 +163,8 @@ struct CurvilinearGrid2d : public dg::Grid2d
      * @param generator must generate an orthogonal grid (class takes ownership of the pointer)
      * @param n number of polynomial coefficients
      * @param Nx number of cells in first coordinate
-     @param Ny number of cells in second coordinate
-     @param bcx boundary condition in first coordinate
+     * @param Ny number of cells in second coordinate
+     * @param bcx boundary condition in first coordinate
      */
     CurvilinearGrid2d( const geo::aGenerator& generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx=dg::DIR):
         dg::Grid2d( 0, generator.width(), 0., generator.height(), n, Nx, Ny, bcx, dg::PER), handle_(generator)
@@ -190,7 +198,6 @@ struct CurvilinearGrid2d : public dg::Grid2d
     const container& g_yy()const{return g_yy_;}
     const container& g_xy()const{return g_xy_;}
     const container& vol()const{return vol2d_;}
-    const container& perpVol()const{return vol2d_;}
     const geo::aGenerator& generator() const{return handle_.get();}
     bool isOrthonormal() const { return generator_->isOrthonormal();}
     bool isOrthogonal() const { return generator_->isOrthogonal();}
diff --git a/inc/dg/geometry/generator.h b/inc/dg/geometry/generator.h
index 0720ae1cb..c3da3fae6 100644
--- a/inc/dg/geometry/generator.h
+++ b/inc/dg/geometry/generator.h
@@ -79,14 +79,14 @@ struct aGridGenerator
 @note in fact it's not completely the identity because we assume that the origin is always (0,0) in the computational space
  @ingroup generators
 */
-struct IdentityGenerator: public aGridGenerator
+struct ShiftedIdentityGenerator: public aGridGenerator
 {
     virtual double width()  const{return lx_;} 
     virtual double height() const{return ly_;}
     virtual bool isOrthonormal() const{return true;}
     virtual bool isOrthogonal() const{return true;}
     virtual bool isConformal()const{return true;}
-    virtual IdentityGenerator* clone() const{return new IdentityGenerator(*this);}
+    virtual ShiftedIdentityGenerator* clone() const{return new ShiftedIdentityGenerator(*this);}
 
     /**
     * @brief Define the 2d box in the physical domain in which to construct the coordinates
@@ -96,7 +96,7 @@ struct IdentityGenerator: public aGridGenerator
     * @param y0 y-coordinate of lower left point
     * @param y1 y-coordainte of upper right point
     */
-    IdentityGenerator( double x0, double x1, double y0, double y1){
+    ShiftedIdentityGenerator( double x0, double x1, double y0, double y1){
         x0_ = x0; lx_ = (x1-x0);
         y0_ = y0; ly_ = (y1-y0);
     }
diff --git a/inc/dg/geometry/geometry_traits.h b/inc/dg/geometry/geometry_traits.h
index 65c9d4182..1f7d9a977 100644
--- a/inc/dg/geometry/geometry_traits.h
+++ b/inc/dg/geometry/geometry_traits.h
@@ -61,30 +61,52 @@ void doDivideVolume( container& inout, const Geometry& g, CurvilinearTag)
     dg::blas1::pointwiseDivide( inout, g.vol(), inout);
 };
 
+//////////////////multiplyPerpVol/////////////
 template <class container, class Geometry>
-void doMultiplyPerpVolume( container& inout, const Geometry& g, OrthonormalTag)
+void doMultiplyPerpVolume( container& inout, const Geometry& g, TwoDimensionalTag)
+{
+    doMultiplyVolume( inout, g, typename dg::GeometryTraits<Geometry>::metric_category());
+};
+template <class container, class Geometry>
+void doMultiplyPerpVolume( container& inout, const Geometry& g, ThreeDimensionalTag)
+{
+    doMultiplyPerpVolume3d( inout, g, typename dg::GeometryTraits<Geometry>::metric_category());
+};
+template <class container, class Geometry>
+void doMultiplyPerpVolume3d( container& inout, const Geometry& g, OrthonormalTag)
 {
 };
 
 template <class container, class Geometry>
-void doMultiplyPerpVolume( container& inout, const Geometry& g, CurvilinearPerpTag)
+void doMultiplyPerpVolume3d( container& inout, const Geometry& g, CurvilinearPerpTag)
 {
     if( !g.isOrthonormal())
         dg::blas1::pointwiseDot( inout, g.perpVol(), inout);
 };
 
+//////////////////dividePerpVol/////////////
 template <class container, class Geometry>
-void doDividePerpVolume( container& inout, const Geometry& g, OrthonormalTag)
+void doDividePerpVolume( container& inout, const Geometry& g, TwoDimensionalTag)
+{
+    doDivideVolume( inout, g, typename dg::GeometryTraits<Geometry>::metric_category());
+};
+template <class container, class Geometry>
+void doDividePerpVolume( container& inout, const Geometry& g, ThreeDimensionalTag)
+{
+    doDividePerpVolume3d( inout, g, typename dg::GeometryTraits<Geometry>::metric_category());
+};
+template <class container, class Geometry>
+void doDividePerpVolume3d( container& inout, const Geometry& g, OrthonormalTag)
 {
 };
-
 template <class container, class Geometry>
-void doDividePerpVolume( container& inout, const Geometry& g, CurvilinearPerpTag)
+void doDividePerpVolume3d( container& inout, const Geometry& g, CurvilinearPerpTag)
 {
     if( !g.isOrthonormal())
         dg::blas1::pointwiseDivide( inout, g.perpVol(), inout);
 };
 
+
 template <class container, class Geometry>
 void doRaisePerpIndex( const container& in1, const container& in2, container& out1, container& out2, const Geometry& g, OrthonormalTag)
 {
@@ -312,14 +334,9 @@ thrust::host_vector<double> doPullback( BinaryOp f, const Geometry& g, Curviline
 }
 
 template< class TernaryOp, class Geometry>
-thrust::host_vector<double> doPullback( TernaryOp f, const Geometry& g, CurvilinearPerpTag, ThreeDimensionalTag, SharedTag)
+thrust::host_vector<double> doPullback( TernaryOp f, const Geometry& g, CurvilinearTag, ThreeDimensionalTag, SharedTag)
 {
-    thrust::host_vector<double> vec( g.size());
-    unsigned size2d = g.n()*g.n()*g.Nx()*g.Ny();
-    for( unsigned k=0; k<g.Nz(); k++)
-        for( unsigned i=0; i<size2d; i++)
-            vec[k*size2d+i] = f( g.r()[i], g.z()[i], g.phi()[k]);
-    return vec;
+    return g.doPullback( f);
 }
 template< class BinaryOp, class Geometry>
 thrust::host_vector<double> doPullback( BinaryOp f, const Geometry& g, OrthonormalTag, TwoDimensionalTag, SharedTag)
diff --git a/inc/dg/geometry/mpi_curvilinear.h b/inc/dg/geometry/mpi_curvilinear.h
index aa2328996..0b8c9415e 100644
--- a/inc/dg/geometry/mpi_curvilinear.h
+++ b/inc/dg/geometry/mpi_curvilinear.h
@@ -13,7 +13,7 @@ namespace dg
 {
 
 ///@cond
-template< class container>
+template< class MPIcontainer>
 struct CurvilinearMPIGrid2d; 
 ///@endcond
 //
@@ -37,8 +37,9 @@ struct CylindricalMPIGrid3d : public dg::MPIGrid3d
      */
     CylindricalMPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, MPI_Comm comm): 
         dg::MPIGrid3d( 0, x1-x0, 0, y1-y0, z0, z1, n, Nx, Ny, Nz, comm),
-        g( new IdentityGenerator(x0,x1,y0,y1), n,Nx,Ny,local().Nz(), bcx,bcy,bcz)
+        handle_( new ShiftedIdentityGenerator(x0,x1,y0,y1), n,Nx,Ny,local().Nz(), bcx,bcy,bcz)
      {
+        CylindricalGrid3d<LocalContainer> g(handle_.get(),n,Nx,Ny,this->Nz());
         divide_and_conquer();
      }
 
@@ -48,10 +49,10 @@ struct CylindricalMPIGrid3d : public dg::MPIGrid3d
      * @note the paramateres given in the constructor are global parameters 
      */
     CylindricalMPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz, MPI_Comm comm):
-        dg::MPIGrid3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, bcx, bcy, bcz, comm),
-        handle_( new IdentityGenerator(x0,x1,y0,y1))
+        dg::MPIGrid3d( 0, x1-x0, 0, y1-y0, z0, z1, n, Nx, Ny, Nz, bcx, bcy, bcz, comm),
+        handle_( new ShiftedIdentityGenerator(x0,x1,y0,y1))
      {
-        CylindricalGrid3d<LocalContainer> g(generator,n,Nx,Ny,this->Nz());
+        CylindricalGrid3d<LocalContainer> g(handle_.get(),n,Nx,Ny,this->Nz());
         divide_and_conquer(g);
      }
 
@@ -65,9 +66,16 @@ struct CylindricalMPIGrid3d : public dg::MPIGrid3d
 
     perpendicular_grid perp_grid() const { return perpendicular_grid(*this);}
 
-    const thrust::host_vector<double>& r()const{return r_;}
-    const thrust::host_vector<double>& z()const{return z_;}
-    const thrust::host_vector<double>& phi()const{return phi_;}
+    template<class TernaryOp>
+    dg::MPI_Vector<thrust::host_vector<double> > doPullback(TernaryOp f)const{
+        thrust::host_vector<double> vec( g.size());
+        unsigned size2d = g.n()*g.n()*g.Nx()*g.Ny();
+        for( unsigned k=0; k<g.Nz(); k++)
+            for( unsigned i=0; i<size2d; i++)
+                vec[k*size2d+i] = f( g.r()[i], g.z()[i], g.phi()[k]);
+        MPI_Vector<thrust::host_vector<double> > v( vec, g.communicator());
+        return v;
+    }
     const dg::MPI_Vector<thrust::host_vector<double> >& xr()const{return xr_;}
     const dg::MPI_Vector<thrust::host_vector<double> >& yr()const{return yr_;}
     const dg::MPI_Vector<thrust::host_vector<double> >& xz()const{return xz_;}
@@ -154,7 +162,7 @@ struct CurvilinearMPIGrid2d : public dg::MPIGrid2d
      * @note the paramateres given in the constructor are global parameters 
      */
     CurvilinearMPIGrid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, MPI_Comm comm): dg::MPIGrid2d( x0, x1, y0, y1, n, Nx, Ny, comm),
-    g_(new IdentityGenerator(x0,x1,y0,y1),n,Nx,Ny){
+    g_(new ShiftedIdentityGenerator(x0,x1,y0,y1),n,Nx,Ny){
         divide_and_conquer();
     
     }
@@ -165,7 +173,7 @@ struct CurvilinearMPIGrid2d : public dg::MPIGrid2d
      * @note the paramateres given in the constructor are global parameters 
      */
     CurvilinearMPIGrid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx, bc bcy, MPI_Comm comm):dg::MPIGrid2d( 0, x1-x0, 0, y1-y0, n, Nx, Ny,bcx, bcy, comm),
-    handle_(new IdentityGenerator(x0,x1,y0,y1)){
+    handle_(new ShiftedIdentityGenerator(x0,x1,y0,y1)){
         dg::CurvilinearGrid2d<thrust::host_vector<double> > g(generator, n, Nx, Ny);
         divide_and_conquer(g);
     }
@@ -208,7 +216,6 @@ struct CurvilinearMPIGrid2d : public dg::MPIGrid2d
     const MPIContainer& g_yy()const{return g_yy_;}
     const MPIContainer& g_xy()const{return g_xy_;}
     const MPIContainer& vol()const{return vol2d_;}
-    const MPIContainer& perpVol()const{return vol2d_;}
     const geo::aGenerator& generator() const{return g.generator();}
     bool isOrthonormal() const { return g.isOrthonormal();}
     bool isOrthogonal() const { return g.isOrthogonal();}
diff --git a/inc/dg/geometry/mpi_grids.h b/inc/dg/geometry/mpi_grids.h
index 5b08f36f0..8862e93dd 100644
--- a/inc/dg/geometry/mpi_grids.h
+++ b/inc/dg/geometry/mpi_grids.h
@@ -84,15 +84,9 @@ MPI_Vector< thrust::host_vector<double> > doPullback( BinaryOp f, const Geometry
 }
 
 template< class TernaryOp, class Geometry>
-MPI_Vector< thrust::host_vector<double> > doPullback( TernaryOp f, const Geometry& g, CurvilinearPerpTag, ThreeDimensionalTag, MPITag)
+MPI_Vector< thrust::host_vector<double> > doPullback( TernaryOp f, const Geometry& g, CurvilinearTag, ThreeDimensionalTag, MPITag)
 {
-    thrust::host_vector<double> vec( g.size());
-    unsigned size2d = g.n()*g.n()*g.Nx()*g.Ny();
-    for( unsigned k=0; k<g.Nz(); k++)
-        for( unsigned i=0; i<size2d; i++)
-            vec[k*size2d+i] = f( g.r()[i], g.z()[i], g.phi()[k]);
-    MPI_Vector<thrust::host_vector<double> > v( vec, g.communicator());
-    return v;
+    return g.doPullback(f);
 }
 template< class BinaryOp, class Geometry>
 MPI_Vector< thrust::host_vector<double> > doPullback( BinaryOp f, const Geometry& g, OrthonormalTag, TwoDimensionalTag, MPITag)
-- 
GitLab


From 879da22f2891c6a8bbcbc9649ffed3ecfbeb3d45 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Fri, 28 Jul 2017 22:08:06 +0200
Subject: [PATCH 092/453] added tensor.h, restored elliptic.h and corrected
 geometry_traits

---
 inc/dg/elliptic.h                 |   8 +-
 inc/dg/geometry/geometry_traits.h |  18 ++---
 inc/dg/geometry/tensor.h          | 117 ++++++++++++++++++++++++++++++
 3 files changed, 130 insertions(+), 13 deletions(-)
 create mode 100644 inc/dg/geometry/tensor.h

diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index 81054cbe9..331228bfa 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -147,10 +147,10 @@ class Elliptic
     void symv( const Vector& x, Vector& y) 
     {
         //compute gradient
-        dg::blas2::gemv( rightx, x, gradx); //R_x*f 
-        dg::blas2::gemv( righty, x, y); //R_y*f
+        dg::blas2::gemv( rightx, x, tempx); //R_x*f 
+        dg::blas2::gemv( righty, x, tempy); //R_y*f
 
-        dg::geo::volRaisePerpIndex( gradx, y, gradx, y, g_);
+        dg::geo::volRaisePerpIndex( tempx, tempy, gradx, y, g_);
 
         //multiply with chi 
         dg::blas1::pointwiseDot( xchi, gradx, gradx); //Chi*R_x*x 
@@ -651,7 +651,7 @@ struct TensorElliptic
     template<class ChiRR, class ChiRZ, class ChiZZ>
     void set( ChiRR chiRR, ChiRZ chiRZ, ChiZZ chiZZ)
     {
-        Vector chiXX, chiXY, chiYY;
+        typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector chiXX, chiXY, chiYY;
         dg::geo::pushForwardPerp( chiRR, chiRZ, chiZZ, chiXX, chiXY, chiYY, g_);
         set( chiXX, chiXY, chiYY);
     }
diff --git a/inc/dg/geometry/geometry_traits.h b/inc/dg/geometry/geometry_traits.h
index 1f7d9a977..50f75d57b 100644
--- a/inc/dg/geometry/geometry_traits.h
+++ b/inc/dg/geometry/geometry_traits.h
@@ -110,16 +110,16 @@ void doDividePerpVolume3d( container& inout, const Geometry& g, CurvilinearPerpT
 template <class container, class Geometry>
 void doRaisePerpIndex( const container& in1, const container& in2, container& out1, container& out2, const Geometry& g, OrthonormalTag)
 {
-    out1=in1;//the container type should check for self-assignment
-    out2=in2;//the container type should check for self-assignment
+    out1.swap(in1);
+    out2.swap(in2);
 };
 template <class container, class Geometry>
 void doRaisePerpIndex( const container& in1, const container& in2, container& out1, container& out2, const Geometry& g, CurvilinearPerpTag)
 {
     if( g.isOrthonormal())
     {
-        out1=in1;//the container type should check for self-assignment
-        out2=in2;//the container type should check for self-assignment
+        out1.swap(in1);
+        out2.swap(in2);
         return;
     }
     if( g.isOrthogonal())
@@ -135,18 +135,18 @@ void doRaisePerpIndex( const container& in1, const container& in2, container& ou
 };
 
 template <class container, class Geometry>
-void doVolRaisePerpIndex( const container& in1, const container& in2, container& out1, container& out2, const Geometry& g, OrthonormalTag)
+void doVolRaisePerpIndex( container& in1, container& in2, container& out1, container& out2, const Geometry& g, OrthonormalTag)
 {
-    out1=in1;//the container type should check for self-assignment
-    out2=in2;//the container type should check for self-assignment
+    out1.swap(in1);
+    out2.swap(in2);
 };
 template <class container, class Geometry>
 void doVolRaisePerpIndex( const container& in1, const container& in2, container& out1, container& out2, const Geometry& g, CurvilinearPerpTag)
 {
     if( g.isConformal())
     {
-        out1=in1;//the container type should check for self-assignment
-        out2=in2;//the container type should check for self-assignment
+        out1.swap(in1);
+        out2.swap(in2);
         return;
     }
     if( g.isOrthogonal())
diff --git a/inc/dg/geometry/tensor.h b/inc/dg/geometry/tensor.h
new file mode 100644
index 000000000..4b16aa25b
--- /dev/null
+++ b/inc/dg/geometry/tensor.h
@@ -0,0 +1,117 @@
+#pragma once 
+
+#include <cusp/coo_matrix.h>
+#include "dg/backend/operator.h"
+#include "dg/blas1.h"
+
+namespace dg
+{
+///sparse matrix with implicit elements either 1 (diagonal) or 0 (off-diagonal)
+///the size is always 3x3 
+template<class container>
+struct SparseUnitTensor
+{
+    /// default size is 3 and no values are set
+    SparseUnitTensor( ):idx_(3,-1),values_(9){}
+    ///swap all data with other tensor
+    void swap( SparseUnitTensor& other){
+        std::swap(idx_,other.idx_);
+        values_.swap( other.values_);
+    }
+    /**
+    * @brief get an index into the values array
+    * @param i row index 0<i<2
+    * @param j col index 0<j<2
+    * @return index (if <0 then value is assumed implicitly)
+    */
+    int operator()(size_t i, size_t j)const{return idx_(i,j);}
+    /**
+    * @brief set an index into the values array (if <0 then value is assumed implicitly)
+
+    * @param i row index 0<i<2
+    * @param j col index 0<j<2
+    * @return reference to an index     
+    */
+    int& operator()(size_t i, size_t j){return idx_(i,j);}
+    ///any value between 0 and 8
+    const container& get(int i)const{return values_[i];}
+    ///any value between 0 and 8
+    container& get(int i){return values_[i];}
+    private:
+    dg::Operator<int>& idx_;
+    std::vector<container> values_;
+
+};
+namespace geo
+{
+
+///container sizes must match each other and the tensor container size
+///may destroy input (either swap in or use as temporary storage)
+///no alias allowed
+template<class container>
+void multiply( const SparseUnitTensor& t, container& in1, container& in2, container& out1, container& out2)
+{
+    if( tensor(0,0)<0 && tensor(0,1)<0 && tensor(1,0)<0 && tensor(1,1)<0)
+    {
+        out1.swap(in1);
+        out2.swap(in2);
+    }
+    if( tensor(0,0)<0) out1=in1;
+    else 
+        dg::blas1::pointwiseDot( tensor.get(tensor(0,0)), in1, out1); //gxx*v_x
+    if(!tensor( 0,1)<0)
+        dg::blas1::pointwiseDot( 1., tensor.get(tensor(0,1)), in2, 1., out1);//gxy*v_y
+
+    if( tensor(1,1)<0) out2=in2;
+    else
+        dg::blas1::pointwiseDot( tensor.get(tensor(1,1)), in2, out2); //gyy*v_y
+    if(!tensor(1,0)<0)
+        dg::blas1::pointwiseDot( 1., tensor.get(tensor(1,0)), in1, 1., out2); //gyx*v_x
+}
+
+template<class container>
+void multiply( const SparseUnitTensor& t, container& in1, container& in2, container& in3, container& out1, container& out2, container& out3)
+{
+    bool empty = true;
+    for( unsigned i=0; i<3; i++)
+        for( unsigned j=0; j<3; j++)
+            if(!tensor(i,j)<0) 
+                empty=false;
+
+    if( empty)
+    {
+        in1.swap(out1);
+        in2.swap(out2);
+        in3.swap(out3);
+    }
+    if( tensor(0,0)<0) out1=in1;
+    if(!tensor( 0,0)<0)
+        dg::blas1::pointwiseDot( tensor.get(tensor(0,0)), in1, out1); 
+    if(!tensor( 0,1)<0)
+        dg::blas1::pointwiseDot( 1., tensor.get(tensor(0,1)), in2, 1., out1);
+    if(!tensor( 0,2)<0)
+        dg::blas1::pointwiseDot( 1., tensor.get(tensor(0,2)), in3, 1., out1);
+
+    if( tensor(1,1)<0) out2=in2;
+    if(!tensor(1,1)<0)
+        dg::blas1::pointwiseDot( tensor.get(tensor(1,1)), in2, out2);
+    if(!tensor(1,0)<0)
+        dg::blas1::pointwiseDot( 1., tensor.get(tensor(1,0)), in1, 1., out2);
+    if(!tensor(1,2)<0)
+        dg::blas1::pointwiseDot( 1., tensor.get(tensor(1,2)), in1, 1., out2);
+
+    if( tensor(2,2)<0) out3=in3;
+    if(!tensor(2,2)<0)
+        dg::blas1::pointwiseDot( tensor.get(tensor(2,2)), in3, out3);
+    if(!tensor(2,1)<0)
+        dg::blas1::pointwiseDot( 1., tensor.get(tensor(2,1)), in2, 1., out3);
+    if(!tensor(2,0)<0)
+        dg::blas1::pointwiseDot( 1., tensor.get(tensor(2,0)), in1, 1., out3);
+}
+
+}//namespace geo
+
+
+
+
+}//namespace dg
-- 
GitLab


From 4fcf595ff81568b0bb2728b4e2f4e811956dfeb6 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Fri, 28 Jul 2017 22:41:13 +0200
Subject: [PATCH 093/453] added point map and form struct

---
 inc/dg/geometry/tensor.h | 36 ++++++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/inc/dg/geometry/tensor.h b/inc/dg/geometry/tensor.h
index 4b16aa25b..5786810a9 100644
--- a/inc/dg/geometry/tensor.h
+++ b/inc/dg/geometry/tensor.h
@@ -42,6 +42,42 @@ struct SparseUnitTensor
     std::vector<container> values_;
 
 };
+
+template<class container>
+struct SparseForm
+{
+    SparseForm():isSet(false){}
+    SparseForm(const container& value):isSet_(true), value_(value){}
+    bool isSet()const{return isSet_;}
+    const container& get()const{return value_;}
+    void set(const container& value){ 
+        isSet_=true;
+        value_ = value; 
+    }
+    private:
+    bool isSet_;
+    container value_;
+};
+
+//needs to construct two/three coordinates from given index
+//probably needs to know the coordinate sizes to make simplified versions
+struct PointMap2d
+{
+    double first( int i);
+    double second( int i);
+    private:
+    thrust::host_vector<double> first_, second_;
+};
+
+struct PointMap3d
+{
+    double first( int i);
+    double second( int i);
+    double third( int i);
+    private:
+    thrust::host_vector<double> first_, second_, third_;
+};
+
 namespace geo
 {
 
-- 
GitLab


From ec1f33638f3ba058e5bf950ada8376e8adc8c598 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sat, 29 Jul 2017 19:47:53 +0200
Subject: [PATCH 094/453] work on sparse tensor struct

---
 inc/dg/geometry/generator.h |  31 +++++-
 inc/dg/geometry/tensor.h    | 202 +++++++++++++++++++++---------------
 2 files changed, 146 insertions(+), 87 deletions(-)

diff --git a/inc/dg/geometry/generator.h b/inc/dg/geometry/generator.h
index c3da3fae6..829664147 100644
--- a/inc/dg/geometry/generator.h
+++ b/inc/dg/geometry/generator.h
@@ -4,6 +4,20 @@ namespace dg
 {
 namespace geo
 {
+
+///The analytically given continuous representation of the real world coordinates and metric
+enum PhysicalSpaceCoordinates{
+    cartesian=0;///2d or 3d X,Y,Z
+    cylindrical=1;///3d R,Z,P
+};
+///The type of discrete coordinates and metric
+enum ComputationalSpaceCoordinates{
+    orthonormal=0; ///coordinate lines are orthogonal (only unity diagonal metric elements)
+    conformal=1; ///only 2d coordinates can be conformal (volume and metric elements are all the same)
+    orthogonal=2; ///coordinate lines are orthogonal (only diagonal metric elements)
+    curvilinear=3; ///non-orthogonal coordinate lines
+};
+
 /**
 * @brief The abstract generator base class 
 
@@ -17,9 +31,19 @@ struct aGridGenerator
 {
     virtual double width()  const=0; //!<length in \f$ \zeta\f$ of the computational space
     virtual double height() const=0; //!<length in \f$ \eta\f$ of the computational space
-    virtual bool isOrthonormal() const{return false;} //!< true if coordinate system is orthonormal (false by default)
-    virtual bool isOrthogonal() const{return false;} //!< true if coordinate system is orthogonal (false by default)
-    virtual bool isConformal()const{return false;} //!< true if coordinate system is conformal (false by default)
+    /**
+    * @brief This is the analytical coordinate system we transform to
+    * @return type of physical space coordinates (default is dg::geo::cartesian)
+    */
+    virtual enum PhysicalSpaceCoordinates physical()const{ return cartesian;} 
+    /**
+    * @brief The type of coordinate system of the computational space
+    * @return default is dg::geo::orthonormal
+    * @note This is a performance hint for the computation and storage of metric elements.
+    *    We believe that you do not lie about what you generate.
+    */
+    virtual enum ComputationalSpaceCoordinates computational()const{ return orthonormal;} 
+
     /**
     * @brief Generate grid points and elements of the Jacobian 
     *
@@ -49,6 +73,7 @@ struct aGridGenerator
         zetaX = zetaY = etaX = etaY =x ;
         generate( zeta1d, eta1d, x,y,zetaX,zetaY,etaX,etaY);
     }
+
     /**
     * @brief Abstract clone method that returns a copy on the heap
     *
diff --git a/inc/dg/geometry/tensor.h b/inc/dg/geometry/tensor.h
index 5786810a9..9f799377d 100644
--- a/inc/dg/geometry/tensor.h
+++ b/inc/dg/geometry/tensor.h
@@ -6,76 +6,113 @@
 
 namespace dg
 {
-///sparse matrix with implicit elements either 1 (diagonal) or 0 (off-diagonal)
-///the size is always 3x3 
+/**
+* @brief Abstract base class for Sparse matrices of containers
+*
+* The size is always assumed 3x3.
+* The elements are implicitly assumed either 1 (diagonal) or 0 (off-diagonal)
+*/
 template<class container>
 struct SparseUnitTensor
 {
-    /// default size is 3 and no values are set
-    SparseUnitTensor( ):idx_(3,-1),values_(9){}
-    ///swap all data with other tensor
-    void swap( SparseUnitTensor& other){
-        std::swap(idx_,other.idx_);
-        values_.swap( other.values_);
-    }
     /**
-    * @brief get an index into the values array
+    * @brief check if an index is set or not
     * @param i row index 0<i<2
     * @param j col index 0<j<2
-    * @return index (if <0 then value is assumed implicitly)
+    * @return true if container is non-empty, false if value is assumed implicitly
     */
-    int operator()(size_t i, size_t j)const{return idx_(i,j);}
+    bool isSet(size_t i, size_t j)const{
+        if( get_value_idx(i,j) <0) return false;
+        return true;
+    }
     /**
     * @brief set an index into the values array (if <0 then value is assumed implicitly)
-
     * @param i row index 0<i<2
     * @param j col index 0<j<2
     * @return reference to an index     
     */
-    int& operator()(size_t i, size_t j){return idx_(i,j);}
-    ///any value between 0 and 8
-    const container& get(int i)const{return values_[i];}
-    ///any value between 0 and 8
-    container& get(int i){return values_[i];}
+    int& valueIdx(size_t i, size_t j){ return get_value_idx(i,j); }
+    /**
+    * @brief get an index into the values array (if <0 then value is assumed implicitly)
+    * @param i row index 0<i<2
+    * @param j col index 0<j<2
+    * @return reference to an index     
+    */
+    int valueIdx(size_t i, size_t j)const{ return get_value_idx(i,j); }
+    const container& getValue(size_t i, size_t j)const{ 
+        int k = get_value_idx(i,j);
+        if(k<0) return container();
+        return getValue(k); 
+    }
+    const container& getValue(size_t i)const{ return values_[i]; }
+    ///set a value for the given idx
+    container& getValue(int i){ return values_[i]; }
+    protected:
+    ///by protecting the constructor we fix the size of the values 
+    ///array associated with the types of derived classes
+    SparseUnitTensor( size_t values_size=9): values_(values_size),idx_(3,-1){}
+    ///we disallow changeing the size
+    SparseUnitTensor( const SparseUnitTensor& src): values_(src.values_), idx_(src.idx_){}
+    ///we disallow changeing the size
+    SparseUnitTensor& operator=()(SparseUnitTensor src)
+    {
+        values_.swap( src.values_);
+        std::swap( idx_, src.idx_)
+        return *this;
+    }
+    ///rule of three
+    ~SparseUnitTensor(){}
+    template<class otherContainer>
+    SparseUnitTensor( const SparseUnitTensor<otherContainer>& src): values_(src.values_.size()), idx_(src.idx_){
+        dg::blas1::transfer( src.values_, values_);
+    }
     private:
-    dg::Operator<int>& idx_;
+    int get_value_idx(size_t i, size_t j)const{return idx_(i,j);}
+    int& get_value_idx(size_t i, size_t j){return idx_(i,j);}
     std::vector<container> values_;
-
+    dg::Operator<int>& idx_;
 };
 
+///The metric tensor is a SparseTensor 
 template<class container>
-struct SparseForm
+struct MetricTensor: public SparseUnitTensor<container>
 {
-    SparseForm():isSet(false){}
-    SparseForm(const container& value):isSet_(true), value_(value){}
-    bool isSet()const{return isSet_;}
-    const container& get()const{return value_;}
-    void set(const container& value){ 
-        isSet_=true;
-        value_ = value; 
+    /// default size is 3 and no values are set
+    MetricTensor( ):SparseUnitTensor( 11){ volIdx_=perpVolIdx_=-1; }
+    bool volIsSet() const{ 
+        if(volIdx_<0)return false;
+        else return true;
+    }
+    template<class otherContainer>
+    MatricTensor( MetricTensor<otherContainer>& src):SparseUnitTensor<container>(src){}
+    bool perpVolIsSet() const{ 
+        if(perpVolIdx_<0)return false;
+        else return true;
+    }
+    int& volIdx(){ return volIdx_;}
+    int& perpVolIdx(){ return perpVolIdx_;}
+    int volIdx()const{ return volIdx_;}
+    int perpVolIdx()const{ return perpVolIdx_;}
+
+    const container& getVol() const{return get(volIdx_);}
+    const container& getPerpVol() const{return get(perpVolIdx_);}
+    void setVol(const container& vol){ 
+        volIdx_=9;
+        this->get(9) = value; 
+    }
+    void setPerpVol(const container& vol){ 
+        perpVolIdx_=10;
+        this->get(10) = value; 
     }
     private:
-    bool isSet_;
-    container value_;
-};
-
-//needs to construct two/three coordinates from given index
-//probably needs to know the coordinate sizes to make simplified versions
-struct PointMap2d
-{
-    double first( int i);
-    double second( int i);
-    private:
-    thrust::host_vector<double> first_, second_;
+    int volIdx_, perpVolIdx_;
 };
 
-struct PointMap3d
+template<class container>
+struct CoordinateTransformation: public SparseUnitTensor<container>
 {
-    double first( int i);
-    double second( int i);
-    double third( int i);
-    private:
-    thrust::host_vector<double> first_, second_, third_;
+    /// the coordinates have index 9, 10 and 11
+    CoordinateTransformation():SparseUnitTensor( 12){}
 };
 
 namespace geo
@@ -87,22 +124,23 @@ namespace geo
 template<class container>
 void multiply( const SparseUnitTensor& t, container& in1, container& in2, container& out1, container& out2)
 {
-    if( tensor(0,0)<0 && tensor(0,1)<0 && tensor(1,0)<0 && tensor(1,1)<0)
+    if( !tensor.isSet(0,0) && !tensor.isSet(0,1) && !tensor.isSet(1,0) && !tensor.isSet(1,1))
     {
         out1.swap(in1);
         out2.swap(in2);
+        return;
     }
-    if( tensor(0,0)<0) out1=in1;
+    if( !tensor.isSet(0,0)) out1=in1;
     else 
-        dg::blas1::pointwiseDot( tensor.get(tensor(0,0)), in1, out1); //gxx*v_x
-    if(!tensor( 0,1)<0)
-        dg::blas1::pointwiseDot( 1., tensor.get(tensor(0,1)), in2, 1., out1);//gxy*v_y
+        dg::blas1::pointwiseDot( tensor.getValue(0,0), in1, out1); //gxx*v_x
+    if(tensor.isSet( 0,1))
+        dg::blas1::pointwiseDot( 1., tensor.getValue(0,1), in2, 1., out1);//gxy*v_y
 
-    if( tensor(1,1)<0) out2=in2;
+    if( !tensor.isSet(1,1)) out2=in2;
     else
-        dg::blas1::pointwiseDot( tensor.get(tensor(1,1)), in2, out2); //gyy*v_y
-    if(!tensor(1,0)<0)
-        dg::blas1::pointwiseDot( 1., tensor.get(tensor(1,0)), in1, 1., out2); //gyx*v_x
+        dg::blas1::pointwiseDot( tensor.getValue(1,1), in2, out2); //gyy*v_y
+    if(tensor.isSet(1,0))
+        dg::blas1::pointwiseDot( 1., tensor.getValue(1,0), in1, 1., out2); //gyx*v_x
 }
 
 template<class container>
@@ -111,43 +149,39 @@ void multiply( const SparseUnitTensor& t, container& in1, container& in2, contai
     bool empty = true;
     for( unsigned i=0; i<3; i++)
         for( unsigned j=0; j<3; j++)
-            if(!tensor(i,j)<0) 
+            if(tensor.isSet(i,j)) 
                 empty=false;
-
     if( empty)
     {
         in1.swap(out1);
         in2.swap(out2);
         in3.swap(out3);
+        return;
     }
-    if( tensor(0,0)<0) out1=in1;
-    if(!tensor( 0,0)<0)
-        dg::blas1::pointwiseDot( tensor.get(tensor(0,0)), in1, out1); 
-    if(!tensor( 0,1)<0)
-        dg::blas1::pointwiseDot( 1., tensor.get(tensor(0,1)), in2, 1., out1);
-    if(!tensor( 0,2)<0)
-        dg::blas1::pointwiseDot( 1., tensor.get(tensor(0,2)), in3, 1., out1);
-
-    if( tensor(1,1)<0) out2=in2;
-    if(!tensor(1,1)<0)
-        dg::blas1::pointwiseDot( tensor.get(tensor(1,1)), in2, out2);
-    if(!tensor(1,0)<0)
-        dg::blas1::pointwiseDot( 1., tensor.get(tensor(1,0)), in1, 1., out2);
-    if(!tensor(1,2)<0)
-        dg::blas1::pointwiseDot( 1., tensor.get(tensor(1,2)), in1, 1., out2);
-
-    if( tensor(2,2)<0) out3=in3;
-    if(!tensor(2,2)<0)
-        dg::blas1::pointwiseDot( tensor.get(tensor(2,2)), in3, out3);
-    if(!tensor(2,1)<0)
-        dg::blas1::pointwiseDot( 1., tensor.get(tensor(2,1)), in2, 1., out3);
-    if(!tensor(2,0)<0)
-        dg::blas1::pointwiseDot( 1., tensor.get(tensor(2,0)), in1, 1., out3);
+    if( !tensor.isSet(0,0)) out1=in1;
+    if(tensor.isSet( 0,0))
+        dg::blas1::pointwiseDot( tensor.getValue(0,0), in1, out1); 
+    if(tensor.isSet( 0,1))
+        dg::blas1::pointwiseDot( 1., tensor.getValue(0,1), in2, 1., out1);
+    if(tensor.isSet( 0,2))
+        dg::blas1::pointwiseDot( 1., tensor.getValue(0,2), in3, 1., out1);
+
+    if( !tensor.isSet(1,1)) out2=in2;
+    if(tensor.isSet(1,1))
+        dg::blas1::pointwiseDot( tensor.getValue(1,1), in2, out2);
+    if(tensor.isSet(1,0))
+        dg::blas1::pointwiseDot( 1., tensor.getValue(1,0), in1, 1., out2);
+    if(tensor.isSet(1,2))
+        dg::blas1::pointwiseDot( 1., tensor.getValue(1,2), in1, 1., out2);
+
+    if(!tensor.isSet(2,2)) out3=in3;
+    if(tensor.isSet(2,2))
+        dg::blas1::pointwiseDot( tensor.getValue(2,2), in3, out3);
+    if(tensor.isSet(2,1))
+        dg::blas1::pointwiseDot( 1., tensor.getValue(2,1), in2, 1., out3);
+    if(tensor.isSet(2,0))
+        dg::blas1::pointwiseDot( 1., tensor.getValue(2,0), in1, 1., out3);
 }
 
 }//namespace geo
-
-
-
-
 }//namespace dg
-- 
GitLab


From 2f0a432e73f1038663d1a2e140764439dd4dcf2d Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sat, 29 Jul 2017 22:08:40 +0200
Subject: [PATCH 095/453] changed interface of the shared tensor class

---
 inc/dg/geometry/tensor.h | 89 ++++++++++++++++++++++------------------
 1 file changed, 48 insertions(+), 41 deletions(-)

diff --git a/inc/dg/geometry/tensor.h b/inc/dg/geometry/tensor.h
index 9f799377d..4fc1db051 100644
--- a/inc/dg/geometry/tensor.h
+++ b/inc/dg/geometry/tensor.h
@@ -7,13 +7,20 @@
 namespace dg
 {
 /**
-* @brief Abstract base class for Sparse matrices of containers
+* @brief Abstract base class for matrices and vectors sharing or implicitly assuming elements 
 *
-* The size is always assumed 3x3.
-* The elements are implicitly assumed either 1 (diagonal) or 0 (off-diagonal)
+* The goal of this class is to enable shared access to stored containers 
+* or not store them at all since the storage of (and computation with) a container is expensive.
+
+* This class contains both a (dense) matrix and a (dense) vector of integers.
+* If positive or zero, the integer represents a gather index into an array of containers, 
+if negative the value of the container is assumed to be 1, except for the off-diagonal entries
+    in the matrix where it is assumed to be 0.
+* We then only need to store non-trivial and non-repetitive containers.
+* @tparam container container class
 */
 template<class container>
-struct SparseUnitTensor
+struct SharedContainers
 {
     /**
     * @brief check if an index is set or not
@@ -25,66 +32,66 @@ struct SparseUnitTensor
         if( get_value_idx(i,j) <0) return false;
         return true;
     }
-    /**
-    * @brief set an index into the values array (if <0 then value is assumed implicitly)
-    * @param i row index 0<i<2
-    * @param j col index 0<j<2
-    * @return reference to an index     
-    */
-    int& valueIdx(size_t i, size_t j){ return get_value_idx(i,j); }
-    /**
-    * @brief get an index into the values array (if <0 then value is assumed implicitly)
-    * @param i row index 0<i<2
-    * @param j col index 0<j<2
-    * @return reference to an index     
-    */
-    int valueIdx(size_t i, size_t j)const{ return get_value_idx(i,j); }
+    bool isSet(size_t i) const{
+        if( get_value_idx(i)<0) return false;
+        return true;
+    }
+    /// Access the underlying container
+    /// @return if !isSet(i,j) the default constructor of container is called, otherwise values[mat_idx(i,j)] is returned
     const container& getValue(size_t i, size_t j)const{ 
         int k = get_value_idx(i,j);
         if(k<0) return container();
-        return getValue(k); 
+        return values_[k];
+    }
+    /// Access the underlying container
+    /// @return if !isSet(i) the default constructor of container is called, otherwise values[vec_idx(i)] is returned
+    const container& getValue(size_t i)const{ 
+        int k = get_value_idx(i);
+        if(k<0) return container();
+        return values_[k];
     }
-    const container& getValue(size_t i)const{ return values_[i]; }
-    ///set a value for the given idx
-    container& getValue(int i){ return values_[i]; }
     protected:
-    ///by protecting the constructor we fix the size of the values 
-    ///array associated with the types of derived classes
-    SparseUnitTensor( size_t values_size=9): values_(values_size),idx_(3,-1){}
+    ///by protecting the constructor we disallow changing anything through a base class reference
+    SharedContainers( const dg::Operator<int>& mat_idx, std::vector<int>& vec_idx, std::vector<container>& values ): mat_idx_(mat_idx), vec_idx(vec_idx), values_(values){}
     ///we disallow changeing the size
-    SparseUnitTensor( const SparseUnitTensor& src): values_(src.values_), idx_(src.idx_){}
+    SharedContainers( const SharedContainers& src):  mat_idx_(src.mat_idx_), vec_idx_(src.vec_idx_), values_(src.values_){}
     ///we disallow changeing the size
-    SparseUnitTensor& operator=()(SparseUnitTensor src)
+    SharedContainers& operator=()(SharedContainers src)
     {
         values_.swap( src.values_);
-        std::swap( idx_, src.idx_)
+        std::swap( mat_idx_, src.mat_idx_)
+        std::swap( vec_idx_, src.vec_idx_)
         return *this;
     }
     ///rule of three
-    ~SparseUnitTensor(){}
+    ~SharedContainers(){}
     template<class otherContainer>
-    SparseUnitTensor( const SparseUnitTensor<otherContainer>& src): values_(src.values_.size()), idx_(src.idx_){
-        dg::blas1::transfer( src.values_, values_);
+    SharedContainers( const SharedContainers<otherContainer>& src): mat_idx_(src.mat_idx()), vec_idx_(src.vec_idx()), values_(src.values().size()), idx_(src.idx_){
+        dg::blas1::transfer( src.values(), values_);
     }
+    const dg::Operator<int>& mat_idx() const {return mat_idx_;}
+    const std::vector<int>& vec_idx() const {return vec_idx_;}
+    const std::vector<container>& values() const{return values_;}
     private:
-    int get_value_idx(size_t i, size_t j)const{return idx_(i,j);}
-    int& get_value_idx(size_t i, size_t j){return idx_(i,j);}
+    int get_value_idx(size_t i, size_t j)const{return mat_idx_(i,j);}
+    int get_value_idx(size_t i)const{return vec_idx_[i];}
+    dg::Operator<int> mat_idx_;
+    std::vector<int> vec_idx_;
     std::vector<container> values_;
-    dg::Operator<int>& idx_;
 };
 
 ///The metric tensor is a SparseTensor 
 template<class container>
-struct MetricTensor: public SparseUnitTensor<container>
+struct MetricTensor: public SharedContainers<container>
 {
     /// default size is 3 and no values are set
-    MetricTensor( ):SparseUnitTensor( 11){ volIdx_=perpVolIdx_=-1; }
+    MetricTensor( ):SharedContainers( 11){ volIdx_=perpVolIdx_=-1; }
     bool volIsSet() const{ 
         if(volIdx_<0)return false;
         else return true;
     }
     template<class otherContainer>
-    MatricTensor( MetricTensor<otherContainer>& src):SparseUnitTensor<container>(src){}
+    MatricTensor( MetricTensor<otherContainer>& src):SharedContainers<container>(src){}
     bool perpVolIsSet() const{ 
         if(perpVolIdx_<0)return false;
         else return true;
@@ -109,10 +116,10 @@ struct MetricTensor: public SparseUnitTensor<container>
 };
 
 template<class container>
-struct CoordinateTransformation: public SparseUnitTensor<container>
+struct CoordinateTransformation: public SharedContainers<container>
 {
     /// the coordinates have index 9, 10 and 11
-    CoordinateTransformation():SparseUnitTensor( 12){}
+    CoordinateTransformation():SharedContainers( 12){}
 };
 
 namespace geo
@@ -122,7 +129,7 @@ namespace geo
 ///may destroy input (either swap in or use as temporary storage)
 ///no alias allowed
 template<class container>
-void multiply( const SparseUnitTensor& t, container& in1, container& in2, container& out1, container& out2)
+void multiply( const SharedContainers& t, container& in1, container& in2, container& out1, container& out2)
 {
     if( !tensor.isSet(0,0) && !tensor.isSet(0,1) && !tensor.isSet(1,0) && !tensor.isSet(1,1))
     {
@@ -144,7 +151,7 @@ void multiply( const SparseUnitTensor& t, container& in1, container& in2, contai
 }
 
 template<class container>
-void multiply( const SparseUnitTensor& t, container& in1, container& in2, container& in3, container& out1, container& out2, container& out3)
+void multiply( const SharedContainers& t, container& in1, container& in2, container& in3, container& out1, container& out2, container& out3)
 {
     bool empty = true;
     for( unsigned i=0; i<3; i++)
-- 
GitLab


From 3ef3290af19c5162562a48d4d82231810d142aec Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 31 Jul 2017 08:17:51 -0700
Subject: [PATCH 096/453] replaced Message and Ooops error classes with
 std::runtime_exception

---
 inc/dg/backend/dlt.h         |  2 +-
 inc/dg/backend/operator.h    | 12 ++----------
 inc/dg/backend/operator_t.cu |  2 +-
 inc/dg/enums.h               |  4 ++--
 inc/dg/exceptions.h          | 24 ------------------------
 inc/dg/nullstelle.h          |  2 +-
 6 files changed, 7 insertions(+), 39 deletions(-)

diff --git a/inc/dg/backend/dlt.h b/inc/dg/backend/dlt.h
index 196b01052..6670110e2 100644
--- a/inc/dg/backend/dlt.h
+++ b/inc/dg/backend/dlt.h
@@ -187,7 +187,7 @@ DLT<T>::DLT( unsigned n):a_(n), w_(n), forw_(n*n), back_(n*n),backEQ_(n*n)
     for( unsigned j=0; j<n*n; j++) stream >> forw_[j];
     for( unsigned j=0; j<n*n; j++) stream >> backEQ_[j];
     if( !stream.good())
-        throw dg::Ooops( "Error: n>20 not allowed!");
+        throw std::invalid_argument( "Error: n>20 not allowed!");
 }
 
 
diff --git a/inc/dg/backend/operator.h b/inc/dg/backend/operator.h
index a26e92659..fb6ed647a 100644
--- a/inc/dg/backend/operator.h
+++ b/inc/dg/backend/operator.h
@@ -363,14 +363,6 @@ namespace create
 namespace detail
 {
 
-struct Message : public std::exception
-{
-    Message( const char * m) : message(m){}
-    char const*  what() const throw(){return message;}
-    private:
-    const char * message;
-};
-
 /*! @brief LU Decomposition with partial pivoting
  *
  * @tparam T value type
@@ -417,7 +409,7 @@ T lr_pivot( dg::Operator<T>& m, std::vector<unsigned>& p)
 
         }
         else 
-            throw Message( "Matrix is singular!!");
+            throw std::runtime_error( "Matrix is singular!!");
     }
     if( numberOfSwaps % 2 != 0)
         determinant*=-1.;
@@ -481,7 +473,7 @@ dg::Operator<T> invert( const dg::Operator<T>& in)
     dg::Operator<T> lr(in);
     T determinant = detail::lr_pivot( lr, pivot);
     if( fabs(determinant ) < 1e-14) 
-        throw detail::Message( "Determinant zero!!");
+        throw std::runtime_error( "Determinant zero!");
     for( unsigned i=0; i<n; i++)
     {
         std::vector<T> unit(n, 0);
diff --git a/inc/dg/backend/operator_t.cu b/inc/dg/backend/operator_t.cu
index f6ac293ad..5ae1ffabf 100644
--- a/inc/dg/backend/operator_t.cu
+++ b/inc/dg/backend/operator_t.cu
@@ -28,7 +28,7 @@ int main()
     std::cout << "Multiplication\n"<<inv_op*op<<"\n";
 
     op.zero();
-    op(0,2) = op(1,1) = op(2,0) = 1;// op(3,3)= 1;
+    op(0,2) = op(1,1) = op(2,0) = 0;// op(3,3)= 1;
     std::cout << "Operator\n"<<op<<"\n";
     inv_op = dg::create::invert(op);
     lr = op;
diff --git a/inc/dg/enums.h b/inc/dg/enums.h
index a56c027ca..e1f689b69 100644
--- a/inc/dg/enums.h
+++ b/inc/dg/enums.h
@@ -62,7 +62,7 @@ std::string bc2str( bc bcx)
  * - NEU_DIR and neu_dir to dg::NEU_DIR
  * - DIR_NEU and dir_neu to dg::DIR_NEU
  *
- * or throws an Ooops if string doesn't match any of these
+ * or throws a runtime_exception if string doesn't match any of these
  * @param s the input string
  * @ingroup creation
  */
@@ -78,7 +78,7 @@ bc str2bc( std::string s)
         return NEU_DIR;
     if( s=="DIR_NEU"||s=="dir_neu" )
         return DIR_NEU;
-    throw Ooops( "No matching boundary condition!");
+    throw std::runtime_exception( "No matching boundary condition!");
 }
 
 /**
diff --git a/inc/dg/exceptions.h b/inc/dg/exceptions.h
index 5bd81f042..1de985579 100644
--- a/inc/dg/exceptions.h
+++ b/inc/dg/exceptions.h
@@ -41,28 +41,4 @@ struct Fail : public std::exception
     double eps;
 };
 
-/**
- * @brief Class you might want to throw in case something goes wrong
- *
- * @ingroup numerical0
- */
-struct Ooops : public std::exception
-{
-
-    /**
-     * @brief Construct from error string
-     *
-     * @param c error string
-     */
-    Ooops( const char * c): c_( c) {}
-    /**
-     * @brief What string
-     *
-     * @return error string
-     */
-    char const* what() const throw(){ return c_;}
-  private:
-    const char* c_;
-};
-
 }//namespace dg
diff --git a/inc/dg/nullstelle.h b/inc/dg/nullstelle.h
index c5e83f56e..ced590a3b 100644
--- a/inc/dg/nullstelle.h
+++ b/inc/dg/nullstelle.h
@@ -79,7 +79,7 @@ int bisection1d (UnaryOp& funktion, double& x_min, double& x_max, const double a
         else 				                x_min = mitte;
         if((x_max-x_min)<aufloesung)        return j+3; 
     }
-    throw Ooops("Zu viele Schritte bei Nullstellensuche! evtl. j_max aendern");
+    throw std::runtime_exception("Zu viele Schritte bei Nullstellensuche! evtl. j_max aendern");
 }
 
       
-- 
GitLab


From 918517316f122c6c0a68e753db7443f1c5d7a280 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Tue, 1 Aug 2017 08:57:12 -0700
Subject: [PATCH 097/453] added message and Error class to throw

---
 inc/dg/backend/dlt.h      |  6 ++-
 inc/dg/backend/operator.h |  3 +-
 inc/dg/exceptions.h       | 83 +++++++++++++++++++++++++++++++++++++--
 3 files changed, 85 insertions(+), 7 deletions(-)

diff --git a/inc/dg/backend/dlt.h b/inc/dg/backend/dlt.h
index 6670110e2..85533eb25 100644
--- a/inc/dg/backend/dlt.h
+++ b/inc/dg/backend/dlt.h
@@ -23,7 +23,7 @@ class DLT
        * @brief Initialize coefficients
        *
        * The constructor reads the data corresponding to given n from the file dlt.dat. 
-       * @param n # of polynomial coefficients
+       * @param n # of polynomial coefficients (0<n<21)
        */
     DLT( unsigned n);
 
@@ -186,8 +186,10 @@ DLT<T>::DLT( unsigned n):a_(n), w_(n), forw_(n*n), back_(n*n),backEQ_(n*n)
     for( unsigned j=0; j<n*n; j++) stream >> back_[j];
     for( unsigned j=0; j<n*n; j++) stream >> forw_[j];
     for( unsigned j=0; j<n*n; j++) stream >> backEQ_[j];
+    
+    if( n==0) throw Error( Message() << "n==0 not allowed! You typed "<<n,_ping_);
     if( !stream.good())
-        throw std::invalid_argument( "Error: n>20 not allowed!");
+        throw Error( Message()<<"n>20 not allowed! You typed: "<<n, _ping_);
 }
 
 
diff --git a/inc/dg/backend/operator.h b/inc/dg/backend/operator.h
index fb6ed647a..1f11e3414 100644
--- a/inc/dg/backend/operator.h
+++ b/inc/dg/backend/operator.h
@@ -366,6 +366,7 @@ namespace detail
 /*! @brief LU Decomposition with partial pivoting
  *
  * @tparam T value type
+ * @note this function throws a runtime_error when the matrix is singular
  */
 template< class T>
 T lr_pivot( dg::Operator<T>& m, std::vector<unsigned>& p)
@@ -462,7 +463,7 @@ void lr_solve( const dg::Operator<T>& lr, const std::vector<unsigned>& p, std::v
  * @param in input matrix
  *
  * @return the inverse of in if it exists
- * @note throws a message if in is singular
+ * @note throws a std::runtime_error if in is singular
  */
 template<class T>
 dg::Operator<T> invert( const dg::Operator<T>& in)
diff --git a/inc/dg/exceptions.h b/inc/dg/exceptions.h
index 1de985579..6c7461ea2 100644
--- a/inc/dg/exceptions.h
+++ b/inc/dg/exceptions.h
@@ -1,19 +1,93 @@
+/*!
+ * \file 
+ * \author Matthias Wiesenberger
+ * \date 01.08.2017 
+ */
 #pragma once
 
 
 #include <exception>
+#include <iostream>
+#include <sstream>
+
+/*! for the simplified construction of a Message use this Macro*/
+#define _ping_ __FILE__, __LINE__ 
+
+namespace dg
+{
+
+///small class holding a stringstream 
+///@ingroup numerical0
+class Message 
+{
+  private:
+    std::stringstream sstream_;
+    Message( const Message&); //we can't copy ostreams in C++
+    Message& operator=(const Message&);
+  public:
+    Message(){}
+    Message( std::string m){ sstream_<<m;}
+    ~Message(){}
+    /// @note you can't use std::endl or std::flush in here
+    template<class T>
+    Message & operator << (const T& value)
+    {
+        sstream_ << value;
+        return *this;
+    }
+    ///return the message as a string
+    std::string str() const {return sstream_.str();}
+    ///put the sringstream string into the ostream
+    friend std::ostream& operator<<(std::ostream& os, const Message& m)
+    {
+        os<<m.str();
+        return os;
+    }
+};
 
 
-/*!@file 
+/*! @brief class intended for the use in throw statements
  *
- * Error class to thow. Derived from std::exception
+ * The objects of this class store a message (that describes the error when thrown)
+ * that can then be displayed in a catch block
+ * \code
+ * try{ throw Error(Message()<<"This is error number "<<number, _ping_);}
+ * catch( Error& m) {std::cerr << m.what();}
+ * \endcode
+ * numerical0
  */
-namespace dg
+class Error : public std::exception
 {
+  private:
+    std::string m;//with a string the Error is copyable
+    const char* f;
+    const int l;
+  public:
+     
+    /*! @brief Constructor
+     *
+     * @param message An instance of a Message class
+     * @param file The file in which the exception is thrown (contained in the predefined Macro __FILE__)
+     * @param line The line in which the exception is thrown (contained in the predefined Macro __LINE__)
+     * \note The Macro _ping_ combines __FILE__, __LINE__ in one. 
+     */
+    Error(const Message& message, const char* file, const int line): f(file), l(line){
+        Message tmp;
+        tmp << "\n    Message from file **"<<f<<"** in line **" <<l<<"**:\n    "<<message<<"\n";
+        m = tmp.str();
+    }
+    
+    /// @return file, line and the message given in the constructor as a string of char
+    virtual const char* what() const throw()
+    {
+        return m.c_str();
+    }
+    virtual ~Error() throw(){}
+};
+
 
 /**
  * @brief Class you might want to throw in case of a non convergence
- *
  * @ingroup numerical0
  */
 struct Fail : public std::exception
@@ -37,6 +111,7 @@ struct Fail : public std::exception
      * @return string "Failed to converge"
      */
     char const* what() const throw(){ return "Failed to converge";}
+    virtual ~Fail() throw(){}
   private:
     double eps;
 };
-- 
GitLab


From 4718e6a5d21f4e8eec9ad1d7cff8bf8b005a62be Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Tue, 1 Aug 2017 09:03:56 -0700
Subject: [PATCH 098/453] new Message and Error class

---
 inc/dg/enums.h      | 3 ++-
 inc/dg/nullstelle.h | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/inc/dg/enums.h b/inc/dg/enums.h
index e1f689b69..45a82675a 100644
--- a/inc/dg/enums.h
+++ b/inc/dg/enums.h
@@ -62,8 +62,9 @@ std::string bc2str( bc bcx)
  * - NEU_DIR and neu_dir to dg::NEU_DIR
  * - DIR_NEU and dir_neu to dg::DIR_NEU
  *
- * or throws a runtime_exception if string doesn't match any of these
  * @param s the input string
+ * @return a valid boundary condition
+ * \throw std::runtime_exception if string doesn't match any of the above
  * @ingroup creation
  */
 bc str2bc( std::string s)
diff --git a/inc/dg/nullstelle.h b/inc/dg/nullstelle.h
index ced590a3b..3f509dd8a 100644
--- a/inc/dg/nullstelle.h
+++ b/inc/dg/nullstelle.h
@@ -54,7 +54,7 @@ class KeineNST_1D: public std::exception
  * \param aufloesung accuracy of the root finding	
  * \return number of used steps to reach the desired accuracy
  * \throw KeineNST_1D if no root lies between the given boundaries
- * \throw Oooops if after 50 steps the accuracy wasn't reached
+ * \throw std::runtime_exception if after 60 steps the accuracy wasn't reached
  *
  * \code nullstelle_1D(funk, x_min, x_max, aufloesung); \endcode
  * \note If the root is found exactly the x_min = x_max 
-- 
GitLab


From e89571592f4c5722ee46b9f208a99558e4cff760 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Wed, 2 Aug 2017 00:08:33 +0200
Subject: [PATCH 099/453] update tensor.h

---
 inc/dg/geometry/tensor.h | 69 ++++------------------------------------
 1 file changed, 7 insertions(+), 62 deletions(-)

diff --git a/inc/dg/geometry/tensor.h b/inc/dg/geometry/tensor.h
index 4fc1db051..9c8dbbfa6 100644
--- a/inc/dg/geometry/tensor.h
+++ b/inc/dg/geometry/tensor.h
@@ -6,6 +6,7 @@
 
 namespace dg
 {
+
 /**
 * @brief Abstract base class for matrices and vectors sharing or implicitly assuming elements 
 *
@@ -29,42 +30,30 @@ struct SharedContainers
     * @return true if container is non-empty, false if value is assumed implicitly
     */
     bool isSet(size_t i, size_t j)const{
-        if( get_value_idx(i,j) <0) return false;
+        if( mat_idx_(i,j) <0) return false;
         return true;
     }
+
     bool isSet(size_t i) const{
-        if( get_value_idx(i)<0) return false;
+        if( vec_idx_[i]<0) return false;
         return true;
     }
     /// Access the underlying container
     /// @return if !isSet(i,j) the default constructor of container is called, otherwise values[mat_idx(i,j)] is returned
     const container& getValue(size_t i, size_t j)const{ 
-        int k = get_value_idx(i,j);
+        int k = mat_idx(i,j);
         if(k<0) return container();
         return values_[k];
     }
     /// Access the underlying container
     /// @return if !isSet(i) the default constructor of container is called, otherwise values[vec_idx(i)] is returned
     const container& getValue(size_t i)const{ 
-        int k = get_value_idx(i);
+        int k = vec_idx_[i];
         if(k<0) return container();
         return values_[k];
     }
-    protected:
-    ///by protecting the constructor we disallow changing anything through a base class reference
+    SharedContainers( ) {}
     SharedContainers( const dg::Operator<int>& mat_idx, std::vector<int>& vec_idx, std::vector<container>& values ): mat_idx_(mat_idx), vec_idx(vec_idx), values_(values){}
-    ///we disallow changeing the size
-    SharedContainers( const SharedContainers& src):  mat_idx_(src.mat_idx_), vec_idx_(src.vec_idx_), values_(src.values_){}
-    ///we disallow changeing the size
-    SharedContainers& operator=()(SharedContainers src)
-    {
-        values_.swap( src.values_);
-        std::swap( mat_idx_, src.mat_idx_)
-        std::swap( vec_idx_, src.vec_idx_)
-        return *this;
-    }
-    ///rule of three
-    ~SharedContainers(){}
     template<class otherContainer>
     SharedContainers( const SharedContainers<otherContainer>& src): mat_idx_(src.mat_idx()), vec_idx_(src.vec_idx()), values_(src.values().size()), idx_(src.idx_){
         dg::blas1::transfer( src.values(), values_);
@@ -73,55 +62,11 @@ struct SharedContainers
     const std::vector<int>& vec_idx() const {return vec_idx_;}
     const std::vector<container>& values() const{return values_;}
     private:
-    int get_value_idx(size_t i, size_t j)const{return mat_idx_(i,j);}
-    int get_value_idx(size_t i)const{return vec_idx_[i];}
     dg::Operator<int> mat_idx_;
     std::vector<int> vec_idx_;
     std::vector<container> values_;
 };
 
-///The metric tensor is a SparseTensor 
-template<class container>
-struct MetricTensor: public SharedContainers<container>
-{
-    /// default size is 3 and no values are set
-    MetricTensor( ):SharedContainers( 11){ volIdx_=perpVolIdx_=-1; }
-    bool volIsSet() const{ 
-        if(volIdx_<0)return false;
-        else return true;
-    }
-    template<class otherContainer>
-    MatricTensor( MetricTensor<otherContainer>& src):SharedContainers<container>(src){}
-    bool perpVolIsSet() const{ 
-        if(perpVolIdx_<0)return false;
-        else return true;
-    }
-    int& volIdx(){ return volIdx_;}
-    int& perpVolIdx(){ return perpVolIdx_;}
-    int volIdx()const{ return volIdx_;}
-    int perpVolIdx()const{ return perpVolIdx_;}
-
-    const container& getVol() const{return get(volIdx_);}
-    const container& getPerpVol() const{return get(perpVolIdx_);}
-    void setVol(const container& vol){ 
-        volIdx_=9;
-        this->get(9) = value; 
-    }
-    void setPerpVol(const container& vol){ 
-        perpVolIdx_=10;
-        this->get(10) = value; 
-    }
-    private:
-    int volIdx_, perpVolIdx_;
-};
-
-template<class container>
-struct CoordinateTransformation: public SharedContainers<container>
-{
-    /// the coordinates have index 9, 10 and 11
-    CoordinateTransformation():SharedContainers( 12){}
-};
-
 namespace geo
 {
 
-- 
GitLab


From 107778c7e4e719e8b89fcbdc5c8f8eee4746ee8b Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Wed, 2 Aug 2017 17:23:59 +0200
Subject: [PATCH 100/453] first changed Grid to Topology

---
 inc/dg/backend/evaluation.cuh |   8 +-
 inc/dg/backend/grid.h         | 273 ++++++++++++++++++----------------
 inc/dg/enums.h                |   4 +-
 3 files changed, 153 insertions(+), 132 deletions(-)

diff --git a/inc/dg/backend/evaluation.cuh b/inc/dg/backend/evaluation.cuh
index 4fc0a8dd7..5afe0156d 100644
--- a/inc/dg/backend/evaluation.cuh
+++ b/inc/dg/backend/evaluation.cuh
@@ -57,7 +57,7 @@ thrust::host_vector<double> evaluate( double (f)(double), const Grid1d& g)
  * @note if you don't like to copy f then just pass a (const) reference then the type should adapt
  */
 template< class BinaryOp>
-thrust::host_vector<double> evaluate( BinaryOp f, const Grid2d& g)
+thrust::host_vector<double> evaluate( BinaryOp f, const aTopology2d& g)
 {
     unsigned n= g.n();
     //TODO: opens dlt.dat twice...!!
@@ -77,7 +77,7 @@ thrust::host_vector<double> evaluate( BinaryOp f, const Grid2d& g)
     return v;
 };
 ///@cond
-thrust::host_vector<double> evaluate( double(f)(double, double), const Grid2d& g)
+thrust::host_vector<double> evaluate( double(f)(double, double), const aTopology2d& g)
 {
     //return evaluate<double(&)(double, double), n>( f, g );
     return evaluate<double(double, double)>( f, g);
@@ -96,7 +96,7 @@ thrust::host_vector<double> evaluate( double(f)(double, double), const Grid2d& g
  * @note if you don't like to copy f then just pass a (const) reference then the type should adapt
  */
 template< class TernaryOp>
-thrust::host_vector<double> evaluate( TernaryOp f, const Grid3d& g)
+thrust::host_vector<double> evaluate( TernaryOp f, const aTopology3d& g)
 {
     unsigned n= g.n();
     //TODO: opens dlt.dat three times...!!
@@ -118,7 +118,7 @@ thrust::host_vector<double> evaluate( TernaryOp f, const Grid3d& g)
     return v;
 };
 ///@cond
-thrust::host_vector<double> evaluate( double(f)(double, double, double), const Grid3d& g)
+thrust::host_vector<double> evaluate( double(f)(double, double, double), const aTopology3d& g)
 {
     //return evaluate<double(&)(double, double), n>( f, g );
     return evaluate<double(double, double, double)>( f, g);
diff --git a/inc/dg/backend/grid.h b/inc/dg/backend/grid.h
index 6609dd416..114b84aed 100644
--- a/inc/dg/backend/grid.h
+++ b/inc/dg/backend/grid.h
@@ -8,14 +8,14 @@
 
 /*! @file 
   
-  Grid objects
+  aTopology objects
   */
 
 
 namespace dg{
 
-class MPIGrid2d;
-class MPIGrid3d;
+class aMPITopology2d;
+class aMPITopology3d;
 
 
 ///@addtogroup grid
@@ -103,7 +103,7 @@ struct Grid1d
     const DLT<double>& dlt() const {return dlt_;}
     void display( std::ostream& os = std::cout) const
     {
-        os << "Grid parameters are: \n"
+        os << "aTopology parameters are: \n"
             <<"    n  = "<<n_<<"\n"
             <<"    N = "<<Nx_<<"\n"
             <<"    h = "<<hx_<<"\n"
@@ -118,7 +118,7 @@ struct Grid1d
      * @brief Shifts a point coordinate if periodic
      *
      * This function shifts a point coordinate to its value between x0() and x1() if bcx() returns dg::PER
-     * @param x0 arbitrary point (irrelevant for the function, it's there to be consistent with GridX1d)
+     * @param x0 arbitrary point (irrelevant for the function, it's there to be consistent with aTopologyX1d)
      * @param x1 end point (inout)
      */
     void shift_topologic( double x0, double& x1)const
@@ -153,56 +153,17 @@ struct Grid1d
     DLT<double> dlt_;
 };
 
-struct Grid3d; //forward declare 3d version
-
+struct aTopology3d; //forward declare 3d version
 
 /**
- * @brief A 2D grid class 
- *
+ * @brief An abstract base class for two-dimensional grids
  */
-struct Grid2d
+struct aTopology2d
 {
     typedef SharedTag memory_category;
     typedef TwoDimensionalTag dimensionality;
-    /**
-     * @brief Construct a 2D grid
-     *
-     * @param x0 left boundary in x
-     * @param x1 right boundary in x 
-     * @param y0 lower boundary in y
-     * @param y1 upper boundary in y 
-     * @param n  # of polynomial coefficients per dimension
-     * @param Nx # of points in x 
-     * @param Ny # of points in y
-     * @param bcx boundary condition in x
-     * @param bcy boundary condition in y
-     */
-    Grid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx = PER, bc bcy = PER):
-        x0_(x0), x1_(x1), y0_(y0), y1_(y1), 
-        n_(n), Nx_(Nx), Ny_(Ny), bcx_(bcx), bcy_( bcy), dlt_(n)
-    {
-        assert( n != 0);
-        assert( x1 > x0 && y1 > y0);
-        assert( Nx > 0  && Ny > 0);
-        lx_ = (x1_-x0_), ly_ = (y1_-y0_);
-        hx_ = lx_/(double)Nx_, hy_ = ly_/(double)Ny_;
-    }
-    /**
-     * @brief Construct a 2d grid as the product of two 1d grids
-     *
-     * @param gx Grid in x - direction
-     * @param gy Grid in y - direction
-     */
-    Grid2d( const Grid1d& gx, const Grid1d& gy): 
-        x0_(gx.x0()), x1_(gx.x1()), y0_(gy.x0()), y1_(gy.x1()), 
-        n_(gx.n()), Nx_(gx.N()), Ny_(gy.N()), bcx_(gx.bcx()), bcy_( gy.bcx()), dlt_(gx.n())
-    {
-        assert( gx.n() == gy.n() );
-        lx_ = (x1_-x0_), ly_ = (y1_-y0_);
-        hx_ = lx_/(double)Nx_, hy_ = ly_/(double)Ny_;
-    }
 
-    Grid2d( const Grid3d& g);
+    //aTopology2d( const aTopology3d& g);
     /**
      * @brief Left boundary in x
      *
@@ -281,12 +242,6 @@ struct Grid2d
      * @return 
      */
     bc bcy() const {return bcy_;}
-    /**
-     * @brief Return a copy
-     *
-     * @return 
-     */
-    Grid2d local_grid() const {return *this;}
 
     /**
     * @brief Multiply the number of cells with a given factor
@@ -331,7 +286,7 @@ struct Grid2d
      */
     void display( std::ostream& os = std::cout) const
     {
-        os << "Grid parameters are: \n"
+        os << "aTopology parameters are: \n"
             <<"    n  = "<<n_<<"\n"
             <<"    Nx = "<<Nx_<<"\n"
             <<"    Ny = "<<Ny_<<"\n"
@@ -353,8 +308,8 @@ struct Grid2d
      *
      * This function shifts point coordinates to its values inside
      the domain if the respective boundary condition is periodic
-     * @param x0 arbitrary coordinate (irrelevant for the function, it's there to be consistent with GridX2d)
-     * @param y0 arbitrary coordinate (irrelevant for the function, it's there to be consistent with GridX2d)
+     * @param x0 arbitrary coordinate (irrelevant for the function, it's there to be consistent with aTopologyX2d)
+     * @param y0 arbitrary coordinate (irrelevant for the function, it's there to be consistent with aTopologyX2d)
      * @param x1 x-coordinate to shift (inout)
      * @param y1 y-coordinate to shift (inout)
      */
@@ -388,17 +343,60 @@ struct Grid2d
         if( (x>=x0_ && x <= x1_) && (y>=y0_ && y <= y1_)) return true; 
         return false;
     }
-    virtual ~Grid2d(){}
-    Grid2d(const Grid2d& src):dlt_(src.dlt_){*this = src;}
-    Grid2d& operator=(const Grid2d& src){
+    ///allow destruction through base class pointer
+    virtual ~aTopology2d(){}
+    ///deep copy of an object onto the heap
+    virtual aTopology2d* clone()const=0;
+    protected:
+    /**
+     * @brief Construct a 2D grid
+     *
+     * @param x0 left boundary in x
+     * @param x1 right boundary in x 
+     * @param y0 lower boundary in y
+     * @param y1 upper boundary in y 
+     * @param n  # of polynomial coefficients per dimension
+     * @param Nx # of points in x 
+     * @param Ny # of points in y
+     * @param bcx boundary condition in x
+     * @param bcy boundary condition in y
+     */
+    aTopology2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx = PER, bc bcy = PER):
+        x0_(x0), x1_(x1), y0_(y0), y1_(y1), 
+        n_(n), Nx_(Nx), Ny_(Ny), bcx_(bcx), bcy_( bcy), dlt_(n)
+    {
+        assert( n != 0);
+        assert( x1 > x0 && y1 > y0);
+        assert( Nx > 0  && Ny > 0);
+        lx_ = (x1_-x0_), ly_ = (y1_-y0_);
+        hx_ = lx_/(double)Nx_, hy_ = ly_/(double)Ny_;
+    }
+    /**
+     * @brief Construct a 2d grid as the product of two 1d grids
+     *
+     * @param gx aTopology in x - direction
+     * @param gy aTopology in y - direction
+     * @note gx and gy must have the same n
+     */
+    aTopology2d( const Grid1d& gx, const Grid1d& gy): 
+        x0_(gx.x0()), x1_(gx.x1()), y0_(gy.x0()), y1_(gy.x1()), 
+        n_(gx.n()), Nx_(gx.N()), Ny_(gy.N()), bcx_(gx.bcx()), bcy_( gy.bcx()), dlt_(gx.n())
+    {
+        assert( gx.n() == gy.n() );
+        lx_ = (x1_-x0_), ly_ = (y1_-y0_);
+        hx_ = lx_/(double)Nx_, hy_ = ly_/(double)Ny_;
+    }
+
+    aTopology2d(const aTopology2d& src):dlt_(src.dlt_){*this = src;}
+    aTopology2d& operator=(const aTopology2d& src){
         x0_=src.x0_, x1_=src.x1_,y0_=src.y0_,y1_=src.y1_;
         lx_=src.lx_,ly_=src.ly_;
         n_=src.n_,Nx_=src.Nx_,Ny_=src.Ny_;
         hx_=src.hx_,hy_=src.hy_;
         bcx_=src.bcx_,bcy_=src.bcy_;
         dlt_=src.dlt_;
+        return *this;
     }
-    protected:
     virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny) {
         n_ = new_n, Nx_ = new_Nx, Ny_ = new_Ny;
         hx_ = lx_/(double)Nx_, hy_ = ly_/(double)Ny_;
@@ -411,8 +409,8 @@ struct Grid2d
         lx_ = (x1_-x0_);
         hx_ = lx_/(double)Nx_;
     }
-  private:
-    friend class MPIGrid2d;
+    private:
+    friend class aMPITopology2d;
     double x0_, x1_, y0_, y1_;
     double lx_, ly_;
     unsigned n_, Nx_, Ny_;
@@ -421,65 +419,18 @@ struct Grid2d
     DLT<double> dlt_;
 };
 
+
 /**
  * @brief A 3D grid class  for cartesian coordinates
  *
  * In the third dimension only 1 polynomial coefficient is used,
  * not n.
  */
-struct Grid3d
+struct aTopology3d
 {
     typedef SharedTag memory_category;
     typedef ThreeDimensionalTag dimensionality;
-    /**
-     * @brief Construct a 3D grid
-     *
-     * @param x0 left boundary in x
-     * @param x1 right boundary in x 
-     * @param y0 lower boundary in y
-     * @param y1 upper boundary in y 
-     * @param z0 lower boundary in z
-     * @param z1 upper boundary in z 
-     * @param n  # of polynomial coefficients per (x-,y-) dimension
-     * @param Nx # of points in x 
-     * @param Ny # of points in y
-     * @param Nz # of points in z
-     * @param bcx boundary condition in x
-     * @param bcy boundary condition in y
-     * @param bcz boundary condition in z
-     * @attention # of polynomial coefficients in z direction is always 1
-     */
-    Grid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = PER, bc bcy = PER, bc bcz = PER):
-        x0_(x0), x1_(x1), y0_(y0), y1_(y1), z0_(z0), z1_(z1),
-        n_(n), Nx_(Nx), Ny_(Ny), Nz_(Nz), bcx_(bcx), bcy_( bcy), bcz_( bcz), dlt_(n)
-    {
-        assert( n != 0);
-        assert( x1 > x0 && y1 > y0 ); assert( z1 > z0 );         
-        assert( Nx > 0  && Ny > 0); assert( Nz > 0);
-
-        lx_ = (x1-x0), ly_ = (y1-y0), lz_ = (z1-z0);
-        hx_ = lx_/(double)Nx_, hy_ = ly_/(double)Ny_, hz_ = lz_/(double)Nz_;
-    }
-    /**
-     * @brief Construct a 3d cartesian grid as the product of three 1d grids
-     *
-     * @param gx Grid in x - direction
-     * @param gy Grid in y - direction
-     * @param gz Grid in z - direction
-     */
-    Grid3d( const Grid1d& gx, const Grid1d& gy, const Grid1d& gz): 
-        x0_(gx.x0()), x1_(gx.x1()),  
-        y0_(gy.x0()), y1_(gy.x1()),
-        z0_(gz.x0()), z1_(gz.x1()),
-        n_(gx.n()), Nx_(gx.N()), Ny_(gy.N()), Nz_(gz.N()),
-        bcx_(gx.bcx()), bcy_( gy.bcx()), bcz_(gz.bcx()), 
-        dlt_(gx.n())
-    {
-        assert( gx.n() == gy.n() );
-        lx_ = (x1_-x0_), ly_ = (y1_-y0_), lz_ = (z1_-z0_);
-        hx_ = lx_/(double)Nx_, hy_ = ly_/(double)Ny_, hz_ = lz_/(double)Nz_;
-    }
-    ///@copydoc Grid2d::multiplyCellNumber()
+    ///@copydoc aTopology2d::multiplyCellNumber()
     void multiplyCellNumber( double fx, double fy){
         set(n_, floor(fx*(double)Nx_+0.5), floor(fy*(double)Ny_+0.5), Nz());
     }
@@ -494,7 +445,7 @@ struct Grid3d
     void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz) {
         assert( new_n>0); assert( new_Nx > 0  && new_Ny > 0); assert( new_Nz > 0);
         if( new_n == n_ && new_Nx == Nx_ && new_Ny == Ny_ && new_Nz == Nz_) return;
-        do_set(new_n, new_Nx,new_Ny,new_Nz);
+        do_set(new_n,new_Nx,new_Ny,new_Nz);
     }
 
     /**
@@ -634,7 +585,7 @@ struct Grid3d
      */
     void display( std::ostream& os = std::cout) const
     {
-        os << "Grid parameters are: \n"
+        os << "aTopology parameters are: \n"
             <<"    n  = "<<n_<<"\n"
             <<"    Nx = "<<Nx_<<"\n"
             <<"    Ny = "<<Ny_<<"\n"
@@ -664,9 +615,9 @@ struct Grid3d
      *
      * This function shifts point coordinates to its values inside
      the domain if the respective boundary condition is periodic
-     * @param x0 arbitrary x-coordinate (irrelevant for the function, it's there to be consistent with GridX3d)
-     * @param y0 arbitrary y-coordinate (irrelevant for the function, it's there to be consistent with GridX3d)
-     * @param z0 arbitrary z-coordinate (irrelevant for the function, it's there to be consistent with GridX3d)
+     * @param x0 arbitrary x-coordinate (irrelevant for the function, it's there to be consistent with aTopologyX3d)
+     * @param y0 arbitrary y-coordinate (irrelevant for the function, it's there to be consistent with aTopologyX3d)
+     * @param z0 arbitrary z-coordinate (irrelevant for the function, it's there to be consistent with aTopologyX3d)
      * @param x1 x-coordinate to shift (inout)
      * @param y1 y-coordinate to shift (inout)
      * @param z1 z-coordinate to shift (inout)
@@ -709,18 +660,69 @@ struct Grid3d
             return true; 
         return false;
     }
-    virtual ~Grid3d(){}
-    Grid3d(const Grid3d& src):dlt_(src.dlt_){*this = src;}
-    Grid3d& operator=(const Grid3d& src){ //use default in C++11
+    ///allow deletion through base class pointer
+    virtual ~aTopology3d(){}
+    virtual aTopology3d* clone()const =0;
+    protected:
+    /**
+     * @brief Construct a 3D grid
+     *
+     * @param x0 left boundary in x
+     * @param x1 right boundary in x 
+     * @param y0 lower boundary in y
+     * @param y1 upper boundary in y 
+     * @param z0 lower boundary in z
+     * @param z1 upper boundary in z 
+     * @param n  # of polynomial coefficients per (x-,y-) dimension
+     * @param Nx # of points in x 
+     * @param Ny # of points in y
+     * @param Nz # of points in z
+     * @param bcx boundary condition in x
+     * @param bcy boundary condition in y
+     * @param bcz boundary condition in z
+     * @attention # of polynomial coefficients in z direction is always 1
+     */
+    aTopology3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = PER, bc bcy = PER, bc bcz = PER):
+        x0_(x0), x1_(x1), y0_(y0), y1_(y1), z0_(z0), z1_(z1),
+        n_(n), Nx_(Nx), Ny_(Ny), Nz_(Nz), bcx_(bcx), bcy_( bcy), bcz_( bcz), dlt_(n)
+    {
+        assert( n != 0);
+        assert( x1 > x0 && y1 > y0 ); assert( z1 > z0 );         
+        assert( Nx > 0  && Ny > 0); assert( Nz > 0);
+
+        lx_ = (x1-x0), ly_ = (y1-y0), lz_ = (z1-z0);
+        hx_ = lx_/(double)Nx_, hy_ = ly_/(double)Ny_, hz_ = lz_/(double)Nz_;
+    }
+    /**
+     * @brief Construct a 3d cartesian grid as the product of three 1d grids
+     *
+     * @param gx a Grid1d in x - direction
+     * @param gy a Grid1d in y - direction
+     * @param gz a Grid1d in z - direction
+     */
+    aTopology3d( const Grid1d& gx, const Grid1d& gy, const Grid1d& gz): 
+        x0_(gx.x0()), x1_(gx.x1()),  
+        y0_(gy.x0()), y1_(gy.x1()),
+        z0_(gz.x0()), z1_(gz.x1()),
+        n_(gx.n()), Nx_(gx.N()), Ny_(gy.N()), Nz_(gz.N()),
+        bcx_(gx.bcx()), bcy_( gy.bcx()), bcz_(gz.bcx()), 
+        dlt_(gx.n())
+    {
+        assert( gx.n() == gy.n() );
+        lx_ = (x1_-x0_), ly_ = (y1_-y0_), lz_ = (z1_-z0_);
+        hx_ = lx_/(double)Nx_, hy_ = ly_/(double)Ny_, hz_ = lz_/(double)Nz_;
+    }
+    aTopology3d(const aTopology3d& src):dlt_(src.dlt_){*this = src;}
+    aTopology3d& operator=(const aTopology3d& src){ //use default in C++11
         x0_=src.x0_, x1_=src.x1_,y0_=src.y0_,y1_=src.y1_,z0_=src.z0_,z1_=src.z1_;
         lx_=src.lx_,ly_=src.ly_,lz_=src.lz_;
         n_=src.n_,Nx_=src.Nx_,Ny_=src.Ny_,Nz_=src.Nz_;
         hx_=src.hx_,hy_=src.hy_,hz_=src.hz_;
         bcx_=src.bcx_,bcy_=src.bcy_,bcz_=src.bcz_;
         dlt_=src.dlt_;
+        return *this;
     }
-    protected:
-    virtual void do_set(unsigned new_n, unsigned new_Ny, unsigned new_Nz)
+    virtual void do_set(unsigned new_n, unsigned new_Nx,unsigned new_Ny, unsigned new_Nz)
     {
         n_ = new_n, Nx_ = new_Nx, Ny_ = new_Ny, Nz_ = new_Nz;
         hx_ = lx_/(double)Nx_, hy_ = ly_/(double)Ny_, hz_ =lz_/(double)Nz_;
@@ -734,7 +736,7 @@ struct Grid3d
         hx_ = lx_/(double)Nx_;
     }
   private:
-    friend class MPIGrid3d;
+    friend class aMPITopology3d;
     double x0_, x1_, y0_, y1_, z0_, z1_;
     double lx_, ly_, lz_;
     unsigned n_, Nx_, Ny_, Nz_;
@@ -745,8 +747,27 @@ struct Grid3d
 ///@}
 
 ///@cond
-Grid2d::Grid2d( const Grid3d& g) : x0_(g.x0()), x1_(g.x1()), y0_(g.y0()), y1_(g.y1()), n_(g.n()), Nx_(g.Nx()), Ny_(g.Ny()), bcx_(g.bcx()), bcy_(g.bcy()), dlt_(g.n())
-{}
+//aTopology2d::aTopology2d( const aTopology3d& g) : x0_(g.x0()), x1_(g.x1()), y0_(g.y0()), y1_(g.y1()), n_(g.n()), Nx_(g.Nx()), Ny_(g.Ny()), bcx_(g.bcx()), bcy_(g.bcy()), dlt_(g.n())
+//{}
 ///@endcond
 
+struct Grid2d : public aTopology2d
+{
+
+    Grid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx = PER, bc bcy = PER):
+        aTopology2d(x0,x1,y0,y1,n,Nx,Ny,bcx,bcy) { }
+    Grid2d( const Grid1d& gx, const Grid1d& gy): aTopology2d(gx,gy){ }
+    Grid2d* clone() const{return new Grid2d(*this);}
+
+};
+struct Grid3d : public aTopology3d
+{
+
+    Grid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = PER, bc bcy = PER, bc bcz=PER):
+        aTopology3d(x0,x1,y0,y1,z0,z1,n,Nx,Ny,Nz,bcx,bcy,bcz) { }
+    Grid3d( const Grid1d& gx, const Grid1d& gy, const Grid1d& gz): 
+        aTopology3d(gx,gy,gz){ }
+    Grid3d* clone() const{return new Grid3d(*this);}
+};
+
 }// namespace dg
diff --git a/inc/dg/enums.h b/inc/dg/enums.h
index 45a82675a..891338525 100644
--- a/inc/dg/enums.h
+++ b/inc/dg/enums.h
@@ -64,7 +64,7 @@ std::string bc2str( bc bcx)
  *
  * @param s the input string
  * @return a valid boundary condition
- * \throw std::runtime_exception if string doesn't match any of the above
+ * \throw std::runtime_error if string doesn't match any of the above
  * @ingroup creation
  */
 bc str2bc( std::string s)
@@ -79,7 +79,7 @@ bc str2bc( std::string s)
         return NEU_DIR;
     if( s=="DIR_NEU"||s=="dir_neu" )
         return DIR_NEU;
-    throw std::runtime_exception( "No matching boundary condition!");
+    throw std::runtime_error( "No matching boundary condition!");
 }
 
 /**
-- 
GitLab


From 5814cc8a752dae3aa7e6d4b564eb59bc319c0a0b Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Wed, 2 Aug 2017 17:25:02 +0200
Subject: [PATCH 101/453] corrected mistake in runtime_exception

---
 inc/dg/enums.h      | 4 ++--
 inc/dg/nullstelle.h | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/inc/dg/enums.h b/inc/dg/enums.h
index 45a82675a..891338525 100644
--- a/inc/dg/enums.h
+++ b/inc/dg/enums.h
@@ -64,7 +64,7 @@ std::string bc2str( bc bcx)
  *
  * @param s the input string
  * @return a valid boundary condition
- * \throw std::runtime_exception if string doesn't match any of the above
+ * \throw std::runtime_error if string doesn't match any of the above
  * @ingroup creation
  */
 bc str2bc( std::string s)
@@ -79,7 +79,7 @@ bc str2bc( std::string s)
         return NEU_DIR;
     if( s=="DIR_NEU"||s=="dir_neu" )
         return DIR_NEU;
-    throw std::runtime_exception( "No matching boundary condition!");
+    throw std::runtime_error( "No matching boundary condition!");
 }
 
 /**
diff --git a/inc/dg/nullstelle.h b/inc/dg/nullstelle.h
index 3f509dd8a..68a91ef44 100644
--- a/inc/dg/nullstelle.h
+++ b/inc/dg/nullstelle.h
@@ -54,7 +54,7 @@ class KeineNST_1D: public std::exception
  * \param aufloesung accuracy of the root finding	
  * \return number of used steps to reach the desired accuracy
  * \throw KeineNST_1D if no root lies between the given boundaries
- * \throw std::runtime_exception if after 60 steps the accuracy wasn't reached
+ * \throw std::runtime_error if after 60 steps the accuracy wasn't reached
  *
  * \code nullstelle_1D(funk, x_min, x_max, aufloesung); \endcode
  * \note If the root is found exactly the x_min = x_max 
@@ -79,7 +79,7 @@ int bisection1d (UnaryOp& funktion, double& x_min, double& x_max, const double a
         else 				                x_min = mitte;
         if((x_max-x_min)<aufloesung)        return j+3; 
     }
-    throw std::runtime_exception("Zu viele Schritte bei Nullstellensuche! evtl. j_max aendern");
+    throw std::runtime_error("Zu viele Schritte bei Nullstellensuche! evtl. j_max aendern");
 }
 
       
-- 
GitLab


From 64f4a0c580b097afc28c3df43a75e06dca994bd1 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Wed, 2 Aug 2017 18:06:33 +0200
Subject: [PATCH 102/453] edited interface for backend functions using Grid

---
 inc/dg/backend/derivatives.h     | 40 ++++++++++++++++----------------
 inc/dg/backend/interpolation.cuh | 12 +++++-----
 inc/dg/backend/projection.cuh    | 12 +++++-----
 inc/dg/backend/weights.cuh       |  8 +++----
 inc/dg/backend/xspacelib.cuh     |  4 ++--
 inc/geometries/fluxfunctions.h   | 12 ++++++++--
 6 files changed, 48 insertions(+), 40 deletions(-)

diff --git a/inc/dg/backend/derivatives.h b/inc/dg/backend/derivatives.h
index 9ea94814e..6f1454b5a 100644
--- a/inc/dg/backend/derivatives.h
+++ b/inc/dg/backend/derivatives.h
@@ -29,7 +29,7 @@ namespace create{
  *
  * @return A host matrix 
  */
-EllSparseBlockMat<double> dx( const Grid2d& g, bc bcx, direction dir = centered)
+EllSparseBlockMat<double> dx( const aTopology2d& g, bc bcx, direction dir = centered)
 {
     EllSparseBlockMat<double> dx;
     dx = dx_normed( g.n(), g.Nx(), g.hx(), bcx, dir);
@@ -46,7 +46,7 @@ EllSparseBlockMat<double> dx( const Grid2d& g, bc bcx, direction dir = centered)
  *
  * @return A host matrix
  */
-EllSparseBlockMat<double> dx( const Grid2d& g, direction dir = centered) { return dx( g, g.bcx(), dir);}
+EllSparseBlockMat<double> dx( const aTopology2d& g, direction dir = centered) { return dx( g, g.bcx(), dir);}
 
 /**
  * @brief Create 2d derivative in y-direction
@@ -57,7 +57,7 @@ EllSparseBlockMat<double> dx( const Grid2d& g, direction dir = centered) { retur
  *
  * @return A host matrix
  */
-EllSparseBlockMat<double> dy( const Grid2d& g, bc bcy, direction dir = centered)
+EllSparseBlockMat<double> dy( const aTopology2d& g, bc bcy, direction dir = centered)
 {
     EllSparseBlockMat<double> dy;
     dy = dx_normed( g.n(), g.Ny(), g.hy(), bcy, dir);
@@ -74,7 +74,7 @@ EllSparseBlockMat<double> dy( const Grid2d& g, bc bcy, direction dir = centered)
  *
  * @return A host matrix 
  */
-EllSparseBlockMat<double> dy( const Grid2d& g, direction dir = centered){ return dy( g, g.bcy(), dir);}
+EllSparseBlockMat<double> dy( const aTopology2d& g, direction dir = centered){ return dy( g, g.bcy(), dir);}
 
 /**
  * @brief Matrix that contains 2d jump terms in X direction
@@ -84,7 +84,7 @@ EllSparseBlockMat<double> dy( const Grid2d& g, direction dir = centered){ return
  *
  * @return A host matrix 
  */
-EllSparseBlockMat<double> jumpX( const Grid2d& g, bc bcx)
+EllSparseBlockMat<double> jumpX( const aTopology2d& g, bc bcx)
 {
     EllSparseBlockMat<double> jx;
     jx = jump( g.n(), g.Nx(), g.hx(), bcx);
@@ -101,7 +101,7 @@ EllSparseBlockMat<double> jumpX( const Grid2d& g, bc bcx)
  *
  * @return A host matrix 
  */
-EllSparseBlockMat<double> jumpY( const Grid2d& g, bc bcy)
+EllSparseBlockMat<double> jumpY( const aTopology2d& g, bc bcy)
 {
     EllSparseBlockMat<double> jy;
     jy = jump( g.n(), g.Ny(), g.hy(), bcy);
@@ -117,7 +117,7 @@ EllSparseBlockMat<double> jumpY( const Grid2d& g, bc bcy)
  *
  * @return A host matrix 
  */
-EllSparseBlockMat<double> jumpX( const Grid2d& g)
+EllSparseBlockMat<double> jumpX( const aTopology2d& g)
 {
     return jumpX( g, g.bcx());
 }
@@ -129,7 +129,7 @@ EllSparseBlockMat<double> jumpX( const Grid2d& g)
  *
  * @return A host matrix 
  */
-EllSparseBlockMat<double> jumpY( const Grid2d& g)
+EllSparseBlockMat<double> jumpY( const aTopology2d& g)
 {
     return jumpY( g, g.bcy());
 }
@@ -144,7 +144,7 @@ EllSparseBlockMat<double> jumpY( const Grid2d& g)
  *
  * @return A host matrix 
  */
-EllSparseBlockMat<double> jumpX( const Grid3d& g, bc bcx)
+EllSparseBlockMat<double> jumpX( const aTopology3d& g, bc bcx)
 {
     EllSparseBlockMat<double> jx;
     jx = jump( g.n(), g.Nx(), g.hx(), bcx);
@@ -161,7 +161,7 @@ EllSparseBlockMat<double> jumpX( const Grid3d& g, bc bcx)
  *
  * @return A host matrix 
  */
-EllSparseBlockMat<double> jumpY( const Grid3d& g, bc bcy)
+EllSparseBlockMat<double> jumpY( const aTopology3d& g, bc bcy)
 {
     EllSparseBlockMat<double> jy;
     jy = jump( g.n(), g.Ny(), g.hy(), bcy);
@@ -179,7 +179,7 @@ EllSparseBlockMat<double> jumpY( const Grid3d& g, bc bcy)
  *
  * @return A host matrix 
  */
-EllSparseBlockMat<double> jumpZ( const Grid3d& g, bc bcz)
+EllSparseBlockMat<double> jumpZ( const aTopology3d& g, bc bcz)
 {
     EllSparseBlockMat<double> jz;
     jz = jump( 1, g.Nz(), g.hz(), bcz);
@@ -195,7 +195,7 @@ EllSparseBlockMat<double> jumpZ( const Grid3d& g, bc bcz)
  *
  * @return A host matrix
  */
-EllSparseBlockMat<double> jumpX( const Grid3d& g)
+EllSparseBlockMat<double> jumpX( const aTopology3d& g)
 {
     return jumpX( g, g.bcx());
 }
@@ -207,7 +207,7 @@ EllSparseBlockMat<double> jumpX( const Grid3d& g)
  *
  * @return A host matrix
  */
-EllSparseBlockMat<double> jumpY( const Grid3d& g)
+EllSparseBlockMat<double> jumpY( const aTopology3d& g)
 {
     return jumpY( g, g.bcy());
 }
@@ -219,7 +219,7 @@ EllSparseBlockMat<double> jumpY( const Grid3d& g)
  *
  * @return A host matrix
  */
-EllSparseBlockMat<double> jumpZ( const Grid3d& g)
+EllSparseBlockMat<double> jumpZ( const aTopology3d& g)
 {
     return jumpZ( g, g.bcz());
 }
@@ -234,7 +234,7 @@ EllSparseBlockMat<double> jumpZ( const Grid3d& g)
  *
  * @return A host matrix 
  */
-EllSparseBlockMat<double> dx( const Grid3d& g, bc bcx, direction dir = centered)
+EllSparseBlockMat<double> dx( const aTopology3d& g, bc bcx, direction dir = centered)
 {
     EllSparseBlockMat<double> dx;
     dx = dx_normed( g.n(), g.Nx(), g.hx(), bcx, dir);
@@ -251,7 +251,7 @@ EllSparseBlockMat<double> dx( const Grid3d& g, bc bcx, direction dir = centered)
  *
  * @return A host matrix 
  */
-EllSparseBlockMat<double> dx( const Grid3d& g, direction dir = centered) { return dx( g, g.bcx(), dir);}
+EllSparseBlockMat<double> dx( const aTopology3d& g, direction dir = centered) { return dx( g, g.bcx(), dir);}
 
 /**
  * @brief Create 3d derivative in y-direction
@@ -262,7 +262,7 @@ EllSparseBlockMat<double> dx( const Grid3d& g, direction dir = centered) { retur
  *
  * @return A host matrix 
  */
-EllSparseBlockMat<double> dy( const Grid3d& g, bc bcy, direction dir = centered)
+EllSparseBlockMat<double> dy( const aTopology3d& g, bc bcy, direction dir = centered)
 {
     EllSparseBlockMat<double> dy;
     dy = dx_normed( g.n(), g.Ny(), g.hy(), bcy, dir);
@@ -280,7 +280,7 @@ EllSparseBlockMat<double> dy( const Grid3d& g, bc bcy, direction dir = centered)
  *
  * @return A host matrix 
  */
-EllSparseBlockMat<double> dy( const Grid3d& g, direction dir = centered){ return dy( g, g.bcy(), dir);}
+EllSparseBlockMat<double> dy( const aTopology3d& g, direction dir = centered){ return dy( g, g.bcy(), dir);}
 
 /**
  * @brief Create 3d derivative in z-direction
@@ -291,7 +291,7 @@ EllSparseBlockMat<double> dy( const Grid3d& g, direction dir = centered){ return
  *
  * @return A host matrix 
  */
-EllSparseBlockMat<double> dz( const Grid3d& g, bc bcz, direction dir = centered)
+EllSparseBlockMat<double> dz( const aTopology3d& g, bc bcz, direction dir = centered)
 {
     EllSparseBlockMat<double> dz;
     dz = dx_normed( 1, g.Nz(), g.hz(), bcz, dir);
@@ -309,7 +309,7 @@ EllSparseBlockMat<double> dz( const Grid3d& g, bc bcz, direction dir = centered)
  *
  * @return A host matrix 
  */
-EllSparseBlockMat<double> dz( const Grid3d& g, direction dir = centered){ return dz( g, g.bcz(), dir);}
+EllSparseBlockMat<double> dz( const aTopology3d& g, direction dir = centered){ return dz( g, g.bcz(), dir);}
 
 
 
diff --git a/inc/dg/backend/interpolation.cuh b/inc/dg/backend/interpolation.cuh
index 88780f7d4..24f6b22fe 100644
--- a/inc/dg/backend/interpolation.cuh
+++ b/inc/dg/backend/interpolation.cuh
@@ -125,7 +125,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const thrust::ho
  *
  * @return interpolation matrix
  */
-cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const thrust::host_vector<double>& x, const thrust::host_vector<double>& y, const Grid2d& g , dg::bc boundary = dg::NEU)
+cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const thrust::host_vector<double>& x, const thrust::host_vector<double>& y, const aTopology2d& g , dg::bc boundary = dg::NEU)
 {
     assert( x.size() == y.size());
     std::vector<double> gauss_nodes = g.dlt().abscissas(); 
@@ -270,7 +270,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const thrust::ho
  * @return interpolation matrix
  * @attention all points (x, y, z) must lie within or on the boundaries of g
  */
-cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const thrust::host_vector<double>& x, const thrust::host_vector<double>& y, const thrust::host_vector<double>& z, const Grid3d& g, dg::bc boundary= dg::NEU)
+cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const thrust::host_vector<double>& x, const thrust::host_vector<double>& y, const thrust::host_vector<double>& z, const aTopology3d& g, dg::bc boundary= dg::NEU)
 {
     assert( x.size() == y.size());
     assert( y.size() == z.size());
@@ -429,7 +429,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const Grid1d& g_
 /**
  * @copydoc interpolation(const Grid1d&,const Grid1d&)
  */
-cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const Grid2d& g_new, const Grid2d& g_old)
+cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const aTopology2d& g_new, const aTopology2d& g_old)
 {
     //assert both grids are on the same box
     assert( g_new.x0() >= g_old.x0());
@@ -446,7 +446,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const Grid2d& g_
 /**
  * @copydoc interpolation(const Grid1d&,const Grid1d&)
  */
-cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const Grid3d& g_new, const Grid3d& g_old)
+cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const aTopology3d& g_new, const aTopology3d& g_old)
 {
     //assert both grids are on the same box
     assert( g_new.x0() >= g_old.x0());
@@ -473,7 +473,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const Grid3d& g_
  * @ingroup misc
  * @return the vector in LSPACE
  */
-thrust::host_vector<double> forward_transform( const thrust::host_vector<double>& in, const Grid2d& g)
+thrust::host_vector<double> forward_transform( const thrust::host_vector<double>& in, const aTopology2d& g)
 {
     thrust::host_vector<double> out(in.size(), 0);
     dg::Operator<double> forward( g.dlt().forward());
@@ -501,7 +501,7 @@ thrust::host_vector<double> forward_transform( const thrust::host_vector<double>
  * @return interpolated point
  * @note g.contains(x,y) must return true
  */
-double interpolate( double x, double y,  const thrust::host_vector<double>& v, const Grid2d& g )
+double interpolate( double x, double y,  const thrust::host_vector<double>& v, const aTopology2d& g )
 {
     assert( v.size() == g.size());
 
diff --git a/inc/dg/backend/projection.cuh b/inc/dg/backend/projection.cuh
index d80ac1442..a2261ce20 100644
--- a/inc/dg/backend/projection.cuh
+++ b/inc/dg/backend/projection.cuh
@@ -69,7 +69,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const Grid1d& g
 /**
  * @copydoc interpolationT
  */
-cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const Grid2d& g_new, const Grid2d& g_old)
+cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const aTopology2d& g_new, const aTopology2d& g_old)
 {
     cusp::coo_matrix<int, double, cusp::host_memory> temp = interpolation( g_old, g_new), A;
     cusp::transpose( temp, A);
@@ -78,7 +78,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const Grid2d& g
 /**
  * @copydoc interpolationT
  */
-cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const Grid3d& g_new, const Grid3d& g_old)
+cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const aTopology3d& g_new, const aTopology3d& g_old)
 {
     cusp::coo_matrix<int, double, cusp::host_memory> temp = interpolation( g_old, g_new), A;
     cusp::transpose( temp, A);
@@ -133,7 +133,7 @@ cusp::coo_matrix< int, double, cusp::host_memory> projection( const Grid1d& g_ne
 /**
  * @copydoc projection
  */
-cusp::coo_matrix< int, double, cusp::host_memory> projection( const Grid2d& g_new, const Grid2d& g_old)
+cusp::coo_matrix< int, double, cusp::host_memory> projection( const aTopology2d& g_new, const aTopology2d& g_old)
 {
     if( g_old.Nx() % g_new.Nx() != 0) std::cerr << "ATTENTION: you project between incompatible grids in x!!\n";
     if( g_old.Ny() % g_new.Ny() != 0) std::cerr << "ATTENTION: you project between incompatible grids in y!!\n";
@@ -162,7 +162,7 @@ cusp::coo_matrix< int, double, cusp::host_memory> projection( const Grid2d& g_ne
 /**
  * @copydoc projection
  */
-cusp::coo_matrix< int, double, cusp::host_memory> projection( const Grid3d& g_new, const Grid3d& g_old)
+cusp::coo_matrix< int, double, cusp::host_memory> projection( const aTopology3d& g_new, const aTopology3d& g_old)
 {
     if( g_old.Nx() % g_new.Nx() != 0) std::cerr << "ATTENTION: you project between incompatible grids in x!!\n";
     if( g_old.Ny() % g_new.Ny() != 0) std::cerr << "ATTENTION: you project between incompatible grids in y!!\n";
@@ -209,7 +209,7 @@ cusp::coo_matrix< int, double, cusp::host_memory> projection( const Grid3d& g_ne
  * @note The boundaries of the old grid must lie within the boundaries of the new grid
  * @note If the grid are very incompatible the matrix-matrix multiplication can take a while
  */
-cusp::coo_matrix< int, double, cusp::host_memory> transformation( const Grid3d& g_new, const Grid3d& g_old)
+cusp::coo_matrix< int, double, cusp::host_memory> transformation( const aTopology3d& g_new, const aTopology3d& g_old)
 {
     Grid3d g_lcm(g_new.x0(), g_new.x1(), g_new.y0(), g_new.y1(), g_new.z0(), g_new.z1(), 
                  lcm(g_new.n(), g_old.n()), lcm(g_new.Nx(), g_old.Nx()), lcm(g_new.Ny(), g_old.Ny()), 
@@ -223,7 +223,7 @@ cusp::coo_matrix< int, double, cusp::host_memory> transformation( const Grid3d&
 /**
  * @copydoc transformation
  */
-cusp::coo_matrix< int, double, cusp::host_memory> transformation( const Grid2d& g_new, const Grid2d& g_old)
+cusp::coo_matrix< int, double, cusp::host_memory> transformation( const aTopology2d& g_new, const aTopology2d& g_old)
 {
     Grid2d g_lcm(g_new.x0(), g_new.x1(), g_new.y0(), g_new.y1(), 
                  lcm(g_new.n(), g_old.n()), lcm(g_new.Nx(), g_old.Nx()), lcm(g_new.Ny(), g_old.Ny()));
diff --git a/inc/dg/backend/weights.cuh b/inc/dg/backend/weights.cuh
index 527631c6f..a439de3b2 100644
--- a/inc/dg/backend/weights.cuh
+++ b/inc/dg/backend/weights.cuh
@@ -87,7 +87,7 @@ int get_j( unsigned n, unsigned Nx, int idx) { return idx%n;}
 *
 * @return Host Vector
 */
-thrust::host_vector<double> weights( const Grid2d& g)
+thrust::host_vector<double> weights( const aTopology2d& g)
 {
     //choose layout
     thrust::host_vector<double> v( g.size());
@@ -105,7 +105,7 @@ thrust::host_vector<double> weights( const Grid2d& g)
 *
 * @return Host Vector
 */
-thrust::host_vector<double> inv_weights( const Grid2d& g)
+thrust::host_vector<double> inv_weights( const aTopology2d& g)
 {
     thrust::host_vector<double> v = weights( g);
     for( unsigned i=0; i<g.size(); i++)
@@ -120,7 +120,7 @@ thrust::host_vector<double> inv_weights( const Grid2d& g)
 *
 * @return Host Vector
 */
-thrust::host_vector<double> weights( const Grid3d& g)
+thrust::host_vector<double> weights( const aTopology3d& g)
 {
     thrust::host_vector<double> v( g.size());
     for( unsigned i=0; i<g.size(); i++)
@@ -138,7 +138,7 @@ thrust::host_vector<double> weights( const Grid3d& g)
 *
 * @return Host Vector
 */
-thrust::host_vector<double> inv_weights( const Grid3d& g)
+thrust::host_vector<double> inv_weights( const aTopology3d& g)
 {
     thrust::host_vector<double> v = weights( g);
     for( unsigned i=0; i<g.size(); i++)
diff --git a/inc/dg/backend/xspacelib.cuh b/inc/dg/backend/xspacelib.cuh
index 507177b76..e8ccb3fcf 100644
--- a/inc/dg/backend/xspacelib.cuh
+++ b/inc/dg/backend/xspacelib.cuh
@@ -86,7 +86,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> scatter( const thrust::host_vec
  * @return transformation matrix
  * @note this matrix has ~n^4 N^2 entries and is not sorted
  */
-cusp::coo_matrix<int, double, cusp::host_memory> backscatter( const Grid2d& g)
+cusp::coo_matrix<int, double, cusp::host_memory> backscatter( const aTopology2d& g)
 {
     typedef cusp::coo_matrix<int, double, cusp::host_memory> Matrix;
     //create equidistant backward transformation
@@ -116,7 +116,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> backscatter( const Grid2d& g)
  * @return transformation matrix
  * @note this matrix has ~n^4 N^2 entries and is not sorted
  */
-cusp::coo_matrix<int, double, cusp::host_memory> backscatter( const Grid3d& g)
+cusp::coo_matrix<int, double, cusp::host_memory> backscatter( const aTopology3d& g)
 {
     Grid2d g2d( g.x0(), g.x1(), g.y0(), g.y1(), g.n(), g.Nx(), g.Ny(), g.bcx(), g.bcy());
     cusp::coo_matrix<int,double, cusp::host_memory> back2d = backscatter( g2d);
diff --git a/inc/geometries/fluxfunctions.h b/inc/geometries/fluxfunctions.h
index 87d618d3c..49af78431 100644
--- a/inc/geometries/fluxfunctions.h
+++ b/inc/geometries/fluxfunctions.h
@@ -127,7 +127,7 @@ struct BinaryFunctorsLvl1
 /**
 * @brief This struct bundles a function and its first and second derivatives
 */
-struct BinaryFunctorsLvl2 : public BinaryFunctorsLvl1
+struct BinaryFunctorsLvl2 
 {
     /**
     * @copydoc BinaryFunctorsLvl1
@@ -136,12 +136,19 @@ struct BinaryFunctorsLvl2 : public BinaryFunctorsLvl1
     * @param fyy \f$ \partial^2 f / \partial y^2\f$ second derivative in second coordinate
     */
     BinaryFunctorsLvl2( aBinaryFunctor* f, aBinaryFunctor* fx, aBinaryFunctor* fy,
-    aBinaryFunctor* fxx, aBinaryFunctor* fxy, aBinaryFunctor* fyy): BinaryFunctorsLvl1(f,fx,fy) 
+    aBinaryFunctor* fxx, aBinaryFunctor* fxy, aBinaryFunctor* fyy): f(f,fx,fy) 
     {
         p_[0].set( fxx);
         p_[1].set( fxy);
         p_[2].set( fyy);
     }
+    operator BinaryFunctorsLvl1 ()const {return f;}
+    /// \f$ f \f$
+    const aBinaryFunctor& f()const{return f.f();}
+    /// \f$ \partial f / \partial x \f$ 
+    const aBinaryFunctor& dfx()const{return f.dfx();}
+    /// \f$ \partial f / \partial y\f$
+    const aBinaryFunctor& dfy()const{return f.dfy();}
     /// \f$ \partial^2f/\partial x^2\f$
     const aBinaryFunctor& dfxx()const{return p_[0].get();}
     /// \f$ \partial^2 f / \partial x \partial y\f$
@@ -149,6 +156,7 @@ struct BinaryFunctorsLvl2 : public BinaryFunctorsLvl1
     /// \f$ \partial^2f/\partial y^2\f$
     const aBinaryFunctor& dfyy()const{return p_[2].get();}
     private:
+    BinaryFunctorsLvl1 f;
     Handle<aBinaryFunctor> p_[3];
 };
 
-- 
GitLab


From c87fc09ed776c090cc41ad4fdc8939e1bb8b8c61 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Wed, 2 Aug 2017 18:50:51 +0200
Subject: [PATCH 103/453] modified magnetic field as a function container

---
 inc/dg/geometry/generator.h     |  6 ++-
 inc/geometries/fluxfunctions.h  | 66 +++++++++++++++----------
 inc/geometries/guenther.h       | 30 ++++++------
 inc/geometries/magnetic_field.h | 85 ++++++++++++++++-----------------
 inc/geometries/solovev.h        | 28 +++++------
 inc/geometries/taylor.h         | 31 ++++++------
 inc/geometries/toroidal.h       | 14 ++++++
 7 files changed, 140 insertions(+), 120 deletions(-)

diff --git a/inc/dg/geometry/generator.h b/inc/dg/geometry/generator.h
index 829664147..4b3bb8323 100644
--- a/inc/dg/geometry/generator.h
+++ b/inc/dg/geometry/generator.h
@@ -180,11 +180,13 @@ struct Handle
         if(ptr_!=0) delete ptr_;
     }
     const cloneable& get()const {return *ptr_;}
-    void set( cloneable* ptr){ 
+    ///takes ownership of the ptr and deletes the current one if non-empty
+    void reset( cloneable* ptr){ 
         Handle tmp(ptr);
         *this=tmp;
     }
-    void set( const cloneable& src){ 
+    ///clones the src
+    void reset( const cloneable& src){ 
         Handle tmp(src);
         *this=tmp;
     }
diff --git a/inc/geometries/fluxfunctions.h b/inc/geometries/fluxfunctions.h
index 49af78431..9a99641f2 100644
--- a/inc/geometries/fluxfunctions.h
+++ b/inc/geometries/fluxfunctions.h
@@ -102,6 +102,8 @@ aBinaryFunctor* make_aBinaryFunctor(const BinaryFunctor& f){return new BinaryFun
 */
 struct BinaryFunctorsLvl1 
 {
+    ///the access functions are undefined as long as the class remains empty
+    BinaryFunctorsLvl1(){}
     /**
     * @brief Take ownership of newly allocated functors
     *
@@ -109,11 +111,15 @@ struct BinaryFunctorsLvl1
     * @param fx \f$ \partial f / \partial x \f$ its derivative in the first coordinate
     * @param fy \f$ \partial f / \partial y \f$ its derivative in the second coordinate
     */
-    BinaryFunctorsLvl1( aBinaryFunctor* f, aBinaryFunctor* fx, aBinaryFunctor* fy): 
+    BinaryFunctorsLvl1( const aBinaryFunctor* f, const aBinaryFunctor* fx, const aBinaryFunctor* fy)
     {
-        p_[0].set(f);
-        p_[1].set(fx);
-        p_[2].set(fy);
+        reset(f,fx,fy);
+    }
+    void reset( const aBinaryFunctor* f, const aBinaryFunctor* fx, const aBinaryFunctor* fy)
+    {
+        p_[0].reset(f);
+        p_[1].reset(fx);
+        p_[2].reset(fy);
     }
     /// \f$ f \f$
     const aBinaryFunctor& f()const{return p_[0].get();}
@@ -129,18 +135,18 @@ struct BinaryFunctorsLvl1
 */
 struct BinaryFunctorsLvl2 
 {
+    ///the access functions are undefined as long as the class remains empty
+    BinaryFunctorsLvl2(){}
     /**
     * @copydoc BinaryFunctorsLvl1
     * @param fxx \f$ \partial^2 f / \partial x^2\f$ second derivative in first coordinate
     * @param fxy \f$ \partial^2 f / \partial x \partial y\f$ second mixed derivative 
     * @param fyy \f$ \partial^2 f / \partial y^2\f$ second derivative in second coordinate
     */
-    BinaryFunctorsLvl2( aBinaryFunctor* f, aBinaryFunctor* fx, aBinaryFunctor* fy,
-    aBinaryFunctor* fxx, aBinaryFunctor* fxy, aBinaryFunctor* fyy): f(f,fx,fy) 
-    {
-        p_[0].set( fxx);
-        p_[1].set( fxy);
-        p_[2].set( fyy);
+    BinaryFunctorsLvl2( const aBinaryFunctor* f, const aBinaryFunctor* fx, const aBinaryFunctor* fy, const aBinaryFunctor* fxx, const aBinaryFunctor* fxy, const aBinaryFunctor* fyy): f(f,fx,fy), f1(fxx,fxy,fyy) 
+    { }
+    void reset( const aBinaryFunctor* f, const aBinaryFunctor* fx, const aBinaryFunctor* fy, const aBinaryFunctor* fxx, const aBinaryFunctor* fxy, const aBinaryFunctor* fyy){ 
+        f.reset(f,fx,fy), f1.reset(fxx,fxy,fyy) 
     }
     operator BinaryFunctorsLvl1 ()const {return f;}
     /// \f$ f \f$
@@ -150,19 +156,19 @@ struct BinaryFunctorsLvl2
     /// \f$ \partial f / \partial y\f$
     const aBinaryFunctor& dfy()const{return f.dfy();}
     /// \f$ \partial^2f/\partial x^2\f$
-    const aBinaryFunctor& dfxx()const{return p_[0].get();}
+    const aBinaryFunctor& dfxx()const{return f1.f();}
     /// \f$ \partial^2 f / \partial x \partial y\f$
-    const aBinaryFunctor& dfxy()const{return p_[1].get();}
+    const aBinaryFunctor& dfxy()const{return f1.fx();}
     /// \f$ \partial^2f/\partial y^2\f$
-    const aBinaryFunctor& dfyy()const{return p_[2].get();}
+    const aBinaryFunctor& dfyy()const{return f1.fy();}
     private:
-    BinaryFunctorsLvl1 f;
-    Handle<aBinaryFunctor> p_[3];
+    BinaryFunctorsLvl1 f,f1;
 };
 
 /// A symmetric 2d tensor field and its divergence
 struct BinarySymmTensorLvl1
 {
+    BinarySymmTensorLvl1( ){}
     /**
      * @brief Take ownership of newly allocated functors
      *
@@ -173,14 +179,17 @@ struct BinarySymmTensorLvl1
      * @param divChiX \f$ \partial_x \chi^{xx} + \partial_y\chi^{yx}\f$ is the x-component of the divergence of the tensor \f$ \chi\f$
      * @param divChiY \f$ \partial_x \chi^{xy} + \partial_y\chi^{yy}\f$ is the y-component of the divergence of the tensor \f$ \chi \f$
     */
-    BinarySymmTensorLvl1( aBinaryFunctor* chi_xx, aBinaryFunctor* chi_xy, aBinaryFunctor* chi_yy,
-    aBinaryFunctor* divChiX, aBinaryFunctor* divChiY)
+    BinarySymmTensorLvl1( const aBinaryFunctor* chi_xx, const aBinaryFunctor* chi_xy, const aBinaryFunctor* chi_yy, const aBinaryFunctor* divChiX, const aBinaryFunctor* divChiY)
     {
-        p_[0].set( chi_xx);
-        p_[1].set( chi_xy);
-        p_[2].set( chi_yy);
-        p_[3].set( divChiX);
-        p_[4].set( divChiY);
+        reset(chi_xx,chi_xy,chi_yy,divChiX,divChiY);
+    }
+    void reset( const aBinaryFunctor* chi_xx, const aBinaryFunctor* chi_xy, const aBinaryFunctor* chi_yy, const aBinaryFunctor* divChiX, const aBinaryFunctor* divChiY)
+    {
+        p_[0].reset( chi_xx);
+        p_[1].reset( chi_xy);
+        p_[2].reset( chi_yy);
+        p_[3].reset( divChiX);
+        p_[4].reset( divChiY);
     }
     ///xy component \f$ \chi^{xx}\f$ 
     const aBinaryFunctor& xx()const{return p_[0].get();}
@@ -199,11 +208,16 @@ struct BinarySymmTensorLvl1
 /// A vector field with three components that depend only on the first two coordinates
 struct BinaryVectorLvl0
 {
-    BinaryVectorLvl0( aBinaryFunctor* v_x, aBinaryFunctor* v_y, aBinaryFunctor* v_z)
+    BinaryVectorLvl0(){}
+    BinaryVectorLvl0( const aBinaryFunctor* v_x, const aBinaryFunctor* v_y, const aBinaryFunctor* v_z)
+    {
+        reset(v_x,v_y,v_z);
+    }
+    void reset( const aBinaryFunctor* v_x, const aBinaryFunctor* v_y, const aBinaryFunctor* v_z)
     {
-        p_[0].set(v_x);
-        p_[1].set(v_y);
-        p_[2].set(v_z);
+        p_[0].reset(v_x);
+        p_[1].reset(v_y);
+        p_[2].reset(v_z);
     }
     /// x-component of the vector
     const aBinaryFunctor& x()const{return p_[0];}
diff --git a/inc/geometries/guenther.h b/inc/geometries/guenther.h
index 5a79eb38c..1395f754f 100644
--- a/inc/geometries/guenther.h
+++ b/inc/geometries/guenther.h
@@ -135,22 +135,20 @@ struct IpolZ : public aCloneableBinaryFunctor<IpolZ>
     double operator()(double R, double Z) const { return 0; }
 };
 
-/**
- * @brief Contains all guenther fields
- */
-struct MagneticField : public dg::geo::aTokamakMagneticField
-{
-    MagneticField( GeomParameters gp): aTokamakMagneticField(gp.R_0, 
-        new Psip(gp), 
-        new PsipR(gp), 
-        new PsipZ(gp), 
-        new PsipRR(gp), 
-        new PsipRZ(gp), 
-        new PsipZZ(gp), 
-        new Ipol(gp), 
-        new IpolR(gp), 
-        new IpolZ(gp)){}
-};
+BinaryFunctorsLvl2 createPsip( GeomParameters gp)
+{
+    BinaryFunctorsLvl2 psip( new Psip(gp), new PsipR(gp), new PsipZ(gp),new PsipRR(gp), new PsipRZ(gp), new PsipZZ(gp));
+    return psip;
+}
+BinaryFunctorsLvl1 createIpol( GeomParameters gp)
+{
+    BinaryFunctorsLvl1 ipol( new Ipol(gp), new IpolR(gp), new IpolZ(gp))
+    return ipol;
+}
+MagneticField createMagField( GeomParameters gp)
+{
+    return MagneticField( gp.R_0, createPsip(gp), createIpol(gp));
+}
 ///@}
 
 /////////////////////////////////////////These should not be necessary!////////
diff --git a/inc/geometries/magnetic_field.h b/inc/geometries/magnetic_field.h
index 66b1e7107..221d137eb 100644
--- a/inc/geometries/magnetic_field.h
+++ b/inc/geometries/magnetic_field.h
@@ -15,7 +15,7 @@ namespace geo
 ///@addtogroup magnetic
 ///@{
 /**
-* @brief Base class of a tokamak magnetic geometry model
+* @brief container class of R, psi and ipol 
 
  This is the representation of magnetic fields that can be modeled in the form
  \f[
@@ -28,20 +28,17 @@ namespace geo
  direction lies within the
  R-Z planes of a cylindrical grid (The plane \f$ \perp \hat e_\varphi\f$ )
 */
-struct aTokamakMagneticField
+struct TokamakMagneticField
 {
-    //maybe construct using make functions instead of deriving from it?
-    aTokamakMagneticField( double R0,
-        aBinaryFunctor* psip,
-        aBinaryFunctor* psipR,
-        aBinaryFunctor* psipZ,
-        aBinaryFunctor* psipRR,
-        aBinaryFunctor* psipRZ,
-        aBinaryFunctor* psipZZ,
-        aBinaryFunctor* ipol,
-        aBinaryFunctor* ipolR,
-        aBinaryFunctor* ipolZ
-        ): R0_(R0), psip_(psip,psipR,psipZ,psipRR,psipRZ,psipZZ),ipol_(ipol,ipolR,ipolZ){ }
+    ///as long as the field stays empty the access functions are undefined
+    TokamakMagneticField(){}
+    TokamakMagneticField( double R0, const BinaryFunctorsLvl2& psip, const BinaryFunctorsLvl1& ipol): R0_(R0), psip_(psip), ipol_(ipol){}
+    void set( double R0, const BinaryFunctorsLvl2& psip, const BinaryFunctorsLvl1& ipol)
+    {
+        R0_=R0;
+        psip_=psip; 
+        ipol_=ipol;
+    }
     /// \f$ R_0 \f$ 
     double R0()const {return R0_;}
     /// \f$ \psi_p(R,Z)\f$, where R, Z are cylindrical coordinates
@@ -78,7 +75,7 @@ struct aTokamakMagneticField
  */ 
 struct Bmodule : public aCloneableBinaryFunctor<Bmodule>
 {
-    Bmodule( const aTokamakMagneticField& mag): mag_(mag)  { }
+    Bmodule( const TokamakMagneticField& mag): mag_(mag)  { }
     /**
     * @brief \f[   \hat{B} \f]
     */ 
@@ -88,7 +85,7 @@ struct Bmodule : public aCloneableBinaryFunctor<Bmodule>
         return mag_.R0()/R*sqrt(ipol*ipol+psipR*psipR +psipZ*psipZ);
     }
   private:
-    aTokamakMagneticField mag_;
+    TokamakMagneticField mag_;
 };
 
 /**
@@ -96,7 +93,7 @@ struct Bmodule : public aCloneableBinaryFunctor<Bmodule>
  */ 
 struct InvB : public aCloneableBinaryFunctor<InvB>
 {
-    InvB(  const aTokamakMagneticField& mag): mag_(mag){ }
+    InvB(  const TokamakMagneticField& mag): mag_(mag){ }
     /**
     * @brief \f[   \frac{1}{\hat{B}} = 
         \frac{\hat{R}}{\hat{R}_0}\frac{1}{ \sqrt{ \hat{I}^2  + \left(\frac{\partial \hat{\psi}_p }{ \partial \hat{R}}\right)^2
@@ -108,7 +105,7 @@ struct InvB : public aCloneableBinaryFunctor<InvB>
         return R/(mag_.R0()*sqrt(ipol*ipol + psipR*psipR +psipZ*psipZ)) ;
     }
   private:
-    aTokamakMagneticField mag_;
+    TokamakMagneticField mag_;
 };
 
 /**
@@ -116,7 +113,7 @@ struct InvB : public aCloneableBinaryFunctor<InvB>
  */ 
 struct LnB : public aCloneableBinaryFunctor<LnB>
 {
-    LnB(const aTokamakMagneticField& mag): mag_(mag) { }
+    LnB(const TokamakMagneticField& mag): mag_(mag) { }
     /**
      * @brief \f[   \ln{(   \hat{B})} = \ln{\left[
           \frac{\hat{R}_0}{\hat{R}} \sqrt{ \hat{I}^2  + \left(\frac{\partial \hat{\psi}_p }{ \partial \hat{R}}\right)^2
@@ -128,7 +125,7 @@ struct LnB : public aCloneableBinaryFunctor<LnB>
         return log(mag_.R0()/R*sqrt(ipol*ipol + psipR*psipR +psipZ*psipZ)) ;
     }
   private:
-    aTokamakMagneticField mag_;
+    TokamakMagneticField mag_;
 };
 
 /**
@@ -137,7 +134,7 @@ struct LnB : public aCloneableBinaryFunctor<LnB>
 struct BR: public aCloneableBinaryFunctor<BR>
 
 {
-    BR(const aTokamakMagneticField& mag): invB_(mag), mag_(mag) { }
+    BR(const TokamakMagneticField& mag): invB_(mag), mag_(mag) { }
 /**
  * @brief \f[  \frac{\partial \hat{B} }{ \partial \hat{R}} = 
       -\frac{1}{\hat B \hat R}   
@@ -156,7 +153,7 @@ struct BR: public aCloneableBinaryFunctor<BR>
     }
   private:
     InvB invB_;
-    aTokamakMagneticField mag_;
+    TokamakMagneticField mag_;
 };
 
 /**
@@ -164,7 +161,7 @@ struct BR: public aCloneableBinaryFunctor<BR>
  */ 
 struct BZ: public aCloneableBinaryFunctor<BZ>
 {
-    BZ(const aTokamakMagneticField& mag ): mag_(mag), invB_(mag) { }
+    BZ(const TokamakMagneticField& mag ): mag_(mag), invB_(mag) { }
     /**
      * @brief \f[  \frac{\partial \hat{B} }{ \partial \hat{Z}} = 
      \frac{ \hat I \left(\frac{\partial \hat I}{\partial\hat Z}    \right)+
@@ -180,7 +177,7 @@ struct BZ: public aCloneableBinaryFunctor<BZ>
         return (invB_(R,Z)/Rn/Rn)*(mag_.ipol()(R,Z)*mag_.ipolZ(R,Z) + mag_.psipR()(R,Z)*mag_.psipRZ()(R,Z) + mag_.psipZ()(R,Z)*mag_.psipZZ()(R,Z));
     }
   private:
-    aTokamakMagneticField mag_;
+    TokamakMagneticField mag_;
     InvB invB_; 
 };
 
@@ -189,7 +186,7 @@ struct BZ: public aCloneableBinaryFunctor<BZ>
  */ 
 struct CurvatureNablaBR: public aCloneableBinaryFunctor<CurvatureNablaBR>
 {
-    CurvatureNablaBR(const aTokamakMagneticField& mag): invB_(mag), bZ_(mag) { }
+    CurvatureNablaBR(const TokamakMagneticField& mag): invB_(mag), bZ_(mag) { }
     /**
      * @brief \f[ \mathcal{\hat{K}}^{\hat{R}}_{\nabla B} =-\frac{1}{ \hat{B}^2}  \frac{\partial \hat{B}}{\partial \hat{Z}}  \f]
      */ 
@@ -207,7 +204,7 @@ struct CurvatureNablaBR: public aCloneableBinaryFunctor<CurvatureNablaBR>
  */ 
 struct CurvatureNablaBZ: public aCloneableBinaryFunctor<CurvatureNablaBZ>
 {
-    CurvatureNablaBZ( const aTokamakMagneticField& mag): invB_(mag), bR_(mag) { }
+    CurvatureNablaBZ( const TokamakMagneticField& mag): invB_(mag), bR_(mag) { }
     /**
      * @brief \f[  \mathcal{\hat{K}}^{\hat{Z}}_{\nabla B} =\frac{1}{ \hat{B}^2}   \frac{\partial \hat{B}}{\partial \hat{R}} \f]
      */    
@@ -226,7 +223,7 @@ struct CurvatureNablaBZ: public aCloneableBinaryFunctor<CurvatureNablaBZ>
 struct CurvatureKappaR: public aCloneableBinaryFunctor<CurvatureKappaR>
 {
     CurvatureKappaR( ){ }
-    CurvatureKappaR( const aTokamakMagneticField& mag){ }
+    CurvatureKappaR( const TokamakMagneticField& mag){ }
     /**
      * @brief \f[ \mathcal{\hat{K}}^{\hat{R}}_{\vec{\kappa}} =0  \f]
      */ 
@@ -241,7 +238,7 @@ struct CurvatureKappaR: public aCloneableBinaryFunctor<CurvatureKappaR>
  */ 
 struct CurvatureKappaZ: public aCloneableBinaryFunctor<CurvatureKappaZ>
 {
-    CurvatureKappaZ( const aTokamakMagneticField& mag): invB_(mag) { }
+    CurvatureKappaZ( const TokamakMagneticField& mag): invB_(mag) { }
     /**
      * @brief \f[  \mathcal{\hat{K}}^{\hat{Z}}_{\vec{\kappa}} = - \frac{1}{\hat{R} \hat{B}} \f]
      */    
@@ -258,7 +255,7 @@ struct CurvatureKappaZ: public aCloneableBinaryFunctor<CurvatureKappaZ>
  */ 
 struct DivCurvatureKappa: public aCloneableBinaryFunctor<DivCurvatureKappa>
 {
-    DivCurvatureKappa( const aTokamakMagneticField& mag): invB_(mag), bZ_(mag){ }
+    DivCurvatureKappa( const TokamakMagneticField& mag): invB_(mag), bZ_(mag){ }
     /**
      * @brief \f[  \vec{\hat{\nabla}}\cdot \mathcal{\hat{K}}_{\vec{\kappa}}  = \frac{1}{\hat{R}  \hat{B}^2 } \partial_{\hat{Z}} \hat{B}\f]
      */    
@@ -276,7 +273,7 @@ struct DivCurvatureKappa: public aCloneableBinaryFunctor<DivCurvatureKappa>
  */ 
 struct GradLnB: public aCloneableBinaryFunctor<GradLnB>
 {
-    GradLnB( const aTokamakMagneticField& mag): mag_(mag), invB_(mag), bR_(mag), bZ_(mag) { } 
+    GradLnB( const TokamakMagneticField& mag): mag_(mag), invB_(mag), bR_(mag), bZ_(mag) { } 
     /**
      * @brief \f[  \hat{\nabla}_\parallel \ln{(\hat{B})} = \frac{1}{\hat{R}\hat{B}^2 } \left[ \hat{B}, \hat{\psi}_p\right]_{\hat{R}\hat{Z}} \f]
      */ 
@@ -286,7 +283,7 @@ struct GradLnB: public aCloneableBinaryFunctor<GradLnB>
         return mag_.R0()*invB*invB*(bR_(R,Z)*mag_.psipZ()(R,Z)-bZ_(R,Z)*mag_.psipR()(R,Z))/R ;
     }
     private:
-    aTokamakMagneticField mag_;
+    TokamakMagneticField mag_;
     InvB invB_;
     BR bR_;
     BZ bZ_;   
@@ -297,14 +294,14 @@ struct GradLnB: public aCloneableBinaryFunctor<GradLnB>
 */
 struct FieldP: public aCloneableBinaryFunctor<LnB>
 {
-    FieldP( const aTokamakMagneticField& mag): mag_(mag){}
+    FieldP( const TokamakMagneticField& mag): mag_(mag){}
     double operator()( double R, double Z, double phi) const
     {
         return mag.R0()*mag_.ipol()(R,Z)/R/R;
     }
     
     private:
-    aTokamakMagneticField mag_;
+    TokamakMagneticField mag_;
 }; 
 
 /**
@@ -312,13 +309,13 @@ struct FieldP: public aCloneableBinaryFunctor<LnB>
  */
 struct FieldR: public aCloneableBinaryFunctor<FieldR>
 {
-    FieldR( const aTokamakMagneticField& mag): mag_(mag){}
+    FieldR( const TokamakMagneticField& mag): mag_(mag){}
     double operator()( double R, double Z) const
     {
         return  mag.R0()/R*mag_.psipZ()(R,Z);
     }
     private:
-    aTokamakMagneticField mag_;
+    TokamakMagneticField mag_;
    
 };
 
@@ -327,13 +324,13 @@ struct FieldR: public aCloneableBinaryFunctor<FieldR>
  */
 struct FieldZ: public aCloneableBinaryFunctor<FieldZ>
 {
-    FieldZ( const aTokamakMagneticField& mag): mag_(mag){}
+    FieldZ( const TokamakMagneticField& mag): mag_(mag){}
     double operator()( double R, double Z) const
     {
         return -mag_.R0()/R*mag_.psipR()(R,Z);
     }
     private:
-    aTokamakMagneticField mag_;
+    TokamakMagneticField mag_;
 };
 
 /**
@@ -342,7 +339,7 @@ struct FieldZ: public aCloneableBinaryFunctor<FieldZ>
 struct FieldT: public aCloneableBinaryFunctor<FieldT>
 
 {
-    FieldT( const aTokamakMagneticField& mag):  R_0_(mag.R0()), fieldR_(mag), fieldZ_(mag){}
+    FieldT( const TokamakMagneticField& mag):  R_0_(mag.R0()), fieldR_(mag), fieldZ_(mag){}
     /**
      * @brief \f[  B^{\theta} = 
      * B^R\partial_R\theta + B^Z\partial_Z\theta\f]
@@ -364,13 +361,13 @@ struct FieldT: public aCloneableBinaryFunctor<FieldT>
  */
 struct BHatR: public aCloneableBinaryFunctor<BHatR>
 {
-    BHatR( const aTokamakMagneticField& mag): mag_(mag), invB_(mag){ }
+    BHatR( const TokamakMagneticField& mag): mag_(mag), invB_(mag){ }
     double operator()( double R, double Z) const
     {
         return  invB_(R,Z)*mag_.R0()/R*mag_.psipZ()(R,Z);
     }
     private:
-    aTokamakMagneticField mag_;
+    TokamakMagneticField mag_;
     InvB invB_;
 
 };
@@ -380,13 +377,13 @@ struct BHatR: public aCloneableBinaryFunctor<BHatR>
  */
 struct BHatZ: public aCloneableBinaryFunctor<BHatZ>
 {
-    BHatZ( const aTokamakMagneticField& mag): mag_(mag), invB_(mag){ }
+    BHatZ( const TokamakMagneticField& mag): mag_(mag), invB_(mag){ }
     double operator()( double R, double Z) const
     {
         return  -invB_(R,Z)*mag_.R0()/R*mag_.psipR()(R,Z);
     }
     private:
-    aTokamakMagneticField mag_;
+    TokamakMagneticField mag_;
     InvB invB_;
 };
 
@@ -395,14 +392,14 @@ struct BHatZ: public aCloneableBinaryFunctor<BHatZ>
  */
 struct BHatP: public aCloneableBinaryFunctor<BHatP>
 {
-    BHatP( const aTokamakMagneticField& mag): mag_(mag), invB_(mag){ }
+    BHatP( const TokamakMagneticField& mag): mag_(mag), invB_(mag){ }
     double operator()( double R, double Z) const
     {
         return invB_(R,Z)*mag_.R0()*mag_.ipol()(R,Z)/R/R;
     }
     
     private:
-    aTokamakMagneticField mag_;
+    TokamakMagneticField mag_;
     InvB invB_;
 }; 
 
diff --git a/inc/geometries/solovev.h b/inc/geometries/solovev.h
index c5f741284..baf2b2d69 100644
--- a/inc/geometries/solovev.h
+++ b/inc/geometries/solovev.h
@@ -440,22 +440,20 @@ struct IpolZ: public aCloneableBinaryFunctor<IpolZ>
     PsipZ psipZ_;
 };
 
-/**
- * @brief Contains all solovev fields
- */
-struct MagneticField : public dg::geo::aTokamakMagneticField
+BinaryFunctorsLvl2 createPsip( GeomParameters gp)
 {
-    MagneticField( GeomParameters gp): aTokamakMagneticField(gp.R_0, 
-        new Psip(gp), 
-        new PsipR(gp), 
-        new PsipZ(gp), 
-        new PsipRR(gp), 
-        new PsipRZ(gp), 
-        new PsipZZ(gp), 
-        new Ipol(gp), 
-        new IpolR(gp), 
-        new IpolZ(gp)){}
-};
+    BinaryFunctorsLvl2 psip( new Psip(gp), new PsipR(gp), new PsipZ(gp),new PsipRR(gp), new PsipRZ(gp), new PsipZZ(gp));
+    return psip;
+}
+BinaryFunctorsLvl1 createIpol( GeomParameters gp)
+{
+    BinaryFunctorsLvl1 ipol( new Ipol(gp), new IpolR(gp), new IpolZ(gp))
+    return ipol;
+}
+MagneticField createMagField( GeomParameters gp)
+{
+    return MagneticField( gp.R_0, createPsip(gp), createIpol(gp));
+}
 ///@}
 
 ///@cond
diff --git a/inc/geometries/taylor.h b/inc/geometries/taylor.h
index 9f34e6e0a..867897dbb 100644
--- a/inc/geometries/taylor.h
+++ b/inc/geometries/taylor.h
@@ -42,8 +42,7 @@ typedef dg::geo::solovev::GeomParameters GeomParameters; //!< bring GeomParamete
  * @attention When the taylor field is used we need the boost library for special functions
  */
 struct Psip : public aCloneableBinaryFunctor<Psip>
-{
-    /**
+{ /**
      * @brief Construct from given geometric parameters
      *
      * @param gp useful geometric parameters
@@ -311,22 +310,20 @@ struct IpolZ: public aCloneableBinaryFunctor<IpolZ>
     PsipZ psipZ_;
 };
 
-/**
- * @brief Contains all taylor fields 
- */
-struct MagneticField : public dg::geo::aTokamakMagneticField
+BinaryFunctorsLvl2 createPsip( solovev::GeomParameters gp)
 {
-    MagneticField( solovev::GeomParameters gp): aTokamakMagneticField(gp.R_0, 
-        new Psip(gp), 
-        new PsipR(gp), 
-        new PsipZ(gp), 
-        new PsipRR(gp), 
-        new PsipRZ(gp), 
-        new PsipZZ(gp), 
-        new Ipol(gp), 
-        new IpolR(gp), 
-        new IpolZ(gp)){}
-};
+    BinaryFunctorsLvl2 psip( new Psip(gp), new PsipR(gp), new PsipZ(gp),new PsipRR(gp), new PsipRZ(gp), new PsipZZ(gp));
+    return psip;
+}
+BinaryFunctorsLvl1 createIpol( solovev::GeomParameters gp)
+{
+    BinaryFunctorsLvl1 ipol( new Ipol(gp), new IpolR(gp), new IpolZ(gp))
+    return ipol;
+}
+MagneticField createMagField( solovev::GeomParameters gp)
+{
+    return MagneticField( gp.R_0, createPsip(gp), createIpol(gp));
+}
 
 ///@}
 
diff --git a/inc/geometries/toroidal.h b/inc/geometries/toroidal.h
index 5f105e666..3cf21f951 100644
--- a/inc/geometries/toroidal.h
+++ b/inc/geometries/toroidal.h
@@ -26,6 +26,20 @@ struct MagneticField : public dg::geo::aTokamakMagneticField
         new Constant(0), 
         new Constant(0)){}
 };
+BinaryFunctorsLvl2 createPsip( )
+{
+    BinaryFunctorsLvl2 psip( new Constant(1), new Constant(0), new Constant(0),new Constant(0), new Constant(0), new Constant(0));
+    return psip;
+}
+BinaryFunctorsLvl1 createIpol( )
+{
+    BinaryFunctorsLvl1 ipol( new Constant(1), new Constant(0), new Constant(0))
+    return ipol;
+}
+MagneticField createMagField( double R0)
+{
+    return MagneticField( R0, createPsip(), createIpol());
+}
 
 }//namespace toroidal
 }//namespace geo
-- 
GitLab


From 2a5064361e90ed72808bd1cf6751579438c8c0bc Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Wed, 2 Aug 2017 19:27:17 +0200
Subject: [PATCH 104/453] made MPI Grids use an abstract base class

---
 inc/dg/backend/mpi_derivatives.h |  40 ++++----
 inc/dg/backend/mpi_evaluation.h  |   8 +-
 inc/dg/backend/mpi_grid.h        | 152 ++++++++++++++++++-------------
 inc/dg/backend/mpi_precon.h      |   8 +-
 4 files changed, 116 insertions(+), 92 deletions(-)

diff --git a/inc/dg/backend/mpi_derivatives.h b/inc/dg/backend/mpi_derivatives.h
index f853b5fa8..c0282a5e4 100644
--- a/inc/dg/backend/mpi_derivatives.h
+++ b/inc/dg/backend/mpi_derivatives.h
@@ -119,7 +119,7 @@ EllSparseBlockMat<double> distribute_rows( const EllSparseBlockMat<double>& src,
 *
 * @return  A mpi matrix
 */
-RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dx( const MPIGrid2d& g, bc bcx, direction dir = centered)
+RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dx( const aMPITopology2d& g, bc bcx, direction dir = centered)
 {
     EllSparseBlockMat<double> matrix = dg::create::dx( g.global(), bcx, dir);
     int vector_dimensions[] = {(int)(g.n()*g.Nx()), (int)(g.n()*g.Ny()), 1}; //x, y, z
@@ -148,7 +148,7 @@ RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dx( c
 *
 * @return  A mpi matrix
 */
-RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dy( const MPIGrid2d& g, bc bcy, direction dir = centered)
+RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dy( const aMPITopology2d& g, bc bcy, direction dir = centered)
 {
     EllSparseBlockMat<double> matrix = dg::create::dy( g.global(), bcy, dir);
     int vector_dimensions[] = {(int)(g.n()*g.Nx()), (int)(g.n()*g.Ny()), 1}; //x, y, z
@@ -175,7 +175,7 @@ RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dy( c
 *
 * @return  A mpi matrix
 */
-RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpX( const MPIGrid2d& g, bc bcx)
+RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpX( const aMPITopology2d& g, bc bcx)
 {
     EllSparseBlockMat<double> matrix = dg::create::jumpX( g.global(), bcx);
     int vector_dimensions[] = {(int)(g.n()*g.Nx()), (int)(g.n()*g.Ny()), 1}; //x, y, z
@@ -201,7 +201,7 @@ RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpX
 *
 * @return  A mpi matrix
 */
-RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpY( const MPIGrid2d& g, bc bcy)
+RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpY( const aMPITopology2d& g, bc bcy)
 {
     EllSparseBlockMat<double> matrix = dg::create::jumpY( g.global(), bcy);
     int vector_dimensions[] = {(int)(g.n()*g.Nx()), (int)(g.n()*g.Ny()), 1}; //x, y, z
@@ -229,7 +229,7 @@ RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpY
 *
 * @return  A mpi matrix
 */
-RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dx( const MPIGrid3d& g, bc bcx, direction dir = centered)
+RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dx( const aMPITopology3d& g, bc bcx, direction dir = centered)
 {
     EllSparseBlockMat<double> matrix = dg::create::dx( g.global(), bcx, dir);
     int vector_dimensions[] = {(int)(g.n()*g.Nx()), (int)(g.n()*g.Ny()), (int)(g.Nz())}; //x, y, z
@@ -256,7 +256,7 @@ RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dx( c
 *
 * @return  A mpi matrix
 */
-RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dy( const MPIGrid3d& g, bc bcy, direction dir = centered)
+RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dy( const aMPITopology3d& g, bc bcy, direction dir = centered)
 {
     EllSparseBlockMat<double> matrix = dg::create::dy( g.global(), bcy, dir);
     int vector_dimensions[] = {(int)(g.n()*g.Nx()), (int)(g.n()*g.Ny()), (int)(g.Nz())}; //x, y, z
@@ -283,7 +283,7 @@ RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dy( c
 *
 * @return  A mpi matrix
 */
-RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dz( const MPIGrid3d& g, bc bcz, direction dir = centered)
+RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dz( const aMPITopology3d& g, bc bcz, direction dir = centered)
 {
     EllSparseBlockMat<double> matrix = dg::create::dz( g.global(), bcz, dir);
     int vector_dimensions[] = {(int)(g.n()*g.Nx()), (int)(g.n()*g.Ny()), (int)(g.Nz())}; //x, y, z
@@ -310,7 +310,7 @@ RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dz( c
 *
 * @return  A mpi matrix
 */
-RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpX( const MPIGrid3d& g, bc bcx)
+RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpX( const aMPITopology3d& g, bc bcx)
 {
     EllSparseBlockMat<double> matrix = dg::create::jumpX( g.global(), bcx);
     int vector_dimensions[] = {(int)(g.n()*g.Nx()), (int)(g.n()*g.Ny()), (int)(g.Nz())}; //x, y, z
@@ -337,7 +337,7 @@ RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpX
 *
 * @return  A mpi matrix
 */
-RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpY( const MPIGrid3d& g, bc bcy)
+RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpY( const aMPITopology3d& g, bc bcy)
 {
     EllSparseBlockMat<double> matrix = dg::create::jumpY( g.global(), bcy);
     int vector_dimensions[] = {(int)(g.n()*g.Nx()), (int)(g.n()*g.Ny()), (int)(g.Nz())}; //x, y, z
@@ -363,7 +363,7 @@ RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpY
 *
 * @return  A mpi matrix
 */
-RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpZ( const MPIGrid3d& g, bc bcz)
+RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpZ( const aMPITopology3d& g, bc bcz)
 {
     EllSparseBlockMat<double> matrix = dg::create::jumpZ( g.global(), bcz);
     int vector_dimensions[] = {(int)(g.n()*g.Nx()), (int)(g.n()*g.Ny()), (int)(g.Nz())}; //x, y, z
@@ -390,7 +390,7 @@ RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpZ
  *
  * @return A mpi matrix 
  */
-RowColDistMat<EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dx( const MPIGrid2d& g, direction dir = centered)
+RowColDistMat<EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dx( const aMPITopology2d& g, direction dir = centered)
 {
     return dx( g, g.bcx(), dir);
 }
@@ -403,7 +403,7 @@ RowColDistMat<EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dx( co
  *
  * @return A mpi matrix 
  */
-RowColDistMat<EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dx( const MPIGrid3d& g, direction dir = centered)
+RowColDistMat<EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dx( const aMPITopology3d& g, direction dir = centered)
 {
     return dx( g, g.bcx(), dir);
 }
@@ -414,7 +414,7 @@ RowColDistMat<EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dx( co
  *
  * @return A mpi matrix 
  */
-RowColDistMat<EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpX( const MPIGrid2d& g)
+RowColDistMat<EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpX( const aMPITopology2d& g)
 {
     return jumpX( g, g.bcx());
 }
@@ -426,7 +426,7 @@ RowColDistMat<EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpX(
  *
  * @return A mpi matrix 
  */
-RowColDistMat<EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpX( const MPIGrid3d& g)
+RowColDistMat<EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpX( const aMPITopology3d& g)
 {
     return jumpX( g, g.bcx());
 }
@@ -439,7 +439,7 @@ RowColDistMat<EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpX(
  *
  * @return A mpi matrix
  */
-RowColDistMat<EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dy( const MPIGrid2d& g, direction dir = centered)
+RowColDistMat<EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dy( const aMPITopology2d& g, direction dir = centered)
 {
     return dy( g, g.bcy(), dir);
 }
@@ -452,7 +452,7 @@ RowColDistMat<EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dy( co
  *
  * @return A mpi matrix 
  */
-RowColDistMat<EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dy( const MPIGrid3d& g, direction dir = centered)
+RowColDistMat<EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dy( const aMPITopology3d& g, direction dir = centered)
 {
     return dy( g, g.bcy(), dir);
 }
@@ -464,7 +464,7 @@ RowColDistMat<EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dy( co
  *
  * @return A mpi matrix
  */
-RowColDistMat<EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpY( const MPIGrid2d& g)
+RowColDistMat<EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpY( const aMPITopology2d& g)
 {
     return jumpY( g, g.bcy());
 }
@@ -476,7 +476,7 @@ RowColDistMat<EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpY(
  *
  * @return A mpi matrix 
  */
-RowColDistMat<EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpY( const MPIGrid3d& g)
+RowColDistMat<EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpY( const aMPITopology3d& g)
 {
     return jumpY( g, g.bcy());
 }
@@ -489,7 +489,7 @@ RowColDistMat<EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpY(
  *
  * @return A mpi matrix 
  */
-RowColDistMat<EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dz( const MPIGrid3d& g, direction dir = centered)
+RowColDistMat<EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dz( const aMPITopology3d& g, direction dir = centered)
 {
     return dz( g, g.bcz(), dir);
 }
@@ -501,7 +501,7 @@ RowColDistMat<EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dz( co
  *
  * @return A mpi matrix 
  */
-RowColDistMat<EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpZ( const MPIGrid3d& g)
+RowColDistMat<EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpZ( const aMPITopology3d& g)
 {
     return jumpZ( g, g.bcz());
 }
diff --git a/inc/dg/backend/mpi_evaluation.h b/inc/dg/backend/mpi_evaluation.h
index 6a77f2ebb..d174a1292 100644
--- a/inc/dg/backend/mpi_evaluation.h
+++ b/inc/dg/backend/mpi_evaluation.h
@@ -28,7 +28,7 @@ namespace dg
             may be constructed during function call.
  */
 template< class BinaryOp>
-MPI_Vector<thrust::host_vector<double> > evaluate( BinaryOp f, const MPIGrid2d& g)
+MPI_Vector<thrust::host_vector<double> > evaluate( BinaryOp f, const aMPITopology2d& g)
 {
     thrust::host_vector<double> w = evaluate( f, g.local());
     MPI_Vector<thrust::host_vector<double> > v( w, g.communicator());
@@ -36,7 +36,7 @@ MPI_Vector<thrust::host_vector<double> > evaluate( BinaryOp f, const MPIGrid2d&
     return v;
 };
 ///@cond
-MPI_Vector<thrust::host_vector<double> > evaluate( double(f)(double, double), const MPIGrid2d& g)
+MPI_Vector<thrust::host_vector<double> > evaluate( double(f)(double, double), const aMPITopology2d& g)
 {
     return evaluate<double(double, double)>( f, g);
 };
@@ -55,7 +55,7 @@ MPI_Vector<thrust::host_vector<double> > evaluate( double(f)(double, double), co
             may be constructed during function call.
  */
 template< class TernaryOp>
-MPI_Vector<thrust::host_vector<double> > evaluate( TernaryOp f, const MPIGrid3d& g)
+MPI_Vector<thrust::host_vector<double> > evaluate( TernaryOp f, const aMPITopology3d& g)
 {
     thrust::host_vector<double> w = evaluate( f, g.local());
     MPI_Vector<thrust::host_vector<double> > v( w, g.communicator());
@@ -63,7 +63,7 @@ MPI_Vector<thrust::host_vector<double> > evaluate( TernaryOp f, const MPIGrid3d&
     return v;
 };
 ///@cond
-MPI_Vector<thrust::host_vector<double> > evaluate( double(f)(double, double, double), const MPIGrid3d& g)
+MPI_Vector<thrust::host_vector<double> > evaluate( double(f)(double, double, double), const aMPITopology3d& g)
 {
     return evaluate<double(double, double, double)>( f, g);
 };
diff --git a/inc/dg/backend/mpi_grid.h b/inc/dg/backend/mpi_grid.h
index 1d814a792..8447fadc0 100644
--- a/inc/dg/backend/mpi_grid.h
+++ b/inc/dg/backend/mpi_grid.h
@@ -24,31 +24,10 @@ namespace dg
  * The boundaries in the constructors are global boundaries, the boundaries returned by the access functions are local boundaries, this is because the grid represents the information given to one process
  *
  */
-struct MPIGrid2d
+struct aMPITopology2d
 {
     typedef MPITag memory_category;
     typedef TwoDimensionalTag dimensionality;
-    /**
-     * @copydoc dg::Grid2d::Grid2d()
-     * @param comm a two-dimensional Cartesian communicator
-     * @note the paramateres given in the constructor are global parameters 
-     */
-    MPIGrid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, MPI_Comm comm):
-        g( x0, x1, y0, y1, n, Nx, Ny), comm( comm)
-    {
-        check_division( Nx, Ny, g.bcx(), g.bcy());
-    }
-
-    /**
-     * @copydoc dg::Grid2d::Grid2d()
-     * @param comm a two-dimensional Cartesian communicator
-     * @note the paramateres given in the constructor are global parameters 
-     */
-    MPIGrid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx, bc bcy, MPI_Comm comm):
-        g( x0, x1, y0, y1, n, Nx, Ny, bcx, bcy), comm( comm)
-    {
-        check_division( Nx, Ny, bcx, bcy);
-    }
 
     /**
     * @brief Multiply the number of cells with a given factor
@@ -58,7 +37,7 @@ struct MPIGrid2d
     * @param fy new global number of cells is fy*global().Ny()
     */
     void multiplyCellNumber( double fx, double fy){
-        set(n_, floor(fx*(double)global.Nx()+0.5), floor(fy*(double)global.Ny()+0.5));
+        set(g.n(), floor(fx*(double)g.Nx()+0.5), floor(fy*(double)g.Ny()+0.5));
     }
     /**
     * @copydoc Grid2d::set(unsigned,unsigned,unsigned)
@@ -275,7 +254,6 @@ struct MPIGrid2d
         int lIdx0 = gIdx0%(n()*Nx());
         int lIdx1 = gIdx1%(n()*Ny());
         localIdx = lIdx1*n()*Nx() + lIdx0;
-        std::cout<< gIdx0<<" "<<gIdx1<<" "<<coords[0]<<" "<<coords[1]<<" "<<lIdx0<<" "<<lIdx1<<" "<<localIdx<<std::endl;
         if( MPI_Cart_rank( comm, coords, &PID) == MPI_SUCCESS ) 
             return true;
         else
@@ -302,13 +280,26 @@ struct MPIGrid2d
      * This is the grid that we would have to use in a non-MPI implementation.
      * @return non-MPI Grid object
      */
-    const Grid2d& global() const {return g;}
-    virtual ~MPIGrid2d(){}
-    MPIGrid2d(const MPIGrid2d& src):g(src.g),comm(src.comm){}
-    MPIGrid2d& operator=(const MPIGrid2d& src){
+    Grid2d global() const {return g;}
+    virtual ~aMPITopology2d(){}
+    virtual aMPITopology2d* clone()const=0;
+    protected:
+
+    /**
+     * @copydoc dg::Grid2d::Grid2d()
+     * @param comm a two-dimensional Cartesian communicator
+     * @note the paramateres given in the constructor are global parameters 
+     */
+    aMPITopology2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx, bc bcy, MPI_Comm comm):
+        g( x0, x1, y0, y1, n, Nx, Ny, bcx, bcy), comm( comm)
+    {
+        check_division( Nx, Ny, bcx, bcy);
+    }
+    aMPITopology2d(const aMPITopology2d& src):g(src.g),comm(src.comm){}
+    aMPITopology2d& operator=(const aMPITopology2d& src){
         g = src.g; comm = src.comm;
+        return *this;
     }
-    protected:
     virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny) {
         g.set(new_n,new_Nx,new_Ny);
     }
@@ -351,34 +342,13 @@ struct MPIGrid2d
  *
  * @note Note that a single cell is never divided across processes.
  */
-struct MPIGrid3d
+struct aMPITopology3d
 {
     typedef MPITag memory_category;
     typedef ThreeDimensionalTag dimensionality;
-    /**
-     * @copydoc Grid3d::Grid3d()
-     * @param comm a three-dimensional Cartesian communicator
-     * @note the paramateres given in the constructor are global parameters 
-     */
-    MPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, MPI_Comm comm):
-        g( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz), comm( comm)
-    {
-        check_division( Nx, Ny, Nz, bcx(), bcy(), bcz());
-    }
-
-    /**
-     * @copydoc Grid3d::Grid3d()
-     * @param comm a three-dimensional Cartesian communicator
-     * @note the paramateres given in the constructor are global parameters 
-     */
-    MPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz, MPI_Comm comm):
-        g( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, bcx, bcy, bcz), comm( comm)
-    {
-        check_division( Nx, Ny, Nz, bcx, bcy, bcz);
-    }
-    ///@copydoc MPIGrid2d::multiplyCellNumber()
+    ///@copydoc aMPITopology2d::multiplyCellNumber()
     void multiplyCellNumber( double fx, double fy){
-        set(n_, floor(fx*(double)global.Nx()+0.5), floor(fy*(double)global.Ny()+0.5), global.Nz());
+        set(g.n(), floor(fx*(double)g.Nx()+0.5), floor(fy*(double)g.Ny()+0.5), g.Nz());
     }
     /**
      * @copydoc Grid3d::set(unsigned,unsigned,unsigned,unsigned)
@@ -592,7 +562,7 @@ struct MPIGrid3d
      */
     int pidOf( double x, double y, double z) const;
     /**
-    * @copydoc MPIGrid2d::local2globalIdx(int,int,int&)
+    * @copydoc aMPITopology2d::local2globalIdx(int,int,int&)
     */
     bool local2globalIdx( int localIdx, int PID, int& globalIdx)
     {
@@ -610,7 +580,7 @@ struct MPIGrid3d
         return true;
     }
     /**
-    * @copydoc MPIGrid2d::global2localIdx(int,int&,int&)
+    * @copydoc aMPITopology2d::global2localIdx(int,int&,int&)
     */
     bool global2localIdx( int globalIdx, int& localIdx, int& PID)
     {
@@ -632,19 +602,32 @@ struct MPIGrid3d
             return false;
     }
     /**
-     *@copydoc MPIGrid2d::local()const
+     *@copydoc aMPITopology2d::local()const
      */
     Grid3d local() const {return Grid3d(x0(), x1(), y0(), y1(), z0(), z1(), n(), Nx(), Ny(), Nz(), bcx(), bcy(), bcz());}
     /**
-     *@copydoc MPIGrid2d::global()const
+     *@copydoc aMPITopology2d::global()const
+     */
+    Grid3d global() const {return g;}
+    virtual ~aMPITopology3d(){}
+    virtual aMPITopology3d* clone() const=0;
+    protected:
+
+    /**
+     * @copydoc Grid3d::Grid3d()
+     * @param comm a three-dimensional Cartesian communicator
+     * @note the paramateres given in the constructor are global parameters 
      */
-    const Grid3d& global() const {return g;}
-    virtual ~MPIGrid3d(){}
-    MPIGrid3d(const MPIGrid3d& src):g(src.g),comm(src.comm){}
-    MPIGrid3d& operator=(const MPIGrid3d& src){
+    aMPITopology3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz, MPI_Comm comm):
+        g( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, bcx, bcy, bcz), comm( comm)
+    {
+        check_division( Nx, Ny, Nz, bcx, bcy, bcz);
+    }
+    aMPITopology3d(const aMPITopology3d& src):g(src.g),comm(src.comm){}
+    aMPITopology3d& operator=(const aMPITopology3d& src){
         g = src.g; comm = src.comm;
+        return *this;
     }
-    protected:
     virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz) {
         g.set(new_n,new_Nx,new_Ny,new_Nz);
     }
@@ -681,7 +664,7 @@ struct MPIGrid3d
     MPI_Comm comm; //just an integer...
 };
 ///@cond
-int MPIGrid2d::pidOf( double x, double y) const
+int aMPITopology2d::pidOf( double x, double y) const
 {
     int dims[2], periods[2], coords[2];
     MPI_Cart_get( comm, 2, dims, periods, coords);
@@ -696,7 +679,7 @@ int MPIGrid2d::pidOf( double x, double y) const
     else
         return -1;
 }
-int MPIGrid3d::pidOf( double x, double y, double z) const
+int aMPITopology3d::pidOf( double x, double y, double z) const
 {
     int dims[3], periods[3], coords[3];
     MPI_Cart_get( comm, 3, dims, periods, coords);
@@ -715,6 +698,47 @@ int MPIGrid3d::pidOf( double x, double y, double z) const
 }
 ///@endcond
 
+struct MPIGrid2d: public aMPITopology2d
+{
+    /**
+     * @copydoc dg::Grid2d::Grid2d()
+     * @param comm a two-dimensional Cartesian communicator
+     * @note the paramateres given in the constructor are global parameters 
+     */
+    MPIGrid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, MPI_Comm comm):
+        aMPITopology2d( x0,x1,y0,y1,n,Nx,Ny,dg::PER,dg::PER,comm)
+    { }
+
+    /**
+     * @copydoc dg::Grid2d::Grid2d()
+     * @param comm a two-dimensional Cartesian communicator
+     * @note the paramateres given in the constructor are global parameters 
+     */
+    MPIGrid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx, bc bcy, MPI_Comm comm):
+        aMPITopology2d( x0,x1,y0,y1,n,Nx,Ny,bcx,bcy,comm)
+    { }
+    virtual MPIGrid2d* clone()const{return new MPIGrid2d(*this);}
+};
+
+
+
+struct MPIGrid3d : public aMPITopology3d
+{
+    MPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, MPI_Comm comm):
+        aMPITopology3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, dg::PER, dg::PER, dg::PER,comm )
+    { }
+
+    /**
+     * @copydoc Grid3d::Grid3d()
+     * @param comm a three-dimensional Cartesian communicator
+     * @note the paramateres given in the constructor are global parameters 
+     */
+    MPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz, MPI_Comm comm):
+        aMPITopology3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, bcx, bcy, bcz, comm)
+    { }
+    virtual MPIGrid3d* clone()const{return new MPIGrid3d(*this);}
+};
+
 
 ///@}
 }//namespace dg
diff --git a/inc/dg/backend/mpi_precon.h b/inc/dg/backend/mpi_precon.h
index 4aea4088d..6045675a8 100644
--- a/inc/dg/backend/mpi_precon.h
+++ b/inc/dg/backend/mpi_precon.h
@@ -36,7 +36,7 @@ namespace create
 *
 * @return Preconditioner
 */
-MPI_Vector<thrust::host_vector<double> > weights( const MPIGrid2d& g)
+MPI_Vector<thrust::host_vector<double> > weights( const aMPITopology2d& g)
 {
     thrust::host_vector<double> w = create::weights( g.local());
     return MPI_Vector<thrust::host_vector<double> >( w, g.communicator());
@@ -48,7 +48,7 @@ MPI_Vector<thrust::host_vector<double> > weights( const MPIGrid2d& g)
 *
 * @return Preconditioner
 */
-MPI_Vector<thrust::host_vector<double> > inv_weights( const MPIGrid2d& g)
+MPI_Vector<thrust::host_vector<double> > inv_weights( const aMPITopology2d& g)
 {
     thrust::host_vector<double> w = create::inv_weights( g.local());
     return MPI_Vector<thrust::host_vector<double> >( w, g.communicator());
@@ -60,7 +60,7 @@ MPI_Vector<thrust::host_vector<double> > inv_weights( const MPIGrid2d& g)
 *
 * @return Preconditioner
 */
-MPI_Vector<thrust::host_vector<double> > weights( const MPIGrid3d& g)
+MPI_Vector<thrust::host_vector<double> > weights( const aMPITopology3d& g)
 {
     thrust::host_vector<double> w = create::weights( g.local());
     return MPI_Vector<thrust::host_vector<double> >( w, g.communicator());
@@ -72,7 +72,7 @@ MPI_Vector<thrust::host_vector<double> > weights( const MPIGrid3d& g)
 *
 * @return Preconditioner
 */
-MPI_Vector<thrust::host_vector<double> > inv_weights( const MPIGrid3d& g)
+MPI_Vector<thrust::host_vector<double> > inv_weights( const aMPITopology3d& g)
 {
     thrust::host_vector<double> w = create::inv_weights( g.local());
     return MPI_Vector<thrust::host_vector<double> >( w, g.communicator());
-- 
GitLab


From 1c4c9f17978361fc6397a290ef7513d66e483db4 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Wed, 2 Aug 2017 22:46:25 +0200
Subject: [PATCH 105/453] made a separate header for the smart pointer

---
 inc/dg/backend/cloneable.h  | 45 +++++++++++++++++++++++++++++++++++++
 inc/dg/geometry/generator.h | 42 ----------------------------------
 2 files changed, 45 insertions(+), 42 deletions(-)
 create mode 100644 inc/dg/backend/cloneable.h

diff --git a/inc/dg/backend/cloneable.h b/inc/dg/backend/cloneable.h
new file mode 100644
index 000000000..f4e2da171
--- /dev/null
+++ b/inc/dg/backend/cloneable.h
@@ -0,0 +1,45 @@
+#pragma once
+
+namespace dg
+{
+
+//there is probably a better class in boost...
+///Helper class to avoid rule of three in grid classes
+///Actually it's a very very basic smart pointer that implements 
+///a deep copy using the clone() method 
+template<class cloneable>
+struct Handle
+{
+    Handle():ptr_(0){}
+    /// take ownership of the pointer
+    Handle( cloneable* ptr): ptr_(ptr){}
+    /// clone given value
+    Handle( const cloneable& src): ptr_(src.clone()){}
+    Handle( const Handle& src):ptr_(0) {
+        if(ptr_!=0) ptr_ = src.ptr->clone(); //deep copy
+    }
+    Handle& operator=( Handle src) {
+        this->swap( src );
+        return *this;
+    }
+    ~Handle(){
+        if(ptr_!=0) delete ptr_;
+    }
+    const cloneable& get()const {return *ptr_;}
+    ///takes ownership of the ptr and deletes the current one if non-empty
+    void reset( cloneable* ptr){ 
+        Handle tmp(ptr);
+        *this=tmp;
+    }
+    ///clones the src
+    void reset( const cloneable& src){ 
+        Handle tmp(src);
+        *this=tmp;
+    }
+    void swap( Handle& src){
+        std::swap(ptr_,src.ptr_);
+    }
+    private:
+    cloneable* ptr_;
+};
+}
diff --git a/inc/dg/geometry/generator.h b/inc/dg/geometry/generator.h
index 4b3bb8323..88f7c548c 100644
--- a/inc/dg/geometry/generator.h
+++ b/inc/dg/geometry/generator.h
@@ -155,46 +155,4 @@ struct ShiftedIdentityGenerator: public aGridGenerator
 };
 
 }//namespace geo
-///@cond
-
-//there is probably a better class in boost...
-///Helper class to avoid rule of three in grid classes
-///Actually it's a very very basic smart pointer that implements 
-///a deep copy using the clone() method 
-template<class cloneable>
-struct Handle
-{
-    Handle():ptr_(0){}
-    /// take ownership of the pointer
-    Handle( cloneable* ptr): ptr_(ptr){}
-    /// clone given value
-    Handle( const cloneable& src): ptr_(src.clone()){}
-    Handle( const Handle& src):ptr_(0) {
-        if(ptr_!=0) ptr_ = src.ptr->clone(); //deep copy
-    }
-    Handle& operator=( Handle src) {
-        this->swap( src );
-        return *this;
-    }
-    ~Handle(){
-        if(ptr_!=0) delete ptr_;
-    }
-    const cloneable& get()const {return *ptr_;}
-    ///takes ownership of the ptr and deletes the current one if non-empty
-    void reset( cloneable* ptr){ 
-        Handle tmp(ptr);
-        *this=tmp;
-    }
-    ///clones the src
-    void reset( const cloneable& src){ 
-        Handle tmp(src);
-        *this=tmp;
-    }
-    void swap( Handle& src){
-        std::swap(ptr_,src.ptr_);
-    }
-    private:
-    cloneable* ptr_;
-};
-///@endcond
 }//namespace dg
-- 
GitLab


From e3a88d403878dd87c7d469e2f70c220ad32f2c55 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Wed, 2 Aug 2017 23:52:19 +0200
Subject: [PATCH 106/453] placed Buffer class in manage.h alongside Handle

---
 inc/dg/backend/average.cuh           |  13 +--
 inc/dg/backend/average.h             |  10 +-
 inc/dg/backend/cloneable.h           |  45 ---------
 inc/dg/backend/ell_interpolation.cuh |  16 ++--
 inc/dg/backend/manage.h              | 137 +++++++++++++++++++++++++++
 inc/dg/backend/mpi_collective.h      |  22 ++---
 inc/dg/backend/mpi_vector.h          |  41 ++------
 inc/dg/exceptions.h                  |   6 +-
 inc/dg/geometry/tensor.h             |  29 ++++--
 9 files changed, 197 insertions(+), 122 deletions(-)
 delete mode 100644 inc/dg/backend/cloneable.h
 create mode 100644 inc/dg/backend/manage.h

diff --git a/inc/dg/backend/average.cuh b/inc/dg/backend/average.cuh
index abdd11788..6b5dbbd87 100644
--- a/inc/dg/backend/average.cuh
+++ b/inc/dg/backend/average.cuh
@@ -2,6 +2,7 @@
 
 #include "evaluation.cuh"
 #include "xspacelib.cuh"
+#include "cloneable.h"
 #include "../blas1.h"
 
 /*! @file 
@@ -33,7 +34,7 @@ struct PoloidalAverage
      *
      * @param g 2d Grid
      */
-    PoloidalAverage( const Grid2d& g):
+    PoloidalAverage( const aTopology2d& g):
         dummy( g.n()*g.Nx()), 
         helper( g.size()), helper1d( g.n()*g.Nx()), ly_(g.ly())
     {
@@ -95,9 +96,9 @@ struct ToroidalAverage
      *
      * @param g3d 3d Grid
      */
-    ToroidalAverage(const dg::Grid3d& g3d):
+    ToroidalAverage(const dg::aTopology3d& g3d):
         g3d_(g3d),
-        sizeg2d_(g3d_.size()/g3d_.Nz())
+        sizeg2d_(g3d_.get().size()/g3d_.get().Nz())
     {        
     }
     /**
@@ -108,15 +109,15 @@ struct ToroidalAverage
      */
     void operator()(const container& src, container& res)
     {
-        for( unsigned k=0; k<g3d_.Nz(); k++)
+        for( unsigned k=0; k<g3d_.get().Nz(); k++)
         {
             container data2d(src.begin() + k*sizeg2d_,src.begin() + (k+1)*sizeg2d_);
             dg::blas1::axpby(1.0,data2d,1.0,res); 
         }
-        dg::blas1::scal(res,1./g3d_.Nz()); //scale avg
+        dg::blas1::scal(res,1./g3d_.get().Nz()); //scale avg
     }
     private:
-    const dg::Grid3d& g3d_;
+    Handle<dg::aTopology3d> g3d_;
     unsigned sizeg2d_;
 };
 }//namespace dg
diff --git a/inc/dg/backend/average.h b/inc/dg/backend/average.h
index bf1852f92..646a2c3ed 100644
--- a/inc/dg/backend/average.h
+++ b/inc/dg/backend/average.h
@@ -24,14 +24,12 @@ struct PoloidalAverage<MPI_Vector<container>, MPI_Vector<IndexContainer> >
 {
     /**
      * @brief Construct from grid mpi object
-     *
-     * @param g 2d MPIGrid
+     * @param g 2d MPITopology
      */
-    PoloidalAverage( const MPIGrid2d& g): 
+    PoloidalAverage( const aMPITopology2d& g): 
         helper1d_( g.n()*g.Nx()), hhelper1d_(g.n()*g.Nx()),
-        recv_(hhelper1d_), ly_(g.global().ly()),
-        dummy( g.n()*g.Nx()), 
-        helper_( g.size()) 
+        recv_(hhelper1d_),dummy( g.n()*g.Nx()), 
+        helper_( g.size()), ly_(g.global().ly())
     {
         int remain[] = {false, true};
         MPI_Cart_sub( g.communicator(), remain, &comm1d_);
diff --git a/inc/dg/backend/cloneable.h b/inc/dg/backend/cloneable.h
deleted file mode 100644
index f4e2da171..000000000
--- a/inc/dg/backend/cloneable.h
+++ /dev/null
@@ -1,45 +0,0 @@
-#pragma once
-
-namespace dg
-{
-
-//there is probably a better class in boost...
-///Helper class to avoid rule of three in grid classes
-///Actually it's a very very basic smart pointer that implements 
-///a deep copy using the clone() method 
-template<class cloneable>
-struct Handle
-{
-    Handle():ptr_(0){}
-    /// take ownership of the pointer
-    Handle( cloneable* ptr): ptr_(ptr){}
-    /// clone given value
-    Handle( const cloneable& src): ptr_(src.clone()){}
-    Handle( const Handle& src):ptr_(0) {
-        if(ptr_!=0) ptr_ = src.ptr->clone(); //deep copy
-    }
-    Handle& operator=( Handle src) {
-        this->swap( src );
-        return *this;
-    }
-    ~Handle(){
-        if(ptr_!=0) delete ptr_;
-    }
-    const cloneable& get()const {return *ptr_;}
-    ///takes ownership of the ptr and deletes the current one if non-empty
-    void reset( cloneable* ptr){ 
-        Handle tmp(ptr);
-        *this=tmp;
-    }
-    ///clones the src
-    void reset( const cloneable& src){ 
-        Handle tmp(src);
-        *this=tmp;
-    }
-    void swap( Handle& src){
-        std::swap(ptr_,src.ptr_);
-    }
-    private:
-    cloneable* ptr_;
-};
-}
diff --git a/inc/dg/backend/ell_interpolation.cuh b/inc/dg/backend/ell_interpolation.cuh
index 3ba00708d..d8b1790e0 100644
--- a/inc/dg/backend/ell_interpolation.cuh
+++ b/inc/dg/backend/ell_interpolation.cuh
@@ -192,13 +192,13 @@ __launch_bounds__(BLOCK_SIZE, 1) //cuda performance hint macro, (max_threads_per
  * @brief Create an interpolation matrix on the device
  *
  * @param x Vector of x-values
- * @param g Grid on which to interpolate 
+ * @param g aTopology on which to interpolate 
  *
  * @return interpolation matrix
  * @note n must be smaller than 5
  * @attention no range check is performed on the input vectors
  */
-cusp::ell_matrix<double, int, cusp::device_memory> ell_interpolation( const thrust::device_vector<double>& x, const Grid1d& g  )
+cusp::ell_matrix<double, int, cusp::device_memory> ell_interpolation( const thrust::device_vector<double>& x, const aGrid1d& g  )
 {
     assert( g.n()<=4);
     //allocate ell matrix storage
@@ -241,13 +241,13 @@ cusp::ell_matrix<double, int, cusp::device_memory> ell_interpolation( const thru
  *
  * @param x Vector of x-values
  * @param y Vector of y-values
- * @param g 2D Grid on which to interpolate 
+ * @param g 2D aTopology on which to interpolate 
  *
  * @return interpolation matrix
  * @note n must be smaller than 5
  * @attention no range check is performed on the input vectors
  */
-cusp::ell_matrix<int, double, cusp::device_memory> ell_interpolation( const thrust::device_vector<double>& x, const thrust::device_vector<double>& y, const Grid2d& g  )
+cusp::ell_matrix<int, double, cusp::device_memory> ell_interpolation( const thrust::device_vector<double>& x, const thrust::device_vector<double>& y, const aTopology2d& g  )
 {
     assert( x.size() == y.size());
     //allocate ell matrix storage
@@ -292,13 +292,13 @@ cusp::ell_matrix<int, double, cusp::device_memory> ell_interpolation( const thru
  * @param x Vector of x-values
  * @param y Vector of y-values
  * @param z Vector of z-values
- * @param g 3D Grid on which to interpolate z direction is assumed periodic
+ * @param g 3D aTopology on which to interpolate z direction is assumed periodic
  *
  * @return interpolation matrix
  * @note n must be smaller than or equal to 4
  * @attention no range check is performed on the input vectors
  */
-cusp::ell_matrix<int, double, cusp::device_memory> ell_interpolation( const thrust::device_vector<double>& x, const thrust::device_vector<double>& y, const thrust::device_vector<double>& z, const Grid3d& g )
+cusp::ell_matrix<int, double, cusp::device_memory> ell_interpolation( const thrust::device_vector<double>& x, const thrust::device_vector<double>& y, const thrust::device_vector<double>& z, const aTopology3d& g )
 {
     assert( x.size() == y.size());
     assert( x.size() == z.size());
@@ -352,7 +352,7 @@ cusp::ell_matrix<int, double, cusp::device_memory> ell_interpolation( const thru
  * @return Interpolation matrix
  * @note The boundaries of the old brid must lie within the boundaries of the new grid
  */
-cusp::ell_matrix<int, double, cusp::device_memory> ell_interpolation( const Grid3d& g_new, const Grid3d& g_old)
+cusp::ell_matrix<int, double, cusp::device_memory> ell_interpolation( const aTopology3d& g_new, const aTopology3d& g_old)
 {
     assert( g_new.x0() >= g_old.x0());
     assert( g_new.x1() <= g_old.x1());
@@ -377,7 +377,7 @@ cusp::ell_matrix<int, double, cusp::device_memory> ell_interpolation( const Grid
  * @return Interpolation matrix
  * @note The boundaries of the old grid must lie within the boundaries of the new grid
  */
-cusp::ell_matrix<int, double, cusp::device_memory> ell_interpolation( const Grid2d& g_new, const Grid2d& g_old)
+cusp::ell_matrix<int, double, cusp::device_memory> ell_interpolation( const aTopology2d& g_new, const aTopology2d& g_old)
 {
     //assert both grids are on the same box
     assert( g_new.x0() >= g_old.x0());
diff --git a/inc/dg/backend/manage.h b/inc/dg/backend/manage.h
new file mode 100644
index 000000000..5a25304d8
--- /dev/null
+++ b/inc/dg/backend/manage.h
@@ -0,0 +1,137 @@
+#pragma once
+
+namespace dg
+{
+
+//there is probably a better class in boost...
+/*!@brief a manager class that invokes the clone() method on the managed ptr when copied
+*
+*when copied invokes a deep copy using the clone() method 
+* this class is most useful when a class needs to hold a polymorphic, cloneable oject as a variable. 
+@tparam cloneable a type that has the clone() method 
+@ingroup misc
+*/
+template<class cloneable>
+struct Handle
+{
+    ///init an empty Handle
+    Handle():ptr_(0){}
+    /**
+    * @brief take ownership of the pointer
+    * @param ptr a pointer to object to manage
+    */
+    Handle( cloneable* ptr): ptr_(ptr){}
+
+    /**
+    * @brief clone the given value and manage
+    * @param src an object to clone
+    */
+    Handle( const cloneable& src): ptr_(src.clone()){}
+    /**
+    * @brief deep copy the given handle
+    * @param src an oject to copy, clones the contained object if not empty
+    */
+    Handle( const Handle& src):ptr_(0) {
+        if(src.ptr_!=0) ptr_ = src.ptr->clone(); //deep copy
+    }
+    /**
+    * @brief deep copy the given handle
+    * @param src an oject to copy and swap
+    */
+    Handle& operator=( Handle src) {
+        this->swap( src );
+        return *this;
+    }
+    ///delete managed pointer if not empty
+    ~Handle(){ if(ptr_!=0) delete ptr_; }
+
+    /**
+    * @brief Get a constant reference to the object on the heap
+    * @return a reference to the cloneable object
+    */
+    const cloneable& get()const {return *ptr_;}
+
+
+    /**
+    * @brief Non constant access to the object on the heap
+    * @return a non-const reference to the cloneable object
+    */
+    cloneable& get()const {return *ptr_;}
+
+    /**
+    * @brief Take the ownership of the given pointer and delete the currently held one if non-empty
+    * @param ptr a pointer to an object to manage
+    */
+    void reset( cloneable* ptr){ 
+        Handle tmp(ptr);
+        *this=tmp;
+    }
+    /**
+    * @brief Clone the given object and replace the currently held one
+    * @param src a cloneable object 
+    */
+    void reset( const cloneable& src){ 
+        Handle tmp(src);
+        *this=tmp;
+    }
+    /**
+    * @brief swap the managed pointers
+    * @param src another handle
+    */
+    void swap( Handle& src){
+        std::swap(ptr_,src.ptr_);
+    }
+    private:
+    cloneable* ptr_;
+};
+
+//Memory buffer class: data can be written even if the object is const
+/**
+* @brief a manager class that invokes the copy constructor on the managed ptr when copied (deep copy)
+*
+* this class is most useful as a memory buffer for classes that need
+* some workspace to fulfill their task but do otherwise not change their state. A buffer object
+can be declared const while the data it holds are still writeable.
+* @tparam T must be default constructible and copyable
+* @ingroup misc
+*/
+template< class T>
+struct Buffer
+{
+    ///new T
+    Buffer(){
+        ptr = new T;
+    }
+    ///new T(t)
+    Buffer( const T& t){
+        ptr = new T(t);
+    }
+    ///delete managed object
+    ~Buffer(){
+        delete ptr;
+    }
+    Buffer( const Buffer& src){ 
+        ptr = new T(*src.ptr);
+    }
+    Buffer& operator=( const Buffer& src){
+        if( this == &src) return *this;
+        Buffer tmp(src);
+        std::swap( ptr, tmp.ptr);
+        return *this;
+    }
+    /**
+    * @brief Get write access to the data on the heap
+    * @return a reference to the data object
+    * @attention never try to delete this
+    */
+    T& data( )const { return *ptr;}
+    /**
+    * @brief Get a constant reference to the data on the heap
+    * @return a reference to the data object
+    */
+    const T& data( )const { return *ptr;}
+    private:
+    T* ptr;
+};
+
+}//namespace dg
diff --git a/inc/dg/backend/mpi_collective.h b/inc/dg/backend/mpi_collective.h
index 14c796901..f3ae52ba9 100644
--- a/inc/dg/backend/mpi_collective.h
+++ b/inc/dg/backend/mpi_collective.h
@@ -10,7 +10,7 @@
 #include <thrust/device_vector.h>
 #include "thrust_vector_blas.cuh"
 #include "dg/blas1.h"
-#include "mpi_vector.h"
+#include "manage.h"
 
 namespace dg{
 
@@ -187,7 +187,7 @@ struct BijectiveComm
         //we could maybe transpose the Collective object!?
         assert( values.size() == idx_.size());
         //nach PID ordnen
-        thrust::gather( idx_.begin(), idx_.end(), values.begin(), values_.data()->begin());
+        thrust::gather( idx_.begin(), idx_.end(), values.begin(), values_.data().begin());
         //senden
         Vector store( p_.store_size());
         p_.scatter( *values_.data(), store);
@@ -207,7 +207,7 @@ struct BijectiveComm
         //actually this is a gather but we constructed it invertedly
         p_.gather( toScatter, *values_.data());
         //nach PID geordnete Werte wieder umsortieren
-        thrust::scatter( values_.data()->begin(), values_.data()->end(), idx_.begin(), values.begin());
+        thrust::scatter( values_.data().begin(), values_.data().end(), idx_.begin(), values.begin());
     }
 
     /**
@@ -259,7 +259,7 @@ struct BijectiveComm
         for( unsigned i=0; i<distance; i++)
             sendTo[keys[i]] = number[i];
         p_.construct( sendTo, comm);
-        values_.data()->resize( idx_.size());
+        values_.data().resize( idx_.size());
     }
     Buffer<Vector> values_;
     Index idx_;
@@ -322,7 +322,7 @@ struct SurjectiveComm
     Vector global_gather( const Vector& values)const
     {
         //gather values to store
-        thrust::gather( gatherMap_.begin(), gatherMap_.end(), values.begin(), store_.data()->begin());
+        thrust::gather( gatherMap_.begin(), gatherMap_.end(), values.begin(), store_.data().begin());
         //now gather from store into buffer
         Vector buffer( buffer_size_);
         bijectiveComm_.global_scatter_reduce( *store_.data(), buffer);
@@ -333,8 +333,8 @@ struct SurjectiveComm
         //first gather values into store
         Vector store_t = bijectiveComm_.global_gather( toScatter);
         //now perform a local sort, reduce and scatter operation
-        thrust::gather( sortMap_.begin(), sortMap_.end(), store_t.begin(), store_.data()->begin());
-        thrust::reduce_by_key( sortedGatherMap_.begin(), sortedGatherMap_.end(), store_.data()->begin(), keys_.data()->begin(), values.begin());
+        thrust::gather( sortMap_.begin(), sortMap_.end(), store_t.begin(), store_.data().begin());
+        thrust::reduce_by_key( sortedGatherMap_.begin(), sortedGatherMap_.end(), store_.data().begin(), keys_.data().begin(), values.begin());
     }
     unsigned size() const {return buffer_size_;}
     const thrust::host_vector<int> getLocalGatherMap() const {return localGatherMap_;}
@@ -355,8 +355,8 @@ struct SurjectiveComm
         Index gatherMap = bijectiveComm_.global_gather( localGatherMap_d);
         dg::blas1::transfer(gatherMap, gatherMap_);
         store_size_ = gatherMap_.size();
-        store_.data()->resize( store_size_);
-        keys_.data()->resize( store_size_);
+        store_.data().resize( store_size_);
+        keys_.data().resize( store_size_);
 
         //now prepare a reduction map and a scatter map
         thrust::host_vector<int> sortMap(gatherMap);
@@ -414,7 +414,7 @@ struct GeneralComm
     void global_scatter_reduce( const Vector& toScatter, Vector& values)
     {
         surjectiveComm_.global_scatter_reduce( toScatter, *store_.data());
-        thrust::scatter( store_.data()->begin(), store_.data()->end(), scatterMap_.begin(), values.begin());
+        thrust::scatter( store_.data().begin(), store_.data().end(), scatterMap_.begin(), values.begin());
     }
 
     unsigned size() const{return surjectiveComm_.size();}
@@ -435,7 +435,7 @@ struct GeneralComm
             thrust::reduce_by_key( gatherMap.begin(), gatherMap.end(), //sorted!
                 one.begin(), keys.begin(), number.begin() ); 
         unsigned distance = thrust::distance( keys.begin(), new_end.first);
-        store_.data()->resize( distance);
+        store_.data().resize( distance);
         scatterMap_.resize(distance);
         thrust::copy( keys.begin(), keys.begin() + distance, scatterMap_.begin());
     }
diff --git a/inc/dg/backend/mpi_vector.h b/inc/dg/backend/mpi_vector.h
index af367fdfa..28359468f 100644
--- a/inc/dg/backend/mpi_vector.h
+++ b/inc/dg/backend/mpi_vector.h
@@ -5,6 +5,7 @@
 #include <thrust/gather.h>
 #include "vector_traits.h"
 #include "thrust_vector_blas.cuh"
+#include "manage.h"
 
 namespace dg
 {
@@ -139,32 +140,6 @@ struct VectorTraits<const MPI_Vector<container> > {
     typedef MPIVectorTag vector_category;
 };
 
-//Memory buffer class: data can be written even if the object is const
-template< class T>
-struct Buffer
-{
-    Buffer(){
-        ptr = new T;
-    }
-    Buffer( const T& t){
-        ptr = new T(t);
-    }
-    ~Buffer(){
-        delete ptr;
-    }
-    Buffer( const Buffer& src){ 
-        ptr = new T(*src.ptr);
-    }
-    Buffer& operator=( const Buffer& src){
-        if( this == &src) return *this;
-        Buffer tmp(src);
-        std::swap( ptr, tmp.ptr);
-        return *this;
-    }
-    T* data( )const { return ptr;}
-    private:
-    T* ptr;
-};
 ///@endcond
 
 /////////////////////////////communicator exchanging columns//////////////////
@@ -331,9 +306,9 @@ void NearestNeighborComm<I,V>::construct( int n, const int dimensions[3], MPI_Co
     }
     gather_map1 =hbgather1, gather_map2 =hbgather2;
     scatter_map1=hbscattr1, scatter_map2=hbscattr2;
-    values.data()->resize( size());
-    buffer1.data()->resize( buffer_size()), buffer2.data()->resize( buffer_size());
-    rb1.data()->resize( buffer_size()), rb2.data()->resize( buffer_size());
+    values.data().resize( size());
+    buffer1.data().resize( buffer_size()), buffer2.data().resize( buffer_size());
+    rb1.data().resize( buffer_size()), rb2.data().resize( buffer_size());
 }
 
 template<class I, class V>
@@ -368,8 +343,8 @@ const V& NearestNeighborComm<I,V>::global_gather( const V& input) const
         //dg::Timer t;
         //t.tic();
     //gather values from input into sendbuffer
-    thrust::gather( gather_map1.begin(), gather_map1.end(), input.begin(), buffer1.data()->begin());
-    thrust::gather( gather_map2.begin(), gather_map2.end(), input.begin(), buffer2.data()->begin());
+    thrust::gather( gather_map1.begin(), gather_map1.end(), input.begin(), buffer1.data().begin());
+    thrust::gather( gather_map2.begin(), gather_map2.end(), input.begin(), buffer2.data().begin());
         //t.toc();
         //if(rank==0)std::cout << "Gather       took "<<t.diff()<<"s\n";
         //t.tic();
@@ -379,8 +354,8 @@ const V& NearestNeighborComm<I,V>::global_gather( const V& input) const
         //if(rank==0)std::cout << "MPI sendrecv took "<<t.diff()<<"s\n";
         //t.tic();
     //scatter received values into values array
-    thrust::scatter( rb1.data()->begin(), rb1.data()->end(), scatter_map1.begin(), values.data()->begin());
-    thrust::scatter( rb2.data()->begin(), rb2.data()->end(), scatter_map2.begin(), values.data()->begin());
+    thrust::scatter( rb1.data().begin(), rb1.data().end(), scatter_map1.begin(), values.data().begin());
+    thrust::scatter( rb2.data().begin(), rb2.data().end(), scatter_map2.begin(), values.data().begin());
         //t.toc();
         //if(rank==0)std::cout << "Scatter      took "<<t.diff()<<"s\n";
     return *values.data();
diff --git a/inc/dg/exceptions.h b/inc/dg/exceptions.h
index 6c7461ea2..4869dd731 100644
--- a/inc/dg/exceptions.h
+++ b/inc/dg/exceptions.h
@@ -17,7 +17,7 @@ namespace dg
 {
 
 ///small class holding a stringstream 
-///@ingroup numerical0
+///@ingroup misc
 class Message 
 {
   private:
@@ -54,7 +54,7 @@ class Message
  * try{ throw Error(Message()<<"This is error number "<<number, _ping_);}
  * catch( Error& m) {std::cerr << m.what();}
  * \endcode
- * numerical0
+ * @ingroup misc
  */
 class Error : public std::exception
 {
@@ -88,7 +88,7 @@ class Error : public std::exception
 
 /**
  * @brief Class you might want to throw in case of a non convergence
- * @ingroup numerical0
+ * @ingroup misc
  */
 struct Fail : public std::exception
 {
diff --git a/inc/dg/geometry/tensor.h b/inc/dg/geometry/tensor.h
index 9c8dbbfa6..ab16d7316 100644
--- a/inc/dg/geometry/tensor.h
+++ b/inc/dg/geometry/tensor.h
@@ -19,10 +19,23 @@ if negative the value of the container is assumed to be 1, except for the off-di
     in the matrix where it is assumed to be 0.
 * We then only need to store non-trivial and non-repetitive containers.
 * @tparam container container class
+* @ingroup misc
 */
 template<class container>
 struct SharedContainers
 {
+    SharedContainers( ) {}
+    SharedContainers( const dg::Operator<int>& mat_idx, std::vector<int>& vec_idx, std::vector<container>& values ): mat_idx_(mat_idx), vec_idx(vec_idx), values_(values){}
+    template<class otherContainer>
+    SharedContainers( const SharedContainers<otherContainer>& src): mat_idx_(src.mat_idx()), vec_idx_(src.vec_idx()), values_(src.values().size()), idx_(src.idx_){
+        dg::blas1::transfer( src.values(), values_);
+    }
+    void set( const dg::Operator<int>& mat_idx, std::vector<int>& vec_idx, std::vector<container>& values ){
+        mat_idx_=mat_idx;
+        vec_idx_=vec_idx;
+        values_=values;
+    }
+
     /**
     * @brief check if an index is set or not
     * @param i row index 0<i<2
@@ -38,26 +51,22 @@ struct SharedContainers
         if( vec_idx_[i]<0) return false;
         return true;
     }
-    /// Access the underlying container
-    /// @return if !isSet(i,j) the default constructor of container is called, otherwise values[mat_idx(i,j)] is returned
+    /*!@brief Access the underlying container
+     * @return if !isSet(i,j) the default constructor of container is called, otherwise values[mat_idx(i,j)] is returned. If the indices fall out of range of mat_idx the result is undefined
+     */
     const container& getValue(size_t i, size_t j)const{ 
         int k = mat_idx(i,j);
         if(k<0) return container();
         return values_[k];
     }
-    /// Access the underlying container
-    /// @return if !isSet(i) the default constructor of container is called, otherwise values[vec_idx(i)] is returned
+    /*!@brief Access the underlying container
+     * @return if !isSet(i) the default constructor of container is called, otherwise values[vec_idx(i)] is returned. If the index falls out of range of vec_idx the result is undefined
+     */
     const container& getValue(size_t i)const{ 
         int k = vec_idx_[i];
         if(k<0) return container();
         return values_[k];
     }
-    SharedContainers( ) {}
-    SharedContainers( const dg::Operator<int>& mat_idx, std::vector<int>& vec_idx, std::vector<container>& values ): mat_idx_(mat_idx), vec_idx(vec_idx), values_(values){}
-    template<class otherContainer>
-    SharedContainers( const SharedContainers<otherContainer>& src): mat_idx_(src.mat_idx()), vec_idx_(src.vec_idx()), values_(src.values().size()), idx_(src.idx_){
-        dg::blas1::transfer( src.values(), values_);
-    }
     const dg::Operator<int>& mat_idx() const {return mat_idx_;}
     const std::vector<int>& vec_idx() const {return vec_idx_;}
     const std::vector<container>& values() const{return values_;}
-- 
GitLab


From e1bf47812f837f53ceccd5d364ca46859e80dfdc Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Thu, 3 Aug 2017 05:54:55 -0700
Subject: [PATCH 107/453] implemented topologies in terms of Grid1d

---
 inc/dg/backend/dlt.h          |   3 +-
 inc/dg/backend/evaluation.cuh |  12 +-
 inc/dg/backend/grid.h         | 460 ++++++++++++++++------------------
 3 files changed, 220 insertions(+), 255 deletions(-)

diff --git a/inc/dg/backend/dlt.h b/inc/dg/backend/dlt.h
index 85533eb25..5683749c8 100644
--- a/inc/dg/backend/dlt.h
+++ b/inc/dg/backend/dlt.h
@@ -19,13 +19,14 @@ template< class T>
 class DLT
 {
   public:
+
       /**
        * @brief Initialize coefficients
        *
        * The constructor reads the data corresponding to given n from the file dlt.dat. 
        * @param n # of polynomial coefficients (0<n<21)
        */
-    DLT( unsigned n);
+    DLT( unsigned n=3);
 
     /**
      * @brief Return Gauss-Legendre weights
diff --git a/inc/dg/backend/evaluation.cuh b/inc/dg/backend/evaluation.cuh
index 5afe0156d..849b58439 100644
--- a/inc/dg/backend/evaluation.cuh
+++ b/inc/dg/backend/evaluation.cuh
@@ -60,9 +60,8 @@ template< class BinaryOp>
 thrust::host_vector<double> evaluate( BinaryOp f, const aTopology2d& g)
 {
     unsigned n= g.n();
-    //TODO: opens dlt.dat twice...!!
-    Grid1d gx( g.x0(), g.x1(), n, g.Nx()); 
-    Grid1d gy( g.y0(), g.y1(), n, g.Ny());
+    Grid1d gx = g.gx();
+    Grid1d gy = g.gy();
     thrust::host_vector<double> absx = create::abscissas( gx);
     thrust::host_vector<double> absy = create::abscissas( gy);
 
@@ -99,10 +98,9 @@ template< class TernaryOp>
 thrust::host_vector<double> evaluate( TernaryOp f, const aTopology3d& g)
 {
     unsigned n= g.n();
-    //TODO: opens dlt.dat three times...!!
-    Grid1d gx( g.x0(), g.x1(), n, g.Nx()); 
-    Grid1d gy( g.y0(), g.y1(), n, g.Ny());
-    Grid1d gz( g.z0(), g.z1(), 1, g.Nz());
+    Grid1d gx = g.gx();
+    Grid1d gy = g.gy();
+    Grid1d gz = g.gz();
     thrust::host_vector<double> absx = create::abscissas( gx);
     thrust::host_vector<double> absy = create::abscissas( gy);
     thrust::host_vector<double> absz = create::abscissas( gz);
diff --git a/inc/dg/backend/grid.h b/inc/dg/backend/grid.h
index 114b84aed..c6a715be3 100644
--- a/inc/dg/backend/grid.h
+++ b/inc/dg/backend/grid.h
@@ -14,10 +14,6 @@
 
 namespace dg{
 
-class aMPITopology2d;
-class aMPITopology3d;
-
-
 ///@addtogroup grid
 ///@{
 /**
@@ -28,6 +24,11 @@ struct Grid1d
 {
     typedef SharedTag memory_category;
     typedef OneDimensionalTag dimensionality;
+    /**
+     * @brief construct an empty grid
+     * this leaves the access functions undefined
+     */
+    Grid1d(){}
     /**
      * @brief 1D grid
      * 
@@ -37,16 +38,12 @@ struct Grid1d
      @param N # of cells
      @param bcx boundary conditions
      */
-    Grid1d( double x0, double x1, unsigned n, unsigned N, bc bcx = PER):
-        x0_(x0), x1_(x1),
-        n_(n), Nx_(N), bcx_(bcx), dlt_(n)
+    Grid1d( double x0, double x1, unsigned n, unsigned N, bc bcx = PER)
     {
-        assert( x1 > x0 );
-        assert( N > 0  );
-        assert( n != 0 );
-        lx_ = (x1-x0);
-        hx_ = lx_/(double)Nx_;
+        set(x0,x1,bcx);
+        set(n,N);
     }
+    //////////////////////////////////////////get/////////////////////////////
     /**
      * @brief left boundary
      *
@@ -64,13 +61,13 @@ struct Grid1d
      *
      * @return 
      */
-    double lx() const {return lx_;}
+    double lx() const {return x1_-x0_;}
     /**
      * @brief cell size
      *
      * @return 
      */
-    double h() const {return hx_;}
+    double h() const {return lx()/(double)Nx_;}
     /**
      * @brief number of cells
      *
@@ -94,6 +91,47 @@ struct Grid1d
      *
      * @return n*Nx
      */
+    //////////////////////////////////////////set/////////////////////////////
+    /**
+     * @brief reset the boundaries of the grid
+     *
+     * @param x0 new left boundary
+     * @param x1 new right boundary ( > x0)
+     * @param bcx new boundary condition
+     */
+    void set(double x0, double x1, bc bcx)
+    {
+        assert( x1 > x0 );
+        x0_=x0, x1_=x1;
+        bcx_=bcx;
+    }
+    /**
+     * @brief reset the cell numbers in the grid
+     *
+     * @param n new # of polynomial coefficients (0<n<21)
+     * @param N new # of cells (>0)
+     */
+    void set( unsigned n, unsigned N)
+    {
+        assert( N > 0  );
+        Nx_=N; n_=n;
+        dlt_=DLT<double>(n);
+    }
+    /**
+     * @brief Reset all values of the grid
+     *
+     * @param x0 new left boundary
+     * @param x1 new right boundary
+     * @param n new # of polynomial coefficients
+     * @param N new # of cells
+     * @param bcx new boundary condition
+     */
+    void set( double x0, double x1, unsigned n, unsigned N, bc bcx)
+    {
+        set(x0,x1,bcx);
+        set(n,N);
+    }
+    /////////////////////////////////////////convencience//////////////////////////////
     unsigned size() const { return n_*Nx_;}
     /**
      * @brief the discrete legendre transformation
@@ -106,10 +144,10 @@ struct Grid1d
         os << "aTopology parameters are: \n"
             <<"    n  = "<<n_<<"\n"
             <<"    N = "<<Nx_<<"\n"
-            <<"    h = "<<hx_<<"\n"
+            <<"    h = "<<h()<<"\n"
             <<"    x0 = "<<x0_<<"\n"
             <<"    x1 = "<<x1_<<"\n"
-            <<"    lx = "<<lx_<<"\n"
+            <<"    lx = "<<lx()<<"\n"
             <<"Boundary conditions in x are: \n"
             <<"    "<<bc2str(bcx_)<<"\n";
     }
@@ -118,7 +156,7 @@ struct Grid1d
      * @brief Shifts a point coordinate if periodic
      *
      * This function shifts a point coordinate to its value between x0() and x1() if bcx() returns dg::PER
-     * @param x0 arbitrary point (irrelevant for the function, it's there to be consistent with aTopologyX1d)
+     * @param x0 arbitrary point (irrelevant for the function, it's there to be consistent with GridX1d)
      * @param x1 end point (inout)
      */
     void shift_topologic( double x0, double& x1)const
@@ -126,9 +164,9 @@ struct Grid1d
         double deltaX;
         if( x1 > x0_) deltaX = x1 -x0_;
         else deltaX = x1_ - x1;
-        unsigned N = floor(deltaX/lx_);
-        if( x1  > x1_ && bcx_ == dg::PER) x1 -= N*lx_;
-        if( x1  < x0_ && bcx_ == dg::PER) x1 += N*lx_;
+        unsigned N = floor(deltaX/lx());
+        if( x1  > x1_ && bcx_ == dg::PER) x1 -= N*lx();
+        if( x1  < x0_ && bcx_ == dg::PER) x1 += N*lx();
     }
 
     /**
@@ -144,17 +182,14 @@ struct Grid1d
         if( (x>=x0_ && x <= x1_)) return true; 
         return false;
     }
+
   private:
     double x0_, x1_;
-    double lx_;
     unsigned n_, Nx_;
-    double hx_;
     bc bcx_;
     DLT<double> dlt_;
 };
 
-struct aTopology3d; //forward declare 3d version
-
 /**
  * @brief An abstract base class for two-dimensional grids
  */
@@ -163,95 +198,104 @@ struct aTopology2d
     typedef SharedTag memory_category;
     typedef TwoDimensionalTag dimensionality;
 
-    //aTopology2d( const aTopology3d& g);
     /**
      * @brief Left boundary in x
      *
      * @return 
      */
-    double x0() const {return x0_;}
+    double x0() const {return gx_.x0();}
     /**
      * @brief Right boundary in x
      *
      * @return 
      */
-    double x1() const {return x1_;}
+    double x1() const {return gx_.x1();}
     /**
      * @brief left boundary in y
      *
      * @return 
      */
-    double y0() const {return y0_;}
+    double y0() const {return gy_.x0();}
     /**
      * @brief Right boundary in y 
      *
      * @return 
      */
-    double y1() const {return y1_;}
+    double y1() const {return gy_.x1();}
     /**
      * @brief length of x 
      *
      * @return 
      */
-    double lx() const {return lx_;}
+    double lx() const {return gx_.lx();}
     /**
      * @brief length of y
      *
      * @return 
      */
-    double ly() const {return ly_;}
+    double ly() const {return gy_.lx();}
     /**
      * @brief cell size in x 
      *
      * @return 
      */
-    double hx() const {return hx_;}
+    double hx() const {return gx_.h();}
     /**
      * @brief cell size in y
      *
      * @return 
      */
-    double hy() const {return hy_;}
+    double hy() const {return gy_.h();}
     /**
      * @brief number of polynomial coefficients in x and y
      *
      * @return 
      */
-    unsigned n() const {return n_;}
+    unsigned n() const {return gx_.n();}
     /**
      * @brief number of cells in x
      *
      * @return 
      */
-    unsigned Nx() const {return Nx_;}
+    unsigned Nx() const {return gx_.N();}
     /**
      * @brief number of cells in y
      *
      * @return 
      */
-    unsigned Ny() const {return Ny_;}
+    unsigned Ny() const {return gy_.N();}
     /**
      * @brief boundary conditions in x
      *
      * @return 
      */
-    bc bcx() const {return bcx_;}
+    bc bcx() const {return gx_.bcx();}
     /**
      * @brief boundary conditions in y
      *
      * @return 
      */
-    bc bcy() const {return bcy_;}
+    bc bcy() const {return gy_.bcx();}
+    /**
+     * @brief discrete legendre trafo
+     *
+     * @return 
+     */
+    const DLT<double>& dlt() const{return gx_.dlt();}
+
+    Grid1d gx()const{return gx_; }
+    Grid1d gy()const{return gy_; }
 
     /**
     * @brief Multiply the number of cells with a given factor
     *
     * With this function you can resize the grid ignorantly of its current size
-    * @param fx new number of cells is fx*Nx()
-    * @param fy new number of cells is fy*Ny()
+    * the number of polynomial coefficients is left as is
+    * @param fx new number of cells is the nearest integer to fx*Nx()
+    * @param fy new number of cells is the nearest integer to fy*Ny()
     */
     void multiplyCellNumber( double fx, double fy){
-        set(n_, floor(fx*(double)Nx_+0.5), floor(fy*(double)Ny_+0.5));
+        do_set(n(), round(fx*(double)Nx()), round(fy*(double)Ny()));
     }
     /**
     * @brief Set the number of polynomials and cells
@@ -261,24 +305,16 @@ struct aTopology2d
     * @param new_Ny new number of cells in y
     */
     void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny) {
-        assert( new_n> 0 && new_Nx > 0  && new_Ny > 0);
-        if( new_n == n_ && new_Nx == Nx_ && new_Ny == Ny_) return;
         do_set(new_n,new_Nx,new_Ny);
     }
 
 
-    /**
-     * @brief discrete legendre trafo
-     *
-     * @return 
-     */
-    const DLT<double>& dlt() const{return dlt_;}
     /**
      * @brief The total number of points
      *
      * @return n*n*Nx*Ny
      */
-    unsigned size() const { return n_*n_*Nx_*Ny_;}
+    unsigned size() const { return gx_.size()*gy_.size();}
     /**
      * @brief Display 
      *
@@ -287,21 +323,21 @@ struct aTopology2d
     void display( std::ostream& os = std::cout) const
     {
         os << "aTopology parameters are: \n"
-            <<"    n  = "<<n_<<"\n"
-            <<"    Nx = "<<Nx_<<"\n"
-            <<"    Ny = "<<Ny_<<"\n"
-            <<"    hx = "<<hx_<<"\n"
-            <<"    hy = "<<hy_<<"\n"
-            <<"    x0 = "<<x0_<<"\n"
-            <<"    x1 = "<<x1_<<"\n"
-            <<"    y0 = "<<y0_<<"\n"
-            <<"    y1 = "<<y1_<<"\n"
-            <<"    lx = "<<lx_<<"\n"
-            <<"    ly = "<<ly_<<"\n"
+            <<"    n  = "<<n()<<"\n"
+            <<"    Nx = "<<Nx()<<"\n"
+            <<"    Ny = "<<Ny()<<"\n"
+            <<"    hx = "<<hx()<<"\n"
+            <<"    hy = "<<hy()<<"\n"
+            <<"    x0 = "<<x0()<<"\n"
+            <<"    x1 = "<<x1()<<"\n"
+            <<"    y0 = "<<y0()<<"\n"
+            <<"    y1 = "<<y1()<<"\n"
+            <<"    lx = "<<lx()<<"\n"
+            <<"    ly = "<<ly()<<"\n"
             <<"Boundary conditions in x are: \n"
-            <<"    "<<bc2str(bcx_)<<"\n"
+            <<"    "<<bc2str(bcx())<<"\n"
             <<"Boundary conditions in y are: \n"
-            <<"    "<<bc2str(bcy_)<<"\n";
+            <<"    "<<bc2str(bcy())<<"\n";
     }
     /**
      * @brief Shifts point coordinates if periodic
@@ -315,19 +351,8 @@ struct aTopology2d
      */
     void shift_topologic( double x0, double y0, double& x1, double& y1)const
     {
-        double deltaX;
-        if( x1 > x0_) deltaX = (x1 -x0_);
-        else deltaX = x1_ - x1;
-        unsigned N = floor(deltaX/lx_);
-        if( x1  > x1_ && bcx_ == dg::PER) x1 -= N*lx_;
-        if( x1  < x0_ && bcx_ == dg::PER) x1 += N*lx_;
-
-        double deltaY;
-        if( y1 > y0_) deltaY = (y1 -y0_);
-        else deltaY = y1_ - y1;
-        N = floor(deltaY/ly_);
-        if( y1  > y1_ && bcy_ == dg::PER){ y1 -= N*ly_; }
-        if( y1  < y0_ && bcy_ == dg::PER){ y1 += N*ly_; }
+        gx_.shift_topologic( x0,x1);
+        gy_.shift_topologic( y0,y1);
     }
     /**
      * @brief Check if the grid contains a point
@@ -340,14 +365,12 @@ struct aTopology2d
      */
     bool contains( double x, double y)const
     {
-        if( (x>=x0_ && x <= x1_) && (y>=y0_ && y <= y1_)) return true; 
+        if( gx_.contains(x) && gy_.contains(y)) return true; 
         return false;
     }
-    ///allow destruction through base class pointer
-    virtual ~aTopology2d(){}
-    ///deep copy of an object onto the heap
-    virtual aTopology2d* clone()const=0;
     protected:
+    ///disallow destruction through base class pointer
+    ~aTopology2d(){}
     /**
      * @brief Construct a 2D grid
      *
@@ -362,15 +385,7 @@ struct aTopology2d
      * @param bcy boundary condition in y
      */
     aTopology2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx = PER, bc bcy = PER):
-        x0_(x0), x1_(x1), y0_(y0), y1_(y1), 
-        n_(n), Nx_(Nx), Ny_(Ny), bcx_(bcx), bcy_( bcy), dlt_(n)
-    {
-        assert( n != 0);
-        assert( x1 > x0 && y1 > y0);
-        assert( Nx > 0  && Ny > 0);
-        lx_ = (x1_-x0_), ly_ = (y1_-y0_);
-        hx_ = lx_/(double)Nx_, hy_ = ly_/(double)Ny_;
-    }
+        gx_(x0,x1,n,Nx,bcx), gy_(y0,y1,n,Ny,bcy) { }
     /**
      * @brief Construct a 2d grid as the product of two 1d grids
      *
@@ -378,46 +393,27 @@ struct aTopology2d
      * @param gy aTopology in y - direction
      * @note gx and gy must have the same n
      */
-    aTopology2d( const Grid1d& gx, const Grid1d& gy): 
-        x0_(gx.x0()), x1_(gx.x1()), y0_(gy.x0()), y1_(gy.x1()), 
-        n_(gx.n()), Nx_(gx.N()), Ny_(gy.N()), bcx_(gx.bcx()), bcy_( gy.bcx()), dlt_(gx.n())
+    aTopology2d( const Grid1d& gx, const Grid1d& gy): gx_(gx),gy_(gy)
     {
         assert( gx.n() == gy.n() );
-        lx_ = (x1_-x0_), ly_ = (y1_-y0_);
-        hx_ = lx_/(double)Nx_, hy_ = ly_/(double)Ny_;
     }
 
-    aTopology2d(const aTopology2d& src):dlt_(src.dlt_){*this = src;}
+    aTopology2d(const aTopology2d& src){gx_=src.gx_, gy_=src.gy_;}
     aTopology2d& operator=(const aTopology2d& src){
-        x0_=src.x0_, x1_=src.x1_,y0_=src.y0_,y1_=src.y1_;
-        lx_=src.lx_,ly_=src.ly_;
-        n_=src.n_,Nx_=src.Nx_,Ny_=src.Ny_;
-        hx_=src.hx_,hy_=src.hy_;
-        bcx_=src.bcx_,bcy_=src.bcy_;
-        dlt_=src.dlt_;
+        gx_=src.gx_;
+        gy_=src.gy_;
         return *this;
     }
-    virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny) {
-        n_ = new_n, Nx_ = new_Nx, Ny_ = new_Ny;
-        hx_ = lx_/(double)Nx_, hy_ = ly_/(double)Ny_;
-        dlt_ = DLT<double>( new_n);
-    }
-    virtual void init_X_boundaries( double x0, double x1)
-    {
-        x0_ = x0, x1_ = x1;
-        assert( x1 > x0 );
-        lx_ = (x1_-x0_);
-        hx_ = lx_/(double)Nx_;
-    }
+    virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny)=0;
     private:
-    friend class aMPITopology2d;
-    double x0_, x1_, y0_, y1_;
-    double lx_, ly_;
-    unsigned n_, Nx_, Ny_;
-    double hx_, hy_;
-    bc bcx_, bcy_;
-    DLT<double> dlt_;
+    Grid1d gx_, gy_; 
 };
+void aTopology2d::do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny)
+{
+    gx_.set(new_n, new_Nx);
+    gy_.set(new_n, new_Ny);
+}
+
 
 
 /**
@@ -432,7 +428,7 @@ struct aTopology3d
     typedef ThreeDimensionalTag dimensionality;
     ///@copydoc aTopology2d::multiplyCellNumber()
     void multiplyCellNumber( double fx, double fy){
-        set(n_, floor(fx*(double)Nx_+0.5), floor(fy*(double)Ny_+0.5), Nz());
+        set(n(), round(fx*(double)Nx()), round(fy*(double)Ny()), Nz());
     }
     /**
     * @brief Set the number of polynomials and cells
@@ -443,8 +439,6 @@ struct aTopology3d
     * @param new_Nz new number of cells in z
     */
     void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz) {
-        assert( new_n>0); assert( new_Nx > 0  && new_Ny > 0); assert( new_Nz > 0);
-        if( new_n == n_ && new_Nx == Nx_ && new_Ny == Ny_ && new_Nz == Nz_) return;
         do_set(new_n,new_Nx,new_Ny,new_Nz);
     }
 
@@ -453,131 +447,149 @@ struct aTopology3d
      *
      * @return 
      */
-    double x0() const {return x0_;}
+    double x0() const {return gx_.x0();}
     /**
      * @brief right boundary in x
      *
      * @return 
      */
-    double x1() const {return x1_;}
+    double x1() const {return gx_.x1();}
 
     /**
      * @brief left boundary in y 
      *
      * @return 
      */
-    double y0() const {return y0_;}
+    double y0() const {return gy_.x0();}
     /**
      * @brief right boundary in y
      *
      * @return 
      */
-    double y1() const {return y1_;}
+    double y1() const {return gy_.x1();}
 
     /**
      * @brief left boundary in z
      *
      * @return 
      */
-    double z0() const {return z0_;}
+    double z0() const {return gz_.x0();}
     /**
      * @brief right boundary in z
      *
      * @return 
      */
-    double z1() const {return z1_;}
+    double z1() const {return gz_.x1();}
 
     /**
      * @brief length in x
      *
      * @return 
      */
-    double lx() const {return lx_;}
+    double lx() const {return gx_.lx();}
     /**
      * @brief length in y
      *
      * @return 
      */
-    double ly() const {return ly_;}
+    double ly() const {return gy_.lx();}
     /**
      * @brief length in z
      *
      * @return 
      */
-    double lz() const {return lz_;}
+    double lz() const {return gz_.lx();}
     
     /**
      * @brief cell size in x
      *
      * @return 
      */
-    double hx() const {return hx_;}
+    double hx() const {return gx_.h();}
     /**
      * @brief cell size in y
      *
      * @return 
      */
-    double hy() const {return hy_;}
+    double hy() const {return gy_.h();}
     /**
      * @brief cell size in z
      *
      * @return 
      */
-    double hz() const {return hz_;}
+    double hz() const {return gz_.h();}
     /**
      * @brief number of polynomial coefficients in x and y
      *
      * @return 
      */
-    unsigned n() const {return n_;}
+    unsigned n() const {return gx_.n();}
     /**
      * @brief number of points in x
      *
      * @return 
      */
-    unsigned Nx() const {return Nx_;}
+    unsigned Nx() const {return gx_.N();}
     /**
      * @brief number of points in y
      *
      * @return 
      */
-    unsigned Ny() const {return Ny_;}
+    unsigned Ny() const {return gy_.N();}
     /**
      * @brief number of points in z
      *
      * @return 
      */
-    unsigned Nz() const {return Nz_;}
+    unsigned Nz() const {return gz_.N();}
     /**
      * @brief boundary conditions in x 
      *
      * @return 
      */
-    bc bcx() const {return bcx_;}
+    bc bcx() const {return gx_.bcx();}
     /**
      * @brief boundary conditions in y
      *
      * @return 
      */
-    bc bcy() const {return bcy_;}
+    bc bcy() const {return gy_.bcx();}
     /**
      * @brief boundary conditions in z 
      *
      * @return 
      */
-    bc bcz() const {return bcz_;}
+    bc bcz() const {return gz_.bcx();}
     /**
      * @brief discrete legendre transformation
      *
      * @return 
      */
-    const DLT<double>& dlt() const{return dlt_;}
+    const DLT<double>& dlt() const{return gx_.dlt();}
+    /**
+     * @brief Return a 1d grid in x
+     *
+     * @return 
+     */
+    Grid1d gx()const{return gx_; }
+    /**
+     * @brief Return a 1d grid in y
+     *
+     * @return 
+     */
+    Grid1d gy()const{return gy_; }
+    /**
+     * @brief Return a 1d grid in z
+     *
+     * @return 
+     */
+    Grid1d gz()const{return gz_; }
     /**
      * @brief The total number of points
      *
      * @return n*n*Nx*Ny*Nz
      */
-    unsigned size() const { return n_*n_*Nx_*Ny_*Nz_;}
+    unsigned size() const { return gx_.size()*gy_.size()*gz_.size();}
     /**
      * @brief Display 
      *
@@ -586,28 +598,28 @@ struct aTopology3d
     void display( std::ostream& os = std::cout) const
     {
         os << "aTopology parameters are: \n"
-            <<"    n  = "<<n_<<"\n"
-            <<"    Nx = "<<Nx_<<"\n"
-            <<"    Ny = "<<Ny_<<"\n"
-            <<"    Nz = "<<Nz_<<"\n"
-            <<"    hx = "<<hx_<<"\n"
-            <<"    hy = "<<hy_<<"\n"
-            <<"    hz = "<<hz_<<"\n"
-            <<"    x0 = "<<x0_<<"\n"
-            <<"    x1 = "<<x1_<<"\n"
-            <<"    y0 = "<<y0_<<"\n"
-            <<"    y1 = "<<y1_<<"\n"
-            <<"    z0 = "<<z0_<<"\n"
-            <<"    z1 = "<<z1_<<"\n"
-            <<"    lx = "<<lx_<<"\n"
-            <<"    ly = "<<ly_<<"\n"
-            <<"    lz = "<<lz_<<"\n"
+            <<"    n  = "<<n()<<"\n"
+            <<"    Nx = "<<Nx()<<"\n"
+            <<"    Ny = "<<Ny()<<"\n"
+            <<"    Nz = "<<Nz()<<"\n"
+            <<"    hx = "<<hx()<<"\n"
+            <<"    hy = "<<hy()<<"\n"
+            <<"    hz = "<<hz()<<"\n"
+            <<"    x0 = "<<x0()<<"\n"
+            <<"    x1 = "<<x1()<<"\n"
+            <<"    y0 = "<<y0()<<"\n"
+            <<"    y1 = "<<y1()<<"\n"
+            <<"    z0 = "<<z0()<<"\n"
+            <<"    z1 = "<<z1()<<"\n"
+            <<"    lx = "<<lx()<<"\n"
+            <<"    ly = "<<ly()<<"\n"
+            <<"    lz = "<<lz()<<"\n"
             <<"Boundary conditions in x are: \n"
-            <<"    "<<bc2str(bcx_)<<"\n"
+            <<"    "<<bc2str(bcx())<<"\n"
             <<"Boundary conditions in y are: \n"
-            <<"    "<<bc2str(bcy_)<<"\n"
+            <<"    "<<bc2str(bcy())<<"\n"
             <<"Boundary conditions in z are: \n"
-            <<"    "<<bc2str(bcz_)<<"\n";
+            <<"    "<<bc2str(bcz())<<"\n";
     }
 
     /**
@@ -624,24 +636,9 @@ struct aTopology3d
      */
     void shift_topologic( double x0, double y0, double z0, double& x1, double& y1, double& z1)const
     {
-        double deltaX;
-        if( x1 > x0_) deltaX = (x1 -x0_);
-        else deltaX = x1_ - x1;
-        unsigned N = floor(deltaX/lx_);
-        if( x1  > x1_ && bcx_ == dg::PER) x1 -= N*lx_;
-        if( x1  < x0_ && bcx_ == dg::PER) x1 += N*lx_;
-        double deltaY;
-        if( y1 > y0_) deltaY = (y1 -y0_);
-        else deltaY = y1_ - y1;
-        N = floor(deltaY/ly_);
-        if( y1  > y1_ && bcy_ == dg::PER) y1 -= N*ly_;
-        if( y1  < y0_ && bcy_ == dg::PER) y1 += N*ly_;
-        double deltaZ;
-        if( z1 > z0_) deltaZ = (z1 -z0_);
-        else deltaZ = z1_ - z1;
-        N = floor(deltaZ/lz_);
-        if( z1  > z1_ && bcz_ == dg::PER) z1 -= N*lz_;
-        if( z1  < z0_ && bcz_ == dg::PER) z1 += N*lz_;
+        gx_.shift_topologic( x0,x1);
+        gy_.shift_topologic( y0,y1);
+        gz_.shift_topologic( z0,z1);
     }
 
     /**
@@ -656,16 +653,15 @@ struct aTopology3d
      */
     bool contains( double x, double y, double z)const
     {
-        if( (x>=x0_ && x <= x1_) && (y>=y0_ && y <= y1_) && (z>=z0_ && z<=z1_)) 
+        if( gx_.contains(x) && gy_.contains(y) && gz_.contains(z)) 
             return true; 
         return false;
     }
-    ///allow deletion through base class pointer
-    virtual ~aTopology3d(){}
-    virtual aTopology3d* clone()const =0;
     protected:
+    ///disallow deletion through base class pointer
+    ~aTopology3d(){}
     /**
-     * @brief Construct a 3D grid
+     * @brief Construct a 3D topology
      *
      * @param x0 left boundary in x
      * @param x1 right boundary in x 
@@ -682,92 +678,62 @@ struct aTopology3d
      * @param bcz boundary condition in z
      * @attention # of polynomial coefficients in z direction is always 1
      */
-    aTopology3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = PER, bc bcy = PER, bc bcz = PER):
-        x0_(x0), x1_(x1), y0_(y0), y1_(y1), z0_(z0), z1_(z1),
-        n_(n), Nx_(Nx), Ny_(Ny), Nz_(Nz), bcx_(bcx), bcy_( bcy), bcz_( bcz), dlt_(n)
-    {
-        assert( n != 0);
-        assert( x1 > x0 && y1 > y0 ); assert( z1 > z0 );         
-        assert( Nx > 0  && Ny > 0); assert( Nz > 0);
-
-        lx_ = (x1-x0), ly_ = (y1-y0), lz_ = (z1-z0);
-        hx_ = lx_/(double)Nx_, hy_ = ly_/(double)Ny_, hz_ = lz_/(double)Nz_;
-    }
+    aTopology3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = PER, bc bcy = PER, bc bcz = PER): 
+        gx_(x0,x1,n,Nx,bcx),
+        gy_(y0,y1,n,Ny,bcy),
+        gz_(z0,z1,1,Nz,bcz){}
     /**
-     * @brief Construct a 3d cartesian grid as the product of three 1d grids
+     * @brief Construct a 3d topology as the product of three 1d grids
      *
      * @param gx a Grid1d in x - direction
      * @param gy a Grid1d in y - direction
      * @param gz a Grid1d in z - direction
+     * @note gx and gy must have the same n and gz.n() must return 1
      */
-    aTopology3d( const Grid1d& gx, const Grid1d& gy, const Grid1d& gz): 
-        x0_(gx.x0()), x1_(gx.x1()),  
-        y0_(gy.x0()), y1_(gy.x1()),
-        z0_(gz.x0()), z1_(gz.x1()),
-        n_(gx.n()), Nx_(gx.N()), Ny_(gy.N()), Nz_(gz.N()),
-        bcx_(gx.bcx()), bcy_( gy.bcx()), bcz_(gz.bcx()), 
-        dlt_(gx.n())
-    {
-        assert( gx.n() == gy.n() );
-        lx_ = (x1_-x0_), ly_ = (y1_-y0_), lz_ = (z1_-z0_);
-        hx_ = lx_/(double)Nx_, hy_ = ly_/(double)Ny_, hz_ = lz_/(double)Nz_;
+    aTopology3d( const Grid1d& gx, const Grid1d& gy, const Grid1d& gz): gx_(gx),gy_(gy),gz_(gz){
+        assert( gx.n() == gy.n());
+        assert( gz.n() == 1);
     }
-    aTopology3d(const aTopology3d& src):dlt_(src.dlt_){*this = src;}
+    aTopology3d(const aTopology3d& src):gx_(src.gx_),gy_(src.gy_),gz_(src.gz_){}
     aTopology3d& operator=(const aTopology3d& src){ //use default in C++11
-        x0_=src.x0_, x1_=src.x1_,y0_=src.y0_,y1_=src.y1_,z0_=src.z0_,z1_=src.z1_;
-        lx_=src.lx_,ly_=src.ly_,lz_=src.lz_;
-        n_=src.n_,Nx_=src.Nx_,Ny_=src.Ny_,Nz_=src.Nz_;
-        hx_=src.hx_,hy_=src.hy_,hz_=src.hz_;
-        bcx_=src.bcx_,bcy_=src.bcy_,bcz_=src.bcz_;
-        dlt_=src.dlt_;
+        gx_=src.gx_; gy_=src.gy_; gz_=src.gz_;
         return *this;
     }
-    virtual void do_set(unsigned new_n, unsigned new_Nx,unsigned new_Ny, unsigned new_Nz)
-    {
-        n_ = new_n, Nx_ = new_Nx, Ny_ = new_Ny, Nz_ = new_Nz;
-        hx_ = lx_/(double)Nx_, hy_ = ly_/(double)Ny_, hz_ =lz_/(double)Nz_;
-        dlt_ = DLT<double>( new_n);
-    }
-    virtual void init_X_boundaries( double x0, double x1)
-    {
-        x0_ = x0, x1_ = x1;
-        assert( x1 > x0 );
-        lx_ = (x1_-x0_);
-        hx_ = lx_/(double)Nx_;
-    }
+    virtual void do_set(unsigned new_n, unsigned new_Nx,unsigned new_Ny, unsigned new_Nz)=0;
   private:
-    friend class aMPITopology3d;
-    double x0_, x1_, y0_, y1_, z0_, z1_;
-    double lx_, ly_, lz_;
-    unsigned n_, Nx_, Ny_, Nz_;
-    double hx_, hy_, hz_;
-    bc bcx_, bcy_, bcz_;
-    DLT<double> dlt_;
+    Grid1d gx_,gy_,gz_;
 };
+void aTopology3d::do_set(unsigned new_n, unsigned new_Nx,unsigned new_Ny, unsigned new_Nz)
+{
+    gx_.set(new_n, new_Nx);
+    gy_.set(new_n, new_Ny);
+    gz_.set(1,new_Nz);
+}
 ///@}
 
-///@cond
-//aTopology2d::aTopology2d( const aTopology3d& g) : x0_(g.x0()), x1_(g.x1()), y0_(g.y0()), y1_(g.y1()), n_(g.n()), Nx_(g.Nx()), Ny_(g.Ny()), bcx_(g.bcx()), bcy_(g.bcy()), dlt_(g.n())
-//{}
-///@endcond
-
 struct Grid2d : public aTopology2d
 {
 
     Grid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx = PER, bc bcy = PER):
         aTopology2d(x0,x1,y0,y1,n,Nx,Ny,bcx,bcy) { }
     Grid2d( const Grid1d& gx, const Grid1d& gy): aTopology2d(gx,gy){ }
-    Grid2d* clone() const{return new Grid2d(*this);}
+    explicit Grid2d( const aTopology2d& src): aTopology2d(src){}
+    private:
+    virtual void do_set( unsigned n, unsigned Nx, unsigned Ny){ 
+        aTopology2d::do_set(n,Nx,Ny);
+    }
 
 };
 struct Grid3d : public aTopology3d
 {
-
     Grid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = PER, bc bcy = PER, bc bcz=PER):
         aTopology3d(x0,x1,y0,y1,z0,z1,n,Nx,Ny,Nz,bcx,bcy,bcz) { }
-    Grid3d( const Grid1d& gx, const Grid1d& gy, const Grid1d& gz): 
-        aTopology3d(gx,gy,gz){ }
-    Grid3d* clone() const{return new Grid3d(*this);}
+    Grid3d( const Grid1d& gx, const Grid1d& gy, const Grid1d& gz): aTopology3d(gx,gy,gz){ }
+    explicit Grid3d( const aTopology3d& src): aTopology3d(src){ }
+    private:
+    virtual void do_set( unsigned n, unsigned Nx, unsigned Ny, unsigned Nz){ 
+        aTopology3d::do_set(n,Nx,Ny,Nz);
+    }
 };
 
 }// namespace dg
-- 
GitLab


From da1dfdce33094d91cde2b568c8124c61a9bf110f Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Thu, 3 Aug 2017 06:15:16 -0700
Subject: [PATCH 108/453] added manage_t program

---
 inc/dg/backend/manage.h    |  9 ++----
 inc/dg/backend/manage_t.cu | 61 ++++++++++++++++++++++++++++++++++++++
 inc/dg/backend/mpi_grid.h  |  8 -----
 3 files changed, 63 insertions(+), 15 deletions(-)
 create mode 100644 inc/dg/backend/manage_t.cu

diff --git a/inc/dg/backend/manage.h b/inc/dg/backend/manage.h
index 5a25304d8..434c39a17 100644
--- a/inc/dg/backend/manage.h
+++ b/inc/dg/backend/manage.h
@@ -32,7 +32,7 @@ struct Handle
     * @param src an oject to copy, clones the contained object if not empty
     */
     Handle( const Handle& src):ptr_(0) {
-        if(src.ptr_!=0) ptr_ = src.ptr->clone(); //deep copy
+        if(src.ptr_!=0) ptr_ = src.ptr_->clone(); //deep copy
     }
     /**
     * @brief deep copy the given handle
@@ -56,7 +56,7 @@ struct Handle
     * @brief Non constant access to the object on the heap
     * @return a non-const reference to the cloneable object
     */
-    cloneable& get()const {return *ptr_;}
+    cloneable& get() {return *ptr_;}
 
     /**
     * @brief Take the ownership of the given pointer and delete the currently held one if non-empty
@@ -125,11 +125,6 @@ struct Buffer
     * @attention never try to delete this
     */
     T& data( )const { return *ptr;}
-    /**
-    * @brief Get a constant reference to the data on the heap
-    * @return a reference to the data object
-    */
-    const T& data( )const { return *ptr;}
     private:
     T* ptr;
 };
diff --git a/inc/dg/backend/manage_t.cu b/inc/dg/backend/manage_t.cu
new file mode 100644
index 000000000..fd26213ea
--- /dev/null
+++ b/inc/dg/backend/manage_t.cu
@@ -0,0 +1,61 @@
+#include <iostream>
+
+#include "manage.h"
+
+struct aAnimal
+{
+    virtual void speak()const = 0;
+    virtual aAnimal* clone() const=0;
+    virtual ~aAnimal(){
+        std::cout << "Animal Destructor\n";
+    }
+};
+
+//Yes pure virtual functions can have a definition
+void aAnimal::speak()const{ std::cout << "I am ";}
+
+struct Dog: public aAnimal
+{
+    virtual void speak()const{
+        aAnimal::speak();
+        std::cout << " a dog!\n";
+    }
+    virtual Dog* clone()const{return new Dog(*this);}
+};
+
+struct Cat : public aAnimal
+{
+    virtual void speak()const { 
+        aAnimal::speak();
+        std::cout << " a cat!\n";
+    }
+    virtual Cat* clone()const{return new Cat(*this);}
+};
+
+
+int main()
+{
+    {
+        dg::Handle<aAnimal> h0, h1(new Dog());
+        aAnimal* ptr = new Cat();
+        h0.reset(ptr);
+        std::cout << "Test correct behaviour of handle: cat and dog\n";
+        h0.get().speak();
+        h1.get().speak();
+        std::cout << "Now copy handle: cat \n";
+        h1 = h0;
+        h1.get().speak();
+    }
+    {
+        dg::Buffer<Dog> buffer;
+        std::cout << "Test correct behaviour of buffer class with dog\n";
+        buffer.data().speak();
+        dg::Buffer<Dog> buffer2 = buffer;
+        buffer2.data().speak();
+    }
+
+
+
+
+    return 0;
+}
diff --git a/inc/dg/backend/mpi_grid.h b/inc/dg/backend/mpi_grid.h
index 8447fadc0..322a3ca0c 100644
--- a/inc/dg/backend/mpi_grid.h
+++ b/inc/dg/backend/mpi_grid.h
@@ -303,10 +303,6 @@ struct aMPITopology2d
     virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny) {
         g.set(new_n,new_Nx,new_Ny);
     }
-    void init_X_boundaries( double global_x0, double global_x1)
-    {
-        g.init_X_boundaries(global_x0, global_x1);
-    }
     private:
     void check_division( unsigned Nx, unsigned Ny, bc bcx, bc bcy)
     {
@@ -631,10 +627,6 @@ struct aMPITopology3d
     virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz) {
         g.set(new_n,new_Nx,new_Ny,new_Nz);
     }
-    void init_X_boundaries( double global_x0, double global_x1)
-    {
-        g.init_X_boundaries(global_x0, global_x1);
-    }
     private:
     void check_division( unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz)
     {
-- 
GitLab


From 752dbcbb61f12a83ac5b6b817f256b3b810151c6 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Thu, 3 Aug 2017 06:26:55 -0700
Subject: [PATCH 109/453] corrected mpi grid class, made destructor protected

---
 inc/dg/backend/mpi_grid.h   | 37 +++++++++++++++++++++++--------------
 inc/dg/backend/mpi_vector.h |  6 +++---
 2 files changed, 26 insertions(+), 17 deletions(-)

diff --git a/inc/dg/backend/mpi_grid.h b/inc/dg/backend/mpi_grid.h
index 322a3ca0c..0653c0658 100644
--- a/inc/dg/backend/mpi_grid.h
+++ b/inc/dg/backend/mpi_grid.h
@@ -15,7 +15,7 @@ namespace dg
 
 
 /**
- * @brief 2D MPI Grid class 
+ * @brief 2D MPI abstract grid class 
  *
  * Represents the local grid coordinates and the process topology. 
  * It just divides the given (global) box into nonoverlapping (local) subboxes that are attributed to each process
@@ -281,9 +281,9 @@ struct aMPITopology2d
      * @return non-MPI Grid object
      */
     Grid2d global() const {return g;}
-    virtual ~aMPITopology2d(){}
-    virtual aMPITopology2d* clone()const=0;
     protected:
+    ///disallow deletion through base class pointer
+    ~aMPITopology2d(){}
 
     /**
      * @copydoc dg::Grid2d::Grid2d()
@@ -300,9 +300,7 @@ struct aMPITopology2d
         g = src.g; comm = src.comm;
         return *this;
     }
-    virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny) {
-        g.set(new_n,new_Nx,new_Ny);
-    }
+    virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny)=0;
     private:
     void check_division( unsigned Nx, unsigned Ny, bc bcx, bc bcy)
     {
@@ -328,6 +326,7 @@ struct aMPITopology2d
 
 };
 
+
 /**
  * @brief 3D MPI Grid class 
  *
@@ -344,7 +343,7 @@ struct aMPITopology3d
     typedef ThreeDimensionalTag dimensionality;
     ///@copydoc aMPITopology2d::multiplyCellNumber()
     void multiplyCellNumber( double fx, double fy){
-        set(g.n(), floor(fx*(double)g.Nx()+0.5), floor(fy*(double)g.Ny()+0.5), g.Nz());
+        set(g.n(), round(fx*(double)g.Nx()), round(fy*(double)g.Ny()), g.Nz());
     }
     /**
      * @copydoc Grid3d::set(unsigned,unsigned,unsigned,unsigned)
@@ -605,9 +604,9 @@ struct aMPITopology3d
      *@copydoc aMPITopology2d::global()const
      */
     Grid3d global() const {return g;}
-    virtual ~aMPITopology3d(){}
-    virtual aMPITopology3d* clone() const=0;
     protected:
+    ///disallow deletion through base class pointer
+    ~aMPITopology3d(){}
 
     /**
      * @copydoc Grid3d::Grid3d()
@@ -624,9 +623,7 @@ struct aMPITopology3d
         g = src.g; comm = src.comm;
         return *this;
     }
-    virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz) {
-        g.set(new_n,new_Nx,new_Ny,new_Nz);
-    }
+    virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz)=0; 
     private:
     void check_division( unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz)
     {
@@ -688,6 +685,12 @@ int aMPITopology3d::pidOf( double x, double y, double z) const
     else
         return -1;
 }
+void aMPITopology2d::do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny) {
+    g.set(new_n,new_Nx,new_Ny);
+}
+void aMPITopology3d::do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz) {
+    g.set(new_n,new_Nx,new_Ny,new_Nz);
+}
 ///@endcond
 
 struct MPIGrid2d: public aMPITopology2d
@@ -709,7 +712,10 @@ struct MPIGrid2d: public aMPITopology2d
     MPIGrid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx, bc bcy, MPI_Comm comm):
         aMPITopology2d( x0,x1,y0,y1,n,Nx,Ny,bcx,bcy,comm)
     { }
-    virtual MPIGrid2d* clone()const{return new MPIGrid2d(*this);}
+    private:
+    virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny){
+        aMPITopology2d::do_set(new_n,new_Nx,new_Ny);
+    }
 };
 
 
@@ -728,7 +734,10 @@ struct MPIGrid3d : public aMPITopology3d
     MPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz, MPI_Comm comm):
         aMPITopology3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, bcx, bcy, bcz, comm)
     { }
-    virtual MPIGrid3d* clone()const{return new MPIGrid3d(*this);}
+    private:
+    virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz){
+        aMPITopology3d::do_set(new_n,new_Nx,new_Ny,new_Nz);
+    }
 };
 
 
diff --git a/inc/dg/backend/mpi_vector.h b/inc/dg/backend/mpi_vector.h
index 28359468f..8478946f7 100644
--- a/inc/dg/backend/mpi_vector.h
+++ b/inc/dg/backend/mpi_vector.h
@@ -337,7 +337,7 @@ int NearestNeighborComm<I,V>::buffer_size() const
 template<class I, class V>
 const V& NearestNeighborComm<I,V>::global_gather( const V& input) const
 {
-    if( silent_) return *values.data();
+    if( silent_) return values.data();
         //int rank;
         //MPI_Comm_rank( MPI_COMM_WORLD, &rank);
         //dg::Timer t;
@@ -349,7 +349,7 @@ const V& NearestNeighborComm<I,V>::global_gather( const V& input) const
         //if(rank==0)std::cout << "Gather       took "<<t.diff()<<"s\n";
         //t.tic();
     //mpi sendrecv
-    sendrecv( *buffer1.data(), *buffer2.data(), *rb1.data(), *rb2.data());
+    sendrecv( buffer1.data(), buffer2.data(), rb1.data(), rb2.data());
         //t.toc();
         //if(rank==0)std::cout << "MPI sendrecv took "<<t.diff()<<"s\n";
         //t.tic();
@@ -358,7 +358,7 @@ const V& NearestNeighborComm<I,V>::global_gather( const V& input) const
     thrust::scatter( rb2.data().begin(), rb2.data().end(), scatter_map2.begin(), values.data().begin());
         //t.toc();
         //if(rank==0)std::cout << "Scatter      took "<<t.diff()<<"s\n";
-    return *values.data();
+    return values.data();
 }
 
 template<class I, class V>
-- 
GitLab


From 6ef5a7beaf3754a4ca2cd66b5f1f510da26a6fca Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Thu, 3 Aug 2017 06:39:05 -0700
Subject: [PATCH 110/453] update documentation on grids

---
 inc/dg/backend/grid.h     | 44 ++++++++++++++++++++++++++-------------
 inc/dg/backend/mpi_grid.h |  2 ++
 2 files changed, 32 insertions(+), 14 deletions(-)

diff --git a/inc/dg/backend/grid.h b/inc/dg/backend/grid.h
index c6a715be3..1695ae8f2 100644
--- a/inc/dg/backend/grid.h
+++ b/inc/dg/backend/grid.h
@@ -192,6 +192,7 @@ struct Grid1d
 
 /**
  * @brief An abstract base class for two-dimensional grids
+ * @note although it is abstract objects are not meant to be hold on the heap via a base class pointer ( we protected the destructor)
  */
 struct aTopology2d
 {
@@ -389,8 +390,8 @@ struct aTopology2d
     /**
      * @brief Construct a 2d grid as the product of two 1d grids
      *
-     * @param gx aTopology in x - direction
-     * @param gy aTopology in y - direction
+     * @param gx a Grid in x - direction
+     * @param gy a Grid in y - direction
      * @note gx and gy must have the same n
      */
     aTopology2d( const Grid1d& gx, const Grid1d& gy): gx_(gx),gy_(gy)
@@ -408,11 +409,6 @@ struct aTopology2d
     private:
     Grid1d gx_, gy_; 
 };
-void aTopology2d::do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny)
-{
-    gx_.set(new_n, new_Nx);
-    gy_.set(new_n, new_Ny);
-}
 
 
 
@@ -421,6 +417,7 @@ void aTopology2d::do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny)
  *
  * In the third dimension only 1 polynomial coefficient is used,
  * not n.
+ * @note although it is abstract objects are not meant to be hold on the heap via a base class pointer ( we protected the destructor)
  */
 struct aTopology3d
 {
@@ -703,20 +700,19 @@ struct aTopology3d
   private:
     Grid1d gx_,gy_,gz_;
 };
-void aTopology3d::do_set(unsigned new_n, unsigned new_Nx,unsigned new_Ny, unsigned new_Nz)
-{
-    gx_.set(new_n, new_Nx);
-    gy_.set(new_n, new_Ny);
-    gz_.set(1,new_Nz);
-}
-///@}
 
+/**
+ * @brief The simplest implementation of aTopology2d
+ */
 struct Grid2d : public aTopology2d
 {
 
+    ///@copydoc aTopology2d::aTopology2d()
     Grid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx = PER, bc bcy = PER):
         aTopology2d(x0,x1,y0,y1,n,Nx,Ny,bcx,bcy) { }
+    ///@copydoc aTopology2d::aTopology2d(const Grid1d&,const Grid1d&)
     Grid2d( const Grid1d& gx, const Grid1d& gy): aTopology2d(gx,gy){ }
+    ///@allow explicit type conversion from any other topology
     explicit Grid2d( const aTopology2d& src): aTopology2d(src){}
     private:
     virtual void do_set( unsigned n, unsigned Nx, unsigned Ny){ 
@@ -724,16 +720,36 @@ struct Grid2d : public aTopology2d
     }
 
 };
+/**
+ * @brief The simplest implementation of aTopology3d
+ */
 struct Grid3d : public aTopology3d
 {
+    ///@copydoc aTopology3d::aTopology3d()
     Grid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = PER, bc bcy = PER, bc bcz=PER):
         aTopology3d(x0,x1,y0,y1,z0,z1,n,Nx,Ny,Nz,bcx,bcy,bcz) { }
+    ///@copydoc aTopology3d::aTopology3d(const Grid1d&,const Grid1d&,const Grid1d&)
     Grid3d( const Grid1d& gx, const Grid1d& gy, const Grid1d& gz): aTopology3d(gx,gy,gz){ }
+    ///@allow explicit type conversion from any other topology
     explicit Grid3d( const aTopology3d& src): aTopology3d(src){ }
     private:
     virtual void do_set( unsigned n, unsigned Nx, unsigned Ny, unsigned Nz){ 
         aTopology3d::do_set(n,Nx,Ny,Nz);
     }
 };
+///@}
+///@cond
+void aTopology2d::do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny)
+{
+    gx_.set(new_n, new_Nx);
+    gy_.set(new_n, new_Ny);
+}
+void aTopology3d::do_set(unsigned new_n, unsigned new_Nx,unsigned new_Ny, unsigned new_Nz)
+{
+    gx_.set(new_n, new_Nx);
+    gy_.set(new_n, new_Ny);
+    gz_.set(1,new_Nz);
+}
+///@endcond
 
 }// namespace dg
diff --git a/inc/dg/backend/mpi_grid.h b/inc/dg/backend/mpi_grid.h
index 0653c0658..4e437b538 100644
--- a/inc/dg/backend/mpi_grid.h
+++ b/inc/dg/backend/mpi_grid.h
@@ -20,6 +20,7 @@ namespace dg
  * Represents the local grid coordinates and the process topology. 
  * It just divides the given (global) box into nonoverlapping (local) subboxes that are attributed to each process
  * @note a single cell is never divided across processes.
+ * @note although it is abstract objects are not meant to be hold on the heap via a base class pointer ( we protected the destructor)
  * @attention
  * The boundaries in the constructors are global boundaries, the boundaries returned by the access functions are local boundaries, this is because the grid represents the information given to one process
  *
@@ -336,6 +337,7 @@ struct aMPITopology2d
  * The boundaries in the constructors are global boundaries, the boundaries returned by the access functions are local boundaries, this is because the grid represents the information given to one process
  *
  * @note Note that a single cell is never divided across processes.
+ * @note although it is abstract objects are not meant to be hold on the heap via a base class pointer ( we protected the destructor)
  */
 struct aMPITopology3d
 {
-- 
GitLab


From 9b70da866c78f3e933e8c56ce6fce99dc3e62549 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Thu, 3 Aug 2017 07:57:22 -0700
Subject: [PATCH 111/453] added geometry2d and geometry3d interface

---
 inc/dg/geometry/cartesian.h   | 20 ++--------
 inc/dg/geometry/curvilinear.h |  9 +++--
 inc/dg/geometry/geometry.h    | 75 +++++++++++++++++++++++++++++++++++
 inc/dg/geometry/tensor.h      |  9 +++--
 4 files changed, 89 insertions(+), 24 deletions(-)
 create mode 100644 inc/dg/geometry/geometry.h

diff --git a/inc/dg/geometry/cartesian.h b/inc/dg/geometry/cartesian.h
index 2b44e8096..4f36ec602 100644
--- a/inc/dg/geometry/cartesian.h
+++ b/inc/dg/geometry/cartesian.h
@@ -1,6 +1,7 @@
 #pragma once
 
 #include "../backend/grid.h"
+#include "geometry.h"
 #include "geometry_traits.h"
 
 namespace dg
@@ -8,26 +9,11 @@ namespace dg
 
 ///@addtogroup basicgrids
 ///@{
-/**
- * @brief one-dimensional Grid with Cartesian metric
- */
-struct CartesianGrid1d: public dg::Grid1d
-{
-    typedef OrthonormalTag metric_category; 
-    ///@copydoc Grid1d::Grid1d()
-    CartesianGrid1d( double x0, double x1, unsigned n, unsigned N, bc bcx = PER): dg::Grid1d(x0,x1,n,N,bcx){}
-    /**
-     * @brief Construct from existing topology
-     *
-     * @param grid existing grid class
-     */
-    CartesianGrid1d( const dg::Grid1d& grid):dg::Grid1d(grid){}
-};
 
 /**
  * @brief two-dimensional Grid with Cartesian metric
  */
-struct CartesianGrid2d: public dg::Grid2d
+struct CartesianGrid2d: public dg::aGeometry2d
 {
     typedef OrthonormalTag metric_category; 
     ///@copydoc Grid2d::Grid2d()
@@ -43,7 +29,7 @@ struct CartesianGrid2d: public dg::Grid2d
 /**
  * @brief three-dimensional Grid with Cartesian metric
  */
-struct CartesianGrid3d: public dg::Grid3d
+struct CartesianGrid3d: public dg::aGeometry3d
 {
     typedef OrthonormalTag metric_category; 
     ///@copydoc Grid3d::Grid3d()
diff --git a/inc/dg/geometry/curvilinear.h b/inc/dg/geometry/curvilinear.h
index 082c5cee9..67e02605a 100644
--- a/inc/dg/geometry/curvilinear.h
+++ b/inc/dg/geometry/curvilinear.h
@@ -2,12 +2,13 @@
 
 #include "dg/backend/grid.h"
 #include "dg/blas1.h"
-#include "dg/geometry/geometry_traits.h"
+#include "geometry.h"
+#include "geometry_traits.h"
 #include "generator.h"
 
 namespace dg
 {
-///@addtogroup grids
+///@addtogroup basicgrids
 ///@{
 
 ///@cond
@@ -24,7 +25,7 @@ struct CurvilinearGrid2d;
  @tparam container models aContainer
  */
 template< class container>
-struct CylindricalGrid3d : public dg::Grid3d
+struct CylindricalGrid3d : public dg::aGeometry3d
 {
     typedef dg::CurvilinearPerpTag metric_category;
     typedef CurvilinearGrid2d<container> perpendicular_grid;
@@ -149,7 +150,7 @@ struct CylindricalGrid3d : public dg::Grid3d
  * @brief A three-dimensional grid based on curvilinear coordinates
  */
 template< class container>
-struct CurvilinearGrid2d : public dg::Grid2d
+struct CurvilinearGrid2d : public dg::aGeometry2d
 {
     typedef dg::CurvilinearPerpTag metric_category;
 
diff --git a/inc/dg/geometry/geometry.h b/inc/dg/geometry/geometry.h
new file mode 100644
index 000000000..03a99941f
--- /dev/null
+++ b/inc/dg/geometry/geometry.h
@@ -0,0 +1,75 @@
+#pragma once
+
+#include "dg/backend/grid.h"
+#include "tensor.h"
+
+namespace dg
+{
+
+///@addtogroup geometry
+///@{
+
+/**
+ * @brief This is the abstract interface class for a two-dimensional Geometry
+ */
+struct aGeometry2d : public aTopology2d
+{
+    const SharedContainers<thrust::host_vector<double> >& map()const{return map_;}
+    SharedContainers<thrust::host_vector<double> > compute_metric()const {
+        return do_compute_metric();
+    }
+    ///allow deletion through base class pointer
+    virtual ~aGeometry2d(){}
+    protected:
+    aGeometry2d(const SharedContainers<thrust::host_vector<double> >& map, const SharedContainers<container >& metric): map_(map), metric_(metric){}
+    aGeometry2d( const aGeometry2d& src):map_(src.map_), metric_(src.metric_){}
+    aGeometry2d& operator=( const aGeometry2d& src){
+        map_=src.map_;
+        metric_=src.metric_;
+    }
+    SharedContainers<thrust::host_vector<double> >& map(){return map_;}
+    private:
+    SharedContainers<thrust::host_vector<double> > map_;
+    virtual SharedContainers<thrust::host_vector<double> > do_compute_metric()const=0;
+};
+
+/**
+ * @brief This is the abstract interface class for a two-dimensional Geometry
+ */
+struct aGeometry3d : public aTopology3d
+{
+    const SharedContainers<thrust::host_vector<double> >& map()const{return map_;}
+    SharedContainers<thrust::host_vector<double> > compute_metric()const {
+        return do_compute_metric();
+    }
+    ///allow deletion through base class pointer
+    virtual ~aGeometry3d(){}
+    protected:
+    aGeometry3d(const SharedContainers<thrust::host_vector<double> >& map): map_(map){}
+    aGeometry3d( const aGeometry2d& src):map_(src.map_){}
+    aGeometry3d& operator=( const aGeometry3d& src){
+        map_=src.map_;
+    }
+    SharedContainers<thrust::host_vector<double> >& map(){return map_;}
+    private:
+    SharedContainers<thrust::host_vector<double> > map_;
+    virtual SharedContainers<thrust::host_vector<double> > do_compute_metric()const=0;
+};
+
+namespace create
+{
+
+SharedContainers<thrust::host_vector<double> > metric( const aGeometry2d& g)
+{
+    return g.compute_metric();
+}
+SharedContainers<thrust::host_vector<double> > metric( const aGeometry3d& g)
+{
+    return g.compute_metric();
+}
+
+}//namespace create
+
+
+///@}
+} //namespace dg
diff --git a/inc/dg/geometry/tensor.h b/inc/dg/geometry/tensor.h
index ab16d7316..1542cc545 100644
--- a/inc/dg/geometry/tensor.h
+++ b/inc/dg/geometry/tensor.h
@@ -47,6 +47,11 @@ struct SharedContainers
         return true;
     }
 
+    /**
+    * @brief check if an index is set or not
+    * @param i row index 0<i<2
+    * @return true if container is non-empty, false if value is assumed implicitly
+    */
     bool isSet(size_t i) const{
         if( vec_idx_[i]<0) return false;
         return true;
@@ -67,6 +72,7 @@ struct SharedContainers
         if(k<0) return container();
         return values_[k];
     }
+
     const dg::Operator<int>& mat_idx() const {return mat_idx_;}
     const std::vector<int>& vec_idx() const {return vec_idx_;}
     const std::vector<container>& values() const{return values_;}
@@ -76,8 +82,6 @@ struct SharedContainers
     std::vector<container> values_;
 };
 
-namespace geo
-{
 
 ///container sizes must match each other and the tensor container size
 ///may destroy input (either swap in or use as temporary storage)
@@ -144,5 +148,4 @@ void multiply( const SharedContainers& t, container& in1, container& in2, contai
         dg::blas1::pointwiseDot( 1., tensor.getValue(2,0), in1, 1., out3);
 }
 
-}//namespace geo
 }//namespace dg
-- 
GitLab


From 03f97427d244cd09e33665268fad9899d05d44d2 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Fri, 4 Aug 2017 00:08:39 +0200
Subject: [PATCH 112/453] rewrite the geometry backend functions

---
 inc/dg/backend/grid.h               |   4 +-
 inc/dg/backend/operator.h           |   6 +-
 inc/dg/backend/topological_traits.h |  19 +-
 inc/dg/backend/vector_traits.h      |   2 +-
 inc/dg/blas1.h                      |   3 +-
 inc/dg/functors.h                   |   2 +-
 inc/dg/geometry.h                   | 216 ++++-------------
 inc/dg/geometry/geometry_traits.h   | 355 ----------------------------
 inc/dg/geometry/mpi_grids.h         |  47 +---
 inc/dg/geometry/tensor.h            | 124 ++++++++--
 inc/dg/geometry/transform.h         | 187 +++++++++++++++
 11 files changed, 357 insertions(+), 608 deletions(-)
 delete mode 100644 inc/dg/geometry/geometry_traits.h
 create mode 100644 inc/dg/geometry/transform.h

diff --git a/inc/dg/backend/grid.h b/inc/dg/backend/grid.h
index 1695ae8f2..67d8f80f9 100644
--- a/inc/dg/backend/grid.h
+++ b/inc/dg/backend/grid.h
@@ -196,8 +196,8 @@ struct Grid1d
  */
 struct aTopology2d
 {
-    typedef SharedTag memory_category;
-    typedef TwoDimensionalTag dimensionality;
+    typedef SharedTag memory_category; //!< tag for choosing default host vector type
+    typedef TwoDimensionalTag dimensionality; 
 
     /**
      * @brief Left boundary in x
diff --git a/inc/dg/backend/operator.h b/inc/dg/backend/operator.h
index 1f11e3414..6f0af0366 100644
--- a/inc/dg/backend/operator.h
+++ b/inc/dg/backend/operator.h
@@ -32,7 +32,7 @@ class Operator
     /**
      * @brief allocate storage for nxn matrix
      *
-     * @param n
+     * @param n size
      */
     Operator( const unsigned n): n_(n), data_(n_*n_){}
     /**
@@ -107,9 +107,9 @@ class Operator
     }
 
     /**
-     * @brief Size of the Operator
+     * @brief Size n of the Operator
      *
-     * @return 
+     * @return n
      */
     unsigned size() const { return n_;}
     /**
diff --git a/inc/dg/backend/topological_traits.h b/inc/dg/backend/topological_traits.h
index fafce97ae..d71fb0180 100644
--- a/inc/dg/backend/topological_traits.h
+++ b/inc/dg/backend/topological_traits.h
@@ -3,12 +3,21 @@
 namespace dg
 {
 
+///@cond
+template <class Topology>
+struct TopologyTraits{
+    typedef typename Topology::memory_category memory_category; //either shared or distributed
+    typedef typename Topology::dimensionality dimensionality; //either shared or distributed
+};
+///@endcond
+
 //memory categories
-struct MPITag{};
-struct SharedTag{};
+struct MPITag{}; //!< distributed memory  system
+struct SharedTag{}; //!<  shared memory system
 
 //dimensionality 
-struct OneDimensionalTag{};
-struct TwoDimensionalTag{};
-struct ThreeDimensionalTag{};
+struct OneDimensionalTag{}; //!< 1d
+struct TwoDimensionalTag{}; //!< 2d
+struct ThreeDimensionalTag{}; //!< 3d
+
 }//namespace dg
diff --git a/inc/dg/backend/vector_traits.h b/inc/dg/backend/vector_traits.h
index 9c16b0763..0b90d9a0a 100644
--- a/inc/dg/backend/vector_traits.h
+++ b/inc/dg/backend/vector_traits.h
@@ -6,7 +6,7 @@
 
 namespace dg{
 
-    ///@cond
+///@cond
 template< class Vector>
 struct VectorTraits {
     typedef typename Vector::value_type value_type;
diff --git a/inc/dg/blas1.h b/inc/dg/blas1.h
index 555719ed7..91fca9dac 100644
--- a/inc/dg/blas1.h
+++ b/inc/dg/blas1.h
@@ -119,8 +119,7 @@ inline void axpby( typename VectorTraits<Vector>::value_type alpha, const Vector
 /*! @brief "new" BLAS 1 routine transform
  *
  * This routine computes \f[ y_i = op(x_i) \f] 
- * This is actually not a BLAS routine since f can be a nonlinear function.
- * It is rather the first step towards a more general library conception.
+ * This is strictly speaking not a BLAS routine since f can be a nonlinear function.
  * @param x Vector x may equal y
  * @param y Vector y contains result, may equal x
  * @param op unary Operator to use on every element
diff --git a/inc/dg/functors.h b/inc/dg/functors.h
index dd80fb21c..a1006d442 100644
--- a/inc/dg/functors.h
+++ b/inc/dg/functors.h
@@ -1148,7 +1148,7 @@ struct PLUS
 };
 
 /**
- * @brief Invert the given value
+ * @brief %Invert the given value
  * \f[ f(x) = 1/x \f]
  *
  * @tparam T value type
diff --git a/inc/dg/geometry.h b/inc/dg/geometry.h
index bb4daac70..5910f0570 100644
--- a/inc/dg/geometry.h
+++ b/inc/dg/geometry.h
@@ -17,6 +17,7 @@
 #include "geometry/mpi_grids.h"
 #include "geometry/mpi_curvilinear.h"
 #endif//MPI_VERSION
+#include "geometry/tensor.h"
 
 
 /*!@file 
@@ -24,16 +25,6 @@
  * geometry functions
  */
 
-/*
-Comment: we want to use template functions for the geometry routines
-because we fundamentally have to distinguish between Grid and MPIGrid. 
-A change in hardware (from shared to distributed memory) always 
-necessitates a recompilation. 
-We try to avoid recompilation when we change the grid 
-at least for the cases in which you don't have to change the model anyways.
-The point is that these functions are not really performance critical 
-so we shouldn't make the distinctions too fine grained.
-*/
 namespace dg{
 
 /*! @brief Geometry routines 
@@ -51,14 +42,14 @@ namespace geo{
  *
  * Computes \f$ f = \sqrt{g}f\f$ 
  * @tparam container container class 
- * @tparam Geometry Geometry class
  * @param inout input (contains result on output)
- * @param g the geometry object
+ * @param metric the metric object
  */
-template<class container, class Geometry>
-void multiplyVolume( container& inout, const Geometry& g)
+template<class container>
+void multiplyVolume( container& inout, const SharedContainers<container>& metric)
 {
-    dg::geo::detail::doMultiplyVolume( inout, g, typename dg::GeometryTraits<Geometry>::metric_category());
+    if( metric.isSet(0))
+        dg::blas1::pointwiseDot( metric.getValue(0), inout, inout);
 }
 
 /**
@@ -66,169 +57,73 @@ void multiplyVolume( container& inout, const Geometry& g)
  *
  * Computes \f$ v = v/ \sqrt{g}\f$ 
  * @tparam container container class 
- * @tparam Geometry Geometry class
  * @param inout input (contains result on output)
- * @param g the geometry object
+ * @param metric the metric
  */
-template<class container, class Geometry>
-void divideVolume( container& inout, const Geometry& g)
+template<class container>
+void divideVolume( container& inout, const SharedContainers<container>& metric)
 {
-    dg::geo::detail::doDivideVolume( inout, g, typename dg::GeometryTraits<Geometry>::metric_category());
+    if( metric.isSet(0))
+        dg::blas1::pointwiseDivide( inout, metric.getValue(0), inout);
 }
 
 /**
  * @brief Raises the index of a covariant vector with the help of the projection tensor in 2d
  *
- * The projection tensor is defined as \f[ h^{ij} := g^{ij} - b^ib^ĵ \f]
- * where \f$ b^i\f$ are the contravariant components of a unit vector.
- * Here, we assume that the metric forms a 2x1 product space and b only has one component \f$ b^3b^3 = g_{33}^{-1} = g^{33}\f$:
- * Compute \f$ v^i = g^{ij}v_j\f$ for \f$ i,j\in \{1,2\}\f$ in the two dimensions of a 2x1 product space
+ * Compute \f$ v^i = g^{ij}v_j\f$ for \f$ i,j\in \{1,2\}\f$ in the first two dimensions 
  * @tparam container the container class
- * @tparam Geometry the geometry class
- * @param covX (input) covariant first component 
- * @param covY (input) covariant second component
- * @param contraX (output) contravariant first component (can be the same as covX)
- * @param contraY (output) contravariant second component (can be the same as covY)
- * @param g The geometry object
- * @note if contraX==covX and/or contraY==covY the transformation is done in-place
+ * @param covX (input) covariant first component (undefined content on output)
+ * @param covY (input) covariant second component (undefined content on output)
+ * @param contraX (output) contravariant first component 
+ * @param contraY (output) contravariant second component 
+ * @param metric The metric
+ * @note no alias allowed 
  */
-template<class container, class Geometry>
-void raisePerpIndex( const container& covX, const container& covY, container& contraX, container& contraY, const Geometry& g)
+template<class container>
+void raisePerpIndex( container& covX, container& covY, container& contraX, container& contraY, const SharedContainers<container>& metric)
 {
     assert( &covX != &contraX);
     assert( &covY != &contraY);
     assert( &covY != &covX);
-    dg::geo::detail::doRaisePerpIndex( covX, covY, contraX, contraY, g, typename dg::GeometryTraits<Geometry>::metric_category());
-
-}
-/**
- * @brief Raises the index of a covariant vector in 2d with the help of the projection tensor in 2d and multiplies the perpendicular volume
- *
- * The projection tensor is defined as \f[ h^{ij} := g^{ij} - b^ib^ĵ \f]
- * where \f$ b^i\f$ are the contravariant components of a unit vector.
- * Here, we assume that the metric forms a 2x1 product space and b only has one component \f$ b^3b^3 = g_{33}^{-1} = g^{33}\f$:
- * Compute \f$ v^i = \sqrt{g/g_{33}} g^{ij}v_j\f$ for \f$ i,j\in \{1,2\}\f$ in the two dimensions of a 2x1 product space. This special 
- * form occurs in the discretization of elliptic operators which is why it get's a special function.
- * @tparam container the container class
- * @tparam Geometry the geometry class
- * @param covX (input) covariant first component 
- * @param covY (input) covariant second component
- * @param contraX (output) contravariant first component (can be the same as covX)
- * @param contraY (output) contravariant second component (can be the same as covY)
- * @param g The geometry object
- * @note if contraX==covX and/or contraY==covY the transformation is done in-place
- */
-template<class container, class Geometry>
-void volRaisePerpIndex( container& covX, container& covY, container& contraX, container& contraY, const Geometry& g)
-{
-    assert( &covX != &contraX);
-    assert( &covY != &contraY);
-    assert( &covY != &covX);
-    dg::geo::detail::doVolRaisePerpIndex( covX, covY, contraX, contraY, g, typename dg::GeometryTraits<Geometry>::metric_category());
-
+    dg::detail::multiply( metric, covX, covY, contraX, contraY);
 }
 
 /**
  * @brief Multiplies the two-dimensional volume element
  *
- * Computes \f$ f = \sqrt{g/g_{zz}}f\f$ on a 2x1 product space
+ * Computes \f$ f = \sqrt{g_\perp}f\f$ where the perpendicualar volume is computed from the 2x2 submatrix of g in the first two coordinates.
  * @tparam container the container class
- * @tparam Geometry the geometry class
  * @param inout input (contains result on output)
- * @param g The geometry object
- * @note if g is two-dimensional this function will redirect to multiplyVolume()
+ * @param metric The metric  
+ * @note if metric is two-dimensional this function will have the same result as multiplyVolume()
  */
-template<class container, class Geometry>
-void multiplyPerpVolume( container& inout, const Geometry& g)
+template<class container>
+void multiplyPerpVolume( container& inout, const SharedContainers<container>& metric)
 {
-    dg::geo::detail::doMultiplyPerpVolume( inout, g, typename GeometryTraits<Geometry>::dimensionality());
+    if( metric.isSet(1))
+        dg::blas1::pointwiseDot( metric.getValue(1), inout, inout);
 }
 
 /**
  * @brief Divides the two-dimensional volume element
  *
- * Computes \f$ f = f /\sqrt{g/g_{zz}}\f$ on a 2x1 product space
+ * Computes \f$ f = f /\sqrt{g_\perp}\f$ where the perpendicualar volume is computed from the 2x2 submatrix of g in the first two coordinates.
  * @tparam container the container class
- * @tparam Geometry the geometry class
  * @param inout input (contains result on output)
- * @param g The geometry object
- * @note if g is two-dimensional this function will redirect to divideVolume()
+ * @param metric The metric tensor
+ * @note if metric is two-dimensional this function will have the same result as divideVolume()
  */
-template<class container, class Geometry>
-void dividePerpVolume( container& inout, const Geometry& g)
-{
-    dg::geo::detail::doDividePerpVolume( inout, g, typename GeometryTraits<Geometry>::dimensionality());
-}
-
-/**
- * @brief Push forward a vector from cylindrical or Cartesian to a new coordinate system
- *
- * Computes \f[ v^x(x,y) = x_R (x,y) v^R(R(x,y), Z(x,y)) + x_Z v^Z(R(x,y), Z(x,y)) \\
-               v^y(x,y) = y_R (x,y) v^R(R(x,y), Z(x,y)) + y_Z v^Z(R(x,y), Z(x,y)) \f]
-   where \f$ x_R = \frac{\partial x}{\partial R}\f$, ... 
- * @tparam Geometry The Geometry class
- * @param vR input R-component in cylindrical coordinates
- * @param vZ input Z-component in cylindrical coordinates
- * @param vx x-component of vector (gets properly resized)
- * @param vy y-component of vector (gets properly resized)
- * @param g The geometry object
- */
-template<class Functor1, class Functor2, class container, class Geometry> 
-void pushForwardPerp( Functor1 vR, Functor2 vZ, 
-        container& vx, container& vy,
-        const Geometry& g)
-{
-    dg::geo::detail::doPushForwardPerp( vR, vZ, vx, vy, g, typename GeometryTraits<Geometry>::metric_category() ); 
-}
-
-///@cond
-template<class container, class Geometry> 
-void pushForwardPerp(
-        double(fR)(double,double,double), double(fZ)(double, double, double), 
-        container& out1, container& out2,
-        const Geometry& g)
-{
-    pushForwardPerp<double(double, double, double), double(double, double, double), Geometry>( fR, fZ, out1, out2, g); 
-}
-///@endcond
-template<class Functor1, class Functor2, class Functor3 class container, class Geometry> 
-void pushForward( Functor1 vR, Functor2 vZ, Functor3 vPhi,
-        container& vx, container& vy, container& vz,
-        const Geometry& g)
+template<class container>
+void dividePerpVolume( container& inout, const SharedContainers<container>& metric)
 {
-    dg::geo::detail::doPushForward( vR, vZ,vPhi,vx, vy,vz, g, typename GeometryTraits<Geometry>::metric_category() ); 
+    if( metric.isSet(1))
+        dg::blas1::pointwiseDivide( metric.getValue(1), inout, inout);
 }
 
-/**
- * @brief Push forward a symmetric 2d tensor from cylindrical or Cartesian to a new coordinate system
- *
- * Computes \f[ 
- \chi^{xx}(x,y) = x_R x_R \chi^{RR} + 2x_Rx_Z \chi^{RZ} + x_Zx_Z\chi^{ZZ} \\
- \chi^{xy}(x,y) = x_R x_R \chi^{RR} + (x_Ry_Z+y_Rx_Z) \chi^{RZ} + x_Zx_Z\chi^{ZZ} \\
- \chi^{yy}(x,y) = y_R y_R \chi^{RR} + 2y_Ry_Z \chi^{RZ} + y_Zy_Z\chi^{ZZ} \\
-               \f]
-   where \f$ x_R = \frac{\partial x}{\partial R}\f$, ... 
- * @tparam Geometry The Geometry class
- * @param chiRR input RR-component in cylindrical coordinates
- * @param chiRZ input RZ-component in cylindrical coordinates
- * @param chiZZ input ZZ-component in cylindrical coordinates
- * @param chixx xx-component of tensor (gets properly resized)
- * @param chixy xy-component of tensor (gets properly resized)
- * @param chiyy yy-component of tensor (gets properly resized)
- * @param g The geometry object
- */
-template<class FunctorRR, class FunctorRZ, class FunctorZZ, class container, class Geometry> 
-void pushForwardPerp( FunctorRR chiRR, FunctorRZ chiRZ, FunctorZZ chiZZ,
-        container& chixx, container& chixy, container& chiyy,
-        const Geometry& g)
-{
-    dg::geo::detail::doPushForwardPerp( chiRR, chiRZ, chiZZ, chixx, chixy, chiyy, g, typename GeometryTraits<Geometry>::metric_category() ); 
-}
 
 ///@}
 }//namespace geo
 
-
 namespace create{
 ///@addtogroup geometry
 ///@{
@@ -237,54 +132,39 @@ namespace create{
  * @brief Create the volume element on the grid (including weights!!)
  *
  * This is the same as the weights multiplied by the volume form \f$ \sqrt{g}\f$
- * @tparam Geometry Geometry class
+ * @tparam Geometry any Geometry class
  * @param g Geometry object
  *
  * @return  The volume form
  */
 template< class Geometry>
-typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector volume( const Geometry& g)
+typename HostVec< typename TopologyTraits<Geometry>::memory_category>::host_vector volume( const Geometry& g)
 {
-    return detail::doCreateVolume( g, typename GeometryTraits<Geometry>::metric_category());
+    typedef typename HostVec< typename TopologyTraits<Geometry>::memory_category>::host_vector host_vector;
+    SharedContainers<host_vector> metric = g.compute_metric();
+    host_vector temp = dg::create::weights( g);
+    dg::geo::multiplyVolume( temp, metric);
+    return temp;
 }
 
 /**
  * @brief Create the inverse volume element on the grid (including weights!!)
  *
  * This is the same as the inv_weights divided by the volume form \f$ \sqrt{g}\f$
- * @tparam Geometry Geometry class
+ * @tparam Geometry any Geometry class
  * @param g Geometry object
  *
  * @return  The inverse volume form
  */
 template< class Geometry>
-typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector inv_volume( const Geometry& g)
+typename HostVec< typename TopologyTraits<Geometry>::memory_category>::host_vector inv_volume( const Geometry& g)
 {
-    return detail::doCreateInvVolume( g, typename GeometryTraits<Geometry>::metric_category());
+    typedef typename HostVec< typename TopologyTraits<Geometry>::memory_category>::host_vector host_vector;
+    host_vector temp = volume(g);
+    dg::blas1::transform(temp,temp,dg::INVERT<double>());
+    return temp;
 }
 
 ///@}
 }//namespace create
-
-/**
- * @brief This function pulls back a function defined in cartesian coordinates to the curvilinear coordinate system
- *
- * @ingroup geometry
- * i.e. F(x,y) = f(R(x,y), Z(x,y)) in 2d 
- * @tparam Functor The (binary or ternary) function object 
- * @param f The function defined in cartesian coordinates
- * @param g The grid
- * @note Template deduction will fail if you overload functions with different dimensionality (e.g. double sine( double x) and double sine(double x, double y) )
- * You will want to rename those uniquely
- *
- * @return A set of points representing F
- */
-template< class Functor, class Geometry>
-typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector 
-    pullback( Functor f, const Geometry& g)
-{
-    return dg::detail::doPullback( f, g, typename GeometryTraits<Geometry>::metric_category(), typename GeometryTraits<Geometry>::dimensionality(), typename GeometryTraits<Geometry>::memory_category() );
-}
-
-
 }//namespace dg
diff --git a/inc/dg/geometry/geometry_traits.h b/inc/dg/geometry/geometry_traits.h
deleted file mode 100644
index 50f75d57b..000000000
--- a/inc/dg/geometry/geometry_traits.h
+++ /dev/null
@@ -1,355 +0,0 @@
-#pragma once
-
-#include "../blas1.h"
-
-namespace dg
-{
-//categories
-//
-struct CurvilinearTag{};  //! For completeness only (in fact we don't have a true 3d curvilinear grid right now)
-struct CurvilinearPerpTag: public CurvilinearTag{}; //! perpVol, vol(), g_xx, g_xy, g_yy (2x1 product grid, or 2d curvilinear)
-struct OrthonormalTag: public CurvilinearPerpTag{}; //! Cartesian grids  (3d or 2d)
-
-//memory_category and dimensionality Tags are already defined in grid.h
-
-///@cond
-template <class Geometry>
-struct GeometryTraits{
-    typedef typename Geometry::metric_category metric_category;
-    typedef typename Geometry::memory_category memory_category; //either shared or distributed
-    typedef typename Geometry::dimensionality dimensionality; //either shared or distributed
-};
-
-template<class MemoryTag>
-struct HostVec {
-};
-template<>
-struct HostVec< SharedTag>
-{
-    typedef thrust::host_vector<double> host_vector;
-};
-
-#ifdef MPI_VERSION
-template<>
-struct HostVec< MPITag>
-{
-    typedef MPI_Vector<thrust::host_vector<double> > host_vector;
-};
-#endif //MPI_VERSION
-
-namespace geo{
-namespace detail{
-
-template <class container, class Geometry>
-void doMultiplyVolume( container& inout, const Geometry& g, OrthonormalTag)
-{
-};
-
-template <class container, class Geometry>
-void doMultiplyVolume( container& inout, const Geometry& g, CurvilinearTag)
-{
-    dg::blas1::pointwiseDot( inout, g.vol(), inout);
-};
-template <class container, class Geometry>
-void doDivideVolume( container& inout, const Geometry& g, OrthonormalTag)
-{
-};
-
-template <class container, class Geometry>
-void doDivideVolume( container& inout, const Geometry& g, CurvilinearTag)
-{
-    dg::blas1::pointwiseDivide( inout, g.vol(), inout);
-};
-
-//////////////////multiplyPerpVol/////////////
-template <class container, class Geometry>
-void doMultiplyPerpVolume( container& inout, const Geometry& g, TwoDimensionalTag)
-{
-    doMultiplyVolume( inout, g, typename dg::GeometryTraits<Geometry>::metric_category());
-};
-template <class container, class Geometry>
-void doMultiplyPerpVolume( container& inout, const Geometry& g, ThreeDimensionalTag)
-{
-    doMultiplyPerpVolume3d( inout, g, typename dg::GeometryTraits<Geometry>::metric_category());
-};
-template <class container, class Geometry>
-void doMultiplyPerpVolume3d( container& inout, const Geometry& g, OrthonormalTag)
-{
-};
-
-template <class container, class Geometry>
-void doMultiplyPerpVolume3d( container& inout, const Geometry& g, CurvilinearPerpTag)
-{
-    if( !g.isOrthonormal())
-        dg::blas1::pointwiseDot( inout, g.perpVol(), inout);
-};
-
-//////////////////dividePerpVol/////////////
-template <class container, class Geometry>
-void doDividePerpVolume( container& inout, const Geometry& g, TwoDimensionalTag)
-{
-    doDivideVolume( inout, g, typename dg::GeometryTraits<Geometry>::metric_category());
-};
-template <class container, class Geometry>
-void doDividePerpVolume( container& inout, const Geometry& g, ThreeDimensionalTag)
-{
-    doDividePerpVolume3d( inout, g, typename dg::GeometryTraits<Geometry>::metric_category());
-};
-template <class container, class Geometry>
-void doDividePerpVolume3d( container& inout, const Geometry& g, OrthonormalTag)
-{
-};
-template <class container, class Geometry>
-void doDividePerpVolume3d( container& inout, const Geometry& g, CurvilinearPerpTag)
-{
-    if( !g.isOrthonormal())
-        dg::blas1::pointwiseDivide( inout, g.perpVol(), inout);
-};
-
-
-template <class container, class Geometry>
-void doRaisePerpIndex( const container& in1, const container& in2, container& out1, container& out2, const Geometry& g, OrthonormalTag)
-{
-    out1.swap(in1);
-    out2.swap(in2);
-};
-template <class container, class Geometry>
-void doRaisePerpIndex( const container& in1, const container& in2, container& out1, container& out2, const Geometry& g, CurvilinearPerpTag)
-{
-    if( g.isOrthonormal())
-    {
-        out1.swap(in1);
-        out2.swap(in2);
-        return;
-    }
-    if( g.isOrthogonal())
-    {
-        dg::blas1::pointwiseDot( g.g_xx(), in1, out1); //gxx*v_x
-        dg::blas1::pointwiseDot( g.g_yy(), in2, out2); //gyy*v_y
-        return;
-    }
-    dg::blas1::pointwiseDot( g.g_xx(), in1, out1); //gxx*v_x
-    dg::blas1::pointwiseDot( g.g_xy(), in1, out2); //gyx*v_x
-    dg::blas1::pointwiseDot( 1., g.g_xy(), in2, 1., out1);//gxy*v_y
-    dg::blas1::pointwiseDot( 1., g.g_yy(), in2, 1., out2); //gyy*v_y
-};
-
-template <class container, class Geometry>
-void doVolRaisePerpIndex( container& in1, container& in2, container& out1, container& out2, const Geometry& g, OrthonormalTag)
-{
-    out1.swap(in1);
-    out2.swap(in2);
-};
-template <class container, class Geometry>
-void doVolRaisePerpIndex( const container& in1, const container& in2, container& out1, container& out2, const Geometry& g, CurvilinearPerpTag)
-{
-    if( g.isConformal())
-    {
-        out1.swap(in1);
-        out2.swap(in2);
-        return;
-    }
-    if( g.isOrthogonal())
-    {
-        dg::blas1::pointwiseDot( g.g_xx(), in1, out1); //gxx*v_x
-        dg::blas1::pointwiseDot( g.g_yy(), in2, out2); //gyy*v_y
-        dg::blas1::pointwiseDot( out1, g.perpVol(), out1);
-        dg::blas1::pointwiseDot( out2, g.perpVol(), out2);
-        return;
-    }
-    dg::blas1::pointwiseDot( g.g_xx(), in1, out1); //gxx*v_x
-    dg::blas1::pointwiseDot( g.g_xy(), in1, out2); //gyx*v_x
-    dg::blas1::pointwiseDot( 1., g.g_xy(), in2, 1., out1);//gxy*v_y
-    dg::blas1::pointwiseDot( 1., g.g_yy(), in2, 1., out2); //gyy*v_y
-    dg::blas1::pointwiseDot( out1, g.perpVol(), out1);
-    dg::blas1::pointwiseDot( out2, g.perpVol(), out2);
-};
-
-
-template<class TernaryOp1, class TernaryOp2, class container, class Geometry> 
-void doPushForwardPerp( TernaryOp1 f1, TernaryOp2 f2, 
-        container& vx, container& vy,
-        const Geometry& g, OrthonormalTag)
-{
-    typedef typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector host_vec;
-    host_vec out1 = evaluate( f1, g);
-    host_vec out2 = evaluate( f2, g);
-    dg::blas1::transfer( out1, vx);
-    dg::blas1::transfer( out2, vy);
-}
-
-template<class TernaryOp1, class TernaryOp2, class container, class Geometry> 
-void doPushForwardPerp( TernaryOp1 f1, TernaryOp2 f2, 
-        container& vx, container& vy,
-        const Geometry& g, CurvilinearPerpTag)
-{
-    typedef typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector host_vec;
-    host_vec out1 = pullback( f1, g), temp1(out1);
-    host_vec out2 = pullback( f2, g), temp2(out2);
-    dg::blas1::pointwiseDot( g.xr(), temp1, out1);
-    dg::blas1::pointwiseDot( 1., g.xz(), temp2, 1., out1);
-    dg::blas1::pointwiseDot( g.yr(), temp1, out2);
-    dg::blas1::pointwiseDot( 1., g.yz(), temp2, 1., out2);
-    dg::blas1::transfer( out1, vx);
-    dg::blas1::transfer( out2, vy);
-}
-
-template<class TernaryOp1, class TernaryOp2, class TernaryOp3 class container, class Geometry> 
-void doPushForward( TernaryOp1 f1, TernaryOp2 f2, TernaryOp3,
-        container& vx, container& vy, container& vz,
-        const Geometry& g, CurvilinearPerpTag)
-{
-    typedef typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector host_vec;
-    doPushForwardPerp(f1,f2,vx,vy,CurvilinearPerpTag());
-    host_vec out3 = pullback( f3, g), temp3(out3);
-    dg::blas1::pointwiseDot( g.zP(), temp3, out3);
-    dg::blas1::transfer( out3, vz);
-}
-template<class TernaryOp1, class TernaryOp2, class TernaryOp3 class container, class Geometry> 
-void doPushForward( TernaryOp1 f1, TernaryOp2 f2, TernaryOp3,
-        container& vx, container& vy, container& vz,
-        const Geometry& g, OrthonormalTag)
-{
-    doPushForwardPerp( f1,f2,f3,vx,vy,vz,OrthonormalTag());
-}
-
-template<class FunctorRR, class FunctorRZ, class FunctorZZ, class container, class Geometry> 
-void doPushForwardPerp( FunctorRR chiRR, FunctorRZ chiRZ, FunctorZZ chiZZ,
-        container& chixx, container& chixy, container& chiyy,
-        const Geometry& g, OrthonormalTag)
-{
-    typedef typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector host_vec;
-    host_vec chixx_ = evaluate( chiRR, g);
-    host_vec chixy_ = evaluate( chiRZ, g);
-    host_vec chiyy_ = evaluate( chiZZ, g);
-    dg::blas1::transfer( chixx_, chixx);
-    dg::blas1::transfer( chixy_, chixy);
-    dg::blas1::transfer( chiyy_, chiyy);
-}
-
-template<class FunctorRR, class FunctorRZ, class FunctorZZ, class container, class Geometry> 
-void doPushForwardPerp( FunctorRR chiRR_, FunctorRZ chiRZ_, FunctorZZ chiZZ_,
-        container& chixx, container& chixy, container& chiyy,
-        const Geometry& g, CurvilinearPerpTag)
-{
-    //compute the rhs
-    typedef typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector host_vec;
-    host_vec chixx_ = pullback( chiRR_, g), chiRR(chixx_);
-    host_vec chixy_ = pullback( chiRZ_, g), chiRZ(chixy_);
-    host_vec chiyy_ = pullback( chiZZ_, g), chiZZ(chiyy_);
-    //compute the transformation matrix
-    host_vec t00(chixx), t01(t00), t02(t00), t10(t00), t11(t00), t12(t00), t20(t00), t21(t00), t22(t00);
-    dg::blas1::pointwiseDot( g.xr(), g.xr(), t00);
-    dg::blas1::pointwiseDot( g.xr(), g.xz(), t01);
-    dg::blas1::scal( t01, 2.);
-    dg::blas1::pointwiseDot( g.xz(), g.xz(), t02);
-
-    dg::blas1::pointwiseDot( g.xr(), g.yr(), t10);
-    dg::blas1::pointwiseDot( g.xr(), g.yz(), t11);
-    dg::blas1::pointwiseDot( 1., g.yr(), g.xz(), 1., t11);
-    dg::blas1::pointwiseDot( g.xz(), g.yz(), t12);
-
-    dg::blas1::pointwiseDot( g.yr(), g.yr(), t20);
-    dg::blas1::pointwiseDot( g.yr(), g.yz(), t21);
-    dg::blas1::scal( t21, 2.);
-    dg::blas1::pointwiseDot( g.yz(), g.yz(), t22);
-    //now multiply
-    dg::blas1::pointwiseDot(     t00, chiRR, chixx_);
-    dg::blas1::pointwiseDot( 1., t01, chiRZ, 1., chixx_);
-    dg::blas1::pointwiseDot( 1., t02, chiZZ, 1., chixx_);
-    dg::blas1::pointwiseDot(     t10, chiRR, chixy_);
-    dg::blas1::pointwiseDot( 1., t11, chiRZ, 1., chixy_);
-    dg::blas1::pointwiseDot( 1., t12, chiZZ, 1., chixy_);
-    dg::blas1::pointwiseDot(     t20, chiRR, chiyy_);
-    dg::blas1::pointwiseDot( 1., t21, chiRZ, 1., chiyy_);
-    dg::blas1::pointwiseDot( 1., t22, chiZZ, 1., chiyy_);
-    dg::blas1::transfer( chixx_, chixx);
-    dg::blas1::transfer( chixy_, chixy);
-    dg::blas1::transfer( chiyy_, chiyy);
-
-}
-
-
-}//namespace detail 
-}//namespace geo
-
-namespace create{
-namespace detail{
-
-template< class Geometry>
-typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector doCreateVolume( const Geometry& g, OrthonormalTag)
-{
-    return dg::create::weights( g);
-}
-
-template< class Geometry>
-typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector doCreateInvVolume( const Geometry& g, OrthonormalTag)
-{
-    return dg::create::inv_weights( g);
-}
-template< class Geometry>
-typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector doCreateVolume( const Geometry& g, CurvilinearTag)
-{
-    typedef typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector host_vector;
-    host_vector temp = dg::create::weights( g), vol(temp);
-    dg::blas1::transfer( g.vol(), vol); //g.vol might be on device
-    dg::blas1::pointwiseDot( vol, temp, temp);
-    return temp;
-}
-
-template< class Geometry>
-typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector doCreateInvVolume( const Geometry& g, CurvilinearTag)
-{
-    typedef typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector host_vector;
-    host_vector temp = dg::create::inv_weights(g), vol(temp);
-    dg::blas1::transfer( g.vol(), vol); //g.vol might be on device
-    dg::blas1::pointwiseDivide( temp, vol, temp);
-    return temp;
-}
-
-}//namespace detail
-}//namespace create
-
-namespace detail
-{
-////pullbacks
-//template< class Geometry>
-//thrust::host_vector<double> doPullback( double(f)(double,double), const Geometry& g, CurvilinearTag, TwoDimensionalTag, SharedTag)
-//{
-//    return doPullback<double(double,double), Geometry>( f, g);
-//}
-//template< class Geometry>
-//thrust::host_vector<double> pullback( double(f)(double,double,double), const Geometry& g, CurvilinearTag, ThreeDimensionalTag, SharedTag)
-//{
-//    return doPullback<double(double,double,double), Geometry>( f, g);
-//}
-
-template< class BinaryOp, class Geometry>
-thrust::host_vector<double> doPullback( BinaryOp f, const Geometry& g, CurvilinearTag, TwoDimensionalTag, SharedTag)
-{
-    thrust::host_vector<double> vec( g.size());
-    for( unsigned i=0; i<g.size(); i++)
-        vec[i] = f( g.r()[i], g.z()[i]);
-    return vec;
-}
-
-template< class TernaryOp, class Geometry>
-thrust::host_vector<double> doPullback( TernaryOp f, const Geometry& g, CurvilinearTag, ThreeDimensionalTag, SharedTag)
-{
-    return g.doPullback( f);
-}
-template< class BinaryOp, class Geometry>
-thrust::host_vector<double> doPullback( BinaryOp f, const Geometry& g, OrthonormalTag, TwoDimensionalTag, SharedTag)
-{
-    return evaluate( f, g);
-}
-template< class TernaryOp, class Geometry>
-thrust::host_vector<double> doPullback( TernaryOp f, const Geometry& g, OrthonormalTag, ThreeDimensionalTag, SharedTag)
-{
-    return evaluate( f,g);
-}
-
-}//namespace detail
-///@endcond
-
-} //namespace dg
diff --git a/inc/dg/geometry/mpi_grids.h b/inc/dg/geometry/mpi_grids.h
index 8862e93dd..9448e9897 100644
--- a/inc/dg/geometry/mpi_grids.h
+++ b/inc/dg/geometry/mpi_grids.h
@@ -14,7 +14,7 @@ namespace dg
 /**
  * @brief The mpi version of a cartesian grid
  */
-struct CartesianMPIGrid2d : public dg::MPIGrid2d
+struct CartesianMPIGrid2d : public aMPIGeometry2d
 {
     typedef OrthonormalTag metric_category; 
 
@@ -37,7 +37,7 @@ struct CartesianMPIGrid2d : public dg::MPIGrid2d
 /**
  * @brief The mpi version of a cartesian grid
  */
-struct CartesianMPIGrid3d : public dg::MPIGrid3d
+struct CartesianMPIGrid3d : public aMPIGeometry3d
 {
     typedef OrthonormalTag metric_category; 
 
@@ -59,47 +59,4 @@ struct CartesianMPIGrid3d : public dg::MPIGrid3d
 
 ///@}
 
-///@cond
-/////////////////////////////////////////////////////MPI pullbacks/////////////////////////////////////////////////
-namespace detail{
-template< class Geometry>
-MPI_Vector< thrust::host_vector<double> > doPullback( double(f)(double,double), const Geometry& g, CurvilinearTag, TwoDimensionalTag, MPITag)
-{
-    return doPullback<double(double,double), Geometry>( f, g);
-}
-template< class Geometry>
-MPI_Vector< thrust::host_vector<double> > pullback( double(f)(double,double,double), const Geometry& g, CurvilinearTag, ThreeDimensionalTag, MPITag)
-{
-    return doPullback<double(double,double,double), Geometry>( f, g);
-}
-
-template< class BinaryOp, class Geometry>
-MPI_Vector< thrust::host_vector<double> > doPullback( BinaryOp f, const Geometry& g, CurvilinearTag, TwoDimensionalTag, MPITag)
-{
-    thrust::host_vector<double> vec( g.size());
-    for( unsigned i=0; i<g.size(); i++)
-        vec[i] = f( g.r()[i], g.z()[i]);
-    MPI_Vector<thrust::host_vector<double> > v( vec, g.communicator());
-    return v;
-}
-
-template< class TernaryOp, class Geometry>
-MPI_Vector< thrust::host_vector<double> > doPullback( TernaryOp f, const Geometry& g, CurvilinearTag, ThreeDimensionalTag, MPITag)
-{
-    return g.doPullback(f);
-}
-template< class BinaryOp, class Geometry>
-MPI_Vector< thrust::host_vector<double> > doPullback( BinaryOp f, const Geometry& g, OrthonormalTag, TwoDimensionalTag, MPITag)
-{
-    return evaluate( f, g);
-}
-template< class TernaryOp, class Geometry>
-MPI_Vector< thrust::host_vector<double> > doPullback( TernaryOp f, const Geometry& g, OrthonormalTag, ThreeDimensionalTag, MPITag)
-{
-    return evaluate( f,g);
-}
-
-} //namespace detail
-///@endcond
-
 }//namespace dg
diff --git a/inc/dg/geometry/tensor.h b/inc/dg/geometry/tensor.h
index 1542cc545..e07c93626 100644
--- a/inc/dg/geometry/tensor.h
+++ b/inc/dg/geometry/tensor.h
@@ -25,7 +25,16 @@ template<class container>
 struct SharedContainers
 {
     SharedContainers( ) {}
+
+    /**
+    * @brief 
+    *
+    * @param mat_idx
+    * @param vec_idx
+    * @param values The contained containers must all have the same size
+    */
     SharedContainers( const dg::Operator<int>& mat_idx, std::vector<int>& vec_idx, std::vector<container>& values ): mat_idx_(mat_idx), vec_idx(vec_idx), values_(values){}
+
     template<class otherContainer>
     SharedContainers( const SharedContainers<otherContainer>& src): mat_idx_(src.mat_idx()), vec_idx_(src.vec_idx()), values_(src.values().size()), idx_(src.idx_){
         dg::blas1::transfer( src.values(), values_);
@@ -47,6 +56,21 @@ struct SharedContainers
         return true;
     }
 
+
+    /**
+    * @brief Test the matrix for emptiness
+    * @return  true if no value in the matrix is set
+    */
+    bool empty()const{
+        bool epty = true;
+        for( unsigned i=0; i<mat_idx_.size(); i++)
+            for( unsigned j=0; j<mat_idx_.size(); j++)
+                if(isSet(i,j)) 
+                    epty=false;
+        return epty;
+    }
+     
+
     /**
     * @brief check if an index is set or not
     * @param i row index 0<i<2
@@ -57,7 +81,8 @@ struct SharedContainers
         return true;
     }
     /*!@brief Access the underlying container
-     * @return if !isSet(i,j) the default constructor of container is called, otherwise values[mat_idx(i,j)] is returned. If the indices fall out of range of mat_idx the result is undefined
+     * @return if !isSet(i,j) the default constructor of container is called, otherwise values[mat_idx(i,j)] is returned. 
+     * @note If the indices fall out of range of mat_idx the result is undefined
      */
     const container& getValue(size_t i, size_t j)const{ 
         int k = mat_idx(i,j);
@@ -65,7 +90,8 @@ struct SharedContainers
         return values_[k];
     }
     /*!@brief Access the underlying container
-     * @return if !isSet(i) the default constructor of container is called, otherwise values[vec_idx(i)] is returned. If the index falls out of range of vec_idx the result is undefined
+     * @return if !isSet(i) the default constructor of container is called, otherwise values[vec_idx(i)] is returned. 
+     * @note If the index falls out of range of vec_idx the result is undefined
      */
     const container& getValue(size_t i)const{ 
         int k = vec_idx_[i];
@@ -73,50 +99,41 @@ struct SharedContainers
         return values_[k];
     }
 
-    const dg::Operator<int>& mat_idx() const {return mat_idx_;}
-    const std::vector<int>& vec_idx() const {return vec_idx_;}
-    const std::vector<container>& values() const{return values_;}
     private:
     dg::Operator<int> mat_idx_;
     std::vector<int> vec_idx_;
     std::vector<container> values_;
 };
 
-
-///container sizes must match each other and the tensor container size
-///may destroy input (either swap in or use as temporary storage)
-///no alias allowed
+///@cond
+namespace detail
+{
 template<class container>
-void multiply( const SharedContainers& t, container& in1, container& in2, container& out1, container& out2)
+void multiply( const SharedContainers<container>& t, container& in1, container& in2, container& out1, container& out2)
 {
-    if( !tensor.isSet(0,0) && !tensor.isSet(0,1) && !tensor.isSet(1,0) && !tensor.isSet(1,1))
+    if( t.empty())
     {
-        out1.swap(in1);
-        out2.swap(in2);
+        in1.swap(out1);
+        in2.swap(out2);
         return;
     }
     if( !tensor.isSet(0,0)) out1=in1;
-    else 
-        dg::blas1::pointwiseDot( tensor.getValue(0,0), in1, out1); //gxx*v_x
+    if(tensor.isSet( 0,0))
+        dg::blas1::pointwiseDot( tensor.getValue(0,0), in1, out1); 
     if(tensor.isSet( 0,1))
-        dg::blas1::pointwiseDot( 1., tensor.getValue(0,1), in2, 1., out1);//gxy*v_y
+        dg::blas1::pointwiseDot( 1., tensor.getValue(0,1), in2, 1., out1);
 
     if( !tensor.isSet(1,1)) out2=in2;
-    else
-        dg::blas1::pointwiseDot( tensor.getValue(1,1), in2, out2); //gyy*v_y
+    if(tensor.isSet(1,1))
+        dg::blas1::pointwiseDot( tensor.getValue(1,1), in2, out2);
     if(tensor.isSet(1,0))
-        dg::blas1::pointwiseDot( 1., tensor.getValue(1,0), in1, 1., out2); //gyx*v_x
+        dg::blas1::pointwiseDot( 1., tensor.getValue(1,0), in1, 1., out2);
 }
 
 template<class container>
-void multiply( const SharedContainers& t, container& in1, container& in2, container& in3, container& out1, container& out2, container& out3)
+void multiply( const SharedContainers<container>& t, container& in1, container& in2, container& in3, container& out1, container& out2, container& out3)
 {
-    bool empty = true;
-    for( unsigned i=0; i<3; i++)
-        for( unsigned j=0; j<3; j++)
-            if(tensor.isSet(i,j)) 
-                empty=false;
-    if( empty)
+    if( t.empty())
     {
         in1.swap(out1);
         in2.swap(out2);
@@ -148,4 +165,59 @@ void multiply( const SharedContainers& t, container& in1, container& in2, contai
         dg::blas1::pointwiseDot( 1., tensor.getValue(2,0), in1, 1., out3);
 }
 
+template<class container>
+void sandwich( const SharedContainers<container>& t, container& chiRR, container& chiRZ, container& chiZZ, container& chixx, container& chixy, container& chiyy)
+{
+    if(t.empty())
+    {
+        chiRR.swap(chixx);
+        chiRZ.swap(chixy);
+        chiZZ.swap(chiyy);
+        return;
+    }
+    
+    //this is a default implementation  (add cases if optimization is necessary)
+    //compute the transformation matrix
+    container t00(chixx), t01(t00), t02(t00), t10(t00), t11(t00), t12(t00), t20(t00), t21(t00), t22(t00);
+    container xr(chixx), xz(xr), yr(xr), yz(xz);
+    //fill values for easier computations
+    if(t.isSet(0,0)) dg::blas1::transfer( t.getValue(0,0), xr);
+    else             dg::blas1::transform( xr,xr, dg::CONSTANT(1));
+    if(t.isSet(0,1)) dg::blas1::transfer( t.getValue(0,1), xz);
+    else             dg::blas1::transform( xz,xz, dg::CONSTANT(0));
+    if(t.isSet(1,0)) dg::blas1::transfer( t.getValue(1,0), yr);
+    else             dg::blas1::transform( yr,yr, dg::CONSTANT(0));
+    if(t.isSet(1,1)) dg::blas1::transfer( t.getValue(1,1), yz);
+    else             dg::blas1::transform( yz,yz, dg::CONSTANT(1));
+
+    dg::blas1::pointwiseDot( xr, xr, t00);
+    dg::blas1::pointwiseDot( xr, xz, t01);
+    dg::blas1::scal( t01, 2.);
+    dg::blas1::pointwiseDot( xz, g.xz, t02);
+
+    dg::blas1::pointwiseDot( xr, yr, t10);
+    dg::blas1::pointwiseDot( xr, yz, t11);
+    dg::blas1::pointwiseDot( 1., yr, g.xz(), 1., t11);
+    dg::blas1::pointwiseDot( xz, yz, t12);
+
+    dg::blas1::pointwiseDot( yr, yr, t20);
+    dg::blas1::pointwiseDot( yr, yz, t21);
+    dg::blas1::scal( t21, 2.);
+    dg::blas1::pointwiseDot( yz, yz, t22);
+
+    //now multiply
+    dg::blas1::pointwiseDot(     t00, chiRR,     chixx);
+    dg::blas1::pointwiseDot( 1., t01, chiRZ, 1., chixx);
+    dg::blas1::pointwiseDot( 1., t02, chiZZ, 1., chixx);
+    dg::blas1::pointwiseDot(     t10, chiRR,     chixy);
+    dg::blas1::pointwiseDot( 1., t11, chiRZ, 1., chixy);
+    dg::blas1::pointwiseDot( 1., t12, chiZZ, 1., chixy);
+    dg::blas1::pointwiseDot(     t20, chiRR,     chiyy);
+    dg::blas1::pointwiseDot( 1., t21, chiRZ, 1., chiyy);
+    dg::blas1::pointwiseDot( 1., t22, chiZZ, 1., chiyy);
+}
+
+}//namespace detail
+///@endcond
+
 }//namespace dg
diff --git a/inc/dg/geometry/transform.h b/inc/dg/geometry/transform.h
new file mode 100644
index 000000000..0b97c80e6
--- /dev/null
+++ b/inc/dg/geometry/transform.h
@@ -0,0 +1,187 @@
+#pragma once
+
+
+namespace dg
+{
+///@cond
+template<class MemoryTag>
+struct HostVec {
+};
+template<>
+struct HostVec< SharedTag>
+{
+    typedef thrust::host_vector<double> host_vector;
+};
+
+///@endcond
+/**
+ * @brief This function pulls back a function defined in cartesian coordinates to the curvilinear coordinate system
+ *
+ * @ingroup geometry
+ * e.g. F(x,y) = f(R(x,y), Z(x,y)) in 2d 
+ * @tparam Functor The binary or ternary function object 
+ * @param f The function defined in cartesian coordinates
+ * @param g a two- or three dimensional Geometry
+ * @note Template deduction for the Functor will fail if you overload functions with different 
+ dimensionality (e.g. double sine( double x) and double sine(double x, double y) )
+ * You will want to rename those uniquely
+ *
+ * @return A set of points representing F
+ */
+template< class Functor>
+thrust::host_vector<double> pullback( Functor f, const aGeometry2d& g)
+{
+    const SharedContainers<thrust::host_vector<double> >& map = g.map();
+    if( !map.isSet(0) && !map.isSet(1)) //implicit
+        return evaluate(f,g);
+    thrust::host_vector<double> vec( g.size());
+    for( unsigned i=0; i<g.size(); i++)
+        vec[i] = f( map.getValue(0)[i], map.getValue(1)[i]);
+    return vec;
+}
+
+///@copydoc pullback(Functor,const aGeometry2d&)
+template< class Functor>
+thrust::host_vector<double> pullback( Functor f, const aGeometry3d& g)
+{
+    const SharedContainers<thrust::host_vector<double> >& map = g.map();
+    if( !map.isSet(0) && !map.isSet(1) && !map.isSet(2)) //implicit
+        return evaluate(f,g);
+
+    thrust::host_vector<double> vec( g.size());
+    for( unsigned i=0; i<g.size(); i++)
+        vec[i] = f( map.getValue(0)[i], map.getValue(1)[i], map.getValue(2)[i]);
+    return vec;
+}
+
+#ifdef MPI_VERSION
+
+///@cond
+template<>
+struct HostVec< MPITag>
+{
+    typedef MPI_Vector<thrust::host_vector<double> > host_vector;
+};
+///@endcond
+
+///@copydoc pullback(Functor,const aGeometry2d&)
+template< class Functor>
+MPI_Vector<thrust::host_vector<double> > pullback( Functor f, const aMPIGeometry2d& g)
+{
+    const SharedContainers<MPI_Vector<thrust::host_vector<double> >& map = g.map();
+    if( !map.isSet(0) && !map.isSet(1)) //implicit
+        return evaluate(f,g);
+    thrust::host_vector<double> vec( g.size());
+    for( unsigned i=0; i<g.size(); i++)
+        vec[i] = f( map.getValue(0).data()[i], map.getValue(1).data()[i]);
+    return vec;
+}
+
+///@copydoc pullback(Functor,const aGeometry2d&)
+template< class Functor>
+MPI_Vector<thrust::host_vector<double> > pullback( Functor f, const aMPIGeometry3d& g)
+{
+    const SharedContainers<MPI_Vector<thrust::host_vector<double> >& map = g.map();
+    if( !map.isSet(0) && !map.isSet(1) && !map.isSet(2)) //implicit
+        return evaluate(f,g);
+
+    thrust::host_vector<double> vec( g.size());
+    for( unsigned i=0; i<g.size(); i++)
+        vec[i] = f( map.getValue(0).data()[i], map.getValue(1).data()[i], map.getValue(2).data()[i]);
+    return vec;
+}
+
+#endif //MPI_VERSION
+
+/**
+ * @brief Push forward a vector from cylindrical or Cartesian to a new coordinate system
+ *
+ * Computes \f[ v^x(x,y) = x_R (x,y) v^R(R(x,y), Z(x,y)) + x_Z v^Z(R(x,y), Z(x,y)) \\
+               v^y(x,y) = y_R (x,y) v^R(R(x,y), Z(x,y)) + y_Z v^Z(R(x,y), Z(x,y)) \f]
+   where \f$ x_R = \frac{\partial x}{\partial R}\f$, ... 
+ * @tparam Functor1 Binary or Ternary functor
+ * @tparam Geometry The Geometry class
+ * @param vR input R-component in cylindrical coordinates
+ * @param vZ input Z-component in cylindrical coordinates
+ * @param vx x-component of vector (gets properly resized)
+ * @param vy y-component of vector (gets properly resized)
+ * @param g The geometry object
+ */
+template<class Functor1, class Functor2, class container, class Geometry> 
+void pushForwardPerp( Functor1 vR, Functor2 vZ, 
+        container& vx, container& vy,
+        const Geometry& g)
+{
+    typedef typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector host_vec;
+    host_vec out1 = pullback( vR, g), temp1(out1);
+    host_vec out2 = pullback( vZ, g), temp2(out2);
+    dg::detail::multiply(g.map(), out1, out2, temp1, temp2);
+    dg::blas1::transfer( out1, vx);
+    dg::blas1::transfer( out2, vy);
+}
+
+/**
+ * @brief Push forward a vector from cylindrical or Cartesian to a new coordinate system
+ *
+ * Computes \f[ v^x(x,y) = x_R (x,y) v^R(R(x,y), Z(x,y)) + x_Z v^Z(R(x,y), Z(x,y)) \\
+               v^y(x,y) = y_R (x,y) v^R(R(x,y), Z(x,y)) + y_Z v^Z(R(x,y), Z(x,y)) \f]
+   where \f$ x_R = \frac{\partial x}{\partial R}\f$, ... 
+ * @tparam Functor1 Binary or Ternary functor
+ * @tparam Geometry The Geometry class
+ * @param vR input R-component in cartesian or cylindrical coordinates
+ * @param vZ input Z-component in cartesian or cylindrical coordinates
+ * @param vPhi input Z-component in cartesian or cylindrical coordinates
+ * @param vx x-component of vector (gets properly resized)
+ * @param vy y-component of vector (gets properly resized)
+ * @param vz z-component of vector (gets properly resized)
+ * @param g The geometry object
+ */
+template<class Functor1, class Functor2, class Functor3 class container, class Geometry> 
+void pushForward( Functor1 vR, Functor2 vZ, Functor3 vPhi,
+        container& vx, container& vy, container& vz,
+        const Geometry& g)
+{
+    typedef typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector host_vec;
+    host_vec out1 = pullback( vR, g), temp1(out1);
+    host_vec out2 = pullback( vZ, g), temp2(out2);
+    host_vec out3 = pullback( vPhi, g), temp3(out3);
+    dg::detail::multiply(g.map(), out1, out2, out3, temp1, temp2, temp3);
+    dg::blas1::transfer( out1, vx);
+    dg::blas1::transfer( out2, vy);
+    dg::blas1::transfer( out3, vz);
+}
+
+/**
+ * @brief Push forward a symmetric 2d tensor from cylindrical or Cartesian to a new coordinate system
+ *
+ * Computes \f[ 
+ \chi^{xx}(x,y) = x_R x_R \chi^{RR} + 2x_Rx_Z \chi^{RZ} + x_Zx_Z\chi^{ZZ} \\
+ \chi^{xy}(x,y) = x_R x_R \chi^{RR} + (x_Ry_Z+y_Rx_Z) \chi^{RZ} + x_Zx_Z\chi^{ZZ} \\
+ \chi^{yy}(x,y) = y_R y_R \chi^{RR} + 2y_Ry_Z \chi^{RZ} + y_Zy_Z\chi^{ZZ} \\
+               \f]
+   where \f$ x_R = \frac{\partial x}{\partial R}\f$, ... 
+ * @tparam Geometry The Geometry class
+ * @param chiRR input RR-component in cylindrical coordinates
+ * @param chiRZ input RZ-component in cylindrical coordinates
+ * @param chiZZ input ZZ-component in cylindrical coordinates
+ * @param chixx xx-component of tensor (gets properly resized)
+ * @param chixy xy-component of tensor (gets properly resized)
+ * @param chiyy yy-component of tensor (gets properly resized)
+ * @param g The geometry object
+ */
+template<class FunctorRR, class FunctorRZ, class FunctorZZ, class container, class Geometry> 
+void pushForwardPerp( FunctorRR chiRR, FunctorRZ chiRZ, FunctorZZ chiZZ,
+        container& chixx, container& chixy, container& chiyy,
+        const Geometry& g)
+{
+    typedef typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector host_vec;
+    host_vec chiRR_ = pullback( chiRR, g), chixx_(chiRR_);
+    host_vec chiRZ_ = pullback( chiRZ, g), chixy_(chiRZ_);
+    host_vec chiZZ_ = pullback( chiZZ, g), chiyy_(chiZZ_);
+    dg::detail::sandwich( g.map(), chiRR_,chiRZ_,chiZZ_, chixx_,chixy_,chiyy_);
+    dg::blas1::transfer( chixx_, chixx);
+    dg::blas1::transfer( chixy_, chixy);
+    dg::blas1::transfer( chiyy_, chiyy);
+}
+
+} //namespace dg
-- 
GitLab


From c65636c04669b04b69f93606930e0a40bffcac11 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Fri, 4 Aug 2017 00:16:12 +0200
Subject: [PATCH 113/453] made aMPIGeometry

---
 inc/dg/geometry.h              |   1 +
 inc/dg/geometry/cartesian.h    |  47 ------------
 inc/dg/geometry/geometry.h     |  35 +++++++++
 inc/dg/geometry/mpi_geometry.h | 127 +++++++++++++++++++++++++++++++++
 inc/dg/geometry/mpi_grids.h    |  62 ----------------
 5 files changed, 163 insertions(+), 109 deletions(-)
 delete mode 100644 inc/dg/geometry/cartesian.h
 create mode 100644 inc/dg/geometry/mpi_geometry.h
 delete mode 100644 inc/dg/geometry/mpi_grids.h

diff --git a/inc/dg/geometry.h b/inc/dg/geometry.h
index 5910f0570..78a4045be 100644
--- a/inc/dg/geometry.h
+++ b/inc/dg/geometry.h
@@ -18,6 +18,7 @@
 #include "geometry/mpi_curvilinear.h"
 #endif//MPI_VERSION
 #include "geometry/tensor.h"
+#include "geometry/transform.h"
 
 
 /*!@file 
diff --git a/inc/dg/geometry/cartesian.h b/inc/dg/geometry/cartesian.h
deleted file mode 100644
index 4f36ec602..000000000
--- a/inc/dg/geometry/cartesian.h
+++ /dev/null
@@ -1,47 +0,0 @@
-#pragma once
-
-#include "../backend/grid.h"
-#include "geometry.h"
-#include "geometry_traits.h"
-
-namespace dg
-{
-
-///@addtogroup basicgrids
-///@{
-
-/**
- * @brief two-dimensional Grid with Cartesian metric
- */
-struct CartesianGrid2d: public dg::aGeometry2d
-{
-    typedef OrthonormalTag metric_category; 
-    ///@copydoc Grid2d::Grid2d()
-    CartesianGrid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx = PER, bc bcy = PER): dg::Grid2d(x0,x1,y0,y1,n,Nx,Ny,bcx,bcy){}
-    /**
-     * @brief Construct from existing topology
-     *
-     * @param grid existing grid class
-     */
-    CartesianGrid2d( const dg::Grid2d& grid):dg::Grid2d(grid){}
-};
-
-/**
- * @brief three-dimensional Grid with Cartesian metric
- */
-struct CartesianGrid3d: public dg::aGeometry3d
-{
-    typedef OrthonormalTag metric_category; 
-    ///@copydoc Grid3d::Grid3d()
-    CartesianGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = PER, bc bcy = PER, bc bcz = PER): dg::Grid3d(x0,x1,y0,y1,z0,z1,n,Nx,Ny,Nz,bcx,bcy,bcz){}
-    /**
-     * @brief Construct from existing topology
-     *
-     * @param grid existing grid class
-     */
-    CartesianGrid3d( const dg::Grid3d& grid):dg::Grid3d(grid){}
-};
-
-///@}
-
-} //namespace dg
diff --git a/inc/dg/geometry/geometry.h b/inc/dg/geometry/geometry.h
index 03a99941f..f75345eb6 100644
--- a/inc/dg/geometry/geometry.h
+++ b/inc/dg/geometry/geometry.h
@@ -70,6 +70,41 @@ SharedContainers<thrust::host_vector<double> > metric( const aGeometry3d& g)
 
 }//namespace create
 
+///@}
+///@addtogroup basicgrids
+///@{
+
+/**
+ * @brief two-dimensional Grid with Cartesian metric
+ */
+struct CartesianGrid2d: public dg::aGeometry2d
+{
+    typedef OrthonormalTag metric_category; 
+    ///@copydoc Grid2d::Grid2d()
+    CartesianGrid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx = PER, bc bcy = PER): dg::Grid2d(x0,x1,y0,y1,n,Nx,Ny,bcx,bcy){}
+    /**
+     * @brief Construct from existing topology
+     *
+     * @param grid existing grid class
+     */
+    CartesianGrid2d( const dg::Grid2d& grid):dg::Grid2d(grid){}
+};
+
+/**
+ * @brief three-dimensional Grid with Cartesian metric
+ */
+struct CartesianGrid3d: public dg::aGeometry3d
+{
+    typedef OrthonormalTag metric_category; 
+    ///@copydoc Grid3d::Grid3d()
+    CartesianGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = PER, bc bcy = PER, bc bcz = PER): dg::Grid3d(x0,x1,y0,y1,z0,z1,n,Nx,Ny,Nz,bcx,bcy,bcz){}
+    /**
+     * @brief Construct from existing topology
+     *
+     * @param grid existing grid class
+     */
+    CartesianGrid3d( const dg::Grid3d& grid):dg::Grid3d(grid){}
+};
 
 ///@}
 } //namespace dg
diff --git a/inc/dg/geometry/mpi_geometry.h b/inc/dg/geometry/mpi_geometry.h
new file mode 100644
index 000000000..7f6e1f577
--- /dev/null
+++ b/inc/dg/geometry/mpi_geometry.h
@@ -0,0 +1,127 @@
+#pragma once
+
+#include "dg/backend/mpi_grid.h"
+#include "tensor.h"
+
+namespace dg
+{
+
+///@addtogroup geometry
+///@{
+
+/**
+ * @brief This is the abstract interface class for a two-dimensional Geometry
+ */
+struct aMPIGeometry2d : public aMPITopology2d
+{
+    typedef MPI_Vector<thrust::host_vector<double> > host_vec;
+    const SharedContainers<host_vec >& map()const{return map_;}
+    SharedContainers<host_vec > compute_metric()const {
+        return do_compute_metric();
+    }
+    ///allow deletion through base class pointer
+    virtual ~aMPIGeometry2d(){}
+    protected:
+    aMPIGeometry2d(const SharedContainers<host_vec >& map, const SharedContainers<container >& metric): map_(map), metric_(metric){}
+    aMPIGeometry2d( const aMPIGeometry2d& src):map_(src.map_), metric_(src.metric_){}
+    aMPIGeometry2d& operator=( const aMPIGeometry2d& src){
+        map_=src.map_;
+        metric_=src.metric_;
+    }
+    SharedContainers<host_vec >& map(){return map_;}
+    private:
+    SharedContainers<host_vec > map_;
+    virtual SharedContainers<host_vec > do_compute_metric()const=0;
+};
+
+/**
+ * @brief This is the abstract interface class for a three-dimensional MPIGeometry
+ */
+struct aMPIGeometry3d : public aMPITopology3d
+{
+    typedef MPI_Vector<thrust::host_vector<double> > host_vec;
+    const SharedContainers<host_vec >& map()const{return map_;}
+    SharedContainers<host_vec > compute_metric()const {
+        return do_compute_metric();
+    }
+    ///allow deletion through base class pointer
+    virtual ~aMPIGeometry3d(){}
+    protected:
+    aMPIGeometry3d(const SharedContainers<host_vec >& map): map_(map){}
+    aMPIGeometry3d( const aMPIGeometry2d& src):map_(src.map_){}
+    aMPIGeometry3d& operator=( const aMPIGeometry3d& src){
+        map_=src.map_;
+    }
+    SharedContainers<host_vec >& map(){return map_;}
+    private:
+    SharedContainers<host_vec > map_;
+    virtual SharedContainers<host_vec > do_compute_metric()const=0;
+};
+
+namespace create
+{
+
+SharedContainers<host_vec > metric( const aMPIGeometry2d& g)
+{
+    return g.compute_metric();
+}
+SharedContainers<host_vec > metric( const aMPIGeometry3d& g)
+{
+    return g.compute_metric();
+}
+
+}//namespace create
+
+
+///@}
+
+///@addtogroup basicgrids
+///@{
+
+/**
+ * @brief The mpi version of a cartesian grid
+ */
+struct CartesianMPIGrid2d : public aMPIGeometry2d
+{
+    /**
+     * @copydoc Grid2d::Grid2d()
+     * @param comm a two-dimensional Cartesian communicator
+     * @note the paramateres given in the constructor are global parameters 
+     */
+    CartesianMPIGrid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, MPI_Comm comm): dg::MPIGrid2d( x0, x1, y0, y1, n, Nx, Ny, comm){}
+
+    /**
+     * @copydoc Grid2d::Grid2d()
+     * @param comm a two-dimensional Cartesian communicator
+     * @note the paramateres given in the constructor are global parameters 
+     */
+    CartesianMPIGrid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx, bc bcy, MPI_Comm comm):dg::MPIGrid2d( x0, x1, y0, y1, n, Nx, Ny,bcx, bcy, comm){}
+    CartesianMPIGrid2d( const dg::MPIGrid2d& grid ):MPIGrid2d( grid){}
+};
+
+/**
+ * @brief The mpi version of a cartesian grid
+ */
+struct CartesianMPIGrid3d : public aMPIGeometry3d
+{
+    typedef OrthonormalTag metric_category; 
+
+    /**
+     * @copydoc Grid3d::Grid3d()
+     * @param comm a three-dimensional Cartesian communicator
+     * @note the paramateres given in the constructor are global parameters 
+     */
+    CartesianMPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, MPI_Comm comm): dg::MPIGrid3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, comm){}
+
+    /**
+     * @copydoc Grid3d::Grid3d()
+     * @param comm a three-dimensional Cartesian communicator
+     * @note the paramateres given in the constructor are global parameters 
+     */
+    CartesianMPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz, MPI_Comm comm):dg::MPIGrid3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, bcx, bcy, bcz, comm){}
+    CartesianMPIGrid3d( const dg::MPIGrid3d& grid ): dg::MPIGrid3d( grid){}
+};
+
+///@}
+
+}//namespace dg
diff --git a/inc/dg/geometry/mpi_grids.h b/inc/dg/geometry/mpi_grids.h
deleted file mode 100644
index 9448e9897..000000000
--- a/inc/dg/geometry/mpi_grids.h
+++ /dev/null
@@ -1,62 +0,0 @@
-#pragma once
-
-#include "geometry_traits.h"
-#include "../backend/mpi_grid.h"
-#include "curvilinear_cylindrical.h"
-
-namespace dg
-{
-
-
-///@addtogroup basicgrids
-///@{
-
-/**
- * @brief The mpi version of a cartesian grid
- */
-struct CartesianMPIGrid2d : public aMPIGeometry2d
-{
-    typedef OrthonormalTag metric_category; 
-
-    /**
-     * @copydoc Grid2d::Grid2d()
-     * @param comm a two-dimensional Cartesian communicator
-     * @note the paramateres given in the constructor are global parameters 
-     */
-    CartesianMPIGrid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, MPI_Comm comm): dg::MPIGrid2d( x0, x1, y0, y1, n, Nx, Ny, comm){}
-
-    /**
-     * @copydoc Grid2d::Grid2d()
-     * @param comm a two-dimensional Cartesian communicator
-     * @note the paramateres given in the constructor are global parameters 
-     */
-    CartesianMPIGrid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx, bc bcy, MPI_Comm comm):dg::MPIGrid2d( x0, x1, y0, y1, n, Nx, Ny,bcx, bcy, comm){}
-    CartesianMPIGrid2d( const dg::MPIGrid2d& grid ):MPIGrid2d( grid){}
-};
-
-/**
- * @brief The mpi version of a cartesian grid
- */
-struct CartesianMPIGrid3d : public aMPIGeometry3d
-{
-    typedef OrthonormalTag metric_category; 
-
-    /**
-     * @copydoc Grid3d::Grid3d()
-     * @param comm a three-dimensional Cartesian communicator
-     * @note the paramateres given in the constructor are global parameters 
-     */
-    CartesianMPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, MPI_Comm comm): dg::MPIGrid3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, comm){}
-
-    /**
-     * @copydoc Grid3d::Grid3d()
-     * @param comm a three-dimensional Cartesian communicator
-     * @note the paramateres given in the constructor are global parameters 
-     */
-    CartesianMPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz, MPI_Comm comm):dg::MPIGrid3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, bcx, bcy, bcz, comm){}
-    CartesianMPIGrid3d( const dg::MPIGrid3d& grid ): dg::MPIGrid3d( grid){}
-};
-
-///@}
-
-}//namespace dg
-- 
GitLab


From dfdbce63260bcb8986a2add9bdeaf3e8415f3ac4 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Fri, 4 Aug 2017 00:24:52 +0200
Subject: [PATCH 114/453] correcte inheritance pattern

---
 inc/dg/geometry/geometry.h        |  4 ++++
 inc/dg/geometry/mpi_curvilinear.h |  5 +++--
 inc/dg/geometry/mpi_geometry.h    |  4 ++++
 inc/dg/geometry/refined_grid.h    | 12 ++++++------
 4 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/inc/dg/geometry/geometry.h b/inc/dg/geometry/geometry.h
index f75345eb6..137f8d15f 100644
--- a/inc/dg/geometry/geometry.h
+++ b/inc/dg/geometry/geometry.h
@@ -18,6 +18,8 @@ struct aGeometry2d : public aTopology2d
     SharedContainers<thrust::host_vector<double> > compute_metric()const {
         return do_compute_metric();
     }
+    ///Geometries are cloneable
+    virtual aGeometry2d* clone()const=0;
     ///allow deletion through base class pointer
     virtual ~aGeometry2d(){}
     protected:
@@ -42,6 +44,8 @@ struct aGeometry3d : public aTopology3d
     SharedContainers<thrust::host_vector<double> > compute_metric()const {
         return do_compute_metric();
     }
+    ///Geometries are cloneable
+    virtual aGeometry3d* clone()const=0;
     ///allow deletion through base class pointer
     virtual ~aGeometry3d(){}
     protected:
diff --git a/inc/dg/geometry/mpi_curvilinear.h b/inc/dg/geometry/mpi_curvilinear.h
index 0b8c9415e..48cbbe0fe 100644
--- a/inc/dg/geometry/mpi_curvilinear.h
+++ b/inc/dg/geometry/mpi_curvilinear.h
@@ -6,6 +6,7 @@
 #include "dg/backend/mpi_grid.h"
 #include "dg/backend/mpi_vector.h"
 #include "curvilinear.h"
+#include "generator.h"
 
 
 
@@ -25,7 +26,7 @@ struct CurvilinearMPIGrid2d;
  * @tparam MPIContainer Vector class that holds metric coefficients
  */
 template<class MPIContainer>
-struct CylindricalMPIGrid3d : public dg::MPIGrid3d
+struct CylindricalMPIGrid3d : public dg::aMPIGeometry3d
 {
     typedef dg::CurvilinearPerpTag metric_category; //!< metric tag
     typedef dg::CylindricalMPIGrid2d<LocalContainer> perpendicular_grid; //!< the two-dimensional grid
@@ -152,7 +153,7 @@ struct CylindricalMPIGrid3d : public dg::MPIGrid3d
  * @tparam MPIContainer Vector class that holds metric coefficients
  */
 template<class MPIContainer>
-struct CurvilinearMPIGrid2d : public dg::MPIGrid2d
+struct CurvilinearMPIGrid2d : public dg::aMPIGeometry2d
 {
     typedef dg::CurvilinearPerpTag metric_category; 
     typedef typename MPIContainer::container_type LocalContainer; //!< the local container type
diff --git a/inc/dg/geometry/mpi_geometry.h b/inc/dg/geometry/mpi_geometry.h
index 7f6e1f577..c3bf96d32 100644
--- a/inc/dg/geometry/mpi_geometry.h
+++ b/inc/dg/geometry/mpi_geometry.h
@@ -19,6 +19,8 @@ struct aMPIGeometry2d : public aMPITopology2d
     SharedContainers<host_vec > compute_metric()const {
         return do_compute_metric();
     }
+    ///Geometries are cloneable
+    virtual aMPIGeometry2d* clone()const=0;
     ///allow deletion through base class pointer
     virtual ~aMPIGeometry2d(){}
     protected:
@@ -44,6 +46,8 @@ struct aMPIGeometry3d : public aMPITopology3d
     SharedContainers<host_vec > compute_metric()const {
         return do_compute_metric();
     }
+    ///Geometries are cloneable
+    virtual aMPIGeometry3d* clone()const=0;
     ///allow deletion through base class pointer
     virtual ~aMPIGeometry3d(){}
     protected:
diff --git a/inc/dg/geometry/refined_grid.h b/inc/dg/geometry/refined_grid.h
index 50a1d9ebe..baa913f18 100644
--- a/inc/dg/geometry/refined_grid.h
+++ b/inc/dg/geometry/refined_grid.h
@@ -259,7 +259,7 @@ struct RefinedGrid3d;
  * @deprecated
  * @ingroup grid
  */
-struct RefinedGrid2d : public dg::Grid2d
+struct RefinedGrid2d : public dg::aTopology2d
 {
     /**
      * @brief Refine a corner of a grid
@@ -275,7 +275,7 @@ struct RefinedGrid2d : public dg::Grid2d
     RefinedGrid2d( unsigned node_x, unsigned node_y, unsigned add_x, unsigned add_y, 
             unsigned howmanyX, unsigned howmanyY,
             double x0, double x1, double y0, double y1, 
-            unsigned n, unsigned Nx, unsigned Ny, bc bcx = dg::PER, bc bcy = dg::PER) : dg::Grid2d( x0, x1, y0, y1, n, n_new(Nx, add_x*howmanyX, bcx), n_new(Ny, add_y*howmanyY, bcy), bcx, bcy), 
+            unsigned n, unsigned Nx, unsigned Ny, bc bcx = dg::PER, bc bcy = dg::PER) : dg::aTopology2d( x0, x1, y0, y1, n, n_new(Nx, add_x*howmanyX, bcx), n_new(Ny, add_y*howmanyY, bcy), bcx, bcy), 
         wx_(size()), wy_(size()), absX_(size()), absY_(size()),
         g_assoc_( x0, x1, y0, y1, n, Nx, Ny, bcx, bcy)
     {
@@ -307,7 +307,7 @@ struct RefinedGrid2d : public dg::Grid2d
      */
     RefinedGrid2d( unsigned multiple_x, unsigned multiple_y,
             double x0, double x1, double y0, double y1, unsigned n,
-            unsigned n_old, unsigned Nx, unsigned Ny, bc bcx = dg::PER, bc bcy = dg::PER) : dg::Grid2d( x0, x1, y0, y1, n, multiple_x*Nx, multiple_y*Ny, bcx, bcy), 
+            unsigned n_old, unsigned Nx, unsigned Ny, bc bcx = dg::PER, bc bcy = dg::PER) : dg::aTopology2d( x0, x1, y0, y1, n, multiple_x*Nx, multiple_y*Ny, bcx, bcy), 
         wx_(size()), wy_(size()), absX_(size()), absY_(size()),
         g_assoc_( x0, x1, y0, y1, n_old, Nx, Ny, bcx, bcy)
     {
@@ -394,7 +394,7 @@ struct RefinedGrid2d : public dg::Grid2d
  * @deprecated
  * @ingroup grid
  */
-struct RefinedGrid3d : public dg::Grid3d
+struct RefinedGrid3d : public dg::aTopology3d
 {
     /**
      * @brief Refine a corner of a grid
@@ -410,7 +410,7 @@ struct RefinedGrid3d : public dg::Grid3d
     RefinedGrid3d( unsigned node_x, unsigned node_y, unsigned add_x, unsigned add_y, 
             unsigned howmanyX, unsigned howmanyY,
             double x0, double x1, double y0, double y1, double z0, double z1, 
-            unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = dg::PER, bc bcy = dg::PER, bc bcz = dg::PER) : dg::Grid3d( x0, x1, y0, y1, z0, z1, n, n_new(Nx, add_x*howmanyX, bcx), n_new(Ny, add_y*howmanyY, bcy), Nz, bcx, bcy, bcz), 
+            unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = dg::PER, bc bcy = dg::PER, bc bcz = dg::PER) : dg::aTopology3d( x0, x1, y0, y1, z0, z1, n, n_new(Nx, add_x*howmanyX, bcx), n_new(Ny, add_y*howmanyY, bcy), Nz, bcx, bcy, bcz), 
         wx_(size()), wy_(size()), absX_(size()), absY_(size()),
         g_assoc_( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, bcx, bcy, bcz)
     {
@@ -445,7 +445,7 @@ struct RefinedGrid3d : public dg::Grid3d
             unsigned n,
             unsigned n_old, unsigned Nx, unsigned Ny, unsigned Nz, 
             bc bcx = dg::PER, bc bcy = dg::PER, bc bcz = dg::PER) : 
-        dg::Grid3d( x0, x1, y0, y1, z0, z1, n, multiple_x*Nx, multiple_y*Ny, Nz, bcx, bcy, bcz), 
+        dg::aTopology3d( x0, x1, y0, y1, z0, z1, n, multiple_x*Nx, multiple_y*Ny, Nz, bcx, bcy, bcz), 
         wx_(size()), wy_(size()), absX_(size()), absY_(size()),
         g_assoc_( x0, x1, y0, y1, z0, z1, n_old, Nx, Ny, Nz, bcx, bcy, bcz)
     {
-- 
GitLab


From f9788838628534d99462dbb331a4f7c7af3215e4 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Fri, 4 Aug 2017 02:32:42 -0700
Subject: [PATCH 115/453] made cartesian and cylindrical grids

---
 inc/dg/backend/grid.h          | 25 +------------
 inc/dg/geometry/curvilinear.h  | 17 ++++-----
 inc/dg/geometry/geometry.h     | 67 +++++++++++++++++++++++++++++-----
 inc/dg/geometry/mpi_geometry.h | 53 +++++++++++++++++++++------
 inc/dg/geometry/tensor.h       |  1 +
 5 files changed, 110 insertions(+), 53 deletions(-)

diff --git a/inc/dg/backend/grid.h b/inc/dg/backend/grid.h
index 67d8f80f9..166d692df 100644
--- a/inc/dg/backend/grid.h
+++ b/inc/dg/backend/grid.h
@@ -284,9 +284,6 @@ struct aTopology2d
      */
     const DLT<double>& dlt() const{return gx_.dlt();}
 
-    Grid1d gx()const{return gx_; }
-    Grid1d gy()const{return gy_; }
-
     /**
     * @brief Multiply the number of cells with a given factor
     *
@@ -385,7 +382,7 @@ struct aTopology2d
      * @param bcx boundary condition in x
      * @param bcy boundary condition in y
      */
-    aTopology2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx = PER, bc bcy = PER):
+    aTopology2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx, bc bcy):
         gx_(x0,x1,n,Nx,bcx), gy_(y0,y1,n,Ny,bcy) { }
     /**
      * @brief Construct a 2d grid as the product of two 1d grids
@@ -563,24 +560,6 @@ struct aTopology3d
      * @return 
      */
     const DLT<double>& dlt() const{return gx_.dlt();}
-    /**
-     * @brief Return a 1d grid in x
-     *
-     * @return 
-     */
-    Grid1d gx()const{return gx_; }
-    /**
-     * @brief Return a 1d grid in y
-     *
-     * @return 
-     */
-    Grid1d gy()const{return gy_; }
-    /**
-     * @brief Return a 1d grid in z
-     *
-     * @return 
-     */
-    Grid1d gz()const{return gz_; }
     /**
      * @brief The total number of points
      *
@@ -675,7 +654,7 @@ struct aTopology3d
      * @param bcz boundary condition in z
      * @attention # of polynomial coefficients in z direction is always 1
      */
-    aTopology3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = PER, bc bcy = PER, bc bcz = PER): 
+    aTopology3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz): 
         gx_(x0,x1,n,Nx,bcx),
         gy_(y0,y1,n,Ny,bcy),
         gz_(z0,z1,1,Nz,bcz){}
diff --git a/inc/dg/geometry/curvilinear.h b/inc/dg/geometry/curvilinear.h
index 67e02605a..c8c37a91c 100644
--- a/inc/dg/geometry/curvilinear.h
+++ b/inc/dg/geometry/curvilinear.h
@@ -1,6 +1,6 @@
 #pragma once
 
-#include "dg/backend/grid.h"
+#include "dg/backend/manage.h"
 #include "dg/blas1.h"
 #include "geometry.h"
 #include "geometry_traits.h"
@@ -25,12 +25,11 @@ struct CurvilinearGrid2d;
  @tparam container models aContainer
  */
 template< class container>
-struct CylindricalGrid3d : public dg::aGeometry3d
+struct CurvilinearGrid3d : public dg::aGeometry3d
 {
-    typedef dg::CurvilinearPerpTag metric_category;
     typedef CurvilinearGrid2d<container> perpendicular_grid;
 
-    CylindricalGrid3d( double R0, double R1, double Z0, double Z1, double phi0, double phi1, unsigned n, unsigned NR, unsigned NZ, unsigned Nphi, bc bcr = PER, bc bcz = PER, bc bcphi = PER): 
+    CurvilinearGrid3d( double R0, double R1, double Z0, double Z1, double phi0, double phi1, unsigned n, unsigned NR, unsigned NZ, unsigned Nphi, bc bcr = PER, bc bcz = PER, bc bcphi = PER): 
         dg::Grid3d(0,R1-R0,0,Z1-Z0,phi0,phi1,n,NR,NZ,Nphi,bcR,bcZ,bcphi)
         {
             handle_.set( new ShiftedIdentityGenerator(R0,R1,Z0,Z1));
@@ -48,7 +47,7 @@ struct CylindricalGrid3d : public dg::aGeometry3d
      @param bcx boundary condition in x
      @note the boundary conditions for y and z are set periodic
      */
-    CylindricalGrid3d( const geo::aGenerator& generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx=dg::DIR):
+    CurvilinearGrid3d( const geo::aGenerator& generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx=dg::DIR):
         dg::Grid3d( 0, generator.width(), 0., generator.height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, dg::PER, dg::PER)
     { 
         handle_ = generator;
@@ -152,8 +151,6 @@ struct CylindricalGrid3d : public dg::aGeometry3d
 template< class container>
 struct CurvilinearGrid2d : public dg::aGeometry2d
 {
-    typedef dg::CurvilinearPerpTag metric_category;
-
     CurvilinearGrid2d( double R0, double R1, double Z0, double Z1, unsigned n, unsigned NR, unsigned NZ, unsigned Nphi, bc bcR = PER, bc bcZ = PER): 
         dg::Grid2d(0,R1-R0,0,Z1-Z0,n,NR,NZ,bcR,bcZ), handle_(new ShiftedIdentityGenerator(R0,R1,Z0,Z1))
         {
@@ -172,7 +169,7 @@ struct CurvilinearGrid2d : public dg::aGeometry2d
     {
         construct( n,Nx,Ny);
     }
-    CurvilinearGrid2d( const CylindricalGrid3d<container>& g):
+    CurvilinearGrid2d( const CurvilinearGrid3d<container>& g):
         dg::Grid2d( g.x0(), g.x1(), g.y0(), g.y1(), g.n(), g.Nx(), g.Ny(), g.bcx(), g.bcy()), handle_(g.generator())
     {
         r_ = g.r();
@@ -211,14 +208,14 @@ struct CurvilinearGrid2d : public dg::aGeometry2d
     }
     void construct( unsigned n, unsigned Nx, unsigned Ny)
     {
-        CylindricalGrid3d<container> g( handle_, n,Nx,Ny,1,bcx());
+        CurvilinearGrid3d<container> g( handle_, n,Nx,Ny,1,bcx());
         r_=g.r(), z_=g.z(), xr_=g.xr(), xz_=g.xz(), yr_=g.yr(), yz_=g.yz();
         g_xx_=g.g_xx(), g_xy_=g.g_xy(), g_yy_=g.g_yy();
         vol2d_=g.perpVol();
     }
     thrust::host_vector<double> r_, z_, xr_, xz_, yr_, yz_;
     container g_xx_, g_xy_, g_yy_, vol2d_;
-    dg::Hanle<geo::aGenerator> handle_;
+    dg::Handle<geo::aGenerator> handle_;
 };
 
 ///@}
diff --git a/inc/dg/geometry/geometry.h b/inc/dg/geometry/geometry.h
index 137f8d15f..5bdd0b637 100644
--- a/inc/dg/geometry/geometry.h
+++ b/inc/dg/geometry/geometry.h
@@ -23,7 +23,11 @@ struct aGeometry2d : public aTopology2d
     ///allow deletion through base class pointer
     virtual ~aGeometry2d(){}
     protected:
-    aGeometry2d(const SharedContainers<thrust::host_vector<double> >& map, const SharedContainers<container >& metric): map_(map), metric_(metric){}
+    /*!
+     * @copydoc aTopology2d::aTopology2d()
+     * @note the default coordinate map will be the identity 
+     */
+    aGeometry2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx, bc bcy):aTopology2d( x0,x1,y0,y1,n,Nx,Ny,bcx,bcy),map_(3){}
     aGeometry2d( const aGeometry2d& src):map_(src.map_), metric_(src.metric_){}
     aGeometry2d& operator=( const aGeometry2d& src){
         map_=src.map_;
@@ -49,8 +53,12 @@ struct aGeometry3d : public aTopology3d
     ///allow deletion through base class pointer
     virtual ~aGeometry3d(){}
     protected:
-    aGeometry3d(const SharedContainers<thrust::host_vector<double> >& map): map_(map){}
-    aGeometry3d( const aGeometry2d& src):map_(src.map_){}
+    /*!
+     * @copydoc aTopology3d::aTopology3d()
+     * @note the default coordinate map will be the identity 
+     */
+    aGeometry3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz): aTopology3d(x0,x1,y0,y1,z0,z1,n,Nx,Ny,Nz,bcx,bcy,bcz){}
+    aGeometry3d( const aGeometry3d& src):map_(src.map_){}
     aGeometry3d& operator=( const aGeometry3d& src){
         map_=src.map_;
     }
@@ -83,15 +91,22 @@ SharedContainers<thrust::host_vector<double> > metric( const aGeometry3d& g)
  */
 struct CartesianGrid2d: public dg::aGeometry2d
 {
-    typedef OrthonormalTag metric_category; 
     ///@copydoc Grid2d::Grid2d()
-    CartesianGrid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx = PER, bc bcy = PER): dg::Grid2d(x0,x1,y0,y1,n,Nx,Ny,bcx,bcy){}
+    CartesianGrid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx = PER, bc bcy = PER): dg::aGeometry2d(x0,x1,y0,y1,n,Nx,Ny,bcx,bcy){}
     /**
      * @brief Construct from existing topology
      *
      * @param grid existing grid class
      */
     CartesianGrid2d( const dg::Grid2d& grid):dg::Grid2d(grid){}
+    virtual CartesianGrid2d* clone()const{return new CartesianGrid2d(*this);}
+    private:
+    virtual SharedContainers<thrust::host_vector<double> > do_compute_metric()const{
+        return SharedContainers<thrust::host_vector<double> >( 3);
+    }
+    virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny){
+        aTopology2d::do_set(new_n,new_Nx,new_Ny);
+    }
 };
 
 /**
@@ -99,16 +114,50 @@ struct CartesianGrid2d: public dg::aGeometry2d
  */
 struct CartesianGrid3d: public dg::aGeometry3d
 {
-    typedef OrthonormalTag metric_category; 
     ///@copydoc Grid3d::Grid3d()
-    CartesianGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = PER, bc bcy = PER, bc bcz = PER): dg::Grid3d(x0,x1,y0,y1,z0,z1,n,Nx,Ny,Nz,bcx,bcy,bcz){}
+    CartesianGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = PER, bc bcy = PER, bc bcz = PER): dg::aGeometry3d(x0,x1,y0,y1,z0,z1,n,Nx,Ny,Nz,bcx,bcy,bcz){}
     /**
-     * @brief Construct from existing topology
-     *
+     * @brief Implicit type conversion from Grid3d
      * @param grid existing grid class
      */
     CartesianGrid3d( const dg::Grid3d& grid):dg::Grid3d(grid){}
+    virtual CartesianGrid3d* clone()const{return new CartesianGrid3d(*this);}
+    private:
+    virtual SharedContainers<thrust::host_vector<double> > do_compute_metric()const{
+        return SharedContainers<thrust::host_vector<double> >( 3);
+    }
+    virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz){
+        aTopology2d::do_set(new_n,new_Nx,new_Ny,new_Nz);
+    }
 };
 
+/**
+ * @brief three-dimensional Grid with Cylindrical metric
+ */
+struct CylindricalGrid3d: public dg::aGeometry3d
+{
+    CylindricalGrid3d( double R0, double R1, double Z0, double Z1, double phi0, double phi1, unsigned n, unsigned NR, unsigned NZ, unsigned Nphi, bc bcR, bc bcZ, bc bcphi = PER): dg::aGeometry3d(R0,R1,Z0,Z1,phi0,phi1,n,NR,NZ,Nphi,bcR,bcZ,bcphi){}
+    virtual CylindricalGrid3d* clone()const{return new CylindricalGrid3d(*this);}
+    private:
+    virtual SharedContainers<thrust::host_vector<double> > do_compute_metric()const{
+
+        std::vector<thrust::host_vector<double> > values(2, size());
+        thrust::host_vector<double> R = dg::evaluate(dg::coo1, *this);
+        unsigned size2d = n()*n()*Nx()*Ny();
+        for( unsigned i = 0; i<Nz(); i++)
+        for( unsigned j = 0; j<size2d; j++)
+        {
+            unsigned idx = i*size2d+j;
+            values[1][idx] = R[j];
+            values[0][idx] = 1./R[j]/R[j];
+        }
+        Operator<int> mat_idx(3,-1); mat_idx(2,2) = 0;
+        std::vector<int> vec_idx(3,-1); vec_idx[0] = 1;
+        return SharedContainers<thrust::host_vector<double> >( mat_idx, vec_idx, values);
+    }
+    virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz){
+        aTopology2d::do_set(new_n,new_Nx,new_Ny,new_Nz);
+    }
+};
 ///@}
 } //namespace dg
diff --git a/inc/dg/geometry/mpi_geometry.h b/inc/dg/geometry/mpi_geometry.h
index c3bf96d32..0f28442ae 100644
--- a/inc/dg/geometry/mpi_geometry.h
+++ b/inc/dg/geometry/mpi_geometry.h
@@ -92,15 +92,15 @@ struct CartesianMPIGrid2d : public aMPIGeometry2d
      * @param comm a two-dimensional Cartesian communicator
      * @note the paramateres given in the constructor are global parameters 
      */
-    CartesianMPIGrid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, MPI_Comm comm): dg::MPIGrid2d( x0, x1, y0, y1, n, Nx, Ny, comm){}
+    CartesianMPIGrid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, MPI_Comm comm): aMPIGeometry2d( x0, x1, y0, y1, n, Nx, Ny, dg::PER,dg::PER,comm){}
 
     /**
      * @copydoc Grid2d::Grid2d()
      * @param comm a two-dimensional Cartesian communicator
      * @note the paramateres given in the constructor are global parameters 
      */
-    CartesianMPIGrid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx, bc bcy, MPI_Comm comm):dg::MPIGrid2d( x0, x1, y0, y1, n, Nx, Ny,bcx, bcy, comm){}
-    CartesianMPIGrid2d( const dg::MPIGrid2d& grid ):MPIGrid2d( grid){}
+    CartesianMPIGrid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx, bc bcy, MPI_Comm comm):dg::aMPIGeometry2d( x0, x1, y0, y1, n, Nx, Ny,bcx, bcy, comm){}
+    CartesianMPIGrid2d( const dg::MPIGrid2d& g): aMPIGeometry2d( g.x0(),g.x1(),g.y0(),g.y1(),g.n(),g.Nx(),g.Ny(),g.bcx(),g.bcy(),g.comm()){}
 };
 
 /**
@@ -108,22 +108,53 @@ struct CartesianMPIGrid2d : public aMPIGeometry2d
  */
 struct CartesianMPIGrid3d : public aMPIGeometry3d
 {
-    typedef OrthonormalTag metric_category; 
-
     /**
-     * @copydoc Grid3d::Grid3d()
+     * @copydoc MPIGrid3d::MPIGrid3d()
      * @param comm a three-dimensional Cartesian communicator
      * @note the paramateres given in the constructor are global parameters 
      */
-    CartesianMPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, MPI_Comm comm): dg::MPIGrid3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, comm){}
+    CartesianMPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, MPI_Comm comm): aMPIGeometry3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, dg::PER,dg::PER,dg::PER comm){}
 
     /**
-     * @copydoc Grid3d::Grid3d()
+     * @copydoc MPIGrid3d::MPIGrid3d()
      * @param comm a three-dimensional Cartesian communicator
-     * @note the paramateres given in the constructor are global parameters 
+     * @note the paramaters given in the constructor are global parameters 
      */
-    CartesianMPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz, MPI_Comm comm):dg::MPIGrid3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, bcx, bcy, bcz, comm){}
-    CartesianMPIGrid3d( const dg::MPIGrid3d& grid ): dg::MPIGrid3d( grid){}
+    CartesianMPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz, MPI_Comm comm):aMPIGeometry3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, bcx, bcy, bcz, comm){}
+
+    CartesianMPIGrid3d( const dg::MPIGrid3d& g): aMPIGeometry3d( g.x0(),g.x1(),g.y0(),g.y1(),g.z0(),g.z1(),g.n(),g.Nx(),g.Ny(),g.Nz(),g.bcx(),g.bcy(),g.bcz(),g.comm()){}
+};
+
+/**
+ * @brief three-dimensional Grid with Cylindrical metric
+ */
+struct CylindricalMPIGrid3d: public aMPIGeometry3d
+{
+    CylindricalMPIGrid3d( double R0, double R1, double Z0, double Z1, double phi0, double phi1, unsigned n, unsigned NR, unsigned NZ, unsigned Nphi, bc bcR, bc bcZ, bc bcphi, MPI_Comm comm): dg::aGeometry3d(R0,R1,Z0,Z1,phi0,phi1,n,NR,NZ,Nphi,bcR,bcZ,bcphi,comm){}
+    ///take PER for bcphi
+    CylindricalMPIGrid3d( double R0, double R1, double Z0, double Z1, double phi0, double phi1, unsigned n, unsigned NR, unsigned NZ, unsigned Nphi, bc bcR, bc bcZ, MPI_Comm comm): dg::aGeometry3d(R0,R1,Z0,Z1,phi0,phi1,n,NR,NZ,Nphi,bcR,bcZ,dg::PER,comm){}
+
+    virtual CylindricalMPIGrid3d* clone()const{return new CylindricalMPIGrid3d(*this);}
+    private:
+    virtual SharedContainers<thrust::host_vector<double> > do_compute_metric()const{
+
+        std::vector<MPI_Vector<thrust::host_vector<double> > > values(2, size());
+        MPI_Vector<thrust::host_vector<double> > R = dg::evaluate(dg::coo1, *this);
+        unsigned size2d = n()*n()*Nx()*Ny();
+        for( unsigned i = 0; i<Nz(); i++)
+        for( unsigned j = 0; j<size2d; j++)
+        {
+            unsigned idx = i*size2d+j;
+            values[1].data()[idx] = R.data()[j];
+            values[0].data()[idx] = 1./R.data()[j]/R.data()[j];
+        }
+        Operator<int> mat_idx(3,-1); mat_idx(2,2) = 0;
+        std::vector<int> vec_idx(3,-1); vec_idx[0] = 1;
+        return SharedContainers<thrust::host_vector<double> >( mat_idx, vec_idx, values);
+    }
+    virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz){
+        aMPITopology2d::do_set(new_n,new_Nx,new_Ny,new_Nz);
+    }
 };
 
 ///@}
diff --git a/inc/dg/geometry/tensor.h b/inc/dg/geometry/tensor.h
index e07c93626..daf7bda39 100644
--- a/inc/dg/geometry/tensor.h
+++ b/inc/dg/geometry/tensor.h
@@ -25,6 +25,7 @@ template<class container>
 struct SharedContainers
 {
     SharedContainers( ) {}
+    SharedContainers( unsigned size):mat_idx_(size,-1), vec_idx_(size,-1){}
 
     /**
     * @brief 
-- 
GitLab


From 68b768dfe818ee10299f88fd8ba423924246059d Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Fri, 4 Aug 2017 02:46:42 -0700
Subject: [PATCH 116/453] renamed headers in geometry

---
 inc/dg/geometry.h                             | 171 ------------
 inc/dg/geometry/base.h                        | 163 +++++++++++
 inc/dg/geometry/geometry.h                    | 263 +++++++++---------
 .../geometry/{mpi_geometry.h => mpi_base.h}   |   0
 4 files changed, 298 insertions(+), 299 deletions(-)
 delete mode 100644 inc/dg/geometry.h
 create mode 100644 inc/dg/geometry/base.h
 rename inc/dg/geometry/{mpi_geometry.h => mpi_base.h} (100%)

diff --git a/inc/dg/geometry.h b/inc/dg/geometry.h
deleted file mode 100644
index 78a4045be..000000000
--- a/inc/dg/geometry.h
+++ /dev/null
@@ -1,171 +0,0 @@
-#pragma once
-
-#include <cassert>
-#include "thrust/host_vector.h"
-#include "backend/evaluation.cuh"
-#include "backend/weights.cuh"
-#ifdef MPI_VERSION
-#include "backend/mpi_vector.h"
-#include "backend/mpi_evaluation.h"
-#include "backend/mpi_precon.h"
-#endif//MPI_VERSION
-#include "geometry/geometry_traits.h"
-#include "geometry/cartesian.h"
-#include "geometry/curvilinear.h"
-#include "geometry/cartesianX.h"
-#ifdef MPI_VERSION
-#include "geometry/mpi_grids.h"
-#include "geometry/mpi_curvilinear.h"
-#endif//MPI_VERSION
-#include "geometry/tensor.h"
-#include "geometry/transform.h"
-
-
-/*!@file 
- *
- * geometry functions
- */
-
-namespace dg{
-
-/*! @brief Geometry routines 
- *
- * @ingroup geometry
- * Only those routines that are actually called need to be implemented.
- * Don't forget to specialize in the dg namespace.
- */
-namespace geo{
-///@addtogroup geometry
-///@{
-
-/**
- * @brief Multiply the input with the volume element without the dG weights!
- *
- * Computes \f$ f = \sqrt{g}f\f$ 
- * @tparam container container class 
- * @param inout input (contains result on output)
- * @param metric the metric object
- */
-template<class container>
-void multiplyVolume( container& inout, const SharedContainers<container>& metric)
-{
-    if( metric.isSet(0))
-        dg::blas1::pointwiseDot( metric.getValue(0), inout, inout);
-}
-
-/**
- * @brief Divide the input vector with the volume element without the dG weights
- *
- * Computes \f$ v = v/ \sqrt{g}\f$ 
- * @tparam container container class 
- * @param inout input (contains result on output)
- * @param metric the metric
- */
-template<class container>
-void divideVolume( container& inout, const SharedContainers<container>& metric)
-{
-    if( metric.isSet(0))
-        dg::blas1::pointwiseDivide( inout, metric.getValue(0), inout);
-}
-
-/**
- * @brief Raises the index of a covariant vector with the help of the projection tensor in 2d
- *
- * Compute \f$ v^i = g^{ij}v_j\f$ for \f$ i,j\in \{1,2\}\f$ in the first two dimensions 
- * @tparam container the container class
- * @param covX (input) covariant first component (undefined content on output)
- * @param covY (input) covariant second component (undefined content on output)
- * @param contraX (output) contravariant first component 
- * @param contraY (output) contravariant second component 
- * @param metric The metric
- * @note no alias allowed 
- */
-template<class container>
-void raisePerpIndex( container& covX, container& covY, container& contraX, container& contraY, const SharedContainers<container>& metric)
-{
-    assert( &covX != &contraX);
-    assert( &covY != &contraY);
-    assert( &covY != &covX);
-    dg::detail::multiply( metric, covX, covY, contraX, contraY);
-}
-
-/**
- * @brief Multiplies the two-dimensional volume element
- *
- * Computes \f$ f = \sqrt{g_\perp}f\f$ where the perpendicualar volume is computed from the 2x2 submatrix of g in the first two coordinates.
- * @tparam container the container class
- * @param inout input (contains result on output)
- * @param metric The metric  
- * @note if metric is two-dimensional this function will have the same result as multiplyVolume()
- */
-template<class container>
-void multiplyPerpVolume( container& inout, const SharedContainers<container>& metric)
-{
-    if( metric.isSet(1))
-        dg::blas1::pointwiseDot( metric.getValue(1), inout, inout);
-}
-
-/**
- * @brief Divides the two-dimensional volume element
- *
- * Computes \f$ f = f /\sqrt{g_\perp}\f$ where the perpendicualar volume is computed from the 2x2 submatrix of g in the first two coordinates.
- * @tparam container the container class
- * @param inout input (contains result on output)
- * @param metric The metric tensor
- * @note if metric is two-dimensional this function will have the same result as divideVolume()
- */
-template<class container>
-void dividePerpVolume( container& inout, const SharedContainers<container>& metric)
-{
-    if( metric.isSet(1))
-        dg::blas1::pointwiseDivide( metric.getValue(1), inout, inout);
-}
-
-
-///@}
-}//namespace geo
-
-namespace create{
-///@addtogroup geometry
-///@{
-
-/**
- * @brief Create the volume element on the grid (including weights!!)
- *
- * This is the same as the weights multiplied by the volume form \f$ \sqrt{g}\f$
- * @tparam Geometry any Geometry class
- * @param g Geometry object
- *
- * @return  The volume form
- */
-template< class Geometry>
-typename HostVec< typename TopologyTraits<Geometry>::memory_category>::host_vector volume( const Geometry& g)
-{
-    typedef typename HostVec< typename TopologyTraits<Geometry>::memory_category>::host_vector host_vector;
-    SharedContainers<host_vector> metric = g.compute_metric();
-    host_vector temp = dg::create::weights( g);
-    dg::geo::multiplyVolume( temp, metric);
-    return temp;
-}
-
-/**
- * @brief Create the inverse volume element on the grid (including weights!!)
- *
- * This is the same as the inv_weights divided by the volume form \f$ \sqrt{g}\f$
- * @tparam Geometry any Geometry class
- * @param g Geometry object
- *
- * @return  The inverse volume form
- */
-template< class Geometry>
-typename HostVec< typename TopologyTraits<Geometry>::memory_category>::host_vector inv_volume( const Geometry& g)
-{
-    typedef typename HostVec< typename TopologyTraits<Geometry>::memory_category>::host_vector host_vector;
-    host_vector temp = volume(g);
-    dg::blas1::transform(temp,temp,dg::INVERT<double>());
-    return temp;
-}
-
-///@}
-}//namespace create
-}//namespace dg
diff --git a/inc/dg/geometry/base.h b/inc/dg/geometry/base.h
new file mode 100644
index 000000000..70ce8d2c9
--- /dev/null
+++ b/inc/dg/geometry/base.h
@@ -0,0 +1,163 @@
+#pragma once
+
+#include "../backend/grid.h"
+#include "tensor.h"
+
+namespace dg
+{
+
+///@addtogroup geometry
+///@{
+
+/**
+ * @brief This is the abstract interface class for a two-dimensional Geometry
+ */
+struct aGeometry2d : public aTopology2d
+{
+    const SharedContainers<thrust::host_vector<double> >& map()const{return map_;}
+    SharedContainers<thrust::host_vector<double> > compute_metric()const {
+        return do_compute_metric();
+    }
+    ///Geometries are cloneable
+    virtual aGeometry2d* clone()const=0;
+    ///allow deletion through base class pointer
+    virtual ~aGeometry2d(){}
+    protected:
+    /*!
+     * @copydoc aTopology2d::aTopology2d()
+     * @note the default coordinate map will be the identity 
+     */
+    aGeometry2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx, bc bcy):aTopology2d( x0,x1,y0,y1,n,Nx,Ny,bcx,bcy),map_(3){}
+    aGeometry2d( const aGeometry2d& src):map_(src.map_), metric_(src.metric_){}
+    aGeometry2d& operator=( const aGeometry2d& src){
+        map_=src.map_;
+        metric_=src.metric_;
+    }
+    SharedContainers<thrust::host_vector<double> >& map(){return map_;}
+    private:
+    SharedContainers<thrust::host_vector<double> > map_;
+    virtual SharedContainers<thrust::host_vector<double> > do_compute_metric()const=0;
+};
+
+/**
+ * @brief This is the abstract interface class for a two-dimensional Geometry
+ */
+struct aGeometry3d : public aTopology3d
+{
+    const SharedContainers<thrust::host_vector<double> >& map()const{return map_;}
+    SharedContainers<thrust::host_vector<double> > compute_metric()const {
+        return do_compute_metric();
+    }
+    ///Geometries are cloneable
+    virtual aGeometry3d* clone()const=0;
+    ///allow deletion through base class pointer
+    virtual ~aGeometry3d(){}
+    protected:
+    /*!
+     * @copydoc aTopology3d::aTopology3d()
+     * @note the default coordinate map will be the identity 
+     */
+    aGeometry3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz): aTopology3d(x0,x1,y0,y1,z0,z1,n,Nx,Ny,Nz,bcx,bcy,bcz){}
+    aGeometry3d( const aGeometry3d& src):map_(src.map_){}
+    aGeometry3d& operator=( const aGeometry3d& src){
+        map_=src.map_;
+    }
+    SharedContainers<thrust::host_vector<double> >& map(){return map_;}
+    private:
+    SharedContainers<thrust::host_vector<double> > map_;
+    virtual SharedContainers<thrust::host_vector<double> > do_compute_metric()const=0;
+};
+
+namespace create
+{
+
+SharedContainers<thrust::host_vector<double> > metric( const aGeometry2d& g)
+{
+    return g.compute_metric();
+}
+SharedContainers<thrust::host_vector<double> > metric( const aGeometry3d& g)
+{
+    return g.compute_metric();
+}
+
+}//namespace create
+
+///@}
+///@addtogroup basicgrids
+///@{
+
+/**
+ * @brief two-dimensional Grid with Cartesian metric
+ */
+struct CartesianGrid2d: public dg::aGeometry2d
+{
+    ///@copydoc Grid2d::Grid2d()
+    CartesianGrid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx = PER, bc bcy = PER): dg::aGeometry2d(x0,x1,y0,y1,n,Nx,Ny,bcx,bcy){}
+    /**
+     * @brief Construct from existing topology
+     *
+     * @param grid existing grid class
+     */
+    CartesianGrid2d( const dg::Grid2d& grid):dg::Grid2d(grid){}
+    virtual CartesianGrid2d* clone()const{return new CartesianGrid2d(*this);}
+    private:
+    virtual SharedContainers<thrust::host_vector<double> > do_compute_metric()const{
+        return SharedContainers<thrust::host_vector<double> >( 3);
+    }
+    virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny){
+        aTopology2d::do_set(new_n,new_Nx,new_Ny);
+    }
+};
+
+/**
+ * @brief three-dimensional Grid with Cartesian metric
+ */
+struct CartesianGrid3d: public dg::aGeometry3d
+{
+    ///@copydoc Grid3d::Grid3d()
+    CartesianGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = PER, bc bcy = PER, bc bcz = PER): dg::aGeometry3d(x0,x1,y0,y1,z0,z1,n,Nx,Ny,Nz,bcx,bcy,bcz){}
+    /**
+     * @brief Implicit type conversion from Grid3d
+     * @param grid existing grid class
+     */
+    CartesianGrid3d( const dg::Grid3d& grid):dg::Grid3d(grid){}
+    virtual CartesianGrid3d* clone()const{return new CartesianGrid3d(*this);}
+    private:
+    virtual SharedContainers<thrust::host_vector<double> > do_compute_metric()const{
+        return SharedContainers<thrust::host_vector<double> >( 3);
+    }
+    virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz){
+        aTopology2d::do_set(new_n,new_Nx,new_Ny,new_Nz);
+    }
+};
+
+/**
+ * @brief three-dimensional Grid with Cylindrical metric
+ */
+struct CylindricalGrid3d: public dg::aGeometry3d
+{
+    CylindricalGrid3d( double R0, double R1, double Z0, double Z1, double phi0, double phi1, unsigned n, unsigned NR, unsigned NZ, unsigned Nphi, bc bcR, bc bcZ, bc bcphi = PER): dg::aGeometry3d(R0,R1,Z0,Z1,phi0,phi1,n,NR,NZ,Nphi,bcR,bcZ,bcphi){}
+    virtual CylindricalGrid3d* clone()const{return new CylindricalGrid3d(*this);}
+    private:
+    virtual SharedContainers<thrust::host_vector<double> > do_compute_metric()const{
+
+        std::vector<thrust::host_vector<double> > values(2, size());
+        thrust::host_vector<double> R = dg::evaluate(dg::coo1, *this);
+        unsigned size2d = n()*n()*Nx()*Ny();
+        for( unsigned i = 0; i<Nz(); i++)
+        for( unsigned j = 0; j<size2d; j++)
+        {
+            unsigned idx = i*size2d+j;
+            values[1][idx] = R[j];
+            values[0][idx] = 1./R[j]/R[j];
+        }
+        Operator<int> mat_idx(3,-1); mat_idx(2,2) = 0;
+        std::vector<int> vec_idx(3,-1); vec_idx[0] = 1;
+        return SharedContainers<thrust::host_vector<double> >( mat_idx, vec_idx, values);
+    }
+    virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz){
+        aTopology2d::do_set(new_n,new_Nx,new_Ny,new_Nz);
+    }
+};
+///@}
+} //namespace dg
diff --git a/inc/dg/geometry/geometry.h b/inc/dg/geometry/geometry.h
index 5bdd0b637..b91273d40 100644
--- a/inc/dg/geometry/geometry.h
+++ b/inc/dg/geometry/geometry.h
@@ -1,163 +1,170 @@
 #pragma once
 
-#include "dg/backend/grid.h"
+#include <cassert>
+#include "thrust/host_vector.h"
+#include "../backend/evaluation.cuh"
+#include "../backend/weights.cuh"
+#ifdef MPI_VERSION
+#include "../backend/mpi_vector.h"
+#include "../backend/mpi_evaluation.h"
+#include "../backend/mpi_precon.h"
+#endif//MPI_VERSION
+#include "base.h"
+#include "curvilinear.h"
+#include "cartesianX.h"
+#ifdef MPI_VERSION
+#include "mpi_base.h"
+#include "mpi_curvilinear.h"
+#endif//MPI_VERSION
 #include "tensor.h"
+#include "transform.h"
 
-namespace dg
-{
 
+/*!@file 
+ *
+ * geometry functions
+ */
+
+namespace dg{
+
+/*! @brief Geometry routines 
+ *
+ * @ingroup geometry
+ * Only those routines that are actually called need to be implemented.
+ * Don't forget to specialize in the dg namespace.
+ */
+namespace geo{
 ///@addtogroup geometry
 ///@{
 
 /**
- * @brief This is the abstract interface class for a two-dimensional Geometry
+ * @brief Multiply the input with the volume element without the dG weights!
+ *
+ * Computes \f$ f = \sqrt{g}f\f$ 
+ * @tparam container container class 
+ * @param inout input (contains result on output)
+ * @param metric the metric object
  */
-struct aGeometry2d : public aTopology2d
+template<class container>
+void multiplyVolume( container& inout, const SharedContainers<container>& metric)
 {
-    const SharedContainers<thrust::host_vector<double> >& map()const{return map_;}
-    SharedContainers<thrust::host_vector<double> > compute_metric()const {
-        return do_compute_metric();
-    }
-    ///Geometries are cloneable
-    virtual aGeometry2d* clone()const=0;
-    ///allow deletion through base class pointer
-    virtual ~aGeometry2d(){}
-    protected:
-    /*!
-     * @copydoc aTopology2d::aTopology2d()
-     * @note the default coordinate map will be the identity 
-     */
-    aGeometry2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx, bc bcy):aTopology2d( x0,x1,y0,y1,n,Nx,Ny,bcx,bcy),map_(3){}
-    aGeometry2d( const aGeometry2d& src):map_(src.map_), metric_(src.metric_){}
-    aGeometry2d& operator=( const aGeometry2d& src){
-        map_=src.map_;
-        metric_=src.metric_;
-    }
-    SharedContainers<thrust::host_vector<double> >& map(){return map_;}
-    private:
-    SharedContainers<thrust::host_vector<double> > map_;
-    virtual SharedContainers<thrust::host_vector<double> > do_compute_metric()const=0;
-};
+    if( metric.isSet(0))
+        dg::blas1::pointwiseDot( metric.getValue(0), inout, inout);
+}
 
 /**
- * @brief This is the abstract interface class for a two-dimensional Geometry
+ * @brief Divide the input vector with the volume element without the dG weights
+ *
+ * Computes \f$ v = v/ \sqrt{g}\f$ 
+ * @tparam container container class 
+ * @param inout input (contains result on output)
+ * @param metric the metric
  */
-struct aGeometry3d : public aTopology3d
+template<class container>
+void divideVolume( container& inout, const SharedContainers<container>& metric)
 {
-    const SharedContainers<thrust::host_vector<double> >& map()const{return map_;}
-    SharedContainers<thrust::host_vector<double> > compute_metric()const {
-        return do_compute_metric();
-    }
-    ///Geometries are cloneable
-    virtual aGeometry3d* clone()const=0;
-    ///allow deletion through base class pointer
-    virtual ~aGeometry3d(){}
-    protected:
-    /*!
-     * @copydoc aTopology3d::aTopology3d()
-     * @note the default coordinate map will be the identity 
-     */
-    aGeometry3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz): aTopology3d(x0,x1,y0,y1,z0,z1,n,Nx,Ny,Nz,bcx,bcy,bcz){}
-    aGeometry3d( const aGeometry3d& src):map_(src.map_){}
-    aGeometry3d& operator=( const aGeometry3d& src){
-        map_=src.map_;
-    }
-    SharedContainers<thrust::host_vector<double> >& map(){return map_;}
-    private:
-    SharedContainers<thrust::host_vector<double> > map_;
-    virtual SharedContainers<thrust::host_vector<double> > do_compute_metric()const=0;
-};
-
-namespace create
+    if( metric.isSet(0))
+        dg::blas1::pointwiseDivide( inout, metric.getValue(0), inout);
+}
+
+/**
+ * @brief Raises the index of a covariant vector with the help of the projection tensor in 2d
+ *
+ * Compute \f$ v^i = g^{ij}v_j\f$ for \f$ i,j\in \{1,2\}\f$ in the first two dimensions 
+ * @tparam container the container class
+ * @param covX (input) covariant first component (undefined content on output)
+ * @param covY (input) covariant second component (undefined content on output)
+ * @param contraX (output) contravariant first component 
+ * @param contraY (output) contravariant second component 
+ * @param metric The metric
+ * @note no alias allowed 
+ */
+template<class container>
+void raisePerpIndex( container& covX, container& covY, container& contraX, container& contraY, const SharedContainers<container>& metric)
 {
+    assert( &covX != &contraX);
+    assert( &covY != &contraY);
+    assert( &covY != &covX);
+    dg::detail::multiply( metric, covX, covY, contraX, contraY);
+}
 
-SharedContainers<thrust::host_vector<double> > metric( const aGeometry2d& g)
+/**
+ * @brief Multiplies the two-dimensional volume element
+ *
+ * Computes \f$ f = \sqrt{g_\perp}f\f$ where the perpendicualar volume is computed from the 2x2 submatrix of g in the first two coordinates.
+ * @tparam container the container class
+ * @param inout input (contains result on output)
+ * @param metric The metric  
+ * @note if metric is two-dimensional this function will have the same result as multiplyVolume()
+ */
+template<class container>
+void multiplyPerpVolume( container& inout, const SharedContainers<container>& metric)
 {
-    return g.compute_metric();
+    if( metric.isSet(1))
+        dg::blas1::pointwiseDot( metric.getValue(1), inout, inout);
 }
-SharedContainers<thrust::host_vector<double> > metric( const aGeometry3d& g)
+
+/**
+ * @brief Divides the two-dimensional volume element
+ *
+ * Computes \f$ f = f /\sqrt{g_\perp}\f$ where the perpendicualar volume is computed from the 2x2 submatrix of g in the first two coordinates.
+ * @tparam container the container class
+ * @param inout input (contains result on output)
+ * @param metric The metric tensor
+ * @note if metric is two-dimensional this function will have the same result as divideVolume()
+ */
+template<class container>
+void dividePerpVolume( container& inout, const SharedContainers<container>& metric)
 {
-    return g.compute_metric();
+    if( metric.isSet(1))
+        dg::blas1::pointwiseDivide( metric.getValue(1), inout, inout);
 }
 
-}//namespace create
 
 ///@}
-///@addtogroup basicgrids
+}//namespace geo
+
+namespace create{
+///@addtogroup geometry
 ///@{
 
 /**
- * @brief two-dimensional Grid with Cartesian metric
+ * @brief Create the volume element on the grid (including weights!!)
+ *
+ * This is the same as the weights multiplied by the volume form \f$ \sqrt{g}\f$
+ * @tparam Geometry any Geometry class
+ * @param g Geometry object
+ *
+ * @return  The volume form
  */
-struct CartesianGrid2d: public dg::aGeometry2d
+template< class Geometry>
+typename HostVec< typename TopologyTraits<Geometry>::memory_category>::host_vector volume( const Geometry& g)
 {
-    ///@copydoc Grid2d::Grid2d()
-    CartesianGrid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx = PER, bc bcy = PER): dg::aGeometry2d(x0,x1,y0,y1,n,Nx,Ny,bcx,bcy){}
-    /**
-     * @brief Construct from existing topology
-     *
-     * @param grid existing grid class
-     */
-    CartesianGrid2d( const dg::Grid2d& grid):dg::Grid2d(grid){}
-    virtual CartesianGrid2d* clone()const{return new CartesianGrid2d(*this);}
-    private:
-    virtual SharedContainers<thrust::host_vector<double> > do_compute_metric()const{
-        return SharedContainers<thrust::host_vector<double> >( 3);
-    }
-    virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny){
-        aTopology2d::do_set(new_n,new_Nx,new_Ny);
-    }
-};
+    typedef typename HostVec< typename TopologyTraits<Geometry>::memory_category>::host_vector host_vector;
+    SharedContainers<host_vector> metric = g.compute_metric();
+    host_vector temp = dg::create::weights( g);
+    dg::geo::multiplyVolume( temp, metric);
+    return temp;
+}
 
 /**
- * @brief three-dimensional Grid with Cartesian metric
+ * @brief Create the inverse volume element on the grid (including weights!!)
+ *
+ * This is the same as the inv_weights divided by the volume form \f$ \sqrt{g}\f$
+ * @tparam Geometry any Geometry class
+ * @param g Geometry object
+ *
+ * @return  The inverse volume form
  */
-struct CartesianGrid3d: public dg::aGeometry3d
+template< class Geometry>
+typename HostVec< typename TopologyTraits<Geometry>::memory_category>::host_vector inv_volume( const Geometry& g)
 {
-    ///@copydoc Grid3d::Grid3d()
-    CartesianGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = PER, bc bcy = PER, bc bcz = PER): dg::aGeometry3d(x0,x1,y0,y1,z0,z1,n,Nx,Ny,Nz,bcx,bcy,bcz){}
-    /**
-     * @brief Implicit type conversion from Grid3d
-     * @param grid existing grid class
-     */
-    CartesianGrid3d( const dg::Grid3d& grid):dg::Grid3d(grid){}
-    virtual CartesianGrid3d* clone()const{return new CartesianGrid3d(*this);}
-    private:
-    virtual SharedContainers<thrust::host_vector<double> > do_compute_metric()const{
-        return SharedContainers<thrust::host_vector<double> >( 3);
-    }
-    virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz){
-        aTopology2d::do_set(new_n,new_Nx,new_Ny,new_Nz);
-    }
-};
+    typedef typename HostVec< typename TopologyTraits<Geometry>::memory_category>::host_vector host_vector;
+    host_vector temp = volume(g);
+    dg::blas1::transform(temp,temp,dg::INVERT<double>());
+    return temp;
+}
 
-/**
- * @brief three-dimensional Grid with Cylindrical metric
- */
-struct CylindricalGrid3d: public dg::aGeometry3d
-{
-    CylindricalGrid3d( double R0, double R1, double Z0, double Z1, double phi0, double phi1, unsigned n, unsigned NR, unsigned NZ, unsigned Nphi, bc bcR, bc bcZ, bc bcphi = PER): dg::aGeometry3d(R0,R1,Z0,Z1,phi0,phi1,n,NR,NZ,Nphi,bcR,bcZ,bcphi){}
-    virtual CylindricalGrid3d* clone()const{return new CylindricalGrid3d(*this);}
-    private:
-    virtual SharedContainers<thrust::host_vector<double> > do_compute_metric()const{
-
-        std::vector<thrust::host_vector<double> > values(2, size());
-        thrust::host_vector<double> R = dg::evaluate(dg::coo1, *this);
-        unsigned size2d = n()*n()*Nx()*Ny();
-        for( unsigned i = 0; i<Nz(); i++)
-        for( unsigned j = 0; j<size2d; j++)
-        {
-            unsigned idx = i*size2d+j;
-            values[1][idx] = R[j];
-            values[0][idx] = 1./R[j]/R[j];
-        }
-        Operator<int> mat_idx(3,-1); mat_idx(2,2) = 0;
-        std::vector<int> vec_idx(3,-1); vec_idx[0] = 1;
-        return SharedContainers<thrust::host_vector<double> >( mat_idx, vec_idx, values);
-    }
-    virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz){
-        aTopology2d::do_set(new_n,new_Nx,new_Ny,new_Nz);
-    }
-};
 ///@}
-} //namespace dg
+}//namespace create
+}//namespace dg
diff --git a/inc/dg/geometry/mpi_geometry.h b/inc/dg/geometry/mpi_base.h
similarity index 100%
rename from inc/dg/geometry/mpi_geometry.h
rename to inc/dg/geometry/mpi_base.h
-- 
GitLab


From ea10ac724d5b392dae317256615f9abd6bbfc392 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Fri, 4 Aug 2017 06:20:42 -0700
Subject: [PATCH 117/453] update documentation

---
 inc/dg/backend/grid.h         |  6 ++++--
 inc/dg/backend/operator.h     |  4 ++--
 inc/dg/dg_doc.h               | 16 ++++++++++------
 inc/dg/geometry/base.h        |  6 ++++--
 inc/dg/geometry/curvilinear.h | 12 ++----------
 inc/dg/geometry/geometry.h    |  2 +-
 inc/dg/geometry/mpi_base.h    |  5 ++++-
 inc/dg/geometry/transform.h   | 14 ++++++++++----
 8 files changed, 37 insertions(+), 28 deletions(-)

diff --git a/inc/dg/backend/grid.h b/inc/dg/backend/grid.h
index 166d692df..7b6ebeafd 100644
--- a/inc/dg/backend/grid.h
+++ b/inc/dg/backend/grid.h
@@ -691,7 +691,7 @@ struct Grid2d : public aTopology2d
         aTopology2d(x0,x1,y0,y1,n,Nx,Ny,bcx,bcy) { }
     ///@copydoc aTopology2d::aTopology2d(const Grid1d&,const Grid1d&)
     Grid2d( const Grid1d& gx, const Grid1d& gy): aTopology2d(gx,gy){ }
-    ///@allow explicit type conversion from any other topology
+    ///allow explicit type conversion from any other topology
     explicit Grid2d( const aTopology2d& src): aTopology2d(src){}
     private:
     virtual void do_set( unsigned n, unsigned Nx, unsigned Ny){ 
@@ -699,6 +699,7 @@ struct Grid2d : public aTopology2d
     }
 
 };
+
 /**
  * @brief The simplest implementation of aTopology3d
  */
@@ -709,7 +710,7 @@ struct Grid3d : public aTopology3d
         aTopology3d(x0,x1,y0,y1,z0,z1,n,Nx,Ny,Nz,bcx,bcy,bcz) { }
     ///@copydoc aTopology3d::aTopology3d(const Grid1d&,const Grid1d&,const Grid1d&)
     Grid3d( const Grid1d& gx, const Grid1d& gy, const Grid1d& gz): aTopology3d(gx,gy,gz){ }
-    ///@allow explicit type conversion from any other topology
+    ///allow explicit type conversion from any other topology
     explicit Grid3d( const aTopology3d& src): aTopology3d(src){ }
     private:
     virtual void do_set( unsigned n, unsigned Nx, unsigned Ny, unsigned Nz){ 
@@ -717,6 +718,7 @@ struct Grid3d : public aTopology3d
     }
 };
 ///@}
+//
 ///@cond
 void aTopology2d::do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny)
 {
diff --git a/inc/dg/backend/operator.h b/inc/dg/backend/operator.h
index 6f0af0366..81ea56dc8 100644
--- a/inc/dg/backend/operator.h
+++ b/inc/dg/backend/operator.h
@@ -366,7 +366,7 @@ namespace detail
 /*! @brief LU Decomposition with partial pivoting
  *
  * @tparam T value type
- * @note this function throws a runtime_error when the matrix is singular
+ * @throw std::runtime_error if the matrix is singular
  */
 template< class T>
 T lr_pivot( dg::Operator<T>& m, std::vector<unsigned>& p)
@@ -463,7 +463,7 @@ void lr_solve( const dg::Operator<T>& lr, const std::vector<unsigned>& p, std::v
  * @param in input matrix
  *
  * @return the inverse of in if it exists
- * @note throws a std::runtime_error if in is singular
+ * @throw std::runtime_error if in is singular
  */
 template<class T>
 dg::Operator<T> invert( const dg::Operator<T>& in)
diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index 28b6d8162..b4417a1eb 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -44,7 +44,7 @@
  *     Objects that store topological information (which point is neighbour of which other point) 
  *     about the grid. 
  *     @{
- *         @defgroup evaluation Function discretization
+ *         @defgroup evaluation evaluate functions
  *             
  *             The function discretisation routines compute the DG discretisation
  *             of analytic functions on a given grid. In 1D the discretisation
@@ -54,10 +54,10 @@
  *             use the product space. We choose x to be the contiguous direction.
  *             The first elements of the resulting vector lie in the cell at (x0,y0) and the last
  *             in (x1, y1).
- *         @defgroup lowlevel Lowlevel helper functions and classes
- *             Low level helper routines.
- *         @defgroup highlevel Weight functions
- *         @defgroup creation Discrete derivatives 
+ *         @defgroup highlevel create weights 
+ *              overloads for the create::weights and create::inv_weights functions for all
+ *              available topologies
+ *         @defgroup creation create derivatives 
  *
  *             High level matrix creation functions
            @defgroup interpolation Interpolation and projection
@@ -68,7 +68,9 @@
           These routines form the heart of our geometry free numerical algorithms. 
           They are called by our geometric operators like the Poisson bracket. 
       @{
-          @defgroup basicgrids Basic grids
+          @defgroup pullback pullback and pushforward
+          @defgroup metric create metric
+          @defgroup basicgrids basic grids
  *        @defgroup utilities Fieldalignment and Averaging
  *            The classes to perform field line integration for DS and averaging classes
       @}
@@ -91,6 +93,8 @@
  *         The functors are useful for either vector transformations or
  *         as init functions in the evaluate routines.
        @defgroup timer Timer class
+ *     @defgroup lowlevel Lowlevel helper functions and classes
+ *         Low level helper routines.
    @}
  * 
  */
diff --git a/inc/dg/geometry/base.h b/inc/dg/geometry/base.h
index 70ce8d2c9..feee2d82d 100644
--- a/inc/dg/geometry/base.h
+++ b/inc/dg/geometry/base.h
@@ -68,9 +68,11 @@ struct aGeometry3d : public aTopology3d
     virtual SharedContainers<thrust::host_vector<double> > do_compute_metric()const=0;
 };
 
+///@}
 namespace create
 {
-
+///@addtogroup metric
+///@{
 SharedContainers<thrust::host_vector<double> > metric( const aGeometry2d& g)
 {
     return g.compute_metric();
@@ -79,10 +81,10 @@ SharedContainers<thrust::host_vector<double> > metric( const aGeometry3d& g)
 {
     return g.compute_metric();
 }
+///@}
 
 }//namespace create
 
-///@}
 ///@addtogroup basicgrids
 ///@{
 
diff --git a/inc/dg/geometry/curvilinear.h b/inc/dg/geometry/curvilinear.h
index c8c37a91c..e13f6cfea 100644
--- a/inc/dg/geometry/curvilinear.h
+++ b/inc/dg/geometry/curvilinear.h
@@ -151,11 +151,6 @@ struct CurvilinearGrid3d : public dg::aGeometry3d
 template< class container>
 struct CurvilinearGrid2d : public dg::aGeometry2d
 {
-    CurvilinearGrid2d( double R0, double R1, double Z0, double Z1, unsigned n, unsigned NR, unsigned NZ, unsigned Nphi, bc bcR = PER, bc bcZ = PER): 
-        dg::Grid2d(0,R1-R0,0,Z1-Z0,n,NR,NZ,bcR,bcZ), handle_(new ShiftedIdentityGenerator(R0,R1,Z0,Z1))
-        {
-            construct( n, NR, NZ);
-        }
     /*!@brief Constructor
     
      * @param generator must generate an orthogonal grid (class takes ownership of the pointer)
@@ -197,13 +192,10 @@ struct CurvilinearGrid2d : public dg::aGeometry2d
     const container& g_xy()const{return g_xy_;}
     const container& vol()const{return vol2d_;}
     const geo::aGenerator& generator() const{return handle_.get();}
-    bool isOrthonormal() const { return generator_->isOrthonormal();}
-    bool isOrthogonal() const { return generator_->isOrthogonal();}
-    bool isConformal() const { return generator_->isConformal();}
     private:
     virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny)
     {
-        dg::Grid2d::do_set( new_n, new_Nx, new_Ny);
+        dg::aTopology2d::do_set( new_n, new_Nx, new_Ny);
         construct( new_n, new_Nx, new_Ny);
     }
     void construct( unsigned n, unsigned Nx, unsigned Ny)
@@ -215,7 +207,7 @@ struct CurvilinearGrid2d : public dg::aGeometry2d
     }
     thrust::host_vector<double> r_, z_, xr_, xz_, yr_, yz_;
     container g_xx_, g_xy_, g_yy_, vol2d_;
-    dg::Handle<geo::aGenerator> handle_;
+    dg::Handle<geo::aGenerator2d> handle_;
 };
 
 ///@}
diff --git a/inc/dg/geometry/geometry.h b/inc/dg/geometry/geometry.h
index b91273d40..2a9e916b7 100644
--- a/inc/dg/geometry/geometry.h
+++ b/inc/dg/geometry/geometry.h
@@ -125,7 +125,7 @@ void dividePerpVolume( container& inout, const SharedContainers<container>& metr
 }//namespace geo
 
 namespace create{
-///@addtogroup geometry
+///@addtogroup metric
 ///@{
 
 /**
diff --git a/inc/dg/geometry/mpi_base.h b/inc/dg/geometry/mpi_base.h
index 0f28442ae..12a433364 100644
--- a/inc/dg/geometry/mpi_base.h
+++ b/inc/dg/geometry/mpi_base.h
@@ -62,8 +62,11 @@ struct aMPIGeometry3d : public aMPITopology3d
     virtual SharedContainers<host_vec > do_compute_metric()const=0;
 };
 
+///@}
 namespace create
 {
+///@addtogroup metric
+///@{
 
 SharedContainers<host_vec > metric( const aMPIGeometry2d& g)
 {
@@ -74,10 +77,10 @@ SharedContainers<host_vec > metric( const aMPIGeometry3d& g)
     return g.compute_metric();
 }
 
+///@}
 }//namespace create
 
 
-///@}
 
 ///@addtogroup basicgrids
 ///@{
diff --git a/inc/dg/geometry/transform.h b/inc/dg/geometry/transform.h
index 0b97c80e6..d93606c0b 100644
--- a/inc/dg/geometry/transform.h
+++ b/inc/dg/geometry/transform.h
@@ -15,9 +15,8 @@ struct HostVec< SharedTag>
 
 ///@endcond
 /**
- * @brief This function pulls back a function defined in cartesian coordinates to the curvilinear coordinate system
+ * @brief This function pulls back a function defined in some basic coordinates to the curvilinear coordinate system
  *
- * @ingroup geometry
  * e.g. F(x,y) = f(R(x,y), Z(x,y)) in 2d 
  * @tparam Functor The binary or ternary function object 
  * @param f The function defined in cartesian coordinates
@@ -27,6 +26,7 @@ struct HostVec< SharedTag>
  * You will want to rename those uniquely
  *
  * @return A set of points representing F
+ * @ingroup pullback
  */
 template< class Functor>
 thrust::host_vector<double> pullback( Functor f, const aGeometry2d& g)
@@ -41,6 +41,7 @@ thrust::host_vector<double> pullback( Functor f, const aGeometry2d& g)
 }
 
 ///@copydoc pullback(Functor,const aGeometry2d&)
+///@ingroup pullback
 template< class Functor>
 thrust::host_vector<double> pullback( Functor f, const aGeometry3d& g)
 {
@@ -65,6 +66,7 @@ struct HostVec< MPITag>
 ///@endcond
 
 ///@copydoc pullback(Functor,const aGeometry2d&)
+///@ingroup pullback
 template< class Functor>
 MPI_Vector<thrust::host_vector<double> > pullback( Functor f, const aMPIGeometry2d& g)
 {
@@ -78,6 +80,7 @@ MPI_Vector<thrust::host_vector<double> > pullback( Functor f, const aMPIGeometry
 }
 
 ///@copydoc pullback(Functor,const aGeometry2d&)
+///@ingroup pullback
 template< class Functor>
 MPI_Vector<thrust::host_vector<double> > pullback( Functor f, const aMPIGeometry3d& g)
 {
@@ -97,7 +100,7 @@ MPI_Vector<thrust::host_vector<double> > pullback( Functor f, const aMPIGeometry
  * @brief Push forward a vector from cylindrical or Cartesian to a new coordinate system
  *
  * Computes \f[ v^x(x,y) = x_R (x,y) v^R(R(x,y), Z(x,y)) + x_Z v^Z(R(x,y), Z(x,y)) \\
-               v^y(x,y) = y_R (x,y) v^R(R(x,y), Z(x,y)) + y_Z v^Z(R(x,y), Z(x,y)) \f]
+                v^y(x,y) = y_R (x,y) v^R(R(x,y), Z(x,y)) + y_Z v^Z(R(x,y), Z(x,y)) \f]
    where \f$ x_R = \frac{\partial x}{\partial R}\f$, ... 
  * @tparam Functor1 Binary or Ternary functor
  * @tparam Geometry The Geometry class
@@ -106,6 +109,7 @@ MPI_Vector<thrust::host_vector<double> > pullback( Functor f, const aMPIGeometry
  * @param vx x-component of vector (gets properly resized)
  * @param vy y-component of vector (gets properly resized)
  * @param g The geometry object
+ * @ingroup pullback
  */
 template<class Functor1, class Functor2, class container, class Geometry> 
 void pushForwardPerp( Functor1 vR, Functor2 vZ, 
@@ -124,7 +128,7 @@ void pushForwardPerp( Functor1 vR, Functor2 vZ,
  * @brief Push forward a vector from cylindrical or Cartesian to a new coordinate system
  *
  * Computes \f[ v^x(x,y) = x_R (x,y) v^R(R(x,y), Z(x,y)) + x_Z v^Z(R(x,y), Z(x,y)) \\
-               v^y(x,y) = y_R (x,y) v^R(R(x,y), Z(x,y)) + y_Z v^Z(R(x,y), Z(x,y)) \f]
+                v^y(x,y) = y_R (x,y) v^R(R(x,y), Z(x,y)) + y_Z v^Z(R(x,y), Z(x,y)) \f]
    where \f$ x_R = \frac{\partial x}{\partial R}\f$, ... 
  * @tparam Functor1 Binary or Ternary functor
  * @tparam Geometry The Geometry class
@@ -135,6 +139,7 @@ void pushForwardPerp( Functor1 vR, Functor2 vZ,
  * @param vy y-component of vector (gets properly resized)
  * @param vz z-component of vector (gets properly resized)
  * @param g The geometry object
+ * @ingroup pullback
  */
 template<class Functor1, class Functor2, class Functor3 class container, class Geometry> 
 void pushForward( Functor1 vR, Functor2 vZ, Functor3 vPhi,
@@ -168,6 +173,7 @@ void pushForward( Functor1 vR, Functor2 vZ, Functor3 vPhi,
  * @param chixy xy-component of tensor (gets properly resized)
  * @param chiyy yy-component of tensor (gets properly resized)
  * @param g The geometry object
+ * @ingroup pullback
  */
 template<class FunctorRR, class FunctorRZ, class FunctorZZ, class container, class Geometry> 
 void pushForwardPerp( FunctorRR chiRR, FunctorRZ chiRZ, FunctorZZ chiZZ,
-- 
GitLab


From b25008bb064c9b70e7c5be4cc080e0df12a3b1e9 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Fri, 4 Aug 2017 08:05:05 -0700
Subject: [PATCH 118/453] corrected elliptic documentation

---
 inc/dg/elliptic.h | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index 331228bfa..5cd8db6e3 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -18,7 +18,7 @@ namespace dg
 {
 
 /**
- * @brief Operator that acts as a 2d negative elliptic differential operator
+ * @brief %Operator that acts as a 2d negative elliptic differential operator
  *
  * @ingroup matrixoperators
  *
@@ -215,7 +215,7 @@ class Elliptic
 
 
 /**
- * @brief Operator that acts as a 3d negative elliptic differential operator
+ * @brief %Operator that acts as a 3d negative elliptic differential operator
  *
  * @ingroup matrixoperators
  *
@@ -225,7 +225,7 @@ class Elliptic
  * \f[ 
  * \begin{align}
  * v = b^x \partial_x f + b^y\partial_y f + b^z \partial_z f \\
- * -\frac{1}{\sqrt{g}} \left(\partial_x(\sqrt{g} b^x v ) + \partial_y(\sqrt{g}b_y v) + \partial_z(\sqrt{g} b_z v)\right)
+ * -\frac{1}{\sqrt{g}} \left(\partial_x(\sqrt{g} b^x v ) + \partial_y(\sqrt{g}b^y v) + \partial_z(\sqrt{g} b^z v)\right)
  *  \end{align}
  *  \f] 
  * is discretized, with \f$ b^i\f$ being the contravariant components of \f$\mathbf b\f$ . 
@@ -418,7 +418,7 @@ struct GeneralElliptic
 };
 
 /**
- * @brief Operator that acts as a 3d negative elliptic differential operator. Is the symmetric of the GeneralElliptic with 
+ * @brief %Operator that acts as a 3d negative elliptic differential operator. Is the symmetric of the GeneralElliptic with 
  * 0.5(D_+ + D_-) or vice versa
  *
  * @ingroup matrixoperators
@@ -429,7 +429,7 @@ struct GeneralElliptic
  * \f[ 
  * \begin{align}
  * v = b^x \partial_x f + b^y\partial_y f + b^z \partial_z f \\
- * -\frac{1}{\sqrt{g}} \left(\partial_x(\sqrt{g} b^x v ) + \partial_y(\sqrt{g}b_y v) + \partial_z(\sqrt{g} b_z v)\right)
+ * -\frac{1}{\sqrt{g}} \left(\partial_x(\sqrt{g} b^x v ) + \partial_y(\sqrt{g}b^y v) + \partial_z(\sqrt{g} b^z v)\right)
  *  \end{align}
  *  \f] 
  * is discretized, with \f$ b^i\f$ being the contravariant components of \f$\mathbf b\f$ . 
@@ -563,7 +563,7 @@ struct GeneralEllipticSym
 };
 
 /**
- * @brief Operator that acts as a 2d negative elliptic differential operator
+ * @brief %Operator that acts as a 2d negative elliptic differential operator
  *
  * @ingroup matrixoperators
  *
-- 
GitLab


From 5cc6c79eea5c22c3b25aecc54fe7213d8d27d22b Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sat, 5 Aug 2017 14:33:12 +0200
Subject: [PATCH 119/453] implemented SparseTensor and SparseElement

---
 inc/dg/geometry/tensor.h    | 377 ++++++++++++++++++++++++++----------
 inc/dg/geometry/transform.h |  37 +++-
 2 files changed, 305 insertions(+), 109 deletions(-)

diff --git a/inc/dg/geometry/tensor.h b/inc/dg/geometry/tensor.h
index daf7bda39..8963dd938 100644
--- a/inc/dg/geometry/tensor.h
+++ b/inc/dg/geometry/tensor.h
@@ -7,6 +7,25 @@
 namespace dg
 {
 
+/*
+1. make SparseTensor (remove vec_idx in SparseTensor)
+2. make SparseElement (a std::vector with 0 or 1 element)
+10  SparseElement det = dg::tensor::determinant(metric) //if not empty make dense and then compute
+3. dg::tensor::sqrt(metric) //inverse sqrt of determinant
+4. if(vol.isSet()) dg::blas1::pointwiseDot( vol, ...)
+5. dg::tensor::scal( metric, vol); //if(isSet) multiply all values elements
+6. dg::tensor::multiply( metric, in1, in2, out1, out2); //sparse matrix dense vector mult
+6. dg::tensor::multiply( metric, in1, in2, in3, out1, out2, out3);  
+7  perp_metric = metric.perp(); //for use in arakawa when computing perp_vol
+8  dense_metric = metric.dens(); //add 0 and 1 to values and rewrite indices
+9  metric.isPerp(), metric.isDense(), metric.isEmpty()
+11 dg::tensor::product( tensor, jac, temp) //make jac dense and then use multiply 3 times (needs write access in getValue(i,j)
+12 dg::tensor::product( jac.transpose(), temp, result)  //then use three elements if symmetric
+13 Geometry has  std::vector<sparseElements>  v[0], v[1], v[2] and Jacobian 
+14 when computing metric believe generator and compute on host for-loop
+*/
+  
+
 /**
 * @brief Abstract base class for matrices and vectors sharing or implicitly assuming elements 
 *
@@ -22,27 +41,24 @@ if negative the value of the container is assumed to be 1, except for the off-di
 * @ingroup misc
 */
 template<class container>
-struct SharedContainers
+struct SparseTensor
 {
-    SharedContainers( ) {}
-    SharedContainers( unsigned size):mat_idx_(size,-1), vec_idx_(size,-1){}
+    SparseTensor( ):mat_idx(3-1) {}
 
     /**
     * @brief 
     *
     * @param mat_idx
-    * @param vec_idx
     * @param values The contained containers must all have the same size
     */
-    SharedContainers( const dg::Operator<int>& mat_idx, std::vector<int>& vec_idx, std::vector<container>& values ): mat_idx_(mat_idx), vec_idx(vec_idx), values_(values){}
+    SparseTensor( const std::vector<container>& values ): mat_idx_(3,-1), values_(values){}
 
     template<class otherContainer>
-    SharedContainers( const SharedContainers<otherContainer>& src): mat_idx_(src.mat_idx()), vec_idx_(src.vec_idx()), values_(src.values().size()), idx_(src.idx_){
+    SparseTensor( const SparseTensor<otherContainer>& src): mat_idx_(src.mat_idx()), values_(src.values().size()), idx_(src.idx_){
         dg::blas1::transfer( src.values(), values_);
     }
-    void set( const dg::Operator<int>& mat_idx, std::vector<int>& vec_idx, std::vector<container>& values ){
-        mat_idx_=mat_idx;
-        vec_idx_=vec_idx;
+    ///sets values leaves indices unchanged (size must be  the same or larger as current 
+    void set( const std::vector<container>& values ){
         values_=values;
     }
 
@@ -57,90 +73,276 @@ struct SharedContainers
         return true;
     }
 
+    int operator()(unsigned i, unsigned j)const{return mat_idx_(i,j);}
+    int& operator()(unsigned i, unsigned j){return mat_idx_(i,j);}
 
     /**
     * @brief Test the matrix for emptiness
     * @return  true if no value in the matrix is set
     */
-    bool empty()const{
-        bool epty = true;
-        for( unsigned i=0; i<mat_idx_.size(); i++)
-            for( unsigned j=0; j<mat_idx_.size(); j++)
-                if(isSet(i,j)) 
-                    epty=false;
-        return epty;
+    bool isEmpty()const{ 
+        bool empty=true;
+        for(unsigned i=0; i<3; i++)
+            for( unsigned j=0; j<3; j++)
+                if( isSet(i,j) ) 
+                    empty=false;
+        return empty;
+    }
+    ///if all elements are set
+    bool isDense()const{
+        bool dense=true;
+        for(unsigned i=0; i<3; i++)
+            for( unsigned j=0; j<3; j++)
+                if( !isSet(i,j) ) 
+                    dense=false;
+        return dense;
+    }
+    ///if all elements in the third dimension are empty
+    bool isPerp() const
+    {
+        bool empty=true;
+        for(unsigned i=0; i<3; i++)
+        {
+            if( isSet(i,2) || isSet(2,i)) 
+                empty=false;
+        }
+        return empty;
     }
      
+     ///return an empty Tensor
+     SparseTensor empty()const{return SparseTensor();}
+     ///copy and fill all unset values
+     SparseTensor dense()const;
+     ///copy and erase all values in the third dimension
+     SparseTensor perp()const;
+     ///unset an index and clears values
+     void erase( unsigned i, unsigned j) {
+         if(!isSet(i,j)) return;
+         mat_idx_(i,j)=-1;
+         clear_unused_values();
+     }
+     ///clear any unused valuse and reset the corresponding indices
+     void clear_unused_values();
 
-    /**
-    * @brief check if an index is set or not
-    * @param i row index 0<i<2
-    * @return true if container is non-empty, false if value is assumed implicitly
-    */
-    bool isSet(size_t i) const{
-        if( vec_idx_[i]<0) return false;
-        return true;
-    }
     /*!@brief Access the underlying container
-     * @return if !isSet(i,j) the default constructor of container is called, otherwise values[mat_idx(i,j)] is returned. 
+     * @return if !isSet(i,j) the result is undefined, otherwise values[mat_idx(i,j)] is returned. 
      * @note If the indices fall out of range of mat_idx the result is undefined
      */
     const container& getValue(size_t i, size_t j)const{ 
-        int k = mat_idx(i,j);
-        if(k<0) return container();
+        int k = mat_idx_(i,j);
         return values_[k];
     }
-    /*!@brief Access the underlying container
-     * @return if !isSet(i) the default constructor of container is called, otherwise values[vec_idx(i)] is returned. 
-     * @note If the index falls out of range of vec_idx the result is undefined
-     */
-    const container& getValue(size_t i)const{ 
-        int k = vec_idx_[i];
-        if(k<0) return container();
+    container& getValue( size_t i, size_t j)
+    {
+        int k = mat_idx_(i,j);
         return values_[k];
     }
+    ///clear all values
+    void clear(){
+        mat_idx_=dg::Operator<int>(3,-1);
+        values_.clear()
+    }
+    SparseTensor transpose()const{
+        SparseTensor tmp(*this);
+        tmp.mat_idx_ = mat_idx_.transpose();
+    }
 
     private:
     dg::Operator<int> mat_idx_;
-    std::vector<int> vec_idx_;
     std::vector<container> values_;
+    void unique_insert(std::vector<int>& indices, int& idx)
+};
+///@cond
+template<class container>
+SparseTensor<container> SparseTensor<container>::dense() const
+{
+    SparseTensor<container> t(*this);
+    if( isEmpty()) return t;
+    container tmp = t.values_[0];
+    //1. diagonal
+    size_t size= values_.size();
+    bool diagonalIsSet=true;
+    for(unsigned i=0; i<3; i++)
+        if(!t.isSet(i,i)) diagonalIsSet = false;
+    dg::blas1::transform( tmp, tmp, dg::CONSTANT(1));
+    if (!diagonalIsSet) t.values_.push_back(tmp);
+    for(unsigned i=0; i<3; i++)
+        if(!t.isSet(i,i)) t.mat_idx_(i,i) = size;
+    //2. off-diagonal
+    size = t.values_.size();
+    bool offIsSet=true;
+    for(unsigned i=0; i<3; i++)
+        for(unsigned j=0; j<3; j++)
+            if( !t.isSet(i,j) ) offIsSet=true;
+    dg::blas1::transform( tmp, tmp, dg::CONSTANT(0));
+    if (!offIsSet) t.values_.push_back(tmp);
+    for(unsigned i=0; i<3; i++)
+        for(unsigned j=0; j<3; j++)
+            if(!t.isSet(i,j) ) t.mat_idx_(i,j) = size;
+    return t;
+}
+
+template<class container>
+SparseTensor<container> SparseTensor<container>::unique_insert(std::vector<int>& indices, int& idx) 
+{
+    bool unique=true;
+    unsigned size=indices.size();
+    for(unsigned i=0; i<size; i++)
+        if(indices[i] == idx) unique=false;
+    if(unique)
+    {
+        indices.push_back(idx);
+        idx=size;
+    }
+}
+
+template<class container>
+SparseTensor<container> SparseTensor<container>::perp() const
+{
+    SparseTensor<container> t(*this);
+    if( isEmpty()) return t;
+    for(unsigned i=0; i<3; i++)
+    {
+        if( t.isSet(2,i)) t.mat_idx_(2,i)=-1;
+        if( t.isSet(i,2)) t.mat_idx_(i,2)=-1;
+    }
+    t.clear_unused_values();
+    return t;
+}
+
+template<class container>
+void SparseTensor<container>::clear_unused_values()
+{
+    //now erase unused elements and redefine indices
+    std::vector<int> unique_idx;
+    for(unsigned i=0; i<3; i++)
+        for(unsigned j=0; j<3; j++)
+            if(isSet(i,j))
+                unique_insert( unique_idx, mat_idx_(i,j));
+
+    std::vector<container> tmp(unique_idx.size());
+    for(unsigned i=0; i<unique_idx.size(); i++)
+    {
+        tmp[i] = values_[unique_idx[i]];
+    }
+    values_.swap(tmp);
+}
+///@endcond
+
+
+//neutral element wrt multiplication represents forms
+template<class container>
+struct SparseElement
+{
+    SparseElement(){}
+    void set( const container& value){ 
+        value_.clear();
+        value_.push_back(value);
+    }
+    const container& getValue( )const { 
+        return value_[0];
+    }
+    container& getValue() {
+        return value_[0];
+    }
+    bool isSet()const{
+        if( value_.empty()) return false;
+        return true;
+    }
+    void clear(){value_.clear();}
+
+    private:
+    std::vector<container> value_;
 };
 
 ///@cond
-namespace detail
+namespace tensor
+{
+
+template<class container>
+void scal( SparseTensor<container>& t, const SparseElement<container>& e)
+{
+    if(!e.isSet()) return;
+    for( unsigned i=0; i<2; i++)
+        for( unsigned j=0; j<2; i++)
+            if(t.isSet(i,j)) dg::blas1::pointwiseDot( e.getValue(), t.getValue(i,j), t.getValue(i,j));
+}
+
+template<class container>
+void multiply( const SparseElement<container>& e, const container& in, container& out)
 {
+    if(e.isSet()) 
+        dg::blas1::pointwiseDot(e.getValue(), in,out);
+    else
+        out=in;
+}
+
+template<class container>
+void divide( const container& in, const SparseElement<container>& e, container& out)
+{
+    if(e.isSet()) 
+        dg::blas1::pointwiseDivide(in, e.getValue(),out);
+    else
+        out=in;
+}
+
+///no aliasing allowed
 template<class container>
-void multiply( const SharedContainers<container>& t, container& in1, container& in2, container& out1, container& out2)
+void multiply( const SparseTensor<container>& t, container& in1, container& in2, container& out1, container& out2)
 {
-    if( t.empty())
+    if( !t.isSet(0,0)&& !t.isSet(0,1) && !t.isSet(1,0) &&!t.isSet(1,1)) 
     {
         in1.swap(out1);
         in2.swap(out2);
         return;
     }
-    if( !tensor.isSet(0,0)) out1=in1;
-    if(tensor.isSet( 0,0))
-        dg::blas1::pointwiseDot( tensor.getValue(0,0), in1, out1); 
-    if(tensor.isSet( 0,1))
-        dg::blas1::pointwiseDot( 1., tensor.getValue(0,1), in2, 1., out1);
+    const_multiply(t,in1,in2,out1,out2);
+}
+///this version keeps the input intact in1 may alias in2
+template<class container>
+void const_multiply( const SparseTensor<container>& t, const container& in1, const container& in2, container& out1, container& out2)
+{
+    if( !t.isSet(0,0)&& !t.isSet(0,1) && !t.isSet(1,0) &&!t.isSet(1,1)) 
+    {
+        out1 = in1;
+        out2 = in2;
+        return;
+    }
+    if( !t.isSet(0,0)) out1=in1;
+    if(t.isSet( 0,0))
+        dg::blas1::pointwiseDot( t.getValue(0,0), in1, out1); 
+    if(t.isSet( 0,1))
+        dg::blas1::pointwiseDot( 1., t.getValue(0,1), in2, 1., out1);
 
-    if( !tensor.isSet(1,1)) out2=in2;
-    if(tensor.isSet(1,1))
-        dg::blas1::pointwiseDot( tensor.getValue(1,1), in2, out2);
-    if(tensor.isSet(1,0))
-        dg::blas1::pointwiseDot( 1., tensor.getValue(1,0), in1, 1., out2);
+    if( !t.isSet(1,1)) out2=in2;
+    if(t.isSet(1,1))
+        dg::blas1::pointwiseDot( t.getValue(1,1), in2, out2);
+    if(t.isSet(1,0))
+        dg::blas1::pointwiseDot( 1., t.getValue(1,0), in1, 1., out2);
 }
 
 template<class container>
-void multiply( const SharedContainers<container>& t, container& in1, container& in2, container& in3, container& out1, container& out2, container& out3)
+void multiply( const SparseTensor<container>& t, container& in1, container& in2, container& in3, container& out1, container& out2, container& out3)
 {
-    if( t.empty())
+    if( t.isEmpty())
     {
         in1.swap(out1);
         in2.swap(out2);
         in3.swap(out3);
         return;
     }
+    const_multiply(t,in1,in2,in3,out1,out2,out3);
+}
+template<class container>
+void const_multiply( const SparseTensor<container>& t, const container& in1, const container& in2, const container& in3, container& out1, container& out2, container& out3)
+{
+    if( t.isEmpty())
+    {
+        out1 = in1;
+        out2 = in2;
+        out3 = in3;
+        return;
+    }
     if( !tensor.isSet(0,0)) out1=in1;
     if(tensor.isSet( 0,0))
         dg::blas1::pointwiseDot( tensor.getValue(0,0), in1, out1); 
@@ -167,58 +369,33 @@ void multiply( const SharedContainers<container>& t, container& in1, container&
 }
 
 template<class container>
-void sandwich( const SharedContainers<container>& t, container& chiRR, container& chiRZ, container& chiZZ, container& chixx, container& chixy, container& chiyy)
+SparseElement<container> determinant( const SparseTensor<container>& t)
 {
-    if(t.empty())
-    {
-        chiRR.swap(chixx);
-        chiRZ.swap(chixy);
-        chiZZ.swap(chiyy);
-        return;
-    }
-    
-    //this is a default implementation  (add cases if optimization is necessary)
-    //compute the transformation matrix
-    container t00(chixx), t01(t00), t02(t00), t10(t00), t11(t00), t12(t00), t20(t00), t21(t00), t22(t00);
-    container xr(chixx), xz(xr), yr(xr), yz(xz);
-    //fill values for easier computations
-    if(t.isSet(0,0)) dg::blas1::transfer( t.getValue(0,0), xr);
-    else             dg::blas1::transform( xr,xr, dg::CONSTANT(1));
-    if(t.isSet(0,1)) dg::blas1::transfer( t.getValue(0,1), xz);
-    else             dg::blas1::transform( xz,xz, dg::CONSTANT(0));
-    if(t.isSet(1,0)) dg::blas1::transfer( t.getValue(1,0), yr);
-    else             dg::blas1::transform( yr,yr, dg::CONSTANT(0));
-    if(t.isSet(1,1)) dg::blas1::transfer( t.getValue(1,1), yz);
-    else             dg::blas1::transform( yz,yz, dg::CONSTANT(1));
-
-    dg::blas1::pointwiseDot( xr, xr, t00);
-    dg::blas1::pointwiseDot( xr, xz, t01);
-    dg::blas1::scal( t01, 2.);
-    dg::blas1::pointwiseDot( xz, g.xz, t02);
-
-    dg::blas1::pointwiseDot( xr, yr, t10);
-    dg::blas1::pointwiseDot( xr, yz, t11);
-    dg::blas1::pointwiseDot( 1., yr, g.xz(), 1., t11);
-    dg::blas1::pointwiseDot( xz, yz, t12);
-
-    dg::blas1::pointwiseDot( yr, yr, t20);
-    dg::blas1::pointwiseDot( yr, yz, t21);
-    dg::blas1::scal( t21, 2.);
-    dg::blas1::pointwiseDot( yz, yz, t22);
-
-    //now multiply
-    dg::blas1::pointwiseDot(     t00, chiRR,     chixx);
-    dg::blas1::pointwiseDot( 1., t01, chiRZ, 1., chixx);
-    dg::blas1::pointwiseDot( 1., t02, chiZZ, 1., chixx);
-    dg::blas1::pointwiseDot(     t10, chiRR,     chixy);
-    dg::blas1::pointwiseDot( 1., t11, chiRZ, 1., chixy);
-    dg::blas1::pointwiseDot( 1., t12, chiZZ, 1., chixy);
-    dg::blas1::pointwiseDot(     t20, chiRR,     chiyy);
-    dg::blas1::pointwiseDot( 1., t21, chiRZ, 1., chiyy);
-    dg::blas1::pointwiseDot( 1., t22, chiZZ, 1., chiyy);
+    if(t.isEmpty())  return SparseElement<container>();
+    SparseTensor<container> d = t.dense();
+    container det = d.getValue(0,0);
+    std::vector<container> sub_det(3,det);
+    dg::blas1::transform( det, det, dg::CONSTANT(0));
+    //first compute the det of three submatrices
+    dg::blas1::pointwiseDot( d(0,0), d(1,1), sub_det[2]);
+    dg::blas1::pointwiseDot( -1., d(1,0), d(0,1), 1. ,sub_det[2]);
+
+    dg::blas1::pointwiseDot( d(0,0), d(2,1), sub_det[1]);
+    dg::blas1::pointwiseDot( -1., d(2,0), d(0,1), 1. ,sub_det[1]);
+
+    dg::blas1::pointwiseDot( d(1,0), d(2,1), sub_det[0]);
+    dg::blas1::pointwiseDot( -1., d(2,0), d(1,1), 1. ,sub_det[0]);
+
+    //now multiply accordint to Laplace expansion
+    dg::blas1::pointwiseDot( 1., d(0,2), sub_det[0], 1.,  det);
+    dg::blas1::pointwiseDot(-1., d(1,2), sub_det[1], 1.,  det);
+    dg::blas1::pointwiseDot( 1., d(2,2), sub_det[2], 1.,  det);
+
+    return SparseElement<container>(det);
 }
 
-}//namespace detail
+
+}//namespace tensor
 ///@endcond
 
 }//namespace dg
diff --git a/inc/dg/geometry/transform.h b/inc/dg/geometry/transform.h
index d93606c0b..8c500f67f 100644
--- a/inc/dg/geometry/transform.h
+++ b/inc/dg/geometry/transform.h
@@ -119,7 +119,7 @@ void pushForwardPerp( Functor1 vR, Functor2 vZ,
     typedef typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector host_vec;
     host_vec out1 = pullback( vR, g), temp1(out1);
     host_vec out2 = pullback( vZ, g), temp2(out2);
-    dg::detail::multiply(g.map(), out1, out2, temp1, temp2);
+    dg::tensor::multiply(g.map(), out1, out2, temp1, temp2);
     dg::blas1::transfer( out1, vx);
     dg::blas1::transfer( out2, vy);
 }
@@ -150,7 +150,7 @@ void pushForward( Functor1 vR, Functor2 vZ, Functor3 vPhi,
     host_vec out1 = pullback( vR, g), temp1(out1);
     host_vec out2 = pullback( vZ, g), temp2(out2);
     host_vec out3 = pullback( vPhi, g), temp3(out3);
-    dg::detail::multiply(g.map(), out1, out2, out3, temp1, temp2, temp3);
+    dg::tensor::multiply(g.map(), out1, out2, out3, temp1, temp2, temp3);
     dg::blas1::transfer( out1, vx);
     dg::blas1::transfer( out2, vy);
     dg::blas1::transfer( out3, vz);
@@ -181,13 +181,32 @@ void pushForwardPerp( FunctorRR chiRR, FunctorRZ chiRZ, FunctorZZ chiZZ,
         const Geometry& g)
 {
     typedef typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector host_vec;
-    host_vec chiRR_ = pullback( chiRR, g), chixx_(chiRR_);
-    host_vec chiRZ_ = pullback( chiRZ, g), chixy_(chiRZ_);
-    host_vec chiZZ_ = pullback( chiZZ, g), chiyy_(chiZZ_);
-    dg::detail::sandwich( g.map(), chiRR_,chiRZ_,chiZZ_, chixx_,chixy_,chiyy_);
-    dg::blas1::transfer( chixx_, chixx);
-    dg::blas1::transfer( chixy_, chixy);
-    dg::blas1::transfer( chiyy_, chiyy);
+    host_vec chiRR_ = pullback( chiRR, g);
+    host_vec chiRZ_ = pullback( chiRZ, g);
+    host_vec chiZZ_ = pullback( chiZZ, g);
+    //transfer to device
+    if(g.map().isEmpty())
+    {
+        chiRR_.swap(chixx);
+        chiRZ_.swap(chixy);
+        chiZZ_.swap(chiyy);
+        return;
+    }
+    const dg::SparseTensor<container> jac = g.map();
+    std::vector<container> values( 3); 
+    values[0] = chiRR_, values[1] = chiRZ_, values[2] = chiZZ_;
+    SparseTensor<container> chi(values);
+    chi(0,0)=0, chi(0,1)=chi(1,0)=1, chi(1,1)=2;
+
+    SparseTensor<container> d = jac.dense(); //now we have a dense tensor
+    container tmp00(d.getValue(0,0)), tmp01(tmp00), tmp10(tmp00), tmp11(tmp00);
+    // multiply Chi*t -> tmp
+    dg::tensor::const_multiply( chi, d.getValue(0,0), d.getValue(1,0), tmp00, tmp10);
+    dg::tensor::const_multiply( chi, d.getValue(0,1), d.getValue(1,1), tmp01, tmp11);
+    // multiply tT * tmp -> Chi
+    SparseTensor<container> transpose = jac.transpose();
+    dg::tensor::multiply( transpose, tmp00, tmp01, chixx, chixy);
+    dg::tensor::multiply( transpose, tmp10, tmp11, chixy, chiyy);
 }
 
 } //namespace dg
-- 
GitLab


From 82115e870543a1d926043f4db8a473046e7cab79 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sun, 6 Aug 2017 13:09:15 +0200
Subject: [PATCH 120/453] optimized inplace tensor multiplication

---
 inc/dg/backend/evaluation.cuh |  10 +-
 inc/dg/geometry/tensor.h      | 166 +++++++++++++++++++---------------
 inc/dg/geometry/tensor_t.cu   |  65 +++++++++++++
 3 files changed, 165 insertions(+), 76 deletions(-)
 create mode 100644 inc/dg/geometry/tensor_t.cu

diff --git a/inc/dg/backend/evaluation.cuh b/inc/dg/backend/evaluation.cuh
index 849b58439..f78f72d2a 100644
--- a/inc/dg/backend/evaluation.cuh
+++ b/inc/dg/backend/evaluation.cuh
@@ -60,8 +60,8 @@ template< class BinaryOp>
 thrust::host_vector<double> evaluate( BinaryOp f, const aTopology2d& g)
 {
     unsigned n= g.n();
-    Grid1d gx = g.gx();
-    Grid1d gy = g.gy();
+    Grid1d gx(g.x0(), g.x1(), g.n(), g.Nx());
+    Grid1d gy(g.y0(), g.y1(), g.n(), g.Ny());
     thrust::host_vector<double> absx = create::abscissas( gx);
     thrust::host_vector<double> absy = create::abscissas( gy);
 
@@ -98,9 +98,9 @@ template< class TernaryOp>
 thrust::host_vector<double> evaluate( TernaryOp f, const aTopology3d& g)
 {
     unsigned n= g.n();
-    Grid1d gx = g.gx();
-    Grid1d gy = g.gy();
-    Grid1d gz = g.gz();
+    Grid1d gx(g.x0(), g.x1(), g.n(), g.Nx());
+    Grid1d gy(g.y0(), g.y1(), g.n(), g.Ny());
+    Grid1d gz(g.z0(), g.z1(), 1, g.Nz());
     thrust::host_vector<double> absx = create::abscissas( gx);
     thrust::host_vector<double> absy = create::abscissas( gy);
     thrust::host_vector<double> absz = create::abscissas( gz);
diff --git a/inc/dg/geometry/tensor.h b/inc/dg/geometry/tensor.h
index 8963dd938..90d3f7aff 100644
--- a/inc/dg/geometry/tensor.h
+++ b/inc/dg/geometry/tensor.h
@@ -2,6 +2,7 @@
 
 #include <cusp/coo_matrix.h>
 #include "dg/backend/operator.h"
+#include "dg/functors.h"
 #include "dg/blas1.h"
 
 namespace dg
@@ -19,7 +20,7 @@ namespace dg
 7  perp_metric = metric.perp(); //for use in arakawa when computing perp_vol
 8  dense_metric = metric.dens(); //add 0 and 1 to values and rewrite indices
 9  metric.isPerp(), metric.isDense(), metric.isEmpty()
-11 dg::tensor::product( tensor, jac, temp) //make jac dense and then use multiply 3 times (needs write access in getValue(i,j)
+11 dg::tensor::product( tensor, jac, temp) //make jac dense and then use multiply 3 times (needs write access in value(i,j)
 12 dg::tensor::product( jac.transpose(), temp, result)  //then use three elements if symmetric
 13 Geometry has  std::vector<sparseElements>  v[0], v[1], v[2] and Jacobian 
 14 when computing metric believe generator and compute on host for-loop
@@ -43,7 +44,8 @@ if negative the value of the container is assumed to be 1, except for the off-di
 template<class container>
 struct SparseTensor
 {
-    SparseTensor( ):mat_idx(3-1) {}
+    SparseTensor( ):mat_idx_(3,-1.) {}
+    SparseTensor( unsigned value_size): mat_idx_(3,-1.), values_(value_size){}
 
     /**
     * @brief 
@@ -51,10 +53,10 @@ struct SparseTensor
     * @param mat_idx
     * @param values The contained containers must all have the same size
     */
-    SparseTensor( const std::vector<container>& values ): mat_idx_(3,-1), values_(values){}
+    SparseTensor( const std::vector<container>& values ): mat_idx_(3,-1.), values_(values){}
 
     template<class otherContainer>
-    SparseTensor( const SparseTensor<otherContainer>& src): mat_idx_(src.mat_idx()), values_(src.values().size()), idx_(src.idx_){
+    SparseTensor( const SparseTensor<otherContainer>& src): mat_idx_(src.mat_idx()), values_(src.values().size()){
         dg::blas1::transfer( src.values(), values_);
     }
     ///sets values leaves indices unchanged (size must be  the same or larger as current 
@@ -73,8 +75,8 @@ struct SparseTensor
         return true;
     }
 
-    int operator()(unsigned i, unsigned j)const{return mat_idx_(i,j);}
-    int& operator()(unsigned i, unsigned j){return mat_idx_(i,j);}
+    int idx(unsigned i, unsigned j)const{return mat_idx_(i,j);}
+    int& idx(unsigned i, unsigned j){return mat_idx_(i,j);}
 
     /**
     * @brief Test the matrix for emptiness
@@ -115,11 +117,9 @@ struct SparseTensor
      SparseTensor dense()const;
      ///copy and erase all values in the third dimension
      SparseTensor perp()const;
-     ///unset an index and clears values
-     void erase( unsigned i, unsigned j) {
-         if(!isSet(i,j)) return;
+     ///unset an index don't clear values
+     void unset( unsigned i, unsigned j) {
          mat_idx_(i,j)=-1;
-         clear_unused_values();
      }
      ///clear any unused valuse and reset the corresponding indices
      void clear_unused_values();
@@ -128,29 +128,31 @@ struct SparseTensor
      * @return if !isSet(i,j) the result is undefined, otherwise values[mat_idx(i,j)] is returned. 
      * @note If the indices fall out of range of mat_idx the result is undefined
      */
-    const container& getValue(size_t i, size_t j)const{ 
+    const container& value(size_t i, size_t j)const{ 
         int k = mat_idx_(i,j);
         return values_[k];
     }
-    container& getValue( size_t i, size_t j)
+    container& value( size_t i)
     {
-        int k = mat_idx_(i,j);
-        return values_[k];
+        return values_[i];
     }
     ///clear all values
     void clear(){
         mat_idx_=dg::Operator<int>(3,-1);
-        values_.clear()
+        values_.clear();
     }
+
     SparseTensor transpose()const{
         SparseTensor tmp(*this);
         tmp.mat_idx_ = mat_idx_.transpose();
+        return tmp;
     }
+    const std::vector<container>& values()const{return values_;}
 
     private:
     dg::Operator<int> mat_idx_;
     std::vector<container> values_;
-    void unique_insert(std::vector<int>& indices, int& idx)
+    void unique_insert(std::vector<int>& indices, int& idx);
 };
 ///@cond
 template<class container>
@@ -183,7 +185,7 @@ SparseTensor<container> SparseTensor<container>::dense() const
 }
 
 template<class container>
-SparseTensor<container> SparseTensor<container>::unique_insert(std::vector<int>& indices, int& idx) 
+void SparseTensor<container>::unique_insert(std::vector<int>& indices, int& idx) 
 {
     bool unique=true;
     unsigned size=indices.size();
@@ -235,14 +237,15 @@ template<class container>
 struct SparseElement
 {
     SparseElement(){}
+    SparseElement(const container& value):value_(1,value){ }
     void set( const container& value){ 
         value_.clear();
         value_.push_back(value);
     }
-    const container& getValue( )const { 
+    const container& value( )const { 
         return value_[0];
     }
-    container& getValue() {
+    container& value() {
         return value_[0];
     }
     bool isSet()const{
@@ -250,6 +253,17 @@ struct SparseElement
         return true;
     }
     void clear(){value_.clear();}
+    SparseElement sqrt(){ 
+        SparseElement tmp(*this);
+        if( isSet()) dg::blas1::transform( tmp.value_, tmp.value_, dg::SQRT<double>());
+        return tmp;
+    }
+    SparseElement invert(){ 
+        SparseElement tmp(*this);
+        if( isSet()) dg::blas1::transform( tmp.value_, tmp.value_, dg::INVERT<double>());
+        return tmp;
+    }
+
 
     private:
     std::vector<container> value_;
@@ -265,14 +279,14 @@ void scal( SparseTensor<container>& t, const SparseElement<container>& e)
     if(!e.isSet()) return;
     for( unsigned i=0; i<2; i++)
         for( unsigned j=0; j<2; i++)
-            if(t.isSet(i,j)) dg::blas1::pointwiseDot( e.getValue(), t.getValue(i,j), t.getValue(i,j));
+            if(t.isSet(i,j)) dg::blas1::pointwiseDot( e.value(), t.value(i,j), t.value(i,j));
 }
 
 template<class container>
 void multiply( const SparseElement<container>& e, const container& in, container& out)
 {
     if(e.isSet()) 
-        dg::blas1::pointwiseDot(e.getValue(), in,out);
+        dg::blas1::pointwiseDot(e.value(), in,out);
     else
         out=in;
 }
@@ -281,48 +295,65 @@ template<class container>
 void divide( const container& in, const SparseElement<container>& e, container& out)
 {
     if(e.isSet()) 
-        dg::blas1::pointwiseDivide(in, e.getValue(),out);
+        dg::blas1::pointwiseDivide(in, e.value(),out);
     else
         out=in;
 }
 
 ///no aliasing allowed
 template<class container>
-void multiply( const SparseTensor<container>& t, container& in1, container& in2, container& out1, container& out2)
+void multiply( const SparseTensor<container>& t, container& inout0, container& inout1, container& workspace)
 {
-    if( !t.isSet(0,0)&& !t.isSet(0,1) && !t.isSet(1,0) &&!t.isSet(1,1)) 
+    if( !t.isSet(1,0) && !t.isSet(0,1))
     {
-        in1.swap(out1);
-        in2.swap(out2);
+        if(t.isSet(0,0)dg::blas1::pointwiseDot( t.value(0,0), inout0,inout0);
+        if(t.isSet(1,1)dg::blas1::pointwiseDot( t.value(1,1), inout1,inout1);
+        //else inout0/1 contain result already
+        return;
+    }
+    if(!t.isSet(1,0)) //decoupled
+    {
+        if(t.isSet(0,0))  dg::blas1::pointwiseDot( t.value(0,0), inout0, inout0);
+        //else inout0=inout0
+        if(t.isSet(0,1)) dg::blas1::pointwiseDot( 1., t.value(0,1), inout1, 1., inout0);
+        if( t.isSet(1,1) dg::blas::pointwiseDot( t.value(1,1), inout1, inout1);
+        return;
+    }
+
+    if(!t.isSet(0,1))
+    {
+        if(t.isSet(1,1))  dg::blas1::pointwiseDot( t.value(1,1), inout1, inout1);
+        //else inout1=inout1
+        if(t.isSet(1,0)) dg::blas1::pointwiseDot( 1., t.value(1,0), inout0, 1., inout1);
+        if( t.isSet(0,0) dg::blas::pointwiseDot( t.value(0,0), inout0, inout0);
         return;
     }
-    const_multiply(t,in1,in2,out1,out2);
+    //else all elements are set
+    dg::blas1::pointwiseDot( t.value(0,1),inout1, workspace);
+    dg::blas1::pointwiseDot( t.value(1,1),inout1, inout1);
+    dg::blas1::pointwiseDot( 1., t.value(1,0),inout0, 1., inout1);
+    dg::blas1::pointwiseDot( 1., t.value(0,0),inout0, 1., workspace);
+    inout0.swap(workspace);
 }
 ///this version keeps the input intact in1 may alias in2
 template<class container>
-void const_multiply( const SparseTensor<container>& t, const container& in1, const container& in2, container& out1, container& out2)
+void multiply( const SparseTensor<container>& t, const container& in1, const container& in2, container& out1, container& out2)
 {
-    if( !t.isSet(0,0)&& !t.isSet(0,1) && !t.isSet(1,0) &&!t.isSet(1,1)) 
-    {
-        out1 = in1;
-        out2 = in2;
-        return;
-    }
     if( !t.isSet(0,0)) out1=in1;
     if(t.isSet( 0,0))
-        dg::blas1::pointwiseDot( t.getValue(0,0), in1, out1); 
+        dg::blas1::pointwiseDot( t.value(0,0), in1, out1); 
     if(t.isSet( 0,1))
-        dg::blas1::pointwiseDot( 1., t.getValue(0,1), in2, 1., out1);
+        dg::blas1::pointwiseDot( 1., t.value(0,1), in2, 1., out1);
 
     if( !t.isSet(1,1)) out2=in2;
     if(t.isSet(1,1))
-        dg::blas1::pointwiseDot( t.getValue(1,1), in2, out2);
+        dg::blas1::pointwiseDot( t.value(1,1), in2, out2);
     if(t.isSet(1,0))
-        dg::blas1::pointwiseDot( 1., t.getValue(1,0), in1, 1., out2);
+        dg::blas1::pointwiseDot( 1., t.value(1,0), in1, 1., out2);
 }
 
 template<class container>
-void multiply( const SparseTensor<container>& t, container& in1, container& in2, container& in3, container& out1, container& out2, container& out3)
+void multiply( const SparseTensor<container>& t, container& inout1, container& inout2, container& inout3, container& out1, container& out2, container& out3)
 {
     if( t.isEmpty())
     {
@@ -334,38 +365,31 @@ void multiply( const SparseTensor<container>& t, container& in1, container& in2,
     const_multiply(t,in1,in2,in3,out1,out2,out3);
 }
 template<class container>
-void const_multiply( const SparseTensor<container>& t, const container& in1, const container& in2, const container& in3, container& out1, container& out2, container& out3)
+void multiply( const SparseTensor<container>& t, const container& in1, const container& in2, const container& in3, container& out1, container& out2, container& out3, container& workspace1)
 {
-    if( t.isEmpty())
-    {
-        out1 = in1;
-        out2 = in2;
-        out3 = in3;
-        return;
-    }
-    if( !tensor.isSet(0,0)) out1=in1;
-    if(tensor.isSet( 0,0))
-        dg::blas1::pointwiseDot( tensor.getValue(0,0), in1, out1); 
-    if(tensor.isSet( 0,1))
-        dg::blas1::pointwiseDot( 1., tensor.getValue(0,1), in2, 1., out1);
-    if(tensor.isSet( 0,2))
-        dg::blas1::pointwiseDot( 1., tensor.getValue(0,2), in3, 1., out1);
-
-    if( !tensor.isSet(1,1)) out2=in2;
-    if(tensor.isSet(1,1))
-        dg::blas1::pointwiseDot( tensor.getValue(1,1), in2, out2);
-    if(tensor.isSet(1,0))
-        dg::blas1::pointwiseDot( 1., tensor.getValue(1,0), in1, 1., out2);
-    if(tensor.isSet(1,2))
-        dg::blas1::pointwiseDot( 1., tensor.getValue(1,2), in1, 1., out2);
-
-    if(!tensor.isSet(2,2)) out3=in3;
-    if(tensor.isSet(2,2))
-        dg::blas1::pointwiseDot( tensor.getValue(2,2), in3, out3);
-    if(tensor.isSet(2,1))
-        dg::blas1::pointwiseDot( 1., tensor.getValue(2,1), in2, 1., out3);
-    if(tensor.isSet(2,0))
-        dg::blas1::pointwiseDot( 1., tensor.getValue(2,0), in1, 1., out3);
+    if( !t.isSet(0,0)) out1=in1;
+    if(t.isSet( 0,0))
+        dg::blas1::pointwiseDot( t.value(0,0), in1, out1); 
+    if(t.isSet( 0,1))
+        dg::blas1::pointwiseDot( 1., t.value(0,1), in2, 1., out1);
+    if(t.isSet( 0,2))
+        dg::blas1::pointwiseDot( 1., t.value(0,2), in3, 1., out1);
+
+    if( !t.isSet(1,1)) out2=in2;
+    if(t.isSet(1,1))
+        dg::blas1::pointwiseDot( t.value(1,1), in2, out2);
+    if(t.isSet(1,0))
+        dg::blas1::pointwiseDot( 1., t.value(1,0), in1, 1., out2);
+    if(t.isSet(1,2))
+        dg::blas1::pointwiseDot( 1., t.value(1,2), in1, 1., out2);
+
+    if(!t.isSet(2,2)) out3=in3;
+    if(t.isSet(2,2))
+        dg::blas1::pointwiseDot( t.value(2,2), in3, out3);
+    if(t.isSet(2,1))
+        dg::blas1::pointwiseDot( 1., t.value(2,1), in2, 1., out3);
+    if(t.isSet(2,0))
+        dg::blas1::pointwiseDot( 1., t.value(2,0), in1, 1., out3);
 }
 
 template<class container>
@@ -373,7 +397,7 @@ SparseElement<container> determinant( const SparseTensor<container>& t)
 {
     if(t.isEmpty())  return SparseElement<container>();
     SparseTensor<container> d = t.dense();
-    container det = d.getValue(0,0);
+    container det = d.value(0,0);
     std::vector<container> sub_det(3,det);
     dg::blas1::transform( det, det, dg::CONSTANT(0));
     //first compute the det of three submatrices
diff --git a/inc/dg/geometry/tensor_t.cu b/inc/dg/geometry/tensor_t.cu
new file mode 100644
index 000000000..bc3575479
--- /dev/null
+++ b/inc/dg/geometry/tensor_t.cu
@@ -0,0 +1,65 @@
+#include <iostream>
+
+#include "tensor.h"
+
+void print( const dg::SparseTensor<thrust::host_vector<double> >& t)
+{
+    for( unsigned i=0; i<3; i++)
+    {
+        for( unsigned j=0; j<3; j++)
+        {
+            if(t.isSet(i,j)) std::cout << t.value(i,j)[0]<<" ";
+            else std::cout <<"XX ";
+        }
+        std::cout << "\n";
+    }
+
+}
+std::ostream& operator<<(std::ostream& os, const dg::SparseElement<thrust::host_vector<double> >& t)
+{
+    if(t.isSet()) os << t.value()[0]<<" ";
+    else os <<"XX ";
+    return os;
+}
+
+int main()
+{
+    thrust::host_vector<double> one(1,11), two(1,22), three(1,33), four(1,44), five(1,55), six(1,66), seven(1,77), eight(1,88), nine(1,99);
+
+    dg::SparseTensor<thrust::host_vector<double> > dense2d(3);
+    dense2d.idx(0,0) = 0, dense2d.idx(0,1) = 1;
+    dense2d.idx(1,0) = 1, dense2d.idx(1,1) = 2;
+    dense2d.value(0) = eight; dense2d.value(1) = two; dense2d.value(2) = nine; 
+    dg::SparseTensor<thrust::host_vector<double> > sparse3d(4);
+    sparse3d.idx(0,0) = 0, sparse3d.idx(0,1) = 1                       ;
+    sparse3d.idx(1,0) = 1                       , sparse3d.idx(1,2) = 3;
+    sparse3d.idx(2,0) = 1                       , sparse3d.idx(2,2) = 3;
+    sparse3d.value(0) = seven; sparse3d.value(1) = three; sparse3d.value(2) = nine, sparse3d.value(3) = one; 
+
+    dg::SparseTensor<thrust::host_vector<double> > empty;
+
+    std::cout << "Test dg::Sparse Tensor class \n";
+    std::cout << "Dense 2d Tensor \n";
+    print( dense2d);
+    std::cout << "dg::Sparse 3d Tensor \n";
+    print( sparse3d);
+    std::cout << "empty Tensor \n";
+    print( empty);
+    std::cout<< "Test dg::SparseElement";
+    dg::SparseElement<thrust::host_vector<double> > e(eight);
+    dg::SparseElement<thrust::host_vector<double> > ee;
+    std::cout<<"\n construct: " <<e<<" "<<ee<<"\n";
+    ee = e;
+    e.set(nine);
+    std::cout << "Assignment and set : "<<e<<" "<<ee<<"\n";
+    dg::SparseElement<thrust::host_vector<double> > sqrt = e.sqrt();
+    std::cout<<"\n sqrt(): "<<sqrt;
+    sqrt = sqrt.invert();
+    std::cout<<"\n invert(): "<<sqrt;
+    sqrt.clear();
+    std::cout<<"\n clear(): "<<sqrt;
+    std::cout <<std::endl;
+    return 0;
+
+
+}
-- 
GitLab


From 15bd6c9a17d2844587588af559619273cd6e9281 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sun, 6 Aug 2017 18:18:22 +0200
Subject: [PATCH 121/453] added cholesky decomposition for SparseTensor

---
 inc/dg/geometry/tensor.h | 306 ++++++++++++++++++++++++++-------------
 1 file changed, 209 insertions(+), 97 deletions(-)

diff --git a/inc/dg/geometry/tensor.h b/inc/dg/geometry/tensor.h
index 90d3f7aff..96194e0e7 100644
--- a/inc/dg/geometry/tensor.h
+++ b/inc/dg/geometry/tensor.h
@@ -132,8 +132,11 @@ struct SparseTensor
         int k = mat_idx_(i,j);
         return values_[k];
     }
+    //always returns a container, if i does not exist yet we resize
     container& value( size_t i)
     {
+        if(i>=values_size());
+        values_.resize(i+1);
         return values_[i];
     }
     ///clear all values
@@ -154,6 +157,135 @@ struct SparseTensor
     std::vector<container> values_;
     void unique_insert(std::vector<int>& indices, int& idx);
 };
+
+//neutral element wrt multiplication represents forms
+template<class container>
+struct SparseElement
+{
+    SparseElement(){}
+    SparseElement(const container& value):value_(1,value){ }
+    const container& value( )const { 
+        return value_[0];
+    }
+    container& value() {
+        if(!isSet()) value_.resize(1);
+        return value_[0];
+    }
+    bool isSet()const{
+        if( value_.empty()) return false;
+        return true;
+    }
+    void clear(){value_.clear();}
+    SparseElement sqrt(){ 
+        SparseElement tmp(*this);
+        if( isSet()) dg::blas1::transform( tmp.value_, tmp.value_, dg::SQRT<double>());
+        return tmp;
+    }
+    SparseElement invert(){ 
+        SparseElement tmp(*this);
+        if( isSet()) dg::blas1::transform( tmp.value_, tmp.value_, dg::INVERT<double>());
+        return tmp;
+    }
+
+
+    private:
+    std::vector<container> value_;
+};
+
+template<class container>
+struct CholeskyTensor
+{
+    CholeskyTensor( const SparseTensor<container>& in)
+    {
+        decompose(in);
+    }
+    void decompose( const SparseTensor<container>& in)
+    {
+        SparseTensor<container> denseIn=in.dense();
+        if(in.isSet(0,0))
+        {
+            diag.idx(0,0)=0;
+            diag.value(0)=in.value(0,0);
+        }
+        if(in.isSet(1,0)
+        {
+            container tmp=in(1,0):
+            if(diag.isSet(0,0)) dg::blas1::pointwiseDivide(tmp,diag.value(0,0),tmp);
+            q_.idx(1,0)=0;
+            q_.value(0)=tmp;
+        }
+        if(in.isSet(2,0))
+        {
+            container tmp=in(2,0):
+            if(diag.isSet(0,0))dg::blas1::pointwiseDivide(tmp,diag.value(0,0),tmp);
+            q_.idx(1,0)=1;
+            q_.value(1)=tmp;
+        }
+
+        if( q_.isSet(1,0) || in.isSet(1,1))
+        {
+            SparseTensor<container> denseL = q_.dense();
+            container tmp=denseL.value(1,0);
+            dg::blas1::pointwiseDot(tmp,tmp,tmp);
+            if(diag.isSet(0,0)) dg::blas1::pointwiseDot(tmp,diag.value(0,0),tmp);
+            dg::blas1::axpby( 1., denseIn(1,1), -1., tmp, tmp);
+            diag.idx(1,1)=1;
+            diag.value(1) = tmp;
+        }
+
+        if( in.isSet(2,1) || (q_.isSet(2,0)&&q_.isSet(1,0)))
+        {
+            SparseTensor<container> denseL = q_.dense();
+            container tmp=denseIn(2,1);
+            dg::blas1::pointwiseDot(denseL.value(2,0), denseL.value(1,0), tmp);
+            if(diag.isSet(0,0))dg::blas1::pointwiseDot(tmp, diag.value(0,0), tmp);
+            dg::blas1::axpby(1., denseIn(2,1),-1.,tmp, tmp);
+            if(diag.isSet(1,1))dg:.blas1::pointwiseDivide(tmp, diag.value(1,1),tmp);
+            q_.idx(2,1)=2;
+            q_.value(2)=tmp;
+        }
+        if( in.isSet(2,2) || q_.isSet(2,0) || q_.isSet(2,1))
+        {
+            SparseTensor<container> denseL = q_.dense();
+            container tmp=denseL(2,0), tmp1=denseL(2,1);
+            dg::blas1::pointwiseDot(tmp,tmp,tmp);
+            if(diag.isSet(0,0))dg::blas1::pointwiseDot(diag.value(0,0),tmp,tmp);
+            dg::blas1::pointwiseDot(tmp1,tmp1,tmp1);
+            if(diag.isSet(1,1))dg::blas1::pointwiseDot(diag.value(1,1),tmp1,tmp1);
+            dg::blas1::axpby(1., denseIn.value(2,2), -1., tmp, tmp);
+            dg::blas1::axpby(1., tmp, -1., tmp1, tmp);
+            diag.idx(2,2)=2;
+            diag.value(2) = tmp;
+        }
+        diag.clear_unused_values();
+        q_.clear_unused_values();
+        lower_=true;
+    }
+    const SparseTensor<container>& lower()const{
+        if(lower_) return q_;
+        std::swap(q_.idx(1,0), q_.idx(0,1));
+        std::swap(q_.idx(2,0), q_.idx(0,2));
+        std::swap(q_.idx(2,1), q_.idx(1,2));
+        lower_=!lower_;
+        return q_;
+
+    }
+    const SparseTensor<container>& upper()const{
+        if(!lower_) return q_;
+        std::swap(q_.idx(1,0), q_.idx(0,1));
+        std::swap(q_.idx(2,0), q_.idx(0,2));
+        std::swap(q_.idx(2,1), q_.idx(1,2));
+        lower_=!lower_;
+        return q_;
+
+    }
+    const SparseTensor<container>& diagonal()const{return diag;}
+
+    private:
+    SparseTensor q_, diag;
+    bool lower_;
+};
+
 ///@cond
 template<class container>
 SparseTensor<container> SparseTensor<container>::dense() const
@@ -232,42 +364,6 @@ void SparseTensor<container>::clear_unused_values()
 ///@endcond
 
 
-//neutral element wrt multiplication represents forms
-template<class container>
-struct SparseElement
-{
-    SparseElement(){}
-    SparseElement(const container& value):value_(1,value){ }
-    void set( const container& value){ 
-        value_.clear();
-        value_.push_back(value);
-    }
-    const container& value( )const { 
-        return value_[0];
-    }
-    container& value() {
-        return value_[0];
-    }
-    bool isSet()const{
-        if( value_.empty()) return false;
-        return true;
-    }
-    void clear(){value_.clear();}
-    SparseElement sqrt(){ 
-        SparseElement tmp(*this);
-        if( isSet()) dg::blas1::transform( tmp.value_, tmp.value_, dg::SQRT<double>());
-        return tmp;
-    }
-    SparseElement invert(){ 
-        SparseElement tmp(*this);
-        if( isSet()) dg::blas1::transform( tmp.value_, tmp.value_, dg::INVERT<double>());
-        return tmp;
-    }
-
-
-    private:
-    std::vector<container> value_;
-};
 
 ///@cond
 namespace tensor
@@ -300,92 +396,71 @@ void divide( const container& in, const SparseElement<container>& e, container&
         out=in;
 }
 
-///no aliasing allowed
+///this version keeps the input intact 
+///aliasing allowed if tensor is either lower, upper or diagonal alias allowed
 template<class container>
-void multiply( const SparseTensor<container>& t, container& inout0, container& inout1, container& workspace)
+void multiply( const SparseTensor<container>& t, const container& in0, const container& in1, container& out0, container& out1)
 {
-    if( !t.isSet(1,0) && !t.isSet(0,1))
-    {
-        if(t.isSet(0,0)dg::blas1::pointwiseDot( t.value(0,0), inout0,inout0);
-        if(t.isSet(1,1)dg::blas1::pointwiseDot( t.value(1,1), inout1,inout1);
-        //else inout0/1 contain result already
-        return;
-    }
-    if(!t.isSet(1,0)) //decoupled
-    {
-        if(t.isSet(0,0))  dg::blas1::pointwiseDot( t.value(0,0), inout0, inout0);
-        //else inout0=inout0
-        if(t.isSet(0,1)) dg::blas1::pointwiseDot( 1., t.value(0,1), inout1, 1., inout0);
-        if( t.isSet(1,1) dg::blas::pointwiseDot( t.value(1,1), inout1, inout1);
-        return;
-    }
-
-    if(!t.isSet(0,1))
+    if(!t.isSet(0,1))//lower triangular
     {
-        if(t.isSet(1,1))  dg::blas1::pointwiseDot( t.value(1,1), inout1, inout1);
-        //else inout1=inout1
-        if(t.isSet(1,0)) dg::blas1::pointwiseDot( 1., t.value(1,0), inout0, 1., inout1);
-        if( t.isSet(0,0) dg::blas::pointwiseDot( t.value(0,0), inout0, inout0);
+        if(t.isSet(1,1))  dg::blas1::pointwiseDot( t.value(1,1), in1, out1);
+        else out1=in1
+        if(t.isSet(1,0)) dg::blas1::pointwiseDot( 1., t.value(1,0), in0, 1., out1);
+        if( t.isSet(0,0)) dg::blas::pointwiseDot( t.value(0,0), in0, out0);
+        else out0=in0;
         return;
     }
-    //else all elements are set
-    dg::blas1::pointwiseDot( t.value(0,1),inout1, workspace);
-    dg::blas1::pointwiseDot( t.value(1,1),inout1, inout1);
-    dg::blas1::pointwiseDot( 1., t.value(1,0),inout0, 1., inout1);
-    dg::blas1::pointwiseDot( 1., t.value(0,0),inout0, 1., workspace);
-    inout0.swap(workspace);
-}
-///this version keeps the input intact in1 may alias in2
-template<class container>
-void multiply( const SparseTensor<container>& t, const container& in1, const container& in2, container& out1, container& out2)
-{
-    if( !t.isSet(0,0)) out1=in1;
-    if(t.isSet( 0,0))
-        dg::blas1::pointwiseDot( t.value(0,0), in1, out1); 
-    if(t.isSet( 0,1))
-        dg::blas1::pointwiseDot( 1., t.value(0,1), in2, 1., out1);
-
-    if( !t.isSet(1,1)) out2=in2;
-    if(t.isSet(1,1))
-        dg::blas1::pointwiseDot( t.value(1,1), in2, out2);
+    //upper triangular
+    if( t.isSet(0,0) ) 
+        dg::blas1::pointwiseDot( t.value(0,0), in0, out0); 
+    else out0=in0;
+    if(t.isSet(0,1))
+        dg::blas1::pointwiseDot( 1.,  t.value(0,1), in1, 1., out0);
+
+    if( t.isSet(1,1) )
+        dg::blas1::pointwiseDot( t.value(1,1), in1, out1);
+    else out1=in1;
     if(t.isSet(1,0))
-        dg::blas1::pointwiseDot( 1., t.value(1,0), in1, 1., out2);
+        dg::blas1::pointwiseDot( 1.,  t.value(1,0), in0, 1., out1);
 }
 
+///aliasing allowed if t is known to be lower or upper triangular
 template<class container>
-void multiply( const SparseTensor<container>& t, container& inout1, container& inout2, container& inout3, container& out1, container& out2, container& out3)
+void multiply( const SparseTensor<container>& t, const container& in1, const container& in2, const container& in3, container& out1, container& out2, container& out3, container& workspace1)
 {
-    if( t.isEmpty())
+    if( !t.isSet(0,1)&&!t.isSet(0,2)&&!t.isSet(1,2))
     {
-        in1.swap(out1);
-        in2.swap(out2);
-        in3.swap(out3);
-        return;
+        //lower triangular
+        if(!t.isSet(2,2)) out3=in3;
+        else dg::blas1::pointwiseDot( t.value(2,2), in3, out3);
+        if(t.isSet(2,1))
+            dg::blas1::pointwiseDot( 1., t.value(2,1), in2, 1., out3);
+        if(t.isSet(2,0))
+            dg::blas1::pointwiseDot( 1., t.value(2,0), in1, 1., out3);
+        if( !t.isSet(1,1)) out2=in2;
+        else dg::blas1::pointwiseDot( t.value(1,1), in2, out2);
+        if(t.isSet(1,0))
+            dg::blas1::pointwiseDot( 1., t.value(1,0), in1, 1., out2);
+        if( !t.isSet(0,0)) out1=in1;
+        else dg::blas1::pointwiseDot( t.value(0,0), in1, out1); 
     }
-    const_multiply(t,in1,in2,in3,out1,out2,out3);
-}
-template<class container>
-void multiply( const SparseTensor<container>& t, const container& in1, const container& in2, const container& in3, container& out1, container& out2, container& out3, container& workspace1)
-{
+    //upper triangular
     if( !t.isSet(0,0)) out1=in1;
-    if(t.isSet( 0,0))
-        dg::blas1::pointwiseDot( t.value(0,0), in1, out1); 
+    else dg::blas1::pointwiseDot( t.value(0,0), in1, out1); 
     if(t.isSet( 0,1))
         dg::blas1::pointwiseDot( 1., t.value(0,1), in2, 1., out1);
     if(t.isSet( 0,2))
         dg::blas1::pointwiseDot( 1., t.value(0,2), in3, 1., out1);
 
     if( !t.isSet(1,1)) out2=in2;
-    if(t.isSet(1,1))
-        dg::blas1::pointwiseDot( t.value(1,1), in2, out2);
+    else dg::blas1::pointwiseDot( t.value(1,1), in2, out2);
     if(t.isSet(1,0))
         dg::blas1::pointwiseDot( 1., t.value(1,0), in1, 1., out2);
     if(t.isSet(1,2))
         dg::blas1::pointwiseDot( 1., t.value(1,2), in1, 1., out2);
 
     if(!t.isSet(2,2)) out3=in3;
-    if(t.isSet(2,2))
-        dg::blas1::pointwiseDot( t.value(2,2), in3, out3);
+    else dg::blas1::pointwiseDot( t.value(2,2), in3, out3);
     if(t.isSet(2,1))
         dg::blas1::pointwiseDot( 1., t.value(2,1), in2, 1., out3);
     if(t.isSet(2,0))
@@ -418,6 +493,43 @@ SparseElement<container> determinant( const SparseTensor<container>& t)
     return SparseElement<container>(det);
 }
 
+//alias always allowed
+template<class container>
+void multiply( const CholeskyTensor<container>& ch, const container& in0, const container& in1, container& out0, container& out1)
+{
+    multiply(ch.upper(), in0, in1, out0, out1);
+    multiply(ch.diag(), out0, out1, out0, out1);
+    multiply(ch.lower(), out0, out1, out0, out1);
+}
+
+template<class container>
+SparseElement<container> determinant( const CholeskyTensor<container>& ch)
+{
+    SparseTensor<container> diag = ch.diag().dense();
+    SparseElement<container> det;
+    if(diag.isEmpty()) return det;
+    else det.value()=diag.value(0,0);
+    dg::blas1::pointwiseDot( det.value(), diag.value(1,1), det.value());
+    dg::blas1::pointwiseDot( det.value(), diag.value(2,2), det.value());
+    return det;
+}
+
+template<class container>
+void scal(const CholeskyTensor<container>& ch, const SparseElement<container>& e)
+{
+    const SparseTensor<container>& diag = ch.diag();
+    if(!e.isSet()) return;
+    unsigned size=diag.values().size();
+    if(!diag.isSet(0,0)|| !diag.isSet(1,1) || !diag.isSet(2,2))
+        diag.value(size) = e.value();
+    for( unsigned i=0; i<2; i++)
+    {
+        if(!diag.isSet(i,i)
+            diag.idx(i,i)=size;
+        else
+            dg::blas1::pointwiseDot( e.value(), diag.value(i,i), diag.value(i,i));
+    }
+}
 
 }//namespace tensor
 ///@endcond
-- 
GitLab


From ffdd54b72ddfdaa5d94dd475a056a74da3865714 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sun, 6 Aug 2017 18:29:08 +0200
Subject: [PATCH 122/453] made tensor_t compile

---
 inc/dg/geometry/tensor.h    | 18 +++++++++---------
 inc/dg/geometry/tensor_t.cu | 27 +++++++++++++++------------
 2 files changed, 24 insertions(+), 21 deletions(-)

diff --git a/inc/dg/geometry/tensor.h b/inc/dg/geometry/tensor.h
index 96194e0e7..b49d76167 100644
--- a/inc/dg/geometry/tensor.h
+++ b/inc/dg/geometry/tensor.h
@@ -135,7 +135,7 @@ struct SparseTensor
     //always returns a container, if i does not exist yet we resize
     container& value( size_t i)
     {
-        if(i>=values_size());
+        if(i>=values_.size());
         values_.resize(i+1);
         return values_[i];
     }
@@ -207,16 +207,16 @@ struct CholeskyTensor
             diag.idx(0,0)=0;
             diag.value(0)=in.value(0,0);
         }
-        if(in.isSet(1,0)
+        if(in.isSet(1,0))
         {
-            container tmp=in(1,0):
+            container tmp=in(1,0);
             if(diag.isSet(0,0)) dg::blas1::pointwiseDivide(tmp,diag.value(0,0),tmp);
             q_.idx(1,0)=0;
             q_.value(0)=tmp;
         }
         if(in.isSet(2,0))
         {
-            container tmp=in(2,0):
+            container tmp=in(2,0);
             if(diag.isSet(0,0))dg::blas1::pointwiseDivide(tmp,diag.value(0,0),tmp);
             q_.idx(1,0)=1;
             q_.value(1)=tmp;
@@ -240,7 +240,7 @@ struct CholeskyTensor
             dg::blas1::pointwiseDot(denseL.value(2,0), denseL.value(1,0), tmp);
             if(diag.isSet(0,0))dg::blas1::pointwiseDot(tmp, diag.value(0,0), tmp);
             dg::blas1::axpby(1., denseIn(2,1),-1.,tmp, tmp);
-            if(diag.isSet(1,1))dg:.blas1::pointwiseDivide(tmp, diag.value(1,1),tmp);
+            if(diag.isSet(1,1))dg::blas1::pointwiseDivide(tmp, diag.value(1,1),tmp);
             q_.idx(2,1)=2;
             q_.value(2)=tmp;
         }
@@ -282,7 +282,7 @@ struct CholeskyTensor
     const SparseTensor<container>& diagonal()const{return diag;}
 
     private:
-    SparseTensor q_, diag;
+    SparseTensor<container> q_, diag;
     bool lower_;
 };
 
@@ -404,9 +404,9 @@ void multiply( const SparseTensor<container>& t, const container& in0, const con
     if(!t.isSet(0,1))//lower triangular
     {
         if(t.isSet(1,1))  dg::blas1::pointwiseDot( t.value(1,1), in1, out1);
-        else out1=in1
+        else out1=in1;
         if(t.isSet(1,0)) dg::blas1::pointwiseDot( 1., t.value(1,0), in0, 1., out1);
-        if( t.isSet(0,0)) dg::blas::pointwiseDot( t.value(0,0), in0, out0);
+        if( t.isSet(0,0)) dg::blas1::pointwiseDot( t.value(0,0), in0, out0);
         else out0=in0;
         return;
     }
@@ -524,7 +524,7 @@ void scal(const CholeskyTensor<container>& ch, const SparseElement<container>& e
         diag.value(size) = e.value();
     for( unsigned i=0; i<2; i++)
     {
-        if(!diag.isSet(i,i)
+        if(!diag.isSet(i,i))
             diag.idx(i,i)=size;
         else
             dg::blas1::pointwiseDot( e.value(), diag.value(i,i), diag.value(i,i));
diff --git a/inc/dg/geometry/tensor_t.cu b/inc/dg/geometry/tensor_t.cu
index bc3575479..35cfa7a39 100644
--- a/inc/dg/geometry/tensor_t.cu
+++ b/inc/dg/geometry/tensor_t.cu
@@ -1,4 +1,5 @@
 #include <iostream>
+#include <cmath>
 
 #include "tensor.h"
 
@@ -9,7 +10,7 @@ void print( const dg::SparseTensor<thrust::host_vector<double> >& t)
         for( unsigned j=0; j<3; j++)
         {
             if(t.isSet(i,j)) std::cout << t.value(i,j)[0]<<" ";
-            else std::cout <<"XX ";
+            else std::cout <<"xx ";
         }
         std::cout << "\n";
     }
@@ -24,7 +25,7 @@ std::ostream& operator<<(std::ostream& os, const dg::SparseElement<thrust::host_
 
 int main()
 {
-    thrust::host_vector<double> one(1,11), two(1,22), three(1,33), four(1,44), five(1,55), six(1,66), seven(1,77), eight(1,88), nine(1,99);
+    thrust::host_vector<double> one(1,10), two(1,20), three(1,30), four(1,40), five(1,50), six(1,60), seven(1,70), eight(1,80), nine(1,90);
 
     dg::SparseTensor<thrust::host_vector<double> > dense2d(3);
     dense2d.idx(0,0) = 0, dense2d.idx(0,1) = 1;
@@ -41,24 +42,26 @@ int main()
     std::cout << "Test dg::Sparse Tensor class \n";
     std::cout << "Dense 2d Tensor \n";
     print( dense2d);
-    std::cout << "dg::Sparse 3d Tensor \n";
+    std::cout << "Sparse 3d Tensor \n";
     print( sparse3d);
     std::cout << "empty Tensor \n";
     print( empty);
+
+
     std::cout<< "Test dg::SparseElement";
     dg::SparseElement<thrust::host_vector<double> > e(eight);
     dg::SparseElement<thrust::host_vector<double> > ee;
     std::cout<<"\n construct: " <<e<<" "<<ee<<"\n";
     ee = e;
-    e.set(nine);
-    std::cout << "Assignment and set : "<<e<<" "<<ee<<"\n";
-    dg::SparseElement<thrust::host_vector<double> > sqrt = e.sqrt();
-    std::cout<<"\n sqrt(): "<<sqrt;
-    sqrt = sqrt.invert();
-    std::cout<<"\n invert(): "<<sqrt;
-    sqrt.clear();
-    std::cout<<"\n clear(): "<<sqrt;
-    std::cout <<std::endl;
+    e.value()=nine;
+    std::cout << "Assignment and set : "<<ee<<" (80) "<<e<<"(90)\n";
+    dg::SparseElement<thrust::host_vector<double> > sqr = e.sqrt();
+    std::cout<<"sqrt(): "<<sqr<<" ("<<std::sqrt(90)<<")\n";
+    sqr = sqr.invert();
+    std::cout<<"invert(): "<<sqr<<"\n";
+    sqr.clear();
+    std::cout<<"clear(): "<<sqr<<"\n";
+    std::cout <<std::flush;
     return 0;
 
 
-- 
GitLab


From deca99b389012308586268588def12109fa5d39b Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 7 Aug 2017 07:13:56 -0700
Subject: [PATCH 123/453] edited tensor functions and added multiply.h header

---
 inc/dg/geometry/multiply.h | 246 +++++++++++++++++++++++++++++++++++++
 inc/dg/geometry/tensor.h   | 177 +-------------------------
 2 files changed, 252 insertions(+), 171 deletions(-)
 create mode 100644 inc/dg/geometry/multiply.h

diff --git a/inc/dg/geometry/multiply.h b/inc/dg/geometry/multiply.h
new file mode 100644
index 000000000..cda9b9cab
--- /dev/null
+++ b/inc/dg/geometry/multiply.h
@@ -0,0 +1,246 @@
+#pragma once 
+
+#include "dg/backend/operator.h"
+#include "dg/functors.h"
+#include "dg/blas1.h"
+
+namespace tensor
+{
+
+template<class container>
+void scal( SparseTensor<container>& t, const SparseElement<container>& e)
+{
+    if(!e.isSet()) return;
+    for( unsigned i=0; i<2; i++)
+        for( unsigned j=0; j<2; i++)
+            if(t.isSet(i,j)) dg::blas1::pointwiseDot( e.value(), t.value(i,j), t.value(i,j));
+}
+
+template<class container>
+void multiply( const SparseElement<container>& e, const container& in, container& out)
+{
+    if(e.isSet()) 
+        dg::blas1::pointwiseDot(e.value(), in,out);
+    else
+        out=in;
+}
+
+template<class container>
+void divide( const container& in, const SparseElement<container>& e, container& out)
+{
+    if(e.isSet()) 
+        dg::blas1::pointwiseDivide(in, e.value(),out);
+    else
+        out=in;
+}
+
+///this version keeps the input intact 
+///aliasing allowed if tensor is either lower, upper or diagonal alias allowed
+template<class container>
+void multiply( const SparseTensor<container>& t, const container& in0, const container& in1, container& out0, container& out1)
+{
+    if(!t.isSet(0,1))//lower triangular
+    {
+        if(t.isSet(1,1))  dg::blas1::pointwiseDot( t.value(1,1), in1, out1);
+        else out1=in1;
+        if(t.isSet(1,0)) dg::blas1::pointwiseDot( 1., t.value(1,0), in0, 1., out1);
+        if( t.isSet(0,0)) dg::blas1::pointwiseDot( t.value(0,0), in0, out0);
+        else out0=in0;
+        return;
+    }
+    //upper triangular
+    if( t.isSet(0,0) ) 
+        dg::blas1::pointwiseDot( t.value(0,0), in0, out0); 
+    else 
+        out0=in0;
+    if(t.isSet(0,1))
+        dg::blas1::pointwiseDot( 1.,  t.value(0,1), in1, 1., out0);
+
+    if( t.isSet(1,1) )
+        dg::blas1::pointwiseDot( t.value(1,1), in1, out1);
+    else 
+        out1=in1;
+    if(t.isSet(1,0))
+        dg::blas1::pointwiseDot( 1.,  t.value(1,0), in0, 1., out1);
+}
+
+template<class container>
+void multiply( const SparseTensor<container>& t, container& inout0, container& inout1, container& workspace)
+{
+    if( t.isSet(0,1) ) dg::blas1::pointwiseDot( t.value(0,1), inout1, workspace);
+    //compute out1 inplace
+    if( t.isSet(1,1)) dg::blas1::pointwiseDot( t.value(1,1), inout1, inout1);
+    if( t.isSet(1,0)) dg::blas1::pointwiseDot( 1., t.value(1,0), inout0, 1., inout1);
+
+    if(t.isSet(0,1)) //workspace is filled
+    {
+        if( !t.isSet(0,0)) dg::blas1::axpby( 1., inout0, 1., workspace, workspace);
+        else dg::blas1::pointwiseDot( 1., t.value(0,0), inout0, 1., workspace); 
+        workspace.swap( inout0);
+    }
+    else
+        if( t.isSet(0,0)) dg::blas1::pointwiseDot( t.value(0,0), inout0, inout0); 
+
+    //needs to load a vector         11 times if every element is set (5 is the optimal symmetric inplace algorithm)
+    //if triangular                  7 (5 optimal)
+    //if diagonal                    4 (optimal)
+    //if unity                       0 (optimal)
+}
+
+///aliasing allowed if t is known to be lower or upper triangular
+template<class container>
+void multiply( const SparseTensor<container>& t, const container& in0, const container& in1, const container& in2, container& out0, container& out1, container& out2)
+{
+    if( !t.isSet(0,1)&&!t.isSet(0,2)&&!t.isSet(1,2))
+    {
+        //lower triangular
+        if(!t.isSet(2,2)) out2=in2;
+        else dg::blas1::pointwiseDot( t.value(2,2), in2, out2);
+        if(t.isSet(2,1))
+            dg::blas1::pointwiseDot( 1., t.value(2,1), in1, 1., out2);
+        if(t.isSet(2,0))
+            dg::blas1::pointwiseDot( 1., t.value(2,0), in0, 1., out2);
+        if( !t.isSet(1,1)) out1=in1;
+        else dg::blas1::pointwiseDot( t.value(1,1), in1, out1);
+        if(t.isSet(1,0))
+            dg::blas1::pointwiseDot( 1., t.value(1,0), in0, 1., out1);
+        if( !t.isSet(0,0)) out0=in0;
+        else dg::blas1::pointwiseDot( t.value(0,0), in0, out0); 
+    }
+    //upper triangular and default
+    if( !t.isSet(0,0)) out0=in0;
+    else dg::blas1::pointwiseDot( t.value(0,0), in0, out0); 
+    if(t.isSet( 0,1))
+        dg::blas1::pointwiseDot( 1., t.value(0,1), in1, 1., out0);
+    if(t.isSet( 0,2))
+        dg::blas1::pointwiseDot( 1., t.value(0,2), in2, 1., out0);
+
+    if( !t.isSet(1,1)) out1=in1;
+    else dg::blas1::pointwiseDot( t.value(1,1), in1, out1);
+    if(t.isSet(1,0))
+        dg::blas1::pointwiseDot( 1., t.value(1,0), in0, 1., out1);
+    if(t.isSet(1,2))
+        dg::blas1::pointwiseDot( 1., t.value(1,2), in0, 1., out1);
+
+    if(!t.isSet(2,2)) out2=in2;
+    else dg::blas1::pointwiseDot( t.value(2,2), in2, out2);
+    if(t.isSet(2,1))
+        dg::blas1::pointwiseDot( 1., t.value(2,1), in1, 1., out2);
+    if(t.isSet(2,0))
+        dg::blas1::pointwiseDot( 1., t.value(2,0), in0, 1., out2);
+}
+
+template<class container>
+void multiply( const SparseTensor<container>& t, container& inout0, container& inout1, container& inout2, container& workspace0, container& workspace1)
+{
+    if( t.isSet(0,1) ) {
+        dg::blas1::pointwiseDot( t.value(0,1), inout1, workspace0);
+        if( t.isSet(0,2) ) dg::blas1::pointwiseDot( 1.,t.value(0,2), inout2, 1.,workspace0);
+    }
+    if(!t.isSet(0,1) && t.isSet(0,2))
+    {
+        dg::blas1::pointwiseDot( t.value(0,2), inout2, workspace0);
+    }
+    //else workspace0 is empty
+    //
+    if( t.isSet(1,0) ) {
+        dg::blas1::pointwiseDot( t.value(1,0), inout0, workspace1);
+        if( t.isSet(1,2) ) dg::blas1::pointwiseDot( 1.,t.value(1,2), inout2, 1.,workspace1);
+    }
+    if(!t.isSet(1,0) && t.isSet(1,2))
+    {
+        dg::blas1::pointwiseDot( t.value(1,2), inout2, workspace1);
+    }
+    //else workspace1 is empty
+    //
+    //compute out2 inplace
+    if( t.isSet(2,2)) dg::blas1::pointwiseDot( t.value(2,2), inout2, inout2);
+    if( t.isSet(2,1)) dg::blas1::pointwiseDot( 1., t.value(2,1), inout1, 1., inout2);
+    if( t.isSet(2,0)) dg::blas1::pointwiseDot( 1., t.value(2,0), inout0, 1., inout2);
+
+    if(t.isSet(0,1) ||t.isSet(0,2) ) //workspace0 is filled
+    {
+        if( !t.isSet(0,0)) dg::blas1::axpby( 1., inout0, 1., workspace0, workspace0);
+        else dg::blas1::pointwiseDot( 1., t.value(0,0), inout0, 1., workspace0); 
+        workspace0.swap( inout0);
+    }
+    else
+        if( t.isSet(0,0)) dg::blas1::pointwiseDot( t.value(0,0), inout0, inout0); 
+    if(t.isSet(1,0) ||t.isSet(1,2) ) //workspace1 is filled
+    {
+        if( !t.isSet(1,1)) dg::blas1::axpby( 1., inout1, 1., workspace1, workspace1);
+        else dg::blas1::pointwiseDot( 1., t.value(1,1), inout1, 1., workspace1); 
+        workspace1.swap( inout1);
+    }
+    else
+        if( t.isSet(1,1)) dg::blas1::pointwiseDot( t.value(1,1), inout1, inout1); 
+    //if everything is set: 28 loads (12 optimal, 9 optimal symmetric)
+    //if effective 2d:      12 (same as 2d algorithm, 5 optimal symmetric)
+
+}
+
+template<class container>
+SparseElement<container> determinant( const SparseTensor<container>& t)
+{
+    if(t.isEmpty())  return SparseElement<container>();
+    SparseTensor<container> d = t.dense();
+    container det = d.value(0,0);
+    std::vector<container> sub_det(3,det);
+    dg::blas1::transform( det, det, dg::CONSTANT(0));
+    //first compute the det of three submatrices
+    dg::blas1::pointwiseDot( d(0,0), d(1,1), sub_det[2]);
+    dg::blas1::pointwiseDot( -1., d(1,0), d(0,1), 1. ,sub_det[2]);
+
+    dg::blas1::pointwiseDot( d(0,0), d(2,1), sub_det[1]);
+    dg::blas1::pointwiseDot( -1., d(2,0), d(0,1), 1. ,sub_det[1]);
+
+    dg::blas1::pointwiseDot( d(1,0), d(2,1), sub_det[0]);
+    dg::blas1::pointwiseDot( -1., d(2,0), d(1,1), 1. ,sub_det[0]);
+
+    //now multiply according to Laplace expansion
+    dg::blas1::pointwiseDot( 1., d(0,2), sub_det[0], 1.,  det);
+    dg::blas1::pointwiseDot(-1., d(1,2), sub_det[1], 1.,  det);
+    dg::blas1::pointwiseDot( 1., d(2,2), sub_det[2], 1.,  det);
+
+    return SparseElement<container>(det);
+}
+
+//alias always allowed
+template<class container>
+void multiply( const CholeskyTensor<container>& ch, const container& in0, const container& in1, container& out0, container& out1)
+{
+    multiply(ch.upper(), in0, in1, out0, out1);
+    multiply(ch.diag(), out0, out1, out0, out1);
+    multiply(ch.lower(), out0, out1, out0, out1);
+}
+
+template<class container>
+SparseElement<container> determinant( const CholeskyTensor<container>& ch)
+{
+    SparseTensor<container> diag = ch.diag().dense();
+    SparseElement<container> det;
+    if(diag.isEmpty()) return det;
+    else det.value()=diag.value(0,0);
+    dg::blas1::pointwiseDot( det.value(), diag.value(1,1), det.value());
+    dg::blas1::pointwiseDot( det.value(), diag.value(2,2), det.value());
+    return det;
+}
+
+template<class container>
+void scal(const CholeskyTensor<container>& ch, const SparseElement<container>& e)
+{
+    const SparseTensor<container>& diag = ch.diag();
+    if(!e.isSet()) return;
+    unsigned size=diag.values().size();
+    if(!diag.isSet(0,0)|| !diag.isSet(1,1) || !diag.isSet(2,2))
+        diag.value(size) = e.value();
+    for( unsigned i=0; i<2; i++)
+    {
+        if(!diag.isSet(i,i))
+            diag.idx(i,i)=size;
+        else
+            dg::blas1::pointwiseDot( e.value(), diag.value(i,i), diag.value(i,i));
+    }
+}
+
+}//namespace tensor
diff --git a/inc/dg/geometry/tensor.h b/inc/dg/geometry/tensor.h
index b49d76167..5cb2f0053 100644
--- a/inc/dg/geometry/tensor.h
+++ b/inc/dg/geometry/tensor.h
@@ -1,6 +1,5 @@
 #pragma once 
 
-#include <cusp/coo_matrix.h>
 #include "dg/backend/operator.h"
 #include "dg/functors.h"
 #include "dg/blas1.h"
@@ -202,6 +201,11 @@ struct CholeskyTensor
     void decompose( const SparseTensor<container>& in)
     {
         SparseTensor<container> denseIn=in.dense();
+        /*
+         * One nice property of positive definite is that the diagonal elements are 
+         * greater than zero.
+         */
+
         if(in.isSet(0,0))
         {
             diag.idx(0,0)=0;
@@ -283,6 +287,7 @@ struct CholeskyTensor
 
     private:
     SparseTensor<container> q_, diag;
+    int pivot_;
     bool lower_;
 };
 
@@ -364,174 +369,4 @@ void SparseTensor<container>::clear_unused_values()
 ///@endcond
 
 
-
-///@cond
-namespace tensor
-{
-
-template<class container>
-void scal( SparseTensor<container>& t, const SparseElement<container>& e)
-{
-    if(!e.isSet()) return;
-    for( unsigned i=0; i<2; i++)
-        for( unsigned j=0; j<2; i++)
-            if(t.isSet(i,j)) dg::blas1::pointwiseDot( e.value(), t.value(i,j), t.value(i,j));
-}
-
-template<class container>
-void multiply( const SparseElement<container>& e, const container& in, container& out)
-{
-    if(e.isSet()) 
-        dg::blas1::pointwiseDot(e.value(), in,out);
-    else
-        out=in;
-}
-
-template<class container>
-void divide( const container& in, const SparseElement<container>& e, container& out)
-{
-    if(e.isSet()) 
-        dg::blas1::pointwiseDivide(in, e.value(),out);
-    else
-        out=in;
-}
-
-///this version keeps the input intact 
-///aliasing allowed if tensor is either lower, upper or diagonal alias allowed
-template<class container>
-void multiply( const SparseTensor<container>& t, const container& in0, const container& in1, container& out0, container& out1)
-{
-    if(!t.isSet(0,1))//lower triangular
-    {
-        if(t.isSet(1,1))  dg::blas1::pointwiseDot( t.value(1,1), in1, out1);
-        else out1=in1;
-        if(t.isSet(1,0)) dg::blas1::pointwiseDot( 1., t.value(1,0), in0, 1., out1);
-        if( t.isSet(0,0)) dg::blas1::pointwiseDot( t.value(0,0), in0, out0);
-        else out0=in0;
-        return;
-    }
-    //upper triangular
-    if( t.isSet(0,0) ) 
-        dg::blas1::pointwiseDot( t.value(0,0), in0, out0); 
-    else out0=in0;
-    if(t.isSet(0,1))
-        dg::blas1::pointwiseDot( 1.,  t.value(0,1), in1, 1., out0);
-
-    if( t.isSet(1,1) )
-        dg::blas1::pointwiseDot( t.value(1,1), in1, out1);
-    else out1=in1;
-    if(t.isSet(1,0))
-        dg::blas1::pointwiseDot( 1.,  t.value(1,0), in0, 1., out1);
-}
-
-///aliasing allowed if t is known to be lower or upper triangular
-template<class container>
-void multiply( const SparseTensor<container>& t, const container& in1, const container& in2, const container& in3, container& out1, container& out2, container& out3, container& workspace1)
-{
-    if( !t.isSet(0,1)&&!t.isSet(0,2)&&!t.isSet(1,2))
-    {
-        //lower triangular
-        if(!t.isSet(2,2)) out3=in3;
-        else dg::blas1::pointwiseDot( t.value(2,2), in3, out3);
-        if(t.isSet(2,1))
-            dg::blas1::pointwiseDot( 1., t.value(2,1), in2, 1., out3);
-        if(t.isSet(2,0))
-            dg::blas1::pointwiseDot( 1., t.value(2,0), in1, 1., out3);
-        if( !t.isSet(1,1)) out2=in2;
-        else dg::blas1::pointwiseDot( t.value(1,1), in2, out2);
-        if(t.isSet(1,0))
-            dg::blas1::pointwiseDot( 1., t.value(1,0), in1, 1., out2);
-        if( !t.isSet(0,0)) out1=in1;
-        else dg::blas1::pointwiseDot( t.value(0,0), in1, out1); 
-    }
-    //upper triangular
-    if( !t.isSet(0,0)) out1=in1;
-    else dg::blas1::pointwiseDot( t.value(0,0), in1, out1); 
-    if(t.isSet( 0,1))
-        dg::blas1::pointwiseDot( 1., t.value(0,1), in2, 1., out1);
-    if(t.isSet( 0,2))
-        dg::blas1::pointwiseDot( 1., t.value(0,2), in3, 1., out1);
-
-    if( !t.isSet(1,1)) out2=in2;
-    else dg::blas1::pointwiseDot( t.value(1,1), in2, out2);
-    if(t.isSet(1,0))
-        dg::blas1::pointwiseDot( 1., t.value(1,0), in1, 1., out2);
-    if(t.isSet(1,2))
-        dg::blas1::pointwiseDot( 1., t.value(1,2), in1, 1., out2);
-
-    if(!t.isSet(2,2)) out3=in3;
-    else dg::blas1::pointwiseDot( t.value(2,2), in3, out3);
-    if(t.isSet(2,1))
-        dg::blas1::pointwiseDot( 1., t.value(2,1), in2, 1., out3);
-    if(t.isSet(2,0))
-        dg::blas1::pointwiseDot( 1., t.value(2,0), in1, 1., out3);
-}
-
-template<class container>
-SparseElement<container> determinant( const SparseTensor<container>& t)
-{
-    if(t.isEmpty())  return SparseElement<container>();
-    SparseTensor<container> d = t.dense();
-    container det = d.value(0,0);
-    std::vector<container> sub_det(3,det);
-    dg::blas1::transform( det, det, dg::CONSTANT(0));
-    //first compute the det of three submatrices
-    dg::blas1::pointwiseDot( d(0,0), d(1,1), sub_det[2]);
-    dg::blas1::pointwiseDot( -1., d(1,0), d(0,1), 1. ,sub_det[2]);
-
-    dg::blas1::pointwiseDot( d(0,0), d(2,1), sub_det[1]);
-    dg::blas1::pointwiseDot( -1., d(2,0), d(0,1), 1. ,sub_det[1]);
-
-    dg::blas1::pointwiseDot( d(1,0), d(2,1), sub_det[0]);
-    dg::blas1::pointwiseDot( -1., d(2,0), d(1,1), 1. ,sub_det[0]);
-
-    //now multiply accordint to Laplace expansion
-    dg::blas1::pointwiseDot( 1., d(0,2), sub_det[0], 1.,  det);
-    dg::blas1::pointwiseDot(-1., d(1,2), sub_det[1], 1.,  det);
-    dg::blas1::pointwiseDot( 1., d(2,2), sub_det[2], 1.,  det);
-
-    return SparseElement<container>(det);
-}
-
-//alias always allowed
-template<class container>
-void multiply( const CholeskyTensor<container>& ch, const container& in0, const container& in1, container& out0, container& out1)
-{
-    multiply(ch.upper(), in0, in1, out0, out1);
-    multiply(ch.diag(), out0, out1, out0, out1);
-    multiply(ch.lower(), out0, out1, out0, out1);
-}
-
-template<class container>
-SparseElement<container> determinant( const CholeskyTensor<container>& ch)
-{
-    SparseTensor<container> diag = ch.diag().dense();
-    SparseElement<container> det;
-    if(diag.isEmpty()) return det;
-    else det.value()=diag.value(0,0);
-    dg::blas1::pointwiseDot( det.value(), diag.value(1,1), det.value());
-    dg::blas1::pointwiseDot( det.value(), diag.value(2,2), det.value());
-    return det;
-}
-
-template<class container>
-void scal(const CholeskyTensor<container>& ch, const SparseElement<container>& e)
-{
-    const SparseTensor<container>& diag = ch.diag();
-    if(!e.isSet()) return;
-    unsigned size=diag.values().size();
-    if(!diag.isSet(0,0)|| !diag.isSet(1,1) || !diag.isSet(2,2))
-        diag.value(size) = e.value();
-    for( unsigned i=0; i<2; i++)
-    {
-        if(!diag.isSet(i,i))
-            diag.idx(i,i)=size;
-        else
-            dg::blas1::pointwiseDot( e.value(), diag.value(i,i), diag.value(i,i));
-    }
-}
-
-}//namespace tensor
-///@endcond
-
 }//namespace dg
-- 
GitLab


From ce5b1a1bf61e6986eba5a917c9718d48eaf29b79 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 7 Aug 2017 08:33:34 -0700
Subject: [PATCH 124/453] updated documentation

---
 inc/dg/exceptions.h         |  11 +-
 inc/dg/geometry/multiply.h  |   7 ++
 inc/dg/geometry/tensor.h    | 236 ++++++++++++++++++++++++++----------
 inc/dg/geometry/tensor_t.cu |   1 +
 4 files changed, 187 insertions(+), 68 deletions(-)

diff --git a/inc/dg/exceptions.h b/inc/dg/exceptions.h
index 4869dd731..5e498425a 100644
--- a/inc/dg/exceptions.h
+++ b/inc/dg/exceptions.h
@@ -16,7 +16,7 @@
 namespace dg
 {
 
-///small class holding a stringstream 
+///@brief small class holding a stringstream 
 ///@ingroup misc
 class Message 
 {
@@ -25,9 +25,15 @@ class Message
     Message( const Message&); //we can't copy ostreams in C++
     Message& operator=(const Message&);
   public:
+    ///construct an empty message
     Message(){}
+    /**
+     * @brief Construct message with string
+     * @param m puts m into stream
+     */
     Message( std::string m){ sstream_<<m;}
     ~Message(){}
+    ///@brief add values to the message stream
     /// @note you can't use std::endl or std::flush in here
     template<class T>
     Message & operator << (const T& value)
@@ -35,9 +41,10 @@ class Message
         sstream_ << value;
         return *this;
     }
-    ///return the message as a string
+    ///return the message contained in the stream as a string
     std::string str() const {return sstream_.str();}
     ///put the sringstream string into the ostream
+    ///@note same as os<<m.str();
     friend std::ostream& operator<<(std::ostream& os, const Message& m)
     {
         os<<m.str();
diff --git a/inc/dg/geometry/multiply.h b/inc/dg/geometry/multiply.h
index cda9b9cab..5f3afe897 100644
--- a/inc/dg/geometry/multiply.h
+++ b/inc/dg/geometry/multiply.h
@@ -7,6 +7,9 @@
 namespace tensor
 {
 
+///@addtogroup geometry
+///@{
+
 template<class container>
 void scal( SparseTensor<container>& t, const SparseElement<container>& e)
 {
@@ -205,6 +208,7 @@ SparseElement<container> determinant( const SparseTensor<container>& t)
     return SparseElement<container>(det);
 }
 
+///@cond
 //alias always allowed
 template<class container>
 void multiply( const CholeskyTensor<container>& ch, const container& in0, const container& in1, container& out0, container& out1)
@@ -242,5 +246,8 @@ void scal(const CholeskyTensor<container>& ch, const SparseElement<container>& e
             dg::blas1::pointwiseDot( e.value(), diag.value(i,i), diag.value(i,i));
     }
 }
+///@endcond
+
+///@}
 
 }//namespace tensor
diff --git a/inc/dg/geometry/tensor.h b/inc/dg/geometry/tensor.h
index 5cb2f0053..124e0060b 100644
--- a/inc/dg/geometry/tensor.h
+++ b/inc/dg/geometry/tensor.h
@@ -7,33 +7,14 @@
 namespace dg
 {
 
-/*
-1. make SparseTensor (remove vec_idx in SparseTensor)
-2. make SparseElement (a std::vector with 0 or 1 element)
-10  SparseElement det = dg::tensor::determinant(metric) //if not empty make dense and then compute
-3. dg::tensor::sqrt(metric) //inverse sqrt of determinant
-4. if(vol.isSet()) dg::blas1::pointwiseDot( vol, ...)
-5. dg::tensor::scal( metric, vol); //if(isSet) multiply all values elements
-6. dg::tensor::multiply( metric, in1, in2, out1, out2); //sparse matrix dense vector mult
-6. dg::tensor::multiply( metric, in1, in2, in3, out1, out2, out3);  
-7  perp_metric = metric.perp(); //for use in arakawa when computing perp_vol
-8  dense_metric = metric.dens(); //add 0 and 1 to values and rewrite indices
-9  metric.isPerp(), metric.isDense(), metric.isEmpty()
-11 dg::tensor::product( tensor, jac, temp) //make jac dense and then use multiply 3 times (needs write access in value(i,j)
-12 dg::tensor::product( jac.transpose(), temp, result)  //then use three elements if symmetric
-13 Geometry has  std::vector<sparseElements>  v[0], v[1], v[2] and Jacobian 
-14 when computing metric believe generator and compute on host for-loop
-*/
-  
-
 /**
-* @brief Abstract base class for matrices and vectors sharing or implicitly assuming elements 
+* @brief Class for 2x2 and 3x3 matrices sharing or implicitly assuming elements 
 *
-* The goal of this class is to enable shared access to stored containers 
+* This class enables shared access to stored containers 
 * or not store them at all since the storage of (and computation with) a container is expensive.
 
-* This class contains both a (dense) matrix and a (dense) vector of integers.
-* If positive or zero, the integer represents a gather index into an array of containers, 
+* This class contains both a (dense) matrix of integers.
+* If positive or zero, the integer represents a gather index into the stored array of containers, 
 if negative the value of the container is assumed to be 1, except for the off-diagonal entries
     in the matrix where it is assumed to be 0.
 * We then only need to store non-trivial and non-repetitive containers.
@@ -43,28 +24,33 @@ if negative the value of the container is assumed to be 1, except for the off-di
 template<class container>
 struct SparseTensor
 {
+    ///no element is set
     SparseTensor( ):mat_idx_(3,-1.) {}
+
+    /**
+     * @brief reserve space for containers in the values array
+     * @param value_size reserve space for this number of containers (default constructor) 
+     */
     SparseTensor( unsigned value_size): mat_idx_(3,-1.), values_(value_size){}
 
     /**
-    * @brief 
-    *
-    * @param mat_idx
+    * @brief pass array of containers
     * @param values The contained containers must all have the same size
     */
     SparseTensor( const std::vector<container>& values ): mat_idx_(3,-1.), values_(values){}
 
-    template<class otherContainer>
-    SparseTensor( const SparseTensor<otherContainer>& src): mat_idx_(src.mat_idx()), values_(src.values().size()){
+    /**
+     * @brief Type conversion from other container types
+     * @tparam OtherContainer calls dg::blas1::transfer to convert OtherContainer to container
+     * @param src the source matrix to convert
+     */
+    template<class OtherContainer>
+    SparseTensor( const SparseTensor<OtherContainer>& src): values_(src.values().size()){
         dg::blas1::transfer( src.values(), values_);
     }
-    ///sets values leaves indices unchanged (size must be  the same or larger as current 
-    void set( const std::vector<container>& values ){
-        values_=values;
-    }
 
     /**
-    * @brief check if an index is set or not
+    * @brief check if a value is set at the given position or not
     * @param i row index 0<i<2
     * @param j col index 0<j<2
     * @return true if container is non-empty, false if value is assumed implicitly
@@ -74,12 +60,74 @@ struct SparseTensor
         return true;
     }
 
+    /**
+    * @brief return index into the values array for the given position
+    * @param i row index 0<i<2
+    * @param j col index 0<j<2
+    * @return -1 if !isSet(i,j), index into values array else
+    */
     int idx(unsigned i, unsigned j)const{return mat_idx_(i,j);}
+    /**
+    * @brief main set function to modify indices
+    *
+    * use this and the value() member to assemble the tensor
+    * @param i row index 0<i<2
+    * @param j col index 0<j<2
+    * @return write access to value index to be set
+    */
     int& idx(unsigned i, unsigned j){return mat_idx_(i,j);}
+     /*! @brief unset an index, does not clear the associated value
+      *
+      * @param i row index 0<i<2
+      * @param j col index 0<j<2
+      */
+     void unset( unsigned i, unsigned j) {
+         mat_idx_(i,j)=-1;
+     }
+     /**
+      * @brief clear any unused values and reset the corresponding indices
+      *
+      * This function erases all values that are unreferenced by any index and appropriately redefines the remaining indices
+      */
+     void clear_unused_values();
+
+    /*!@brief Access the underlying container
+     * @return if !isSet(i,j) the result is undefined, otherwise values[idx(i,j)] is returned. 
+     * @param i row index 0<i<2
+     * @param j col index 0<j<2
+     * @note If the indices fall out of range of index the result is undefined
+     */
+    const container& value(size_t i, size_t j)const{ 
+        int k = mat_idx_(i,j);
+        return values_[k];
+    }
+    /**
+     * @brief Return the container at given position, create one if there isn't one already
+     * @param i index into the values array
+     * @return  always returns a container 
+     */
+    container& value( size_t i)
+    {
+        if(i>=values_.size());
+        values_.resize(i+1);
+        return values_[i];
+    }
+    /**
+     * @brief Return read access to the values array
+     * @return read access to values array
+     */
+    const std::vector<container>& values()const{return values_;}
+    ///clear all values, Tensor is empty after that
+    void clear(){
+        mat_idx_=dg::Operator<int>(3,-1);
+        values_.clear();
+    }
 
     /**
     * @brief Test the matrix for emptiness
-    * @return  true if no value in the matrix is set
+    *
+    * The matrix is empty if !isSet(i,j) for all i and j
+    * @return true if no value in the matrix is set
     */
     bool isEmpty()const{ 
         bool empty=true;
@@ -89,7 +137,13 @@ struct SparseTensor
                     empty=false;
         return empty;
     }
-    ///if all elements are set
+
+    /**
+    * @brief Test the matrix for emptiness
+    *
+    * The matrix is dense if isSet(i,j) for all i and j
+    * @return true if all values in the matrix are set
+    */
     bool isDense()const{
         bool dense=true;
         for(unsigned i=0; i<3; i++)
@@ -98,7 +152,13 @@ struct SparseTensor
                     dense=false;
         return dense;
     }
-    ///if all elements in the third dimension are empty
+
+    /**
+    * @brief Test if the elements in the third dimension are unset
+    *
+    * The matrix is perpendicular if !isSet(i,j) for any i,j=2
+    * @return true if perpenicular
+    * */
     bool isPerp() const
     {
         bool empty=true;
@@ -110,46 +170,24 @@ struct SparseTensor
         return empty;
     }
      
-     ///return an empty Tensor
+     ///construct an empty Tensor
      SparseTensor empty()const{return SparseTensor();}
-     ///copy and fill all unset values
+     ///Construct a tensor filled unset values with explicit 0 or 1
+     /// @note returns an empty matrix if isEmpty() returns true
      SparseTensor dense()const;
      ///copy and erase all values in the third dimension
+     ///@note calls clear_unused_values() to get rid of the elements
      SparseTensor perp()const;
-     ///unset an index don't clear values
-     void unset( unsigned i, unsigned j) {
-         mat_idx_(i,j)=-1;
-     }
-     ///clear any unused valuse and reset the corresponding indices
-     void clear_unused_values();
 
-    /*!@brief Access the underlying container
-     * @return if !isSet(i,j) the result is undefined, otherwise values[mat_idx(i,j)] is returned. 
-     * @note If the indices fall out of range of mat_idx the result is undefined
+    /**
+     * @brief Return the transpose of the currrent tensor
+     * @return swapped rows and columns
      */
-    const container& value(size_t i, size_t j)const{ 
-        int k = mat_idx_(i,j);
-        return values_[k];
-    }
-    //always returns a container, if i does not exist yet we resize
-    container& value( size_t i)
-    {
-        if(i>=values_.size());
-        values_.resize(i+1);
-        return values_[i];
-    }
-    ///clear all values
-    void clear(){
-        mat_idx_=dg::Operator<int>(3,-1);
-        values_.clear();
-    }
-
     SparseTensor transpose()const{
         SparseTensor tmp(*this);
         tmp.mat_idx_ = mat_idx_.transpose();
         return tmp;
     }
-    const std::vector<container>& values()const{return values_;}
 
     private:
     dg::Operator<int> mat_idx_;
@@ -157,29 +195,61 @@ struct SparseTensor
     void unique_insert(std::vector<int>& indices, int& idx);
 };
 
-//neutral element wrt multiplication represents forms
+
+/**
+ * @brief This is a sparse Tensor with only one element i.e. a Form
+ *
+ * @tparam container a container class
+ * @ingroup misc
+ */
 template<class container>
 struct SparseElement
 {
+    ///create empty object
     SparseElement(){}
+    /**
+     * @brief copy construct value
+     * @param value a value
+     */
     SparseElement(const container& value):value_(1,value){ }
+
+    ///@brief Read access
+    ///@return read access to contained value
     const container& value( )const { 
         return value_[0];
     }
+    /**
+     * @brief write access, create a container if there isn't one already
+     * @return write access, always returns a container 
+     */
     container& value() {
         if(!isSet()) value_.resize(1);
         return value_[0];
     }
+
+    /**
+     * @brief check if an element is set or not
+     * @return false if the value array is empty
+     */
     bool isSet()const{
         if( value_.empty()) return false;
         return true;
     }
+    ///@brief Clear contained value
     void clear(){value_.clear();}
+    /**
+     * @brief calls sqrt transform function on value
+     * @return Sparse element containing sqrt
+     */
     SparseElement sqrt(){ 
         SparseElement tmp(*this);
         if( isSet()) dg::blas1::transform( tmp.value_, tmp.value_, dg::SQRT<double>());
         return tmp;
     }
+    /**
+     * @brief calls invert transform function on value
+     * @return Sparse element containing inverse 
+     */
     SparseElement invert(){ 
         SparseElement tmp(*this);
         if( isSet()) dg::blas1::transform( tmp.value_, tmp.value_, dg::INVERT<double>());
@@ -191,13 +261,33 @@ struct SparseElement
     std::vector<container> value_;
 };
 
+/**
+ * @brief data structure to hold the LDL^T decomposition of a symmetric positive definite matrix
+ *
+ * LDL^T stands for a lower triangular matrix L,  a diagonal matrix D and the transpose L^T
+ * @tparam container a valid container class
+ * @attention the tensor in the Elliptic classes actually only need to be positive **semi-definite**
+ * and unfortunately the decomposition is unstable for semi-definite matrices.
+* @ingroup misc
+ */
 template<class container>
 struct CholeskyTensor
 {
+    /**
+     * @brief decompose given tensor
+     *
+     * @param in must be symmetric and positive definite
+     */
     CholeskyTensor( const SparseTensor<container>& in)
     {
         decompose(in);
     }
+    /**
+     * @brief decompose given tensor
+     *
+     * overwrites the existing decomposition
+     * @param in must be symmetric and positive definite
+     */
     void decompose( const SparseTensor<container>& in)
     {
         SparseTensor<container> denseIn=in.dense();
@@ -265,6 +355,11 @@ struct CholeskyTensor
         q_.clear_unused_values();
         lower_=true;
     }
+
+    /**
+     * @brief Returns L
+     * @return a lower triangular matrix with 1 on diagonal
+     */
     const SparseTensor<container>& lower()const{
         if(lower_) return q_;
         std::swap(q_.idx(1,0), q_.idx(0,1));
@@ -274,6 +369,10 @@ struct CholeskyTensor
         return q_;
 
     }
+    /**
+     * @brief Returns L^T
+     * @return a upper triangular matrix with 1 on diagonal
+     */
     const SparseTensor<container>& upper()const{
         if(!lower_) return q_;
         std::swap(q_.idx(1,0), q_.idx(0,1));
@@ -283,6 +382,11 @@ struct CholeskyTensor
         return q_;
 
     }
+
+    /**
+     * @brief Returns D
+     * @return only diagonal elements are set if any
+     */
     const SparseTensor<container>& diagonal()const{return diag;}
 
     private:
diff --git a/inc/dg/geometry/tensor_t.cu b/inc/dg/geometry/tensor_t.cu
index 35cfa7a39..187dc35e8 100644
--- a/inc/dg/geometry/tensor_t.cu
+++ b/inc/dg/geometry/tensor_t.cu
@@ -2,6 +2,7 @@
 #include <cmath>
 
 #include "tensor.h"
+#include "multiply.h"
 
 void print( const dg::SparseTensor<thrust::host_vector<double> >& t)
 {
-- 
GitLab


From 38b573b53b69b1126921f88fbe13159bc39f18e5 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Tue, 8 Aug 2017 00:31:37 +0200
Subject: [PATCH 125/453] debugging tensor.h and multiply.h there is still a
 bug somewhere

---
 inc/dg/backend/dlt.h        |   4 +-
 inc/dg/backend/operator.h   |  10 +--
 inc/dg/exceptions.h         |  20 +++--
 inc/dg/geometry/multiply.h  | 175 +++++++++++++++++++++++++++++-------
 inc/dg/geometry/tensor.h    | 152 +++++++++++++++++--------------
 inc/dg/geometry/tensor_t.cu |  62 ++++++++++---
 6 files changed, 295 insertions(+), 128 deletions(-)

diff --git a/inc/dg/backend/dlt.h b/inc/dg/backend/dlt.h
index 5683749c8..a270995b2 100644
--- a/inc/dg/backend/dlt.h
+++ b/inc/dg/backend/dlt.h
@@ -188,9 +188,9 @@ DLT<T>::DLT( unsigned n):a_(n), w_(n), forw_(n*n), back_(n*n),backEQ_(n*n)
     for( unsigned j=0; j<n*n; j++) stream >> forw_[j];
     for( unsigned j=0; j<n*n; j++) stream >> backEQ_[j];
     
-    if( n==0) throw Error( Message() << "n==0 not allowed! You typed "<<n,_ping_);
+    if( n==0) throw Error( Message(_ping_) << "n==0 not allowed! You typed "<<n);
     if( !stream.good())
-        throw Error( Message()<<"n>20 not allowed! You typed: "<<n, _ping_);
+        throw Error( Message(_ping_)<<"n>20 not allowed! You typed: "<<n );
 }
 
 
diff --git a/inc/dg/backend/operator.h b/inc/dg/backend/operator.h
index 81ea56dc8..fc417d495 100644
--- a/inc/dg/backend/operator.h
+++ b/inc/dg/backend/operator.h
@@ -55,7 +55,7 @@ class Operator
         unsigned n = std::distance( first, last);
         n_ = (unsigned)sqrt( (value_type)n);
 #ifdef DG_DEBUG
-        assert( n_*n_ == n);
+        if( n_*n_!=n) throw Error( Message(_ping_)<<"Too few elements "<<n<<" need "<<n_*n_<<"\n");
 #endif
     }
     /**
@@ -68,7 +68,7 @@ class Operator
         unsigned n = src.size();
         n_ = (unsigned)sqrt( (value_type)n);
 #ifdef DG_DEBUG
-        assert( n_*n_ == n);
+        if( n_*n_!=n) throw Error( Message(_ping_)<<"Wrong number of elements "<<n<<" need "<<n_*n_<<"\n");
 #endif
     }
 
@@ -89,7 +89,7 @@ class Operator
      */
     T& operator()(const size_t i, const size_t j){
 #ifdef DG_DEBUG
-        assert( i<n_ && j < n_);
+        if(!(i<n_&&j<n_)) throw Error( Message(_ping_) << "You tried to access out of range "<<i<<" "<<j<<" size is "<<n_<<"\n");
 #endif
         return data_[ i*n_+j];
     }
@@ -101,7 +101,7 @@ class Operator
      */
     const T& operator()(const size_t i, const size_t j) const {
 #ifdef DG_DEBUG
-        assert( i<n_ && j < n_);
+        if(!(i<n_&&j<n_)) throw Error( Message(_ping_) << "You tried to access out of range "<<i<<" "<<j<<" size is "<<n_<<"\n");
 #endif
         return data_[ i*n_+j];
     }
@@ -134,7 +134,7 @@ class Operator
      */
     void swap_lines( const size_t i, const size_t k)
     {
-        assert( i< n_ && k<n_);
+        if(!( i< n_ && k<n_)) throw Error( Message(_ping_) << "Out of range "<<i<<" "<<k<<" range is "<<n_<<"\n");
         for( size_t j = 0; j<n_; j++)
         {
             std::swap( data_[i*n_+j], data_[k*n_+j]);
diff --git a/inc/dg/exceptions.h b/inc/dg/exceptions.h
index 5e498425a..399c706e4 100644
--- a/inc/dg/exceptions.h
+++ b/inc/dg/exceptions.h
@@ -27,6 +27,15 @@ class Message
   public:
     ///construct an empty message
     Message(){}
+    /*!@brief Initiate message with the file and line it comes from
+
+     * @param file The file in which the exception is thrown (contained in the predefined Macro __FILE__)
+     * @param line The line in which the exception is thrown (contained in the predefined Macro __LINE__)
+     * \note The Macro _ping_ combines __FILE__, __LINE__ in one. 
+     */
+    Message(const char* file, const int line){
+        sstream_ << "\n    Message from file **"<<file<<"** in line **" <<line<<"**:\n    ";
+    }
     /**
      * @brief Construct message with string
      * @param m puts m into stream
@@ -67,21 +76,14 @@ class Error : public std::exception
 {
   private:
     std::string m;//with a string the Error is copyable
-    const char* f;
-    const int l;
   public:
      
     /*! @brief Constructor
      *
      * @param message An instance of a Message class
-     * @param file The file in which the exception is thrown (contained in the predefined Macro __FILE__)
-     * @param line The line in which the exception is thrown (contained in the predefined Macro __LINE__)
-     * \note The Macro _ping_ combines __FILE__, __LINE__ in one. 
      */
-    Error(const Message& message, const char* file, const int line): f(file), l(line){
-        Message tmp;
-        tmp << "\n    Message from file **"<<f<<"** in line **" <<l<<"**:\n    "<<message<<"\n";
-        m = tmp.str();
+    Error(const Message& message){
+        m = message.str();
     }
     
     /// @return file, line and the message given in the constructor as a string of char
diff --git a/inc/dg/geometry/multiply.h b/inc/dg/geometry/multiply.h
index 5f3afe897..8a548caba 100644
--- a/inc/dg/geometry/multiply.h
+++ b/inc/dg/geometry/multiply.h
@@ -3,42 +3,101 @@
 #include "dg/backend/operator.h"
 #include "dg/functors.h"
 #include "dg/blas1.h"
+#include "tensor.h"
 
+namespace dg
+{
 namespace tensor
 {
 
 ///@addtogroup geometry
 ///@{
 
+/**
+ * @brief Scale tensor with a container
+ *
+ * Computes \f$ t^{ij} = \mu t^{ij}\f$ 
+ * @tparam container container class 
+ * @param t input (contains result on output)
+ * @param mu all elements in t are scaled with mu
+ */
 template<class container>
-void scal( SparseTensor<container>& t, const SparseElement<container>& e)
+void scal( SparseTensor<container>& t, const container& mu)
 {
-    if(!e.isSet()) return;
-    for( unsigned i=0; i<2; i++)
-        for( unsigned j=0; j<2; i++)
-            if(t.isSet(i,j)) dg::blas1::pointwiseDot( e.value(), t.value(i,j), t.value(i,j));
+    unsigned size=t.values().size();
+    for( unsigned i=0; i<size; i++)
+        dg::blas1::pointwiseDot( mu, t.value(i), t.value(i));
+    if(!t.isSet(0,0)|| !t.isSet(1,1) || !t.isSet(2,2))
+        t.value(size) = mu;
+    for( unsigned i=0; i<3; i++)
+    {
+        if(!t.isSet(i,i) )
+            t.idx(i,i)=size;
+    }
 }
 
+/**
+ * @brief Scale tensor with a form
+ *
+ * Computes \f$ t^{ij} = \mu t^{ij}\f$ 
+ * @tparam container container class 
+ * @param t input (contains result on output)
+ * @param mu if mu.isEmpty() then nothing happens, else all elements in t are scaled with its value
+ */
 template<class container>
-void multiply( const SparseElement<container>& e, const container& in, container& out)
+void scal( SparseTensor<container>& t, const SparseElement<container>& mu)
 {
-    if(e.isSet()) 
-        dg::blas1::pointwiseDot(e.value(), in,out);
+    if(!mu.isSet()) return;
+    else scal(t,mu.value());
+}
+
+/**
+ * @brief Multiply container with form
+ *
+ * @tparam container container class 
+ * @param mu if mu.isEmpty() then nothing happens, else the input is pointwise multiplied with the value in mu
+ * @param in input vector
+ * @param out output vector (may alias in)
+ */
+template<class container>
+void pointwiseDot( const SparseElement<container>& mu, const container& in, container& out)
+{
+    if(mu.isSet()) 
+        dg::blas1::pointwiseDot(mu.value(), in,out);
     else
         out=in;
 }
 
+/**
+ * @brief Divide container with form
+ *
+ * @tparam container container class 
+ * @param in input vector
+ * @param mu if mu.isEmpty() then nothing happens, else the input is pointwise divided with the value in mu
+ * @param out output vector (may alias in)
+ */
 template<class container>
-void divide( const container& in, const SparseElement<container>& e, container& out)
+void pointwiseDivide( const container& in, const SparseElement<container>& mu, container& out)
 {
-    if(e.isSet()) 
-        dg::blas1::pointwiseDivide(in, e.value(),out);
+    if(mu.isSet()) 
+        dg::blas1::pointwiseDivide(in, mu.value(),out);
     else
         out=in;
 }
 
-///this version keeps the input intact 
-///aliasing allowed if tensor is either lower, upper or diagonal alias allowed
+/**
+ * @brief Multiply a tensor with a vector in 2d
+ *
+ * Compute \f$ w^i = t^{ij}v_j\f$ for \f$ i,j\in \{1,2\}\f$ in the first two dimensions (ignores the 3rd dimension in t)
+ * @tparam container the container class
+ * @param t input Tensor
+ * @param in0 (input) covariant first component 
+ * @param in1 (input) covariant second component
+ * @param out0 (output) contravariant first component 
+ * @param out1 (output) contravariant second component 
+ * @note this version keeps the input intact 
+ * @attention aliasing only allowed if tensor is either lower, or upper triangular 
+ */
 template<class container>
 void multiply( const SparseTensor<container>& t, const container& in0, const container& in1, container& out0, container& out1)
 {
@@ -67,8 +126,20 @@ void multiply( const SparseTensor<container>& t, const container& in0, const con
         dg::blas1::pointwiseDot( 1.,  t.value(1,0), in0, 1., out1);
 }
 
+/**
+ * @brief Multiply a tensor with a vector in 2d inplace
+ *
+ * Compute \f$ v^i = t^{ij}v_j\f$ for \f$ i,j\in \{1,2\}\f$ in the first two dimensions (ignores the 3rd dimension in t)
+ * @tparam container the container class
+ * @param t input Tensor
+ * @param inout0 (input/output) covariant first component 
+ * @param inout1 (input/output) covariant second component
+ * @param workspace (write) optional workspace
+ * @note this version overwrites the input and may or may not write into the workspace
+ * @attention aliasing not allowed
+ */
 template<class container>
-void multiply( const SparseTensor<container>& t, container& inout0, container& inout1, container& workspace)
+void multiply_inplace( const SparseTensor<container>& t, container& inout0, container& inout1, container& workspace)
 {
     if( t.isSet(0,1) ) dg::blas1::pointwiseDot( t.value(0,1), inout1, workspace);
     //compute out1 inplace
@@ -90,7 +161,21 @@ void multiply( const SparseTensor<container>& t, container& inout0, container& i
     //if unity                       0 (optimal)
 }
 
-///aliasing allowed if t is known to be lower or upper triangular
+/**
+ * @brief Multiply a tensor with a vector in 2d
+ *
+ * Compute \f$ w^i = t^{ij}v_j\f$ for \f$ i,j\in \{1,2\}\f$ in the first two dimensions (ignores the 3rd dimension in t)
+ * @tparam container the container class
+ * @param t input Tensor
+ * @param in0 (input) covariant first component 
+ * @param in1 (input) covariant second component
+ * @param in2 (input) covariant third component
+ * @param out0 (output) contravariant first component 
+ * @param out1 (output) contravariant second component 
+ * @param out2 (output) contravariant third component 
+ * @note this version keeps the input intact 
+ * @attention aliasing only allowed if tensor is either lower, or upper triangular 
+ */
 template<class container>
 void multiply( const SparseTensor<container>& t, const container& in0, const container& in1, const container& in2, container& out0, container& out1, container& out2)
 {
@@ -133,8 +218,22 @@ void multiply( const SparseTensor<container>& t, const container& in0, const con
         dg::blas1::pointwiseDot( 1., t.value(2,0), in0, 1., out2);
 }
 
+/**
+ * @brief Multiply a tensor with a vector in 2d inplace
+ *
+ * Compute \f$ v^i = t^{ij}v_j\f$ for \f$ i,j\in \{1,2\}\f$ in the first two dimensions (ignores the 3rd dimension in t)
+ * @tparam container the container class
+ * @param t input Tensor
+ * @param inout0 (input/output) covariant first component 
+ * @param inout1 (input/output) covariant second component
+ * @param inout2 (input/output) covariant second component
+ * @param workspace0 (write) optional workspace
+ * @param workspace1 (write) optional workspace
+ * @note this version overwrites the input and may or may not write into the workspace
+ * @attention aliasing not allowed
+ */
 template<class container>
-void multiply( const SparseTensor<container>& t, container& inout0, container& inout1, container& inout2, container& workspace0, container& workspace1)
+void multiply_inplace( const SparseTensor<container>& t, container& inout0, container& inout1, container& inout2, container& workspace0, container& workspace1)
 {
     if( t.isSet(0,1) ) {
         dg::blas1::pointwiseDot( t.value(0,1), inout1, workspace0);
@@ -182,6 +281,12 @@ void multiply( const SparseTensor<container>& t, container& inout0, container& i
 
 }
 
+/**
+* @brief Compute the determinant of a tensor
+* @tparam container the container class
+* @param t the input tensor 
+* @return the determinant of t as a SparseElement (unset if t is empty)
+*/
 template<class container>
 SparseElement<container> determinant( const SparseTensor<container>& t)
 {
@@ -191,19 +296,19 @@ SparseElement<container> determinant( const SparseTensor<container>& t)
     std::vector<container> sub_det(3,det);
     dg::blas1::transform( det, det, dg::CONSTANT(0));
     //first compute the det of three submatrices
-    dg::blas1::pointwiseDot( d(0,0), d(1,1), sub_det[2]);
-    dg::blas1::pointwiseDot( -1., d(1,0), d(0,1), 1. ,sub_det[2]);
+    dg::blas1::pointwiseDot( d.value(0,0), d.value(1,1), sub_det[2]);
+    dg::blas1::pointwiseDot( -1., d.value(1,0), d.value(0,1), 1. ,sub_det[2]);
 
-    dg::blas1::pointwiseDot( d(0,0), d(2,1), sub_det[1]);
-    dg::blas1::pointwiseDot( -1., d(2,0), d(0,1), 1. ,sub_det[1]);
+    dg::blas1::pointwiseDot( d.value(0,0), d.value(2,1), sub_det[1]);
+    dg::blas1::pointwiseDot( -1., d.value(2,0), d.value(0,1), 1. ,sub_det[1]);
 
-    dg::blas1::pointwiseDot( d(1,0), d(2,1), sub_det[0]);
-    dg::blas1::pointwiseDot( -1., d(2,0), d(1,1), 1. ,sub_det[0]);
+    dg::blas1::pointwiseDot( d.value(1,0), d.value(2,1), sub_det[0]);
+    dg::blas1::pointwiseDot( -1., d.value(2,0), d.value(1,1), 1. ,sub_det[0]);
 
     //now multiply according to Laplace expansion
-    dg::blas1::pointwiseDot( 1., d(0,2), sub_det[0], 1.,  det);
-    dg::blas1::pointwiseDot(-1., d(1,2), sub_det[1], 1.,  det);
-    dg::blas1::pointwiseDot( 1., d(2,2), sub_det[2], 1.,  det);
+    dg::blas1::pointwiseDot( 1., d.value(0,2), sub_det[0], 1.,  det);
+    dg::blas1::pointwiseDot(-1., d.value(1,2), sub_det[1], 1.,  det);
+    dg::blas1::pointwiseDot( 1., d.value(2,2), sub_det[2], 1.,  det);
 
     return SparseElement<container>(det);
 }
@@ -214,8 +319,15 @@ template<class container>
 void multiply( const CholeskyTensor<container>& ch, const container& in0, const container& in1, container& out0, container& out1)
 {
     multiply(ch.upper(), in0, in1, out0, out1);
-    multiply(ch.diag(), out0, out1, out0, out1);
-    multiply(ch.lower(), out0, out1, out0, out1);
+    //multiply( ch.diagonal(), out0, out1, out0, out1);
+    //multiply(ch.lower(), out0, out1, out0, out1);
+}
+template<class container>
+void multiply( const CholeskyTensor<container>& ch, const container& in0, const container& in1, const container& in2, container& out0, container& out1, container& out2)
+{
+    multiply(ch.upper(), in0, in1,in2, out0, out1,out2);
+    multiply(ch.diagonal(), out0, out1,out2, out0, out1,out2);
+    multiply(ch.lower(), out0, out1,out2, out0, out1,out2);
 }
 
 template<class container>
@@ -236,14 +348,14 @@ void scal(const CholeskyTensor<container>& ch, const SparseElement<container>& e
     const SparseTensor<container>& diag = ch.diag();
     if(!e.isSet()) return;
     unsigned size=diag.values().size();
+    for( unsigned i=0; i<size; i++)
+        dg::blas1::pointwiseDot( e.value(), diag.value(i), diag.value(i));
     if(!diag.isSet(0,0)|| !diag.isSet(1,1) || !diag.isSet(2,2))
         diag.value(size) = e.value();
-    for( unsigned i=0; i<2; i++)
+    for( unsigned i=0; i<3; i++)
     {
-        if(!diag.isSet(i,i))
+        if(!diag.isSet(i,i) )
             diag.idx(i,i)=size;
-        else
-            dg::blas1::pointwiseDot( e.value(), diag.value(i,i), diag.value(i,i));
     }
 }
 ///@endcond
@@ -251,3 +363,4 @@ void scal(const CholeskyTensor<container>& ch, const SparseElement<container>& e
 ///@}
 
 }//namespace tensor
+}//namespace dg
diff --git a/inc/dg/geometry/tensor.h b/inc/dg/geometry/tensor.h
index 124e0060b..e23149bbd 100644
--- a/inc/dg/geometry/tensor.h
+++ b/inc/dg/geometry/tensor.h
@@ -45,14 +45,19 @@ struct SparseTensor
      * @param src the source matrix to convert
      */
     template<class OtherContainer>
-    SparseTensor( const SparseTensor<OtherContainer>& src): values_(src.values().size()){
-        dg::blas1::transfer( src.values(), values_);
+    SparseTensor( const SparseTensor<OtherContainer>& src): mat_idx_(3,-1.), values_(src.values().size()){
+        for(unsigned i=0; i<3; i++)
+            for(unsigned j=0; j<3; j++)
+                mat_idx_(i,j)=src.idx(i,j);
+
+        for( unsigned i=0; i<src.values().size(); i++)
+            dg::blas1::transfer( src.values()[i], values_[i]);
     }
 
     /**
     * @brief check if a value is set at the given position or not
-    * @param i row index 0<i<2
-    * @param j col index 0<j<2
+    * @param i row index 0<=i<3
+    * @param j col index 0<=j<3
     * @return true if container is non-empty, false if value is assumed implicitly
     */
     bool isSet(size_t i, size_t j)const{
@@ -61,25 +66,25 @@ struct SparseTensor
     }
 
     /**
-    * @brief return index into the values array for the given position
-    * @param i row index 0<i<2
-    * @param j col index 0<j<2
+    * @brief read index into the values array at the given position
+    * @param i row index 0<=i<3
+    * @param j col index 0<=j<3
     * @return -1 if !isSet(i,j), index into values array else
     */
     int idx(unsigned i, unsigned j)const{return mat_idx_(i,j);}
     /**
-    * @brief main set function to modify indices
+    * @brief write index into the values array at the given position
     *
     * use this and the value() member to assemble the tensor
-    * @param i row index 0<i<2
-    * @param j col index 0<j<2
+    * @param i row index 0<=i<3
+    * @param j col index 0<=j<3
     * @return write access to value index to be set
     */
     int& idx(unsigned i, unsigned j){return mat_idx_(i,j);}
      /*! @brief unset an index, does not clear the associated value
       *
-      * @param i row index 0<i<2
-      * @param j col index 0<j<2
+      * @param i row index 0<=i<3
+      * @param j col index 0<=j<3
       */
      void unset( unsigned i, unsigned j) {
          mat_idx_(i,j)=-1;
@@ -91,16 +96,18 @@ struct SparseTensor
       */
      void clear_unused_values();
 
-    /*!@brief Access the underlying container
+    /*!@brief Read access the underlying container
      * @return if !isSet(i,j) the result is undefined, otherwise values[idx(i,j)] is returned. 
-     * @param i row index 0<i<2
-     * @param j col index 0<j<2
+     * @param i row index 0<=i<3
+     * @param j col index 0<=j<3
      * @note If the indices fall out of range of index the result is undefined
      */
     const container& value(size_t i, size_t j)const{ 
         int k = mat_idx_(i,j);
         return values_[k];
     }
+    //if you're looking for this function: YOU DON'T NEED IT!!ALIASING
+    //container& value(size_t i, size_t j);
     /**
      * @brief Return the container at given position, create one if there isn't one already
      * @param i index into the values array
@@ -108,8 +115,7 @@ struct SparseTensor
      */
     container& value( size_t i)
     {
-        if(i>=values_.size());
-        values_.resize(i+1);
+        if(i>=values_.size() ) values_.resize(i+1);
         return values_[i];
     }
     /**
@@ -172,8 +178,8 @@ struct SparseTensor
      
      ///construct an empty Tensor
      SparseTensor empty()const{return SparseTensor();}
-     ///Construct a tensor filled unset values with explicit 0 or 1
-     /// @note returns an empty matrix if isEmpty() returns true
+     ///Construct a tensor with all unset values filled with explicit 0 or 1
+     /// @note undefined isEmpty() returns true
      SparseTensor dense()const;
      ///copy and erase all values in the third dimension
      ///@note calls clear_unused_values() to get rid of the elements
@@ -212,6 +218,15 @@ struct SparseElement
      * @param value a value
      */
     SparseElement(const container& value):value_(1,value){ }
+    template<class OtherContainer>
+    SparseElement( const SparseElement<OtherContainer>& src)
+    {
+        if(src.isSet())
+        {
+            value_.resize(1);
+            dg::blas1::transfer(src.value(), value_[0]);
+        }
+    }
 
     ///@brief Read access
     ///@return read access to contained value
@@ -282,6 +297,11 @@ struct CholeskyTensor
     {
         decompose(in);
     }
+    template<class OtherContainer>
+    CholeskyTensor( const CholeskyTensor<OtherContainer>& in):q_(in.lower()),diag_(in.diagonal()),upper_(in.upper())
+    { 
+        }
+
     /**
      * @brief decompose given tensor
      *
@@ -298,21 +318,21 @@ struct CholeskyTensor
 
         if(in.isSet(0,0))
         {
-            diag.idx(0,0)=0;
-            diag.value(0)=in.value(0,0);
+            diag_.idx(0,0)=0;
+            diag_.value(0)=in.value(0,0);
         }
         if(in.isSet(1,0))
         {
-            container tmp=in(1,0);
-            if(diag.isSet(0,0)) dg::blas1::pointwiseDivide(tmp,diag.value(0,0),tmp);
+            container tmp=in.value(1,0);
+            if(diag_.isSet(0,0)) dg::blas1::pointwiseDivide(tmp,diag_.value(0,0),tmp);
             q_.idx(1,0)=0;
             q_.value(0)=tmp;
         }
         if(in.isSet(2,0))
         {
-            container tmp=in(2,0);
-            if(diag.isSet(0,0))dg::blas1::pointwiseDivide(tmp,diag.value(0,0),tmp);
-            q_.idx(1,0)=1;
+            container tmp=in.value(2,0);
+            if(diag_.isSet(0,0))dg::blas1::pointwiseDivide(tmp,diag_.value(0,0),tmp);
+            q_.idx(2,0)=1;
             q_.value(1)=tmp;
         }
 
@@ -321,39 +341,39 @@ struct CholeskyTensor
             SparseTensor<container> denseL = q_.dense();
             container tmp=denseL.value(1,0);
             dg::blas1::pointwiseDot(tmp,tmp,tmp);
-            if(diag.isSet(0,0)) dg::blas1::pointwiseDot(tmp,diag.value(0,0),tmp);
-            dg::blas1::axpby( 1., denseIn(1,1), -1., tmp, tmp);
-            diag.idx(1,1)=1;
-            diag.value(1) = tmp;
+            if(diag_.isSet(0,0)) dg::blas1::pointwiseDot(tmp,diag_.value(0,0),tmp);
+            dg::blas1::axpby( 1., denseIn.value(1,1), -1., tmp, tmp);
+            diag_.idx(1,1)=1;
+            diag_.value(1) = tmp;
         }
 
         if( in.isSet(2,1) || (q_.isSet(2,0)&&q_.isSet(1,0)))
         {
             SparseTensor<container> denseL = q_.dense();
-            container tmp=denseIn(2,1);
+            container tmp=denseIn.value(2,1);
             dg::blas1::pointwiseDot(denseL.value(2,0), denseL.value(1,0), tmp);
-            if(diag.isSet(0,0))dg::blas1::pointwiseDot(tmp, diag.value(0,0), tmp);
-            dg::blas1::axpby(1., denseIn(2,1),-1.,tmp, tmp);
-            if(diag.isSet(1,1))dg::blas1::pointwiseDivide(tmp, diag.value(1,1),tmp);
+            if(diag_.isSet(0,0))dg::blas1::pointwiseDot(tmp, diag_.value(0,0), tmp);
+            dg::blas1::axpby(1., denseIn.value(2,1),-1.,tmp, tmp);
+            if(diag_.isSet(1,1))dg::blas1::pointwiseDivide(tmp, diag_.value(1,1),tmp);
             q_.idx(2,1)=2;
             q_.value(2)=tmp;
         }
         if( in.isSet(2,2) || q_.isSet(2,0) || q_.isSet(2,1))
         {
             SparseTensor<container> denseL = q_.dense();
-            container tmp=denseL(2,0), tmp1=denseL(2,1);
+            container tmp=denseL.value(2,0), tmp1=denseL.value(2,1);
             dg::blas1::pointwiseDot(tmp,tmp,tmp);
-            if(diag.isSet(0,0))dg::blas1::pointwiseDot(diag.value(0,0),tmp,tmp);
+            if(diag_.isSet(0,0))dg::blas1::pointwiseDot(diag_.value(0,0),tmp,tmp);
             dg::blas1::pointwiseDot(tmp1,tmp1,tmp1);
-            if(diag.isSet(1,1))dg::blas1::pointwiseDot(diag.value(1,1),tmp1,tmp1);
+            if(diag_.isSet(1,1))dg::blas1::pointwiseDot(diag_.value(1,1),tmp1,tmp1);
             dg::blas1::axpby(1., denseIn.value(2,2), -1., tmp, tmp);
             dg::blas1::axpby(1., tmp, -1., tmp1, tmp);
-            diag.idx(2,2)=2;
-            diag.value(2) = tmp;
+            diag_.idx(2,2)=2;
+            diag_.value(2) = tmp;
         }
-        diag.clear_unused_values();
+        diag_.clear_unused_values();
         q_.clear_unused_values();
-        lower_=true;
+        upper_=q_.transpose();
     }
 
     /**
@@ -361,11 +381,6 @@ struct CholeskyTensor
      * @return a lower triangular matrix with 1 on diagonal
      */
     const SparseTensor<container>& lower()const{
-        if(lower_) return q_;
-        std::swap(q_.idx(1,0), q_.idx(0,1));
-        std::swap(q_.idx(2,0), q_.idx(0,2));
-        std::swap(q_.idx(2,1), q_.idx(1,2));
-        lower_=!lower_;
         return q_;
 
     }
@@ -374,12 +389,7 @@ struct CholeskyTensor
      * @return a upper triangular matrix with 1 on diagonal
      */
     const SparseTensor<container>& upper()const{
-        if(!lower_) return q_;
-        std::swap(q_.idx(1,0), q_.idx(0,1));
-        std::swap(q_.idx(2,0), q_.idx(0,2));
-        std::swap(q_.idx(2,1), q_.idx(1,2));
-        lower_=!lower_;
-        return q_;
+        return upper_;
 
     }
 
@@ -387,11 +397,10 @@ struct CholeskyTensor
      * @brief Returns D
      * @return only diagonal elements are set if any
      */
-    const SparseTensor<container>& diagonal()const{return diag;}
+    const SparseTensor<container>& diagonal()const{return diag_;}
 
     private:
-    SparseTensor<container> q_, diag;
-    int pivot_;
+    SparseTensor<container> q_, diag_, upper_;
     bool lower_;
 };
 
@@ -400,28 +409,32 @@ template<class container>
 SparseTensor<container> SparseTensor<container>::dense() const
 {
     SparseTensor<container> t(*this);
-    if( isEmpty()) return t;
+    if( isEmpty()) throw Error(Message(_ping_)<< "Can't make an empty tensor dense! ") ;
     container tmp = t.values_[0];
     //1. diagonal
     size_t size= values_.size();
     bool diagonalIsSet=true;
     for(unsigned i=0; i<3; i++)
         if(!t.isSet(i,i)) diagonalIsSet = false;
-    dg::blas1::transform( tmp, tmp, dg::CONSTANT(1));
-    if (!diagonalIsSet) t.values_.push_back(tmp);
-    for(unsigned i=0; i<3; i++)
-        if(!t.isSet(i,i)) t.mat_idx_(i,i) = size;
+    if (!diagonalIsSet){
+        dg::blas1::transform( tmp, tmp, dg::CONSTANT(1));
+        t.values_.push_back(tmp);
+        for(unsigned i=0; i<3; i++)
+            if(!t.isSet(i,i)) t.mat_idx_(i,i) = size;
+    }
     //2. off-diagonal
     size = t.values_.size();
     bool offIsSet=true;
     for(unsigned i=0; i<3; i++)
         for(unsigned j=0; j<3; j++)
-            if( !t.isSet(i,j) ) offIsSet=true;
-    dg::blas1::transform( tmp, tmp, dg::CONSTANT(0));
-    if (!offIsSet) t.values_.push_back(tmp);
-    for(unsigned i=0; i<3; i++)
-        for(unsigned j=0; j<3; j++)
-            if(!t.isSet(i,j) ) t.mat_idx_(i,j) = size;
+            if( !t.isSet(i,j) ) offIsSet=false;
+    if (!offIsSet){
+        dg::blas1::transform( tmp, tmp, dg::CONSTANT(0));
+        t.values_.push_back(tmp);
+        for(unsigned i=0; i<3; i++)
+            for(unsigned j=0; j<3; j++)
+                if(!t.isSet(i,j) ) t.mat_idx_(i,j) = size;
+    }
     return t;
 }
 
@@ -431,7 +444,10 @@ void SparseTensor<container>::unique_insert(std::vector<int>& indices, int& idx)
     bool unique=true;
     unsigned size=indices.size();
     for(unsigned i=0; i<size; i++)
-        if(indices[i] == idx) unique=false;
+        if(indices[i] == idx) {
+            unique=false;
+            idx=i;
+        }
     if(unique)
     {
         indices.push_back(idx);
@@ -465,9 +481,7 @@ void SparseTensor<container>::clear_unused_values()
 
     std::vector<container> tmp(unique_idx.size());
     for(unsigned i=0; i<unique_idx.size(); i++)
-    {
         tmp[i] = values_[unique_idx[i]];
-    }
     values_.swap(tmp);
 }
 ///@endcond
diff --git a/inc/dg/geometry/tensor_t.cu b/inc/dg/geometry/tensor_t.cu
index 187dc35e8..140297c6a 100644
--- a/inc/dg/geometry/tensor_t.cu
+++ b/inc/dg/geometry/tensor_t.cu
@@ -4,7 +4,8 @@
 #include "tensor.h"
 #include "multiply.h"
 
-void print( const dg::SparseTensor<thrust::host_vector<double> >& t)
+template<class container>
+void print( const dg::SparseTensor<container >& t)
 {
     for( unsigned i=0; i<3; i++)
     {
@@ -17,41 +18,78 @@ void print( const dg::SparseTensor<thrust::host_vector<double> >& t)
     }
 
 }
-std::ostream& operator<<(std::ostream& os, const dg::SparseElement<thrust::host_vector<double> >& t)
+
+template<class container>
+std::ostream& operator<<(std::ostream& os, const dg::SparseElement<container >& t)
 {
     if(t.isSet()) os << t.value()[0]<<" ";
     else os <<"XX ";
     return os;
 }
 
+thrust::host_vector<double> one(1,10), two(1,20), three(1,30), four(1,40), five(1,50), six(1,60), seven(1,70), eight(1,80), nine(1,90);
+
 int main()
 {
-    thrust::host_vector<double> one(1,10), two(1,20), three(1,30), four(1,40), five(1,50), six(1,60), seven(1,70), eight(1,80), nine(1,90);
+    std::cout << "Test dg::Sparse Tensor class \n";
+    std::cout << "construct empty Tensor \n";
+    dg::SparseTensor<thrust::host_vector<double> > empty;
+    if( empty.isEmpty())print( empty);
 
     dg::SparseTensor<thrust::host_vector<double> > dense2d(3);
     dense2d.idx(0,0) = 0, dense2d.idx(0,1) = 1;
     dense2d.idx(1,0) = 1, dense2d.idx(1,1) = 2;
     dense2d.value(0) = eight; dense2d.value(1) = two; dense2d.value(2) = nine; 
-    dg::SparseTensor<thrust::host_vector<double> > sparse3d(4);
+    std::cout << "Dense 2d Tensor \n";
+    if( !dense2d.isEmpty())print( dense2d);
+    std::vector<thrust::host_vector<double> > values(4);
+    values[0] = seven; values[1] = three; values[2] = nine; values[3] = one; 
+    dg::SparseTensor<thrust::host_vector<double> > sparse3d(values);
     sparse3d.idx(0,0) = 0, sparse3d.idx(0,1) = 1                       ;
     sparse3d.idx(1,0) = 1                       , sparse3d.idx(1,2) = 3;
     sparse3d.idx(2,0) = 1                       , sparse3d.idx(2,2) = 3;
-    sparse3d.value(0) = seven; sparse3d.value(1) = three; sparse3d.value(2) = nine, sparse3d.value(3) = one; 
-
-    dg::SparseTensor<thrust::host_vector<double> > empty;
+    dg::SparseTensor<thrust::device_vector<double> > sparse3d_D=sparse3d;
 
-    std::cout << "Test dg::Sparse Tensor class \n";
-    std::cout << "Dense 2d Tensor \n";
-    print( dense2d);
     std::cout << "Sparse 3d Tensor \n";
     print( sparse3d);
+    std::cout << "unset an element \n";
+    sparse3d.unset( 2,0);
+    print( sparse3d);
+    std::cout << "clear unused values \n";
+    std::cout << "Size before "<<sparse3d.values().size()<<"\n";
+    sparse3d.clear_unused_values();
+    std::cout << "Size after  "<<sparse3d.values().size()<<"\n";
+    std::cout << "tensor after \n";
+    print( sparse3d);
     std::cout << "empty Tensor \n";
-    print( empty);
+    print( sparse3d.empty());
+    std::cout << "explicit dense Tensor \n";
+    dg::SparseTensor<thrust::host_vector<double> > dense3d = sparse3d.dense();
+    if(dense3d.isDense())print( dense3d);
+    sparse3d = sparse3d_D;
+    std::cout << "original stored on device \n";
+    if(!sparse3d.isDense())print( sparse3d);
+    std::cout << "perp \n";
+    if( sparse3d.perp().isPerp()) print( sparse3d.perp());
+    std::cout << "transpose \n";
+    print( sparse3d.transpose());
+
+    sparse3d.idx(2,0)=sparse3d.idx(0,2);
+    sparse3d.idx(1,2)=sparse3d.idx(2,1);
+    sparse3d.idx(1,1)=3;
+    sparse3d.value(3)=nine;
+    std::cout<<"Make a LDL^T decomposition\n";
+    dg::CholeskyTensor<thrust::host_vector<double> > ch(sparse3d);
+    std::cout << "origin\n"; print(sparse3d);
+    std::cout << "lower \n"; print(ch.lower());
+    std::cout << "diag  \n"; print(ch.diagonal());
+    std::cout << "upper \n"; print(ch.upper());
+
 
 
     std::cout<< "Test dg::SparseElement";
     dg::SparseElement<thrust::host_vector<double> > e(eight);
-    dg::SparseElement<thrust::host_vector<double> > ee;
+    dg::SparseElement<thrust::device_vector<double> > ee;
     std::cout<<"\n construct: " <<e<<" "<<ee<<"\n";
     ee = e;
     e.value()=nine;
-- 
GitLab


From aae5b3318f3cd17babe6243f29623bcfab3d6b57 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Tue, 8 Aug 2017 02:44:40 -0700
Subject: [PATCH 126/453] redefine compute_metric as metric

---
 inc/dg/geometry/base.h     | 8 ++++----
 inc/dg/geometry/geometry.h | 2 +-
 inc/dg/geometry/multiply.h | 6 +++---
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/inc/dg/geometry/base.h b/inc/dg/geometry/base.h
index feee2d82d..17d374bdb 100644
--- a/inc/dg/geometry/base.h
+++ b/inc/dg/geometry/base.h
@@ -15,7 +15,7 @@ namespace dg
 struct aGeometry2d : public aTopology2d
 {
     const SharedContainers<thrust::host_vector<double> >& map()const{return map_;}
-    SharedContainers<thrust::host_vector<double> > compute_metric()const {
+    SharedContainers<thrust::host_vector<double> > metric()const {
         return do_compute_metric();
     }
     ///Geometries are cloneable
@@ -45,7 +45,7 @@ struct aGeometry2d : public aTopology2d
 struct aGeometry3d : public aTopology3d
 {
     const SharedContainers<thrust::host_vector<double> >& map()const{return map_;}
-    SharedContainers<thrust::host_vector<double> > compute_metric()const {
+    SharedContainers<thrust::host_vector<double> > metric()const {
         return do_compute_metric();
     }
     ///Geometries are cloneable
@@ -75,11 +75,11 @@ namespace create
 ///@{
 SharedContainers<thrust::host_vector<double> > metric( const aGeometry2d& g)
 {
-    return g.compute_metric();
+    return g.metric();
 }
 SharedContainers<thrust::host_vector<double> > metric( const aGeometry3d& g)
 {
-    return g.compute_metric();
+    return g.metric();
 }
 ///@}
 
diff --git a/inc/dg/geometry/geometry.h b/inc/dg/geometry/geometry.h
index 2a9e916b7..698774e38 100644
--- a/inc/dg/geometry/geometry.h
+++ b/inc/dg/geometry/geometry.h
@@ -141,7 +141,7 @@ template< class Geometry>
 typename HostVec< typename TopologyTraits<Geometry>::memory_category>::host_vector volume( const Geometry& g)
 {
     typedef typename HostVec< typename TopologyTraits<Geometry>::memory_category>::host_vector host_vector;
-    SharedContainers<host_vector> metric = g.compute_metric();
+    SharedContainers<host_vector> metric = g.metric();
     host_vector temp = dg::create::weights( g);
     dg::geo::multiplyVolume( temp, metric);
     return temp;
diff --git a/inc/dg/geometry/multiply.h b/inc/dg/geometry/multiply.h
index 8a548caba..6be7a6d9e 100644
--- a/inc/dg/geometry/multiply.h
+++ b/inc/dg/geometry/multiply.h
@@ -110,19 +110,19 @@ void multiply( const SparseTensor<container>& t, const container& in0, const con
         else out0=in0;
         return;
     }
-    //upper triangular
+    //upper triangular and default
     if( t.isSet(0,0) ) 
         dg::blas1::pointwiseDot( t.value(0,0), in0, out0); 
     else 
         out0=in0;
-    if(t.isSet(0,1))
+    if(t.isSet(0,1)) //true
         dg::blas1::pointwiseDot( 1.,  t.value(0,1), in1, 1., out0);
 
     if( t.isSet(1,1) )
         dg::blas1::pointwiseDot( t.value(1,1), in1, out1);
     else 
         out1=in1;
-    if(t.isSet(1,0))
+    if(t.isSet(1,0)) //if aliasing happens this is wrong
         dg::blas1::pointwiseDot( 1.,  t.value(1,0), in0, 1., out1);
 }
 
-- 
GitLab


From dcbeb76d4ad1cf08e6712bb5e7d9d42f7ab68cac Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Tue, 8 Aug 2017 03:57:49 -0700
Subject: [PATCH 127/453] redefine interface in Geometry2d for metric, jacobian
 and map

---
 inc/dg/geometry/base.h | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/inc/dg/geometry/base.h b/inc/dg/geometry/base.h
index 17d374bdb..e784083e5 100644
--- a/inc/dg/geometry/base.h
+++ b/inc/dg/geometry/base.h
@@ -14,9 +14,14 @@ namespace dg
  */
 struct aGeometry2d : public aTopology2d
 {
-    const SharedContainers<thrust::host_vector<double> >& map()const{return map_;}
-    SharedContainers<thrust::host_vector<double> > metric()const {
-        return do_compute_metric();
+    SharedContainers<thrust::host_vector<double> > jacobian()const{
+        return do_provide_jacobian();
+    }
+    SharedContainers<thrust::host_vector<double> > metric()const { 
+        return do_provide_metric(); 
+    }
+    std::vector<thrust::host_vector<double> > map(){
+        return do_provide_map();
     }
     ///Geometries are cloneable
     virtual aGeometry2d* clone()const=0;
@@ -33,10 +38,10 @@ struct aGeometry2d : public aTopology2d
         map_=src.map_;
         metric_=src.metric_;
     }
-    SharedContainers<thrust::host_vector<double> >& map(){return map_;}
     private:
-    SharedContainers<thrust::host_vector<double> > map_;
     virtual SharedContainers<thrust::host_vector<double> > do_compute_metric()const=0;
+    virtual SharedContainers<thrust::host_vector<double> > do_compute_jacobian()const=0;
+    virtual std::vector<thrust::host_vector<double> > do_compute_map()const=0;
 };
 
 /**
@@ -44,8 +49,8 @@ struct aGeometry2d : public aTopology2d
  */
 struct aGeometry3d : public aTopology3d
 {
-    const SharedContainers<thrust::host_vector<double> >& map()const{return map_;}
-    SharedContainers<thrust::host_vector<double> > metric()const {
+    std::vector<thrust::host_vector<double> >& map(){return map_;}
+    SharedContainers<thrust::host_vector<double> > metric() {
         return do_compute_metric();
     }
     ///Geometries are cloneable
-- 
GitLab


From 3a1fd51c668635d6c4ee48453d8c259823a00265 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Tue, 8 Aug 2017 04:56:53 -0700
Subject: [PATCH 128/453] separate interface from algorithms!!

---
 inc/dg/geometry/geometry.h | 102 +----------------------------
 inc/dg/geometry/multiply.h |  21 ++++++
 inc/dg/geometry/tensor.h   | 128 ++++++++++++++++---------------------
 3 files changed, 79 insertions(+), 172 deletions(-)

diff --git a/inc/dg/geometry/geometry.h b/inc/dg/geometry/geometry.h
index 698774e38..2c86cb69e 100644
--- a/inc/dg/geometry/geometry.h
+++ b/inc/dg/geometry/geometry.h
@@ -18,6 +18,7 @@
 #endif//MPI_VERSION
 #include "tensor.h"
 #include "transform.h"
+#include "multiply.h"
 
 
 /*!@file 
@@ -27,103 +28,6 @@
 
 namespace dg{
 
-/*! @brief Geometry routines 
- *
- * @ingroup geometry
- * Only those routines that are actually called need to be implemented.
- * Don't forget to specialize in the dg namespace.
- */
-namespace geo{
-///@addtogroup geometry
-///@{
-
-/**
- * @brief Multiply the input with the volume element without the dG weights!
- *
- * Computes \f$ f = \sqrt{g}f\f$ 
- * @tparam container container class 
- * @param inout input (contains result on output)
- * @param metric the metric object
- */
-template<class container>
-void multiplyVolume( container& inout, const SharedContainers<container>& metric)
-{
-    if( metric.isSet(0))
-        dg::blas1::pointwiseDot( metric.getValue(0), inout, inout);
-}
-
-/**
- * @brief Divide the input vector with the volume element without the dG weights
- *
- * Computes \f$ v = v/ \sqrt{g}\f$ 
- * @tparam container container class 
- * @param inout input (contains result on output)
- * @param metric the metric
- */
-template<class container>
-void divideVolume( container& inout, const SharedContainers<container>& metric)
-{
-    if( metric.isSet(0))
-        dg::blas1::pointwiseDivide( inout, metric.getValue(0), inout);
-}
-
-/**
- * @brief Raises the index of a covariant vector with the help of the projection tensor in 2d
- *
- * Compute \f$ v^i = g^{ij}v_j\f$ for \f$ i,j\in \{1,2\}\f$ in the first two dimensions 
- * @tparam container the container class
- * @param covX (input) covariant first component (undefined content on output)
- * @param covY (input) covariant second component (undefined content on output)
- * @param contraX (output) contravariant first component 
- * @param contraY (output) contravariant second component 
- * @param metric The metric
- * @note no alias allowed 
- */
-template<class container>
-void raisePerpIndex( container& covX, container& covY, container& contraX, container& contraY, const SharedContainers<container>& metric)
-{
-    assert( &covX != &contraX);
-    assert( &covY != &contraY);
-    assert( &covY != &covX);
-    dg::detail::multiply( metric, covX, covY, contraX, contraY);
-}
-
-/**
- * @brief Multiplies the two-dimensional volume element
- *
- * Computes \f$ f = \sqrt{g_\perp}f\f$ where the perpendicualar volume is computed from the 2x2 submatrix of g in the first two coordinates.
- * @tparam container the container class
- * @param inout input (contains result on output)
- * @param metric The metric  
- * @note if metric is two-dimensional this function will have the same result as multiplyVolume()
- */
-template<class container>
-void multiplyPerpVolume( container& inout, const SharedContainers<container>& metric)
-{
-    if( metric.isSet(1))
-        dg::blas1::pointwiseDot( metric.getValue(1), inout, inout);
-}
-
-/**
- * @brief Divides the two-dimensional volume element
- *
- * Computes \f$ f = f /\sqrt{g_\perp}\f$ where the perpendicualar volume is computed from the 2x2 submatrix of g in the first two coordinates.
- * @tparam container the container class
- * @param inout input (contains result on output)
- * @param metric The metric tensor
- * @note if metric is two-dimensional this function will have the same result as divideVolume()
- */
-template<class container>
-void dividePerpVolume( container& inout, const SharedContainers<container>& metric)
-{
-    if( metric.isSet(1))
-        dg::blas1::pointwiseDivide( metric.getValue(1), inout, inout);
-}
-
-
-///@}
-}//namespace geo
-
 namespace create{
 ///@addtogroup metric
 ///@{
@@ -141,9 +45,9 @@ template< class Geometry>
 typename HostVec< typename TopologyTraits<Geometry>::memory_category>::host_vector volume( const Geometry& g)
 {
     typedef typename HostVec< typename TopologyTraits<Geometry>::memory_category>::host_vector host_vector;
-    SharedContainers<host_vector> metric = g.metric();
+    SparseElement<host_vecotr> vol = dg::tensor::determinant(g.metric());
     host_vector temp = dg::create::weights( g);
-    dg::geo::multiplyVolume( temp, metric);
+    dg::tensor::pointwiseDot( vol, temp, temp);
     return temp;
 }
 
diff --git a/inc/dg/geometry/multiply.h b/inc/dg/geometry/multiply.h
index 6be7a6d9e..3f468f482 100644
--- a/inc/dg/geometry/multiply.h
+++ b/inc/dg/geometry/multiply.h
@@ -12,6 +12,27 @@ namespace tensor
 
 ///@addtogroup geometry
 ///@{
+/**
+ * @brief calls sqrt transform function on value
+ * @param mu if empty, value is assumed 1 and empty element is returned
+ * @return Sparse element containing sqrt of input element
+ */
+SparseElement sqrt(const SparseElement& mu){ 
+    SparseElement mu(*this);
+    if( isSet()) dg::blas1::transform( mu.value(), mu.value(), dg::SQRT<double>());
+    return mu;
+}
+
+/**
+ * @brief calls invert transform function on value
+ * @param mu if empty, value is assumed 1 and empty element is returned
+ * @return Sparse element containing inverse of input element
+ */
+SparseElement invert(const SparseElement& mu){ 
+    SparseElement mu(*this);
+    if( isSet()) dg::blas1::transform( mu.value(), mu.value(), dg::INVERT<double>());
+    return mu;
+}
 
 /**
  * @brief Scale tensor with a container
diff --git a/inc/dg/geometry/tensor.h b/inc/dg/geometry/tensor.h
index e23149bbd..ecf75a10a 100644
--- a/inc/dg/geometry/tensor.h
+++ b/inc/dg/geometry/tensor.h
@@ -6,6 +6,61 @@
 
 namespace dg
 {
+    //separate algorithms from interface!!
+
+/**
+ * @brief This is a sparse Tensor with only one element i.e. a Form
+ *
+ * @tparam container a container class
+ * @ingroup misc
+ */
+template<class container>
+struct SparseElement
+{
+    ///create empty object
+    SparseElement(){}
+    /**
+     * @brief copy construct value
+     * @param value a value
+     */
+    SparseElement(const container& value):value_(1,value){ }
+    template<class OtherContainer>
+    SparseElement( const SparseElement<OtherContainer>& src)
+    {
+        if(src.isSet())
+        {
+            value_.resize(1);
+            dg::blas1::transfer(src.value(), value_[0]);
+        }
+    }
+
+    ///@brief Read access
+    ///@return read access to contained value
+    const container& value( )const { 
+        return value_[0];
+    }
+    /**
+     * @brief write access, create a container if there isn't one already
+     * @return write access, always returns a container 
+     */
+    container& value() {
+        if(!isSet()) value_.resize(1);
+        return value_[0];
+    }
+
+    /**
+     * @brief check if an element is set or not
+     * @return false if the value array is empty
+     */
+    bool isSet()const{
+        if( value_.empty()) return false;
+        return true;
+    }
+    ///@brief Clear contained value
+    void clear(){value_.clear();}
+    private:
+    std::vector<container> value_;
+};
 
 /**
 * @brief Class for 2x2 and 3x3 matrices sharing or implicitly assuming elements 
@@ -202,79 +257,6 @@ struct SparseTensor
 };
 
 
-/**
- * @brief This is a sparse Tensor with only one element i.e. a Form
- *
- * @tparam container a container class
- * @ingroup misc
- */
-template<class container>
-struct SparseElement
-{
-    ///create empty object
-    SparseElement(){}
-    /**
-     * @brief copy construct value
-     * @param value a value
-     */
-    SparseElement(const container& value):value_(1,value){ }
-    template<class OtherContainer>
-    SparseElement( const SparseElement<OtherContainer>& src)
-    {
-        if(src.isSet())
-        {
-            value_.resize(1);
-            dg::blas1::transfer(src.value(), value_[0]);
-        }
-    }
-
-    ///@brief Read access
-    ///@return read access to contained value
-    const container& value( )const { 
-        return value_[0];
-    }
-    /**
-     * @brief write access, create a container if there isn't one already
-     * @return write access, always returns a container 
-     */
-    container& value() {
-        if(!isSet()) value_.resize(1);
-        return value_[0];
-    }
-
-    /**
-     * @brief check if an element is set or not
-     * @return false if the value array is empty
-     */
-    bool isSet()const{
-        if( value_.empty()) return false;
-        return true;
-    }
-    ///@brief Clear contained value
-    void clear(){value_.clear();}
-    /**
-     * @brief calls sqrt transform function on value
-     * @return Sparse element containing sqrt
-     */
-    SparseElement sqrt(){ 
-        SparseElement tmp(*this);
-        if( isSet()) dg::blas1::transform( tmp.value_, tmp.value_, dg::SQRT<double>());
-        return tmp;
-    }
-    /**
-     * @brief calls invert transform function on value
-     * @return Sparse element containing inverse 
-     */
-    SparseElement invert(){ 
-        SparseElement tmp(*this);
-        if( isSet()) dg::blas1::transform( tmp.value_, tmp.value_, dg::INVERT<double>());
-        return tmp;
-    }
-
-
-    private:
-    std::vector<container> value_;
-};
 
 /**
  * @brief data structure to hold the LDL^T decomposition of a symmetric positive definite matrix
-- 
GitLab


From 5169a642d56284b234051d66235d2ba9f1038828 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Tue, 8 Aug 2017 06:31:53 -0700
Subject: [PATCH 129/453] replaced HostVec with GeometryTraits::host_vector

---
 inc/dg/elliptic.h                           |  2 +-
 inc/dg/geometry/{base.h => base_geometry.h} |  2 +-
 inc/dg/geometry/geometry.h                  | 10 +++++-----
 inc/dg/geometry/transform.h                 | 18 ++++++++++++------
 4 files changed, 19 insertions(+), 13 deletions(-)
 rename inc/dg/geometry/{base.h => base_geometry.h} (98%)

diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index 5cd8db6e3..a63c271c5 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -651,7 +651,7 @@ struct TensorElliptic
     template<class ChiRR, class ChiRZ, class ChiZZ>
     void set( ChiRR chiRR, ChiRZ chiRZ, ChiZZ chiZZ)
     {
-        typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector chiXX, chiXY, chiYY;
+        typename GeometryTraits<Geometry>::host_vector chiXX, chiXY, chiYY;
         dg::geo::pushForwardPerp( chiRR, chiRZ, chiZZ, chiXX, chiXY, chiYY, g_);
         set( chiXX, chiXY, chiYY);
     }
diff --git a/inc/dg/geometry/base.h b/inc/dg/geometry/base_geometry.h
similarity index 98%
rename from inc/dg/geometry/base.h
rename to inc/dg/geometry/base_geometry.h
index e784083e5..f9901d662 100644
--- a/inc/dg/geometry/base.h
+++ b/inc/dg/geometry/base_geometry.h
@@ -109,7 +109,7 @@ struct CartesianGrid2d: public dg::aGeometry2d
     virtual CartesianGrid2d* clone()const{return new CartesianGrid2d(*this);}
     private:
     virtual SharedContainers<thrust::host_vector<double> > do_compute_metric()const{
-        return SharedContainers<thrust::host_vector<double> >( 3);
+        return SharedContainers<thrust::host_vector<double> >();
     }
     virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny){
         aTopology2d::do_set(new_n,new_Nx,new_Ny);
diff --git a/inc/dg/geometry/geometry.h b/inc/dg/geometry/geometry.h
index 2c86cb69e..0898af60b 100644
--- a/inc/dg/geometry/geometry.h
+++ b/inc/dg/geometry/geometry.h
@@ -42,10 +42,10 @@ namespace create{
  * @return  The volume form
  */
 template< class Geometry>
-typename HostVec< typename TopologyTraits<Geometry>::memory_category>::host_vector volume( const Geometry& g)
+typename GeometryTraits<Geometry>::host_vector volume( const Geometry& g)
 {
-    typedef typename HostVec< typename TopologyTraits<Geometry>::memory_category>::host_vector host_vector;
-    SparseElement<host_vecotr> vol = dg::tensor::determinant(g.metric());
+    typedef typename GeometryTraits< Geometry>::host_vector host_vec;
+    SparseElement<host_vector> vol = dg::tensor::determinant(g.metric());
     host_vector temp = dg::create::weights( g);
     dg::tensor::pointwiseDot( vol, temp, temp);
     return temp;
@@ -61,9 +61,9 @@ typename HostVec< typename TopologyTraits<Geometry>::memory_category>::host_vect
  * @return  The inverse volume form
  */
 template< class Geometry>
-typename HostVec< typename TopologyTraits<Geometry>::memory_category>::host_vector inv_volume( const Geometry& g)
+typename GeometryTraits<Geometry>::host_vector inv_volume( const Geometry& g)
 {
-    typedef typename HostVec< typename TopologyTraits<Geometry>::memory_category>::host_vector host_vector;
+    typedef typename GeometryTraits< Geometry>::host_vector host_vec;
     host_vector temp = volume(g);
     dg::blas1::transform(temp,temp,dg::INVERT<double>());
     return temp;
diff --git a/inc/dg/geometry/transform.h b/inc/dg/geometry/transform.h
index 8c500f67f..2f3d6500b 100644
--- a/inc/dg/geometry/transform.h
+++ b/inc/dg/geometry/transform.h
@@ -4,11 +4,17 @@
 namespace dg
 {
 ///@cond
+template<class Geometry>
+class GeometryTraits
+{
+    typedef typename MemoryTraits< typename TopologyTraits<Geometry>::memory_category>::host_vector host_vector;
+
+};
 template<class MemoryTag>
-struct HostVec {
+struct MemoryTraits {
 };
 template<>
-struct HostVec< SharedTag>
+struct MemoryTraits< SharedTag>
 {
     typedef thrust::host_vector<double> host_vector;
 };
@@ -59,7 +65,7 @@ thrust::host_vector<double> pullback( Functor f, const aGeometry3d& g)
 
 ///@cond
 template<>
-struct HostVec< MPITag>
+struct MemoryTraits< MPITag>
 {
     typedef MPI_Vector<thrust::host_vector<double> > host_vector;
 };
@@ -116,7 +122,7 @@ void pushForwardPerp( Functor1 vR, Functor2 vZ,
         container& vx, container& vy,
         const Geometry& g)
 {
-    typedef typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector host_vec;
+    typedef typename GeometryTraits< Geometry>::host_vector host_vec;
     host_vec out1 = pullback( vR, g), temp1(out1);
     host_vec out2 = pullback( vZ, g), temp2(out2);
     dg::tensor::multiply(g.map(), out1, out2, temp1, temp2);
@@ -146,7 +152,7 @@ void pushForward( Functor1 vR, Functor2 vZ, Functor3 vPhi,
         container& vx, container& vy, container& vz,
         const Geometry& g)
 {
-    typedef typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector host_vec;
+    typedef typename GeometryTraits< Geometry>::host_vector host_vec;
     host_vec out1 = pullback( vR, g), temp1(out1);
     host_vec out2 = pullback( vZ, g), temp2(out2);
     host_vec out3 = pullback( vPhi, g), temp3(out3);
@@ -180,7 +186,7 @@ void pushForwardPerp( FunctorRR chiRR, FunctorRZ chiRZ, FunctorZZ chiZZ,
         container& chixx, container& chixy, container& chiyy,
         const Geometry& g)
 {
-    typedef typename HostVec< typename GeometryTraits<Geometry>::memory_category>::host_vector host_vec;
+    typedef typename GeometryTraits< Geometry>::host_vector host_vec;
     host_vec chiRR_ = pullback( chiRR, g);
     host_vec chiRZ_ = pullback( chiRZ, g);
     host_vec chiZZ_ = pullback( chiZZ, g);
-- 
GitLab


From 96c4a601a72bc15bbd89639ce057f66030a50a75 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Tue, 8 Aug 2017 06:48:56 -0700
Subject: [PATCH 130/453] finalize base geometry

---
 inc/dg/geometry/base_geometry.h | 108 +++++++++++++++-----------------
 1 file changed, 50 insertions(+), 58 deletions(-)

diff --git a/inc/dg/geometry/base_geometry.h b/inc/dg/geometry/base_geometry.h
index f9901d662..f80b0eb4d 100644
--- a/inc/dg/geometry/base_geometry.h
+++ b/inc/dg/geometry/base_geometry.h
@@ -14,14 +14,14 @@ namespace dg
  */
 struct aGeometry2d : public aTopology2d
 {
-    SharedContainers<thrust::host_vector<double> > jacobian()const{
-        return do_provide_jacobian();
+    SparseTensor<thrust::host_vector<double> > jacobian()const{
+        return do_compute_jacobian();
     }
-    SharedContainers<thrust::host_vector<double> > metric()const { 
-        return do_provide_metric(); 
+    SparseTensor<thrust::host_vector<double> > metric()const { 
+        return do_compute_metric(); 
     }
     std::vector<thrust::host_vector<double> > map(){
-        return do_provide_map();
+        return do_compute_map();
     }
     ///Geometries are cloneable
     virtual aGeometry2d* clone()const=0;
@@ -32,26 +32,39 @@ struct aGeometry2d : public aTopology2d
      * @copydoc aTopology2d::aTopology2d()
      * @note the default coordinate map will be the identity 
      */
-    aGeometry2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx, bc bcy):aTopology2d( x0,x1,y0,y1,n,Nx,Ny,bcx,bcy),map_(3){}
-    aGeometry2d( const aGeometry2d& src):map_(src.map_), metric_(src.metric_){}
+    aGeometry2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx, bc bcy):aTopology2d( x0,x1,y0,y1,n,Nx,Ny,bcx,bcy){}
+    aGeometry2d( const aGeometry2d& src):aTopology2d(src){}
     aGeometry2d& operator=( const aGeometry2d& src){
-        map_=src.map_;
-        metric_=src.metric_;
+        aTopology2d::operator=(src);
+        return *this;
     }
     private:
-    virtual SharedContainers<thrust::host_vector<double> > do_compute_metric()const=0;
-    virtual SharedContainers<thrust::host_vector<double> > do_compute_jacobian()const=0;
-    virtual std::vector<thrust::host_vector<double> > do_compute_map()const=0;
+    virtual SparseTensor<thrust::host_vector<double> > do_compute_metric()const {
+        return SharedContainer<thrust::host_vector<double> >();
+    }
+    virtual SparseTensor<thrust::host_vector<double> > do_compute_jacobian()const {
+        return SharedContainer<thrust::host_vector<double> >();
+    }
+    virtual std::vector<thrust::host_vector<double> > do_compute_map()const{
+        return std::vector<thrust::host_vector<double> >();
+    }
+
+
 };
 
 /**
- * @brief This is the abstract interface class for a two-dimensional Geometry
+ * @brief This is the abstract interface class for a three-dimensional Geometry
  */
 struct aGeometry3d : public aTopology3d
 {
-    std::vector<thrust::host_vector<double> >& map(){return map_;}
-    SharedContainers<thrust::host_vector<double> > metric() {
-        return do_compute_metric();
+    SparseTensor<thrust::host_vector<double> > jacobian()const{
+        return do_compute_jacobian();
+    }
+    SparseTensor<thrust::host_vector<double> > metric()const { 
+        return do_compute_metric(); 
+    }
+    std::vector<thrust::host_vector<double> > map(){
+        return do_compute_map();
     }
     ///Geometries are cloneable
     virtual aGeometry3d* clone()const=0;
@@ -63,32 +76,24 @@ struct aGeometry3d : public aTopology3d
      * @note the default coordinate map will be the identity 
      */
     aGeometry3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz): aTopology3d(x0,x1,y0,y1,z0,z1,n,Nx,Ny,Nz,bcx,bcy,bcz){}
-    aGeometry3d( const aGeometry3d& src):map_(src.map_){}
+    aGeometry3d( const aGeometry3d& src):aTopology3d(src){}
     aGeometry3d& operator=( const aGeometry3d& src){
-        map_=src.map_;
+        aTopology3d::operator=(src);
+        return *this;
     }
-    SharedContainers<thrust::host_vector<double> >& map(){return map_;}
     private:
-    SharedContainers<thrust::host_vector<double> > map_;
-    virtual SharedContainers<thrust::host_vector<double> > do_compute_metric()const=0;
+    virtual SparseTensor<thrust::host_vector<double> > do_compute_metric()const {
+        return SharedContainer<thrust::host_vector<double> >();
+    }
+    virtual SparseTensor<thrust::host_vector<double> > do_compute_jacobian()const {
+        return SharedContainer<thrust::host_vector<double> >();
+    }
+    virtual std::vector<thrust::host_vector<double> > do_compute_map()const{
+        return std::vector<thrust::host_vector<double> >();
+    }
 };
 
 ///@}
-namespace create
-{
-///@addtogroup metric
-///@{
-SharedContainers<thrust::host_vector<double> > metric( const aGeometry2d& g)
-{
-    return g.metric();
-}
-SharedContainers<thrust::host_vector<double> > metric( const aGeometry3d& g)
-{
-    return g.metric();
-}
-///@}
-
-}//namespace create
 
 ///@addtogroup basicgrids
 ///@{
@@ -108,9 +113,6 @@ struct CartesianGrid2d: public dg::aGeometry2d
     CartesianGrid2d( const dg::Grid2d& grid):dg::Grid2d(grid){}
     virtual CartesianGrid2d* clone()const{return new CartesianGrid2d(*this);}
     private:
-    virtual SharedContainers<thrust::host_vector<double> > do_compute_metric()const{
-        return SharedContainers<thrust::host_vector<double> >();
-    }
     virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny){
         aTopology2d::do_set(new_n,new_Nx,new_Ny);
     }
@@ -130,11 +132,8 @@ struct CartesianGrid3d: public dg::aGeometry3d
     CartesianGrid3d( const dg::Grid3d& grid):dg::Grid3d(grid){}
     virtual CartesianGrid3d* clone()const{return new CartesianGrid3d(*this);}
     private:
-    virtual SharedContainers<thrust::host_vector<double> > do_compute_metric()const{
-        return SharedContainers<thrust::host_vector<double> >( 3);
-    }
     virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz){
-        aTopology2d::do_set(new_n,new_Nx,new_Ny,new_Nz);
+        aTopology3d::do_set(new_n,new_Nx,new_Ny,new_Nz);
     }
 };
 
@@ -146,24 +145,17 @@ struct CylindricalGrid3d: public dg::aGeometry3d
     CylindricalGrid3d( double R0, double R1, double Z0, double Z1, double phi0, double phi1, unsigned n, unsigned NR, unsigned NZ, unsigned Nphi, bc bcR, bc bcZ, bc bcphi = PER): dg::aGeometry3d(R0,R1,Z0,Z1,phi0,phi1,n,NR,NZ,Nphi,bcR,bcZ,bcphi){}
     virtual CylindricalGrid3d* clone()const{return new CylindricalGrid3d(*this);}
     private:
-    virtual SharedContainers<thrust::host_vector<double> > do_compute_metric()const{
-
-        std::vector<thrust::host_vector<double> > values(2, size());
+    virtual SparseTensor<thrust::host_vector<double> > do_compute_metric()const{
+        SparseTensor<thrust::host_vector<double> metric(1);
         thrust::host_vector<double> R = dg::evaluate(dg::coo1, *this);
-        unsigned size2d = n()*n()*Nx()*Ny();
-        for( unsigned i = 0; i<Nz(); i++)
-        for( unsigned j = 0; j<size2d; j++)
-        {
-            unsigned idx = i*size2d+j;
-            values[1][idx] = R[j];
-            values[0][idx] = 1./R[j]/R[j];
-        }
-        Operator<int> mat_idx(3,-1); mat_idx(2,2) = 0;
-        std::vector<int> vec_idx(3,-1); vec_idx[0] = 1;
-        return SharedContainers<thrust::host_vector<double> >( mat_idx, vec_idx, values);
+        for( unsigned i = 0; i<size(); i++)
+            R[i] = 1./R[i]/R[i];
+        metric.value(0) = R;
+        metric.idx(2,2)=0;
+        return metric;
     }
     virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz){
-        aTopology2d::do_set(new_n,new_Nx,new_Ny,new_Nz);
+        aTopology3d::do_set(new_n,new_Nx,new_Ny,new_Nz);
     }
 };
 ///@}
-- 
GitLab


From d607a64aa8075081cb20eb57ce572abc78c3ceb5 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Tue, 8 Aug 2017 07:12:21 -0700
Subject: [PATCH 131/453] map is always a dense vector

---
 inc/dg/geometry/base_geometry.h | 11 +++++++++--
 inc/dg/geometry/geometry.h      |  6 +++---
 inc/dg/geometry/transform.h     | 26 ++++++++------------------
 3 files changed, 20 insertions(+), 23 deletions(-)

diff --git a/inc/dg/geometry/base_geometry.h b/inc/dg/geometry/base_geometry.h
index f80b0eb4d..8922a8c2b 100644
--- a/inc/dg/geometry/base_geometry.h
+++ b/inc/dg/geometry/base_geometry.h
@@ -46,7 +46,10 @@ struct aGeometry2d : public aTopology2d
         return SharedContainer<thrust::host_vector<double> >();
     }
     virtual std::vector<thrust::host_vector<double> > do_compute_map()const{
-        return std::vector<thrust::host_vector<double> >();
+        std::vector<thrust::host_vector<double> > map(2);
+        map[0] = dg::evaluate(dg::cooX2d, *this);
+        map[1] = dg::evaluate(dg::cooY2d, *this);
+        return map;
     }
 
 
@@ -89,7 +92,11 @@ struct aGeometry3d : public aTopology3d
         return SharedContainer<thrust::host_vector<double> >();
     }
     virtual std::vector<thrust::host_vector<double> > do_compute_map()const{
-        return std::vector<thrust::host_vector<double> >();
+        std::vector<thrust::host_vector<double> > map(3);
+        map[0] = dg::evaluate(dg::cooX3d, *this);
+        map[1] = dg::evaluate(dg::cooY3d, *this);
+        map[2] = dg::evaluate(dg::cooZ3d, *this);
+        return map;
     }
 };
 
diff --git a/inc/dg/geometry/geometry.h b/inc/dg/geometry/geometry.h
index 0898af60b..f236dd095 100644
--- a/inc/dg/geometry/geometry.h
+++ b/inc/dg/geometry/geometry.h
@@ -10,11 +10,11 @@
 #include "../backend/mpi_precon.h"
 #endif//MPI_VERSION
 #include "base.h"
-#include "curvilinear.h"
-#include "cartesianX.h"
+//#include "curvilinear.h"
+//#include "cartesianX.h"
 #ifdef MPI_VERSION
 #include "mpi_base.h"
-#include "mpi_curvilinear.h"
+//#include "mpi_curvilinear.h"
 #endif//MPI_VERSION
 #include "tensor.h"
 #include "transform.h"
diff --git a/inc/dg/geometry/transform.h b/inc/dg/geometry/transform.h
index 2f3d6500b..8baf8e0b8 100644
--- a/inc/dg/geometry/transform.h
+++ b/inc/dg/geometry/transform.h
@@ -37,12 +37,10 @@ struct MemoryTraits< SharedTag>
 template< class Functor>
 thrust::host_vector<double> pullback( Functor f, const aGeometry2d& g)
 {
-    const SharedContainers<thrust::host_vector<double> >& map = g.map();
-    if( !map.isSet(0) && !map.isSet(1)) //implicit
-        return evaluate(f,g);
+    std::vector<thrust::host_vector<double> > map = g.map();
     thrust::host_vector<double> vec( g.size());
     for( unsigned i=0; i<g.size(); i++)
-        vec[i] = f( map.getValue(0)[i], map.getValue(1)[i]);
+        vec[i] = f( map[0][i], map[1][i]);
     return vec;
 }
 
@@ -51,13 +49,10 @@ thrust::host_vector<double> pullback( Functor f, const aGeometry2d& g)
 template< class Functor>
 thrust::host_vector<double> pullback( Functor f, const aGeometry3d& g)
 {
-    const SharedContainers<thrust::host_vector<double> >& map = g.map();
-    if( !map.isSet(0) && !map.isSet(1) && !map.isSet(2)) //implicit
-        return evaluate(f,g);
-
+    std::vector<thrust::host_vector<double> > map = g.map();
     thrust::host_vector<double> vec( g.size());
     for( unsigned i=0; i<g.size(); i++)
-        vec[i] = f( map.getValue(0)[i], map.getValue(1)[i], map.getValue(2)[i]);
+        vec[i] = f( map[0][i], map[1][i], map[2][i]);
     return vec;
 }
 
@@ -76,12 +71,10 @@ struct MemoryTraits< MPITag>
 template< class Functor>
 MPI_Vector<thrust::host_vector<double> > pullback( Functor f, const aMPIGeometry2d& g)
 {
-    const SharedContainers<MPI_Vector<thrust::host_vector<double> >& map = g.map();
-    if( !map.isSet(0) && !map.isSet(1)) //implicit
-        return evaluate(f,g);
+    std::vector<MPI_Vector<thrust::host_vector<double> > map = g.map();
     thrust::host_vector<double> vec( g.size());
     for( unsigned i=0; i<g.size(); i++)
-        vec[i] = f( map.getValue(0).data()[i], map.getValue(1).data()[i]);
+        vec[i] = f( map[0].data()[i], map[1].data()[i]);
     return vec;
 }
 
@@ -90,13 +83,10 @@ MPI_Vector<thrust::host_vector<double> > pullback( Functor f, const aMPIGeometry
 template< class Functor>
 MPI_Vector<thrust::host_vector<double> > pullback( Functor f, const aMPIGeometry3d& g)
 {
-    const SharedContainers<MPI_Vector<thrust::host_vector<double> >& map = g.map();
-    if( !map.isSet(0) && !map.isSet(1) && !map.isSet(2)) //implicit
-        return evaluate(f,g);
-
+    std::vector<MPI_Vector<thrust::host_vector<double> > map = g.map();
     thrust::host_vector<double> vec( g.size());
     for( unsigned i=0; i<g.size(); i++)
-        vec[i] = f( map.getValue(0).data()[i], map.getValue(1).data()[i], map.getValue(2).data()[i]);
+        vec[i] = f( map[0].data()[i], map[1].data()[i], map[2].data()[i]);
     return vec;
 }
 
-- 
GitLab


From df37f8d9d90d162c4f79a6bf242369f0259ea350 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Tue, 8 Aug 2017 08:44:20 -0700
Subject: [PATCH 132/453] work on mpi_base

---
 inc/dg/geometry/base_geometry.h |   5 +-
 inc/dg/geometry/mpi_base.h      | 122 ++++++++++++++++++--------------
 2 files changed, 70 insertions(+), 57 deletions(-)

diff --git a/inc/dg/geometry/base_geometry.h b/inc/dg/geometry/base_geometry.h
index 8922a8c2b..643c211da 100644
--- a/inc/dg/geometry/base_geometry.h
+++ b/inc/dg/geometry/base_geometry.h
@@ -157,13 +157,12 @@ struct CylindricalGrid3d: public dg::aGeometry3d
         thrust::host_vector<double> R = dg::evaluate(dg::coo1, *this);
         for( unsigned i = 0; i<size(); i++)
             R[i] = 1./R[i]/R[i];
-        metric.value(0) = R;
         metric.idx(2,2)=0;
+        metric.value(0) = R;
         return metric;
     }
     virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz){
         aTopology3d::do_set(new_n,new_Nx,new_Ny,new_Nz);
     }
-};
-///@}
+}; ///@}
 } //namespace dg
diff --git a/inc/dg/geometry/mpi_base.h b/inc/dg/geometry/mpi_base.h
index 12a433364..026168133 100644
--- a/inc/dg/geometry/mpi_base.h
+++ b/inc/dg/geometry/mpi_base.h
@@ -14,26 +14,39 @@ namespace dg
  */
 struct aMPIGeometry2d : public aMPITopology2d
 {
-    typedef MPI_Vector<thrust::host_vector<double> > host_vec;
-    const SharedContainers<host_vec >& map()const{return map_;}
-    SharedContainers<host_vec > compute_metric()const {
+    typedef MPI_Vector<thrust::host_vector<double> > host_vector;
+    SparseTensor<host_vector > jacobian()const {
+        return do_compute_jacobian();
+    }
+    SparseTensor<host_vector > metric()const {
         return do_compute_metric();
     }
+    std::vector<host_vec > map()const{
+        return do_compute_map();
+    }
     ///Geometries are cloneable
     virtual aMPIGeometry2d* clone()const=0;
     ///allow deletion through base class pointer
     virtual ~aMPIGeometry2d(){}
     protected:
-    aMPIGeometry2d(const SharedContainers<host_vec >& map, const SharedContainers<container >& metric): map_(map), metric_(metric){}
-    aMPIGeometry2d( const aMPIGeometry2d& src):map_(src.map_), metric_(src.metric_){}
+    aMPIGeometry2d( const aMPIGeometry2d& src):aMPITopology2d(src){}
     aMPIGeometry2d& operator=( const aMPIGeometry2d& src){
-        map_=src.map_;
-        metric_=src.metric_;
+        aMPITopology2d::operator=(src);
+        return *this;
     }
-    SharedContainers<host_vec >& map(){return map_;}
     private:
-    SharedContainers<host_vec > map_;
-    virtual SharedContainers<host_vec > do_compute_metric()const=0;
+    virtual SparseTensor<host_vector > do_compute_metric()const {
+        return SharedContainer<host_vector >();
+    }
+    virtual SparseTensor<host_vector > do_compute_jacobian()const {
+        return SharedContainer<host_vector >();
+    }
+    virtual std::vector<host_vector > do_compute_map()const{
+        std::vector<host_vector> map(2);
+        map[0] = dg::evaluate(dg::cooX2d, *this);
+        map[1] = dg::evaluate(dg::cooY2d, *this);
+        return map;
+    }
 };
 
 /**
@@ -41,46 +54,43 @@ struct aMPIGeometry2d : public aMPITopology2d
  */
 struct aMPIGeometry3d : public aMPITopology3d
 {
-    typedef MPI_Vector<thrust::host_vector<double> > host_vec;
-    const SharedContainers<host_vec >& map()const{return map_;}
-    SharedContainers<host_vec > compute_metric()const {
-        return do_compute_metric();
+    typedef MPI_Vector<thrust::host_vector<double> > host_vector;
+    SparseTensor<host_vector > jacobian()const{
+        return do_compute_jacobian();
+    }
+    SparseTensor<host_vector > metric()const { 
+        return do_compute_metric(); 
+    }
+    std::vector<host_vector > map(){
+        return do_compute_map();
     }
     ///Geometries are cloneable
     virtual aMPIGeometry3d* clone()const=0;
     ///allow deletion through base class pointer
     virtual ~aMPIGeometry3d(){}
     protected:
-    aMPIGeometry3d(const SharedContainers<host_vec >& map): map_(map){}
-    aMPIGeometry3d( const aMPIGeometry2d& src):map_(src.map_){}
+    aMPIGeometry3d( const aMPIGeometry3d& src):aMPITopology3d(src){}
     aMPIGeometry3d& operator=( const aMPIGeometry3d& src){
-        map_=src.map_;
+        aMPITopology3d::operator=(src);
+        return *this;
     }
-    SharedContainers<host_vec >& map(){return map_;}
     private:
-    SharedContainers<host_vec > map_;
-    virtual SharedContainers<host_vec > do_compute_metric()const=0;
+    virtual SparseTensor<host_vector > do_compute_metric()const {
+        return SharedContainer<host_vector >();
+    }
+    virtual SparseTensor<host_vector > do_compute_jacobian()const {
+        return SharedContainer<host_vector >();
+    }
+    virtual std::vector<host_vector > do_compute_map()const{
+        std::vector<host_vector> map(3);
+        map[0] = dg::evaluate(dg::cooX3d, *this);
+        map[1] = dg::evaluate(dg::cooY3d, *this);
+        map[2] = dg::evaluate(dg::cooZ3d, *this);
+        return map;
+    }
 };
 
 ///@}
-namespace create
-{
-///@addtogroup metric
-///@{
-
-SharedContainers<host_vec > metric( const aMPIGeometry2d& g)
-{
-    return g.compute_metric();
-}
-SharedContainers<host_vec > metric( const aMPIGeometry3d& g)
-{
-    return g.compute_metric();
-}
-
-///@}
-}//namespace create
-
-
 
 ///@addtogroup basicgrids
 ///@{
@@ -104,6 +114,12 @@ struct CartesianMPIGrid2d : public aMPIGeometry2d
      */
     CartesianMPIGrid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx, bc bcy, MPI_Comm comm):dg::aMPIGeometry2d( x0, x1, y0, y1, n, Nx, Ny,bcx, bcy, comm){}
     CartesianMPIGrid2d( const dg::MPIGrid2d& g): aMPIGeometry2d( g.x0(),g.x1(),g.y0(),g.y1(),g.n(),g.Nx(),g.Ny(),g.bcx(),g.bcy(),g.comm()){}
+    virtual CartesianMPIGrid2d* clone()const{return new CartesianMPIGrid2d(*this);}
+    private:
+    virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny){
+        aMPITopology2d::do_set(new_n,new_Nx,new_Ny);
+    }
+
 };
 
 /**
@@ -126,6 +142,11 @@ struct CartesianMPIGrid3d : public aMPIGeometry3d
     CartesianMPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz, MPI_Comm comm):aMPIGeometry3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, bcx, bcy, bcz, comm){}
 
     CartesianMPIGrid3d( const dg::MPIGrid3d& g): aMPIGeometry3d( g.x0(),g.x1(),g.y0(),g.y1(),g.z0(),g.z1(),g.n(),g.Nx(),g.Ny(),g.Nz(),g.bcx(),g.bcy(),g.bcz(),g.comm()){}
+    virtual CartesianMPIGrid3d* clone()const{return new CartesianMPIGrid3d(*this);}
+    private:
+    virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz){
+        aMPITopology3d::do_set(new_n,new_Nx,new_Ny,new_Nz);
+    }
 };
 
 /**
@@ -139,24 +160,17 @@ struct CylindricalMPIGrid3d: public aMPIGeometry3d
 
     virtual CylindricalMPIGrid3d* clone()const{return new CylindricalMPIGrid3d(*this);}
     private:
-    virtual SharedContainers<thrust::host_vector<double> > do_compute_metric()const{
-
-        std::vector<MPI_Vector<thrust::host_vector<double> > > values(2, size());
-        MPI_Vector<thrust::host_vector<double> > R = dg::evaluate(dg::coo1, *this);
-        unsigned size2d = n()*n()*Nx()*Ny();
-        for( unsigned i = 0; i<Nz(); i++)
-        for( unsigned j = 0; j<size2d; j++)
-        {
-            unsigned idx = i*size2d+j;
-            values[1].data()[idx] = R.data()[j];
-            values[0].data()[idx] = 1./R.data()[j]/R.data()[j];
-        }
-        Operator<int> mat_idx(3,-1); mat_idx(2,2) = 0;
-        std::vector<int> vec_idx(3,-1); vec_idx[0] = 1;
-        return SharedContainers<thrust::host_vector<double> >( mat_idx, vec_idx, values);
+    virtual SparseTensor<host_vector > do_compute_metric()const{
+        SparseTensor<host_vector> metric(1);
+        host_vector R = dg::evaluate(dg::coo1, *this);
+        for( unsigned i = 0; i<size(); i++)
+            R.data()[i] = 1./R.data()[i]/R.data()[i];
+        metric.idx(2,2)=0;
+        metric.value(0) = R;
+        return metric;
     }
     virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz){
-        aMPITopology2d::do_set(new_n,new_Nx,new_Ny,new_Nz);
+        aMPITopology3d::do_set(new_n,new_Nx,new_Ny,new_Nz);
     }
 };
 
-- 
GitLab


From a016808eb0bf870ca046366bf7d16e33a16f7023 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Tue, 8 Aug 2017 19:53:52 +0200
Subject: [PATCH 133/453] added multiply_t.cu

---
 inc/dg/geometry/multiply_t.cu | 89 +++++++++++++++++++++++++++++++++++
 1 file changed, 89 insertions(+)
 create mode 100644 inc/dg/geometry/multiply_t.cu

diff --git a/inc/dg/geometry/multiply_t.cu b/inc/dg/geometry/multiply_t.cu
new file mode 100644
index 000000000..e267e4429
--- /dev/null
+++ b/inc/dg/geometry/multiply_t.cu
@@ -0,0 +1,89 @@
+#include <iostream>
+#include <cmath>
+
+#include "tensor.h"
+#include "multiply.h"
+
+template<class container>
+void print( const dg::SparseTensor<container >& t)
+{
+    for( unsigned i=0; i<3; i++)
+    {
+        for( unsigned j=0; j<3; j++)
+        {
+            if(t.isSet(i,j)) std::cout << t.value(i,j)[0]<<" ";
+            else std::cout <<"xx ";
+        }
+        std::cout << "\n";
+    }
+
+}
+
+template<class container>
+std::ostream& operator<<(std::ostream& os, const dg::SparseElement<container >& t)
+{
+    if(t.isSet()) os << t.value()[0]<<" ";
+    else os <<"XX ";
+    return os;
+}
+
+const thrust::host_vector<double> one(1,1), two(1,2), three(1,3), four(1,4), five(1,5), six(1,6), seven(1,7), eight(1,8), nine(1,9);
+
+int main()
+{
+    std::cout << "Test dg::Sparse Tensor class \n";
+    dg::SparseTensor<thrust::host_vector<double> > t(3);
+    t.idx(0,0) = 0, t.idx(0,1)=t.idx(1,0) = 1;
+    t.idx(1,1) = 2;
+    t.value(0)= two, t.value(1) = three, t.value(2)=four;
+    thrust::host_vector<double> inout0=eight, inout1=nine, inout2=two, work0(inout0), work1(inout1), work2(inout2);
+    dg::SparseElement<thrust::host_vector<double> >mu(five), nu;
+    std::cout << "Begin T\n"; print(t);
+    dg::tensor::scal(t,mu); 
+    std::cout<< "Scale with 5 \n";print(t);
+    dg::tensor::scal(t,nu); 
+    std::cout << "Scale with 1 \n";print(t);
+    dg::tensor::scal(t,mu.invert()); 
+    std::cout << "Scale with 1/5 \n";print(t);
+    dg::tensor::scal(t,one); 
+    std::cout << "Scale with 1 \n";print(t);
+
+    std::cout << "Test Element multiplies \n";
+    dg::tensor::pointwiseDot(mu, eight, inout0); 
+    std::cout<< "8*5 = "<<inout0[0]<<"\n";
+    dg::tensor::pointwiseDivide(inout0,nu,inout0); 
+    dg::tensor::pointwiseDivide(inout0,mu,inout0); 
+    std::cout << "Restore 8 = "<<inout0[0]<<"\n";
+    std::cout << "Multiply T with [8,9]\n";
+    dg::tensor::multiply(t, inout0, inout1, work0, work1);
+    std::cout << "Result         is ["<<work0[0]<<" "<<work1[0]<<"]\n";
+    dg::tensor::multiply_inplace(t, inout0, inout1, work1);
+    std::cout << "Result inplace is ["<<inout0[0]<<" "<<inout1[0]<<"]\n";
+    t.idx(0,2) = 2; std::swap( t.idx(1,1), t.idx(2,1));  print(t);
+    std::cout << "Multiply T with [8,9,2]\n";
+    dg::tensor::multiply(t, eight, nine,two, work0, work1, work2);
+    std::cout << "Result         is ["<<work0[0]<<" "<<work1[0]<<" "<<work2[0]<<"]\n";
+    inout0=eight, inout1=nine, inout2=two;
+    dg::tensor::multiply_inplace(t, inout0, inout1,inout2, work1,work2);
+    std::cout << "Result inplace is ["<<inout0[0]<<" "<<inout1[0]<<" "<<inout2[0]<<"]\n";
+    std::cout << "Determinant of T: "<<dg::tensor::determinant(t).value()[0]<<" (41)\n";
+    std::swap(t.idx(2,1), t.idx(2,0)); 
+    t.value(0) = five;
+    t.idx(1,1) = 0;
+    t.idx(2,2) = 0;
+    std::cout<<"Make a LDL^T decomposition\n";
+    dg::CholeskyTensor<thrust::host_vector<double> > ch(t);
+    std::cout << "origin\n"; print(t);
+    std::cout << "lower \n"; print(ch.lower());
+    std::cout << "diag  \n"; print(ch.diagonal());
+    std::cout << "upper \n"; print(ch.upper());
+    std::cout << "Multiply T with [8,9]\n";
+    dg::tensor::multiply(ch, inout0, inout1, work0, work1);
+    std::cout << "Result         is ["<<work0[0]<<" "<<work1[0]<<"]\n";
+    std::cout << "Multiply T with [8,9,2]\n";
+    dg::tensor::multiply(ch, eight, nine,two, work0, work1, work2);
+    std::cout << "Result         is ["<<work0[0]<<" "<<work1[0]<<" "<<work2[0]<<"]\n";
+    return 0;
+
+
+}
-- 
GitLab


From 679bcfc4fe93cfff8847df9ed0089dc939951f5c Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Tue, 8 Aug 2017 20:59:21 +0200
Subject: [PATCH 134/453] debugged multiply.h

---
 inc/dg/geometry/multiply.h    | 49 +++++++++++++++++++----------------
 inc/dg/geometry/multiply_t.cu | 31 ++++++++++++++--------
 inc/dg/geometry/tensor.h      | 19 +++++++++++---
 inc/dg/geometry/tensor_t.cu   | 12 +++------
 4 files changed, 66 insertions(+), 45 deletions(-)

diff --git a/inc/dg/geometry/multiply.h b/inc/dg/geometry/multiply.h
index 3f468f482..2dcdc672f 100644
--- a/inc/dg/geometry/multiply.h
+++ b/inc/dg/geometry/multiply.h
@@ -14,24 +14,24 @@ namespace tensor
 ///@{
 /**
  * @brief calls sqrt transform function on value
- * @param mu if empty, value is assumed 1 and empty element is returned
- * @return Sparse element containing sqrt of input element
+ * @tparam container container class 
+ * @param mu if empty, stays empty, else contains sqrt of input
  */
-SparseElement sqrt(const SparseElement& mu){ 
-    SparseElement mu(*this);
-    if( isSet()) dg::blas1::transform( mu.value(), mu.value(), dg::SQRT<double>());
-    return mu;
+template<class container>
+void sqrt( SparseElement<container>& mu){ 
+    if( mu.isSet()) 
+        dg::blas1::transform( mu.value(), mu.value(), dg::SQRT<double>());
 }
 
 /**
  * @brief calls invert transform function on value
- * @param mu if empty, value is assumed 1 and empty element is returned
- * @return Sparse element containing inverse of input element
+ * @tparam container container class 
+ * @param mu if empty, stays empty, else contains inverse of input
  */
-SparseElement invert(const SparseElement& mu){ 
-    SparseElement mu(*this);
-    if( isSet()) dg::blas1::transform( mu.value(), mu.value(), dg::INVERT<double>());
-    return mu;
+template<class container>
+void invert(SparseElement<container>& mu){ 
+    if(mu.isSet()) 
+        dg::blas1::transform( mu.value(), mu.value(), dg::INVERT<double>());
 }
 
 /**
@@ -156,7 +156,7 @@ void multiply( const SparseTensor<container>& t, const container& in0, const con
  * @param inout0 (input/output) covariant first component 
  * @param inout1 (input/output) covariant second component
  * @param workspace (write) optional workspace
- * @note this version overwrites the input and may or may not write into the workspace
+ * @note this version overwrites the input and the workspace
  * @attention aliasing not allowed
  */
 template<class container>
@@ -215,6 +215,7 @@ void multiply( const SparseTensor<container>& t, const container& in0, const con
             dg::blas1::pointwiseDot( 1., t.value(1,0), in0, 1., out1);
         if( !t.isSet(0,0)) out0=in0;
         else dg::blas1::pointwiseDot( t.value(0,0), in0, out0); 
+        return;
     }
     //upper triangular and default
     if( !t.isSet(0,0)) out0=in0;
@@ -226,10 +227,10 @@ void multiply( const SparseTensor<container>& t, const container& in0, const con
 
     if( !t.isSet(1,1)) out1=in1;
     else dg::blas1::pointwiseDot( t.value(1,1), in1, out1);
-    if(t.isSet(1,0))
-        dg::blas1::pointwiseDot( 1., t.value(1,0), in0, 1., out1);
     if(t.isSet(1,2))
-        dg::blas1::pointwiseDot( 1., t.value(1,2), in0, 1., out1);
+        dg::blas1::pointwiseDot( 1., t.value(1,2), in2, 1., out1);
+    if(t.isSet(1,0)) //wrong if inplace
+        dg::blas1::pointwiseDot( 1., t.value(1,0), in0, 1., out1);
 
     if(!t.isSet(2,2)) out2=in2;
     else dg::blas1::pointwiseDot( t.value(2,2), in2, out2);
@@ -256,6 +257,7 @@ void multiply( const SparseTensor<container>& t, const container& in0, const con
 template<class container>
 void multiply_inplace( const SparseTensor<container>& t, container& inout0, container& inout1, container& inout2, container& workspace0, container& workspace1)
 {
+    //first: store off-diagonals of first two rows
     if( t.isSet(0,1) ) {
         dg::blas1::pointwiseDot( t.value(0,1), inout1, workspace0);
         if( t.isSet(0,2) ) dg::blas1::pointwiseDot( 1.,t.value(0,2), inout2, 1.,workspace0);
@@ -276,11 +278,12 @@ void multiply_inplace( const SparseTensor<container>& t, container& inout0, cont
     }
     //else workspace1 is empty
     //
-    //compute out2 inplace
+    //second: compute out2 inplace
     if( t.isSet(2,2)) dg::blas1::pointwiseDot( t.value(2,2), inout2, inout2);
     if( t.isSet(2,1)) dg::blas1::pointwiseDot( 1., t.value(2,1), inout1, 1., inout2);
     if( t.isSet(2,0)) dg::blas1::pointwiseDot( 1., t.value(2,0), inout0, 1., inout2);
 
+    //third add values stored in workspaces
     if(t.isSet(0,1) ||t.isSet(0,2) ) //workspace0 is filled
     {
         if( !t.isSet(0,0)) dg::blas1::axpby( 1., inout0, 1., workspace0, workspace0);
@@ -339,16 +342,16 @@ SparseElement<container> determinant( const SparseTensor<container>& t)
 template<class container>
 void multiply( const CholeskyTensor<container>& ch, const container& in0, const container& in1, container& out0, container& out1)
 {
-    multiply(ch.upper(), in0, in1, out0, out1);
-    //multiply( ch.diagonal(), out0, out1, out0, out1);
-    //multiply(ch.lower(), out0, out1, out0, out1);
+    multiply(ch.upper(),     in0,  in1,  out0, out1);
+    multiply(ch.diagonal(),  out0, out1, out0, out1);
+    multiply(ch.lower(),     out0, out1, out0, out1);
 }
 template<class container>
 void multiply( const CholeskyTensor<container>& ch, const container& in0, const container& in1, const container& in2, container& out0, container& out1, container& out2)
 {
-    multiply(ch.upper(), in0, in1,in2, out0, out1,out2);
-    multiply(ch.diagonal(), out0, out1,out2, out0, out1,out2);
-    multiply(ch.lower(), out0, out1,out2, out0, out1,out2);
+    multiply(ch.upper(),    in0,  in1, in2,  out0, out1, out2);
+    multiply(ch.diagonal(), out0, out1,out2, out0, out1, out2);
+    multiply(ch.lower(),    out0, out1,out2, out0, out1, out2);
 }
 
 template<class container>
diff --git a/inc/dg/geometry/multiply_t.cu b/inc/dg/geometry/multiply_t.cu
index e267e4429..bbd97ab8f 100644
--- a/inc/dg/geometry/multiply_t.cu
+++ b/inc/dg/geometry/multiply_t.cu
@@ -27,7 +27,7 @@ std::ostream& operator<<(std::ostream& os, const dg::SparseElement<container >&
     return os;
 }
 
-const thrust::host_vector<double> one(1,1), two(1,2), three(1,3), four(1,4), five(1,5), six(1,6), seven(1,7), eight(1,8), nine(1,9);
+const thrust::host_vector<double> one(1,1), two(1,2), three(1,3), four(1,4), five(1,5), six(1,6), seven(1,7), eight(1,8), nine(1,9), zero(1,0);
 
 int main()
 {
@@ -42,18 +42,25 @@ int main()
     dg::tensor::scal(t,mu); 
     std::cout<< "Scale with 5 \n";print(t);
     dg::tensor::scal(t,nu); 
-    std::cout << "Scale with 1 \n";print(t);
-    dg::tensor::scal(t,mu.invert()); 
+    std::cout << "Scale with empty element \n";print(t);
+    dg::tensor::invert(mu);
+    dg::tensor::scal(t,mu); 
     std::cout << "Scale with 1/5 \n";print(t);
-    dg::tensor::scal(t,one); 
-    std::cout << "Scale with 1 \n";print(t);
+    dg::tensor::scal(t,two); 
+    std::cout << "Scale with container(2) \n";print(t);
 
     std::cout << "Test Element multiplies \n";
+    dg::SparseElement<thrust::host_vector<double> > sqr(nine);
+    dg::tensor::sqrt(sqr);
+    std::cout<<"sqrt(): "<<sqr<<" ("<<std::sqrt(9)<<")\n";
+    dg::tensor::invert(sqr);
+    std::cout<<"invert(): "<<sqr<<"\n";
     dg::tensor::pointwiseDot(mu, eight, inout0); 
     std::cout<< "8*5 = "<<inout0[0]<<"\n";
     dg::tensor::pointwiseDivide(inout0,nu,inout0); 
     dg::tensor::pointwiseDivide(inout0,mu,inout0); 
     std::cout << "Restore 8 = "<<inout0[0]<<"\n";
+    std::cout << "Test Tensor multiplies \n";
     std::cout << "Multiply T with [8,9]\n";
     dg::tensor::multiply(t, inout0, inout1, work0, work1);
     std::cout << "Result         is ["<<work0[0]<<" "<<work1[0]<<"]\n";
@@ -66,7 +73,7 @@ int main()
     inout0=eight, inout1=nine, inout2=two;
     dg::tensor::multiply_inplace(t, inout0, inout1,inout2, work1,work2);
     std::cout << "Result inplace is ["<<inout0[0]<<" "<<inout1[0]<<" "<<inout2[0]<<"]\n";
-    std::cout << "Determinant of T: "<<dg::tensor::determinant(t).value()[0]<<" (41)\n";
+    std::cout << "Determinant of T: "<<dg::tensor::determinant(t).value()[0]<<" (320)\n";
     std::swap(t.idx(2,1), t.idx(2,0)); 
     t.value(0) = five;
     t.idx(1,1) = 0;
@@ -75,14 +82,16 @@ int main()
     dg::CholeskyTensor<thrust::host_vector<double> > ch(t);
     std::cout << "origin\n"; print(t);
     std::cout << "lower \n"; print(ch.lower());
-    std::cout << "diag  \n"; print(ch.diagonal());
+    std::cout << "diag  \n"; 
+    if(ch.diagonal().isDiagonal())print(ch.diagonal());
     std::cout << "upper \n"; print(ch.upper());
     std::cout << "Multiply T with [8,9]\n";
-    dg::tensor::multiply(ch, inout0, inout1, work0, work1);
-    std::cout << "Result         is ["<<work0[0]<<" "<<work1[0]<<"]\n";
+    inout0=eight, inout1=nine, inout2=two;
+    dg::tensor::multiply(ch, inout0, inout1, inout0, inout1);
+    std::cout << "Result         is ["<<inout0[0]<<", "<<inout1[0]<<"] ([94, 93])\n";
     std::cout << "Multiply T with [8,9,2]\n";
-    dg::tensor::multiply(ch, eight, nine,two, work0, work1, work2);
-    std::cout << "Result         is ["<<work0[0]<<" "<<work1[0]<<" "<<work2[0]<<"]\n";
+    dg::tensor::multiply(ch, eight,nine,two, work0, work1, work2);
+    std::cout << "Result         is ["<<work0[0]<<" "<<work1[0]<<" "<<work2[0]<<"] (110, 93, 74)\n";
     return 0;
 
 
diff --git a/inc/dg/geometry/tensor.h b/inc/dg/geometry/tensor.h
index ecf75a10a..d90328a41 100644
--- a/inc/dg/geometry/tensor.h
+++ b/inc/dg/geometry/tensor.h
@@ -200,7 +200,7 @@ struct SparseTensor
     }
 
     /**
-    * @brief Test the matrix for emptiness
+    * @brief Test if all elements are set
     *
     * The matrix is dense if isSet(i,j) for all i and j
     * @return true if all values in the matrix are set
@@ -220,8 +220,7 @@ struct SparseTensor
     * The matrix is perpendicular if !isSet(i,j) for any i,j=2
     * @return true if perpenicular
     * */
-    bool isPerp() const
-    {
+    bool isPerp() const {
         bool empty=true;
         for(unsigned i=0; i<3; i++)
         {
@@ -230,6 +229,20 @@ struct SparseTensor
         }
         return empty;
     }
+    /**
+    * @brief Test if no off-diagonals are set
+    *
+    * The matrix is diagonal if no off-diagonal element is set
+    * @return true if no off-diagonal element is set
+    */
+    bool isDiagonal()const{
+        bool diagonal=true;
+        for(unsigned i=0; i<3; i++)
+            for(unsigned j=i+1; j<3; j++)
+                if( isSet(i,j) ||  isSet(j,i))
+                    diagonal=false;
+        return diagonal;
+    }
      
      ///construct an empty Tensor
      SparseTensor empty()const{return SparseTensor();}
diff --git a/inc/dg/geometry/tensor_t.cu b/inc/dg/geometry/tensor_t.cu
index 140297c6a..520bbfdd5 100644
--- a/inc/dg/geometry/tensor_t.cu
+++ b/inc/dg/geometry/tensor_t.cu
@@ -60,9 +60,9 @@ int main()
     sparse3d.clear_unused_values();
     std::cout << "Size after  "<<sparse3d.values().size()<<"\n";
     std::cout << "tensor after \n";
-    print( sparse3d);
+    if(!sparse3d.isDiagonal()) print( sparse3d);
     std::cout << "empty Tensor \n";
-    print( sparse3d.empty());
+    if(sparse3d.empty().isDiagonal()) print( sparse3d.empty());
     std::cout << "explicit dense Tensor \n";
     dg::SparseTensor<thrust::host_vector<double> > dense3d = sparse3d.dense();
     if(dense3d.isDense())print( dense3d);
@@ -94,12 +94,8 @@ int main()
     ee = e;
     e.value()=nine;
     std::cout << "Assignment and set : "<<ee<<" (80) "<<e<<"(90)\n";
-    dg::SparseElement<thrust::host_vector<double> > sqr = e.sqrt();
-    std::cout<<"sqrt(): "<<sqr<<" ("<<std::sqrt(90)<<")\n";
-    sqr = sqr.invert();
-    std::cout<<"invert(): "<<sqr<<"\n";
-    sqr.clear();
-    std::cout<<"clear(): "<<sqr<<"\n";
+    e.clear();
+    std::cout<<"clear(): "<<e<<"\n";
     std::cout <<std::flush;
     return 0;
 
-- 
GitLab


From 5ba3afdd054b7d9d41dabaf9d711153dc7dfeb68 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Tue, 8 Aug 2017 21:26:57 +0200
Subject: [PATCH 135/453] initial work on generators

---
 inc/dg/geometry/curvilinear.h |  41 ++-----------
 inc/dg/geometry/generator.h   | 106 +++++-----------------------------
 2 files changed, 21 insertions(+), 126 deletions(-)

diff --git a/inc/dg/geometry/curvilinear.h b/inc/dg/geometry/curvilinear.h
index e13f6cfea..2888bccc7 100644
--- a/inc/dg/geometry/curvilinear.h
+++ b/inc/dg/geometry/curvilinear.h
@@ -47,7 +47,7 @@ struct CurvilinearGrid3d : public dg::aGeometry3d
      @param bcx boundary condition in x
      @note the boundary conditions for y and z are set periodic
      */
-    CurvilinearGrid3d( const geo::aGenerator& generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx=dg::DIR):
+    CurvilinearGrid3d( const aGenerator& generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx=dg::DIR):
         dg::Grid3d( 0, generator.width(), 0., generator.height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, dg::PER, dg::PER)
     { 
         handle_ = generator;
@@ -69,24 +69,7 @@ struct CurvilinearGrid3d : public dg::aGeometry3d
                 vec[k*size2d+i] = f( r_[i], z_[i], phi_[k]);
         return vec;
     }
-    /// a 3d vector containing dx/dR
-    const thrust::host_vector<double>& xr()const{return xr_;}
-    /// a 3d vector containing dy/dR
-    const thrust::host_vector<double>& yr()const{return yr_;}
-    /// a 3d vector containing dx/dZ
-    const thrust::host_vector<double>& xz()const{return xz_;}
-    /// a 3d vector containing dy/dZ
-    const thrust::host_vector<double>& yz()const{return yz_;}
-    const container& g_xx()const{return g_xx_;}
-    const container& g_yy()const{return g_yy_;}
-    const container& g_xy()const{return g_xy_;}
-    const container& g_pp()const{return g_pp_;}
-    const container& vol()const{return vol_;}
-    const container& perpVol()const{return vol2d_;}
-    const geo::aGenerator & generator() const{return handle_.get()}
-    bool isOrthonormal() const { return handle_.get().isOrthonormal();}
-    bool isOrthogonal() const { return handle_.get().isOrthogonal();}
-    bool isConformal() const { return handle_.get().isConformal();}
+    const aGenerator & generator() const{return handle_.get()}
     private:
     virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny,unsigned new_Nz){
         dg::Grid3d::do_set( new_n, new_Nx, new_Ny,new_Nz);
@@ -142,7 +125,7 @@ struct CurvilinearGrid3d : public dg::aGeometry3d
     thrust::host_vector<double> r_, z_, phi_;  //2d and 1d vectors
     thrust::host_vector<double> xr_, xz_, yr_, yz_;
     container g_xx_, g_xy_, g_yy_, g_pp_, vol_, vol2d_;
-    dg::Handle<geo::aGenerator> handle_;
+    dg::Handle<aGenerator> handle_;
 };
 
 /**
@@ -159,7 +142,7 @@ struct CurvilinearGrid2d : public dg::aGeometry2d
      * @param Ny number of cells in second coordinate
      * @param bcx boundary condition in first coordinate
      */
-    CurvilinearGrid2d( const geo::aGenerator& generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx=dg::DIR):
+    CurvilinearGrid2d( const aGenerator2d& generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx=dg::DIR):
         dg::Grid2d( 0, generator.width(), 0., generator.height(), n, Nx, Ny, bcx, dg::PER), handle_(generator)
     {
         construct( n,Nx,Ny);
@@ -181,17 +164,7 @@ struct CurvilinearGrid2d : public dg::aGeometry2d
         thrust::copy( g.perpVol().begin(), g.perpVol().begin()+s, vol2d_.begin());
     }
 
-    const thrust::host_vector<double>& r()const{return r_;}
-    const thrust::host_vector<double>& z()const{return z_;}
-    const thrust::host_vector<double>& xr()const{return xr_;}
-    const thrust::host_vector<double>& yr()const{return yr_;}
-    const thrust::host_vector<double>& xz()const{return xz_;}
-    const thrust::host_vector<double>& yz()const{return yz_;}
-    const container& g_xx()const{return g_xx_;}
-    const container& g_yy()const{return g_yy_;}
-    const container& g_xy()const{return g_xy_;}
-    const container& vol()const{return vol2d_;}
-    const geo::aGenerator& generator() const{return handle_.get();}
+    const aGenerator& generator() const{return handle_.get();}
     private:
     virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny)
     {
@@ -205,9 +178,7 @@ struct CurvilinearGrid2d : public dg::aGeometry2d
         g_xx_=g.g_xx(), g_xy_=g.g_xy(), g_yy_=g.g_yy();
         vol2d_=g.perpVol();
     }
-    thrust::host_vector<double> r_, z_, xr_, xz_, yr_, yz_;
-    container g_xx_, g_xy_, g_yy_, vol2d_;
-    dg::Handle<geo::aGenerator2d> handle_;
+    dg::Handle<aGenerator2d> handle_;
 };
 
 ///@}
diff --git a/inc/dg/geometry/generator.h b/inc/dg/geometry/generator.h
index 88f7c548c..efaef6c96 100644
--- a/inc/dg/geometry/generator.h
+++ b/inc/dg/geometry/generator.h
@@ -2,21 +2,6 @@
 
 namespace dg
 {
-namespace geo
-{
-
-///The analytically given continuous representation of the real world coordinates and metric
-enum PhysicalSpaceCoordinates{
-    cartesian=0;///2d or 3d X,Y,Z
-    cylindrical=1;///3d R,Z,P
-};
-///The type of discrete coordinates and metric
-enum ComputationalSpaceCoordinates{
-    orthonormal=0; ///coordinate lines are orthogonal (only unity diagonal metric elements)
-    conformal=1; ///only 2d coordinates can be conformal (volume and metric elements are all the same)
-    orthogonal=2; ///coordinate lines are orthogonal (only diagonal metric elements)
-    curvilinear=3; ///non-orthogonal coordinate lines
-};
 
 /**
 * @brief The abstract generator base class 
@@ -27,22 +12,14 @@ is a product space.
 @note the origin of the computational space is assumed to be (0,0)
  @ingroup generators
 */
-struct aGridGenerator
+struct aGenerator2d
 {
-    virtual double width()  const=0; //!<length in \f$ \zeta\f$ of the computational space
-    virtual double height() const=0; //!<length in \f$ \eta\f$ of the computational space
-    /**
-    * @brief This is the analytical coordinate system we transform to
-    * @return type of physical space coordinates (default is dg::geo::cartesian)
-    */
-    virtual enum PhysicalSpaceCoordinates physical()const{ return cartesian;} 
-    /**
-    * @brief The type of coordinate system of the computational space
-    * @return default is dg::geo::orthonormal
-    * @note This is a performance hint for the computation and storage of metric elements.
-    *    We believe that you do not lie about what you generate.
-    */
-    virtual enum ComputationalSpaceCoordinates computational()const{ return orthonormal;} 
+    ///@brief length in \f$ \zeta\f$ of the computational space
+    double width()  const{return do_width();}
+    ///@brief length in \f$ \eta\f$ of the computational space
+    double height() const{return do_height();}
+    ///@brief sparsity pattern for metric
+    bool isOrthogonal() const { return doIsOrthogonal(); }
 
     /**
     * @brief Generate grid points and elements of the Jacobian 
@@ -58,7 +35,7 @@ struct aGridGenerator
     * @note the first (\f$ \zeta\f$) coordinate shall be constructed contiguously in memory, i.e. the resuling lists are \f$ x_0=x(\zeta_0, \eta_0),\ x_1=x(\zeta_1, \eta_0)\ x_2=x(\zeta_2, \eta_0)\dots x_{NM-1}=x(\zeta_{N-1} \eta_{M-1})\f$
     * @note All the resulting vectors are write-only and get properly resized
     */
-    void operator()( 
+    void generate( 
          const thrust::host_vector<double>& zeta1d, 
          const thrust::host_vector<double>& eta1d, 
          thrust::host_vector<double>& x, 
@@ -71,7 +48,7 @@ struct aGridGenerator
         unsigned size = zeta1d.size()*eta1d.size();
         x.resize(size), y.resize(size);
         zetaX = zetaY = etaX = etaY =x ;
-        generate( zeta1d, eta1d, x,y,zetaX,zetaY,etaX,etaY);
+        do_generate( zeta1d, eta1d, x,y,zetaX,zetaY,etaX,etaY);
     }
 
     /**
@@ -80,12 +57,13 @@ struct aGridGenerator
     * @return a copy of *this on the heap
     */
     virtual aGridGenerator* clone() const=0;
+    virtual ~aGridGenerator(){}
 
     protected:
     aGridGenerator(const aGridGenerator& src){}
     aGridGenerator& operator=(const aGridGenerator& src){}
-    ///@copydoc operator()()
-    virtual void generate(
+    private:
+    virtual void do_generate(
          const thrust::host_vector<double>& zeta1d, 
          const thrust::host_vector<double>& eta1d, 
          thrust::host_vector<double>& x, 
@@ -94,65 +72,11 @@ struct aGridGenerator
          thrust::host_vector<double>& zetaY, 
          thrust::host_vector<double>& etaX, 
          thrust::host_vector<double>& etaY) const = 0;
-    
-   virtual ~aGridGenerator(){}
-};
-
-/**
-* @brief The shifted identity coordinate transformation
-
-@note in fact it's not completely the identity because we assume that the origin is always (0,0) in the computational space
- @ingroup generators
-*/
-struct ShiftedIdentityGenerator: public aGridGenerator
-{
-    virtual double width()  const{return lx_;} 
-    virtual double height() const{return ly_;}
-    virtual bool isOrthonormal() const{return true;}
-    virtual bool isOrthogonal() const{return true;}
-    virtual bool isConformal()const{return true;}
-    virtual ShiftedIdentityGenerator* clone() const{return new ShiftedIdentityGenerator(*this);}
+     virtual double do_width() const =0;
+     virtual double do_height() const =0;
+     virtual bool doIsOrthogonal()const{return false;}
 
-    /**
-    * @brief Define the 2d box in the physical domain in which to construct the coordinates
-    *
-    * @param x0 x-coordinate of lower left point
-    * @param x1 x-coordinate of upper right point
-    * @param y0 y-coordinate of lower left point
-    * @param y1 y-coordainte of upper right point
-    */
-    ShiftedIdentityGenerator( double x0, double x1, double y0, double y1){
-        x0_ = x0; lx_ = (x1-x0);
-        y0_ = y0; ly_ = (y1-y0);
-    }
 
-    protected:
-    virtual void generate(
-         const thrust::host_vector<double>& zeta1d, 
-         const thrust::host_vector<double>& eta1d, 
-         thrust::host_vector<double>& x, 
-         thrust::host_vector<double>& y, 
-         thrust::host_vector<double>& zetaX, 
-         thrust::host_vector<double>& zetaY, 
-         thrust::host_vector<double>& etaX, 
-         thrust::host_vector<double>& etaY) const
-     {
-         for(unsigned i=0; i<eta1d.size();i++)
-             for(unsigned j=0; j<zeta1d.size();j++)
-             {
-                 x[i*zeta1d.size()+j] = x0_ + zeta1d[j];
-                 y[i*zeta1d.size()+j] = y0_ + eta1d[i];
-                 zetaX[i*zeta1d.size()+j] = 1;
-                 zetaY[i*zeta1d.size()+j] = 0;
-                 etaX[i*zeta1d.size()+j] = 0.;
-                 etaY[i*zeta1d.size()+j] = 1.;
-             }
-                 
-     }
-     private:
-     double x0_,y0_,lx_,ly_;
-    
 };
 
-}//namespace geo
 }//namespace dg
-- 
GitLab


From fbf23e99ebdc33a238ff70cf3c796b9dda7c3c51 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Tue, 8 Aug 2017 22:39:58 +0200
Subject: [PATCH 136/453] made curvilinear grids

---
 inc/dg/backend/grid.h         |   6 +-
 inc/dg/geometry/curvilinear.h | 174 +++++++++++++++++-----------------
 2 files changed, 90 insertions(+), 90 deletions(-)

diff --git a/inc/dg/backend/grid.h b/inc/dg/backend/grid.h
index 7b6ebeafd..e5eb98ec6 100644
--- a/inc/dg/backend/grid.h
+++ b/inc/dg/backend/grid.h
@@ -303,7 +303,8 @@ struct aTopology2d
     * @param new_Ny new number of cells in y
     */
     void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny) {
-        do_set(new_n,new_Nx,new_Ny);
+        if( !( new_n==n() && new_Nx==Nx() && new_Ny == Ny() ) ) 
+            do_set(new_n,new_Nx,new_Ny);
     }
 
 
@@ -433,7 +434,8 @@ struct aTopology3d
     * @param new_Nz new number of cells in z
     */
     void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz) {
-        do_set(new_n,new_Nx,new_Ny,new_Nz);
+        if(!( new_n==n() && new_Nx ==Nx() && new_Ny == Ny() && new_Nz==Nz())) 
+            do_set(new_n,new_Nx,new_Ny,new_Nz);
     }
 
     /**
diff --git a/inc/dg/geometry/curvilinear.h b/inc/dg/geometry/curvilinear.h
index 2888bccc7..566f19f28 100644
--- a/inc/dg/geometry/curvilinear.h
+++ b/inc/dg/geometry/curvilinear.h
@@ -25,17 +25,10 @@ struct CurvilinearGrid2d;
  @tparam container models aContainer
  */
 template< class container>
-struct CurvilinearGrid3d : public dg::aGeometry3d
+struct CylindricalProductGrid3d : public dg::aGeometry3d
 {
     typedef CurvilinearGrid2d<container> perpendicular_grid;
 
-    CurvilinearGrid3d( double R0, double R1, double Z0, double Z1, double phi0, double phi1, unsigned n, unsigned NR, unsigned NZ, unsigned Nphi, bc bcr = PER, bc bcz = PER, bc bcphi = PER): 
-        dg::Grid3d(0,R1-R0,0,Z1-Z0,phi0,phi1,n,NR,NZ,Nphi,bcR,bcZ,bcphi)
-        {
-            handle_.set( new ShiftedIdentityGenerator(R0,R1,Z0,Z1));
-            construct( n, NR, NZ,Nphi);
-        }
-
     /*!@brief Constructor
     
      * the coordinates of the computational space are called x,y,z
@@ -45,86 +38,88 @@ struct CurvilinearGrid3d : public dg::aGeometry3d
      @param Ny number of cells in y 
      @param Nz  number of cells z
      @param bcx boundary condition in x
-     @note the boundary conditions for y and z are set periodic
+     @param bcy boundary condition in y
+     @param bcz boundary condition in z
      */
-    CurvilinearGrid3d( const aGenerator& generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx=dg::DIR):
-        dg::Grid3d( 0, generator.width(), 0., generator.height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, dg::PER, dg::PER)
+    CylindricalProductGrid3d( const aGenerator& generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx=dg::DIR, bc bcy=dg::PER, bc bcz=dg::PER):
+        dg::aGeometry3d( 0, generator.width(), 0., generator.height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, bcy, bcz)
     { 
+        map_.resize(3);
         handle_ = generator;
-        construct( n, Nx, Ny,Nz);
+        constructPerp( n, Nx, Ny);
+        constructParallel(Nz);
     }
 
     perpendicular_grid perp_grid() const { return perpendicular_grid(*this);}
 
-    /**
-    * @brief Function to do the pullback
-    * @note Don't call! Use the pullback() function
-    */
-    template<class TernaryOp>
-    thrust::host_vector<double> doPullback( TernaryOp f)const {
-        thrust::host_vector<double> vec( size());
-        unsigned size2d = n()*n()*Nx()*Ny();
-        for( unsigned k=0; k<Nz(); k++)
-            for( unsigned i=0; i<size2d; i++)
-                vec[k*size2d+i] = f( r_[i], z_[i], phi_[k]);
-        return vec;
-    }
-    const aGenerator & generator() const{return handle_.get()}
+    const aGenerator2d & generator() const{return handle_.get()}
     private:
     virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny,unsigned new_Nz){
-        dg::Grid3d::do_set( new_n, new_Nx, new_Ny,new_Nz);
-        construct( new_n, new_Nx, new_Ny,new_Nz);
+        dg::aTopology3d::do_set( new_n, new_Nx, new_Ny,new_Nz);
+        if( !( new_n == n() && new_Nx == Nx() && new_Ny == Ny() ) )
+            constructPerp( new_n, new_Nx, new_Ny);
+        constructParallel(new_Nz);
     }
-    void construct( unsigned n, unsigned Nx, unsigned Ny,unsigned Nz)
+    //construct phi and lift rest to 3d
+    void constructParallel(unsigned Nz)
     {
-        dg::Grid1d gY1d( 0., y0(), n, Ny, dg::PER);
-        dg::Grid1d gX1d( 0., x0(), n, Nx);
-        dg::Grid1d gphi( z0(), z1(), 1, Nz);
-        phi_ = dg::create::abscissas( gphi);
-        thrust::host_vector<double> x_vec = dg::evaluate( dg::cooX1d, gX1d);
-        thrust::host_vector<double> y_vec = dg::evaluate( dg::cooX1d, gY1d);
-        handle_.get()( x_vec, y_vec, r_, z_, xr_, xz_, yr_, yz_);
-        //lift to 3D grid
+        map_[2]=dg::evaluate(dg::cooZ3d, *this);
         unsigned size = this->size();
-        xr_.resize(size), yr_.resize( size), xz_.resize( size), yz_.resize(size);
         unsigned size2d = this->n()*this->n()*this->Nx()*this->Ny();
-        for( unsigned k=1; k<this->Nz(); k++)
+        //resize for 3d values
+        for( unsigned r=0; r<4;r++)
+            jac_.value(r).resize(size);
+        map_[0].resize(size);
+        map_[1].resize(size);
+        //lift to 3D grid
+        for( unsigned k=1; k<Nz; k++)
             for( unsigned i=0; i<size2d; i++)
             {
-                xr_[k*size2d+i] = xr_[(k-1)*size2d+i];
-                xz_[k*size2d+i] = xz_[(k-1)*size2d+i];
-                yr_[k*size2d+i] = yr_[(k-1)*size2d+i];
-                yz_[k*size2d+i] = yz_[(k-1)*size2d+i];
+                for(unsigned r=0; r<4; r++)
+                    jac_.value(r)[k*size2d+i] = jac_.value(r)[(k-1)*size2d+i];
+                map_[0][k*size2d+i] = map_[0][(k-1)*size2d+i];
+                map_[1][k*size2d+i] = map_[1][(k-1)*size2d+i];
             }
-        construct_metric();
     }
-    //compute metric elements from xr, xz, yr, yz, r and z
-    void construct_metric( )
+    //construct 2d plane
+    void constructPerp( unsigned n, unsigned Nx, unsigned Ny)
     {
-        thrust::host_vector<double> tempxx( size()), tempxy(size()), tempyy(size()), tempvol(size()), tempvol2d(size()), temppp(size());
-        unsigned size2d = this->n()*this->n()*this->Nx()*this->Ny();
-        for( unsigned i = 0; i<this->Nz(); i++)
-        for( unsigned j = 0; j<size2d; j++)
+        dg::Grid1d gX1d( 0., x0(), n, Nx);
+        dg::Grid1d gY1d( 0., y0(), n, Ny);
+        thrust::host_vector<double> x_vec = dg::evaluate( dg::cooX1d, gX1d);
+        thrust::host_vector<double> y_vec = dg::evaluate( dg::cooX1d, gY1d);
+        handle_.get().generate( x_vec, y_vec, map_[0], map_[1], jac_.value(0), jac_.value(1), jac_.value(2), jac_.value(3));
+        jac_.idx(0,0) = 0, jac_.idx(0,1) = 1, jac_.idx(1,0)=2, jac_.idx(1,1) = 3;
+    }
+    virtual SparseTensor<thrust::host_vector> do_compute_jacobian( ) const {
+        return jac_;
+    }
+    virtual SparseTensor<thrust::host_vector> do_compute_metric( ) const
+    {
+        thrust::host_vector<double> tempxx( size()), tempxy(size()), tempyy(size()), temppp(size());
+        for( unsigned i=0; i<size(); i++)
         {
-            unsigned idx = i*size2d+j;
-            tempxx[idx] = (xr_[idx]*xr_[idx]+xz_[idx]*xz_[idx]);
-            tempxy[idx] = (yr_[idx]*xr_[idx]+yz_[idx]*xz_[idx]);
-            tempyy[idx] = (yr_[idx]*yr_[idx]+yz_[idx]*yz_[idx]);
-            tempvol2d[idx] = 1./sqrt( tempxx[idx]*tempyy[idx] -tempxy[idx]*tempxy[idx] );
-            tempvol[idx] = r_[j]*tempvol2d[idx];
-            temppp[idx] = 1./r_[j]/r_[j];
+            tempxx[i] = (jac_.value(0,0)[i]*jac_.value(0,0)[i]+jac_.value(0,1)[i]*jac_.value(0,1)[i]);
+            tempxy[i] = (jac_.value(0,0)[i]*jac_.value(1,0)[i]+jac_.value(0,1)[i]*jac_.value(1,1)[i]);
+            tempyy[i] = (jac_.value(1,0)[i]*jac_.value(1,0)[i]+jac_.value(1,1)[i]*jac_.value(1,1)[i]);
+            temppp[i] = 1./map_[2][i]/map_[2][i]; //1/R^2
+        }
+        SparseTensor<thrust::host_vector<double> > metric;
+        metric.idx(0,0) = 0;
+        metric.value(0) = tempxx;
+        metric.idx(1,1) = 1;
+        metric.value(1) = tempyy;
+        metric.idx(2,2) = 2;
+        metric.value(2) = temppp;
+        if( !handle_.get().isOrthogonal())
+        {
+            metric.idx(0,1) = metric.idx(1,0) = 3;
+            metric.value(3) = tempxy;
         }
-        //now transfer to device
-        dg::blas1::transfer(tempxx,     g_xx_);
-        dg::blas1::transfer(tempxy,     g_xy_);
-        dg::blas1::transfer(tempyy,     g_yy_);
-        dg::blas1::transfer(tempvol,    vol_);
-        dg::blas1::transfer(tempvol2d,  vol2d_);
-        dg::blas1::transfer(temppp,     g_pp_);
     }
-    thrust::host_vector<double> r_, z_, phi_;  //2d and 1d vectors
-    thrust::host_vector<double> xr_, xz_, yr_, yz_;
-    container g_xx_, g_xy_, g_yy_, g_pp_, vol_, vol2d_;
+    virtual std::vector<thrurst::host_vector<double> > do_compute_map()const{return map_;}
+    std::vector<thrust::host_vector<double> > map_;
+    SparseTensor<thrust::host_vector<double> > jac_;
     dg::Handle<aGenerator> handle_;
 };
 
@@ -141,30 +136,24 @@ struct CurvilinearGrid2d : public dg::aGeometry2d
      * @param Nx number of cells in first coordinate
      * @param Ny number of cells in second coordinate
      * @param bcx boundary condition in first coordinate
+     * @param bcy boundary condition in second coordinate
      */
-    CurvilinearGrid2d( const aGenerator2d& generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx=dg::DIR):
-        dg::Grid2d( 0, generator.width(), 0., generator.height(), n, Nx, Ny, bcx, dg::PER), handle_(generator)
+    CurvilinearGrid2d( const aGenerator2d& generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx=dg::DIR, bc bcy=dg::PER):
+        dg::aGeometry2d( 0, generator.width(), 0., generator.height(), n, Nx, Ny, bcx, dg::PER), handle_(generator)
     {
         construct( n,Nx,Ny);
     }
-    CurvilinearGrid2d( const CurvilinearGrid3d<container>& g):
-        dg::Grid2d( g.x0(), g.x1(), g.y0(), g.y1(), g.n(), g.Nx(), g.Ny(), g.bcx(), g.bcy()), handle_(g.generator())
+    CurvilinearGrid2d( CylindricalProductGrid3d<container> g):
+        dg::aGeometry2d( g.x0(), g.x1(), g.y0(), g.y1(), g.n(), g.Nx(), g.Ny(), g.bcx(), g.bcy()), handle_(g.generator())
     {
-        r_ = g.r();
-        z_ = g.z();
-        unsigned s = this->size();
-        xr_.resize(s), xz_.resize(s), yr_.resize(s), yz_.resize(s);
-        g_xx_.resize( s), g_xy_.resize(s), g_yy_.resize(s), vol2d_.resize(s);
-        for( unsigned i=0; i<s; i++) { 
-            xr_[i]=g.xr()[i], xz_[i]=g.xz()[i], yr_[i]=g.yr()[i], yz_[i]=g.yz()[i];
-        }
-        thrust::copy( g.g_xx().begin(), g.g_xx().begin()+s, g_xx_.begin());
-        thrust::copy( g.g_xy().begin(), g.g_xy().begin()+s, g_xy_.begin());
-        thrust::copy( g.g_yy().begin(), g.g_yy().begin()+s, g_yy_.begin());
-        thrust::copy( g.perpVol().begin(), g.perpVol().begin()+s, vol2d_.begin());
+        g.set( n(), Nx(), Ny(), 1); //shouldn't trigger 2d grid generator
+        map_=g.map();
+        jac_=g.jacobian().perp();
+        metric_=g.metric().perp();
+        map_.pop_back();
     }
 
-    const aGenerator& generator() const{return handle_.get();}
+    const aGenerator2d& generator() const{return handle_.get();}
     private:
     virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny)
     {
@@ -173,11 +162,20 @@ struct CurvilinearGrid2d : public dg::aGeometry2d
     }
     void construct( unsigned n, unsigned Nx, unsigned Ny)
     {
-        CurvilinearGrid3d<container> g( handle_, n,Nx,Ny,1,bcx());
-        r_=g.r(), z_=g.z(), xr_=g.xr(), xz_=g.xz(), yr_=g.yr(), yz_=g.yz();
-        g_xx_=g.g_xx(), g_xy_=g.g_xy(), g_yy_=g.g_yy();
-        vol2d_=g.perpVol();
+        CurvilinearGrid3d<container> g( handle_.get(), n,Nx,Ny,1,bcx());
+        jac_=g.jacobian();
+        map_=g.map();
+        metric_=g.metric();
+    }
+    virtual SparseTensor<thrust::host_vector> do_compute_jacobian( ) const {
+        return jac_;
+    }
+    virtual SparseTensor<thrust::host_vector> do_compute_metric( ) const {
+        return metric_;
     }
+    virtual std::vector<thrurst::host_vector<double> > do_compute_map()const{return map_;}
+    dg::SparseTensor<thrust::host_vector<double> > jac_, metric_;
+    std::vector<thrust::host_vector<double> > map_;
     dg::Handle<aGenerator2d> handle_;
 };
 
-- 
GitLab


From dd0a775e00ebb0cbad101f3ed40064f962547494 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Tue, 8 Aug 2017 22:51:02 +0200
Subject: [PATCH 137/453] begin to modify MPI curvilinear grids

---
 inc/dg/geometry/curvilinear.h     | 28 +++++++++------------
 inc/dg/geometry/mpi_curvilinear.h | 41 ++++++-------------------------
 2 files changed, 18 insertions(+), 51 deletions(-)

diff --git a/inc/dg/geometry/curvilinear.h b/inc/dg/geometry/curvilinear.h
index 566f19f28..467dba94c 100644
--- a/inc/dg/geometry/curvilinear.h
+++ b/inc/dg/geometry/curvilinear.h
@@ -12,7 +12,6 @@ namespace dg
 ///@{
 
 ///@cond
-template< class container>
 struct CurvilinearGrid2d; 
 ///@endcond
 
@@ -22,9 +21,7 @@ struct CurvilinearGrid2d;
 
 /**
  * @brief A three-dimensional grid based on curvilinear coordinates
- @tparam container models aContainer
  */
-template< class container>
 struct CylindricalProductGrid3d : public dg::aGeometry3d
 {
     typedef CurvilinearGrid2d<container> perpendicular_grid;
@@ -35,11 +32,11 @@ struct CylindricalProductGrid3d : public dg::aGeometry3d
      * @param generator must generate a grid
      * @param n number of %Gaussian nodes in x and y
      * @param Nx number of cells in x
-     @param Ny number of cells in y 
-     @param Nz  number of cells z
-     @param bcx boundary condition in x
-     @param bcy boundary condition in y
-     @param bcz boundary condition in z
+     * @param Ny number of cells in y 
+     * @param Nz  number of cells z
+     * @param bcx boundary condition in x
+     * @param bcy boundary condition in y
+     * @param bcz boundary condition in z
      */
     CylindricalProductGrid3d( const aGenerator& generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx=dg::DIR, bc bcy=dg::PER, bc bcz=dg::PER):
         dg::aGeometry3d( 0, generator.width(), 0., generator.height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, bcy, bcz)
@@ -105,15 +102,12 @@ struct CylindricalProductGrid3d : public dg::aGeometry3d
             temppp[i] = 1./map_[2][i]/map_[2][i]; //1/R^2
         }
         SparseTensor<thrust::host_vector<double> > metric;
-        metric.idx(0,0) = 0;
-        metric.value(0) = tempxx;
-        metric.idx(1,1) = 1;
-        metric.value(1) = tempyy;
-        metric.idx(2,2) = 2;
-        metric.value(2) = temppp;
+        metric.idx(0,0) = 0; metric.value(0) = tempxx;
+        metric.idx(1,1) = 1; metric.value(1) = tempyy;
+        metric.idx(2,2) = 2; metric.value(2) = temppp;
         if( !handle_.get().isOrthogonal())
         {
-            metric.idx(0,1) = metric.idx(1,0) = 3;
+            metric.idx(0,1) = metric.idx(1,0) = 3; 
             metric.value(3) = tempxy;
         }
     }
@@ -124,9 +118,8 @@ struct CylindricalProductGrid3d : public dg::aGeometry3d
 };
 
 /**
- * @brief A three-dimensional grid based on curvilinear coordinates
+ * @brief A two-dimensional grid based on curvilinear coordinates
  */
-template< class container>
 struct CurvilinearGrid2d : public dg::aGeometry2d
 {
     /*!@brief Constructor
@@ -154,6 +147,7 @@ struct CurvilinearGrid2d : public dg::aGeometry2d
     }
 
     const aGenerator2d& generator() const{return handle_.get();}
+    virtual CurvilinearGrid2d* clone()const{return new CurvilinearGrid2d(*this);}
     private:
     virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny)
     {
diff --git a/inc/dg/geometry/mpi_curvilinear.h b/inc/dg/geometry/mpi_curvilinear.h
index 48cbbe0fe..7f5362d57 100644
--- a/inc/dg/geometry/mpi_curvilinear.h
+++ b/inc/dg/geometry/mpi_curvilinear.h
@@ -14,7 +14,6 @@ namespace dg
 {
 
 ///@cond
-template< class MPIcontainer>
 struct CurvilinearMPIGrid2d; 
 ///@endcond
 //
@@ -25,11 +24,9 @@ struct CurvilinearMPIGrid2d;
  * This is s 2x1 product space grid
  * @tparam MPIContainer Vector class that holds metric coefficients
  */
-template<class MPIContainer>
-struct CylindricalMPIGrid3d : public dg::aMPIGeometry3d
+struct CylindricalProductMPIGrid3d : public dg::aMPIGeometry3d
 {
-    typedef dg::CurvilinearPerpTag metric_category; //!< metric tag
-    typedef dg::CylindricalMPIGrid2d<LocalContainer> perpendicular_grid; //!< the two-dimensional grid
+    typedef dg::CurvilinearMPIGrid2d<LocalContainer> perpendicular_grid; //!< the two-dimensional grid
     typedef typename MPIContainer::container_type LocalContainer; //!< the local container type
     /**
      * @copydoc Grid3d::Grid3d()
@@ -152,41 +149,18 @@ struct CylindricalMPIGrid3d : public dg::aMPIGeometry3d
 /**
  * @tparam MPIContainer Vector class that holds metric coefficients
  */
-template<class MPIContainer>
 struct CurvilinearMPIGrid2d : public dg::aMPIGeometry2d
 {
-    typedef dg::CurvilinearPerpTag metric_category; 
     typedef typename MPIContainer::container_type LocalContainer; //!< the local container type
-    /**
-     * @copydoc Grid2d::Grid2d()
-     * @param comm a two-dimensional Cartesian communicator
-     * @note the paramateres given in the constructor are global parameters 
-     */
-    CurvilinearMPIGrid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, MPI_Comm comm): dg::MPIGrid2d( x0, x1, y0, y1, n, Nx, Ny, comm),
-    g_(new ShiftedIdentityGenerator(x0,x1,y0,y1),n,Nx,Ny){
-        divide_and_conquer();
-    
-    }
-
-    /**
-     * @copydoc Grid2d::Grid2d()
-     * @param comm a two-dimensional Cartesian communicator
-     * @note the paramateres given in the constructor are global parameters 
-     */
-    CurvilinearMPIGrid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx, bc bcy, MPI_Comm comm):dg::MPIGrid2d( 0, x1-x0, 0, y1-y0, n, Nx, Ny,bcx, bcy, comm),
-    handle_(new ShiftedIdentityGenerator(x0,x1,y0,y1)){
-        dg::CurvilinearGrid2d<thrust::host_vector<double> > g(generator, n, Nx, Ny);
-        divide_and_conquer(g);
-    }
 
     CurvilinearMPIGrid2d( const geo::aGenerator& generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx, MPI_Comm comm2d): 
-        dg::MPIGrid2d( 0, generator.width(), 0., generator.height(), n, Nx, Ny, bcx, dg::PER, comm2d),handle_(generator)
+        dg::aMPIGeometry2d( 0, generator.width(), 0., generator.height(), n, Nx, Ny, bcx, dg::PER, comm2d),handle_(generator)
     {
         dg::CurvilinearGrid2d<thrust::host_vector<double> > g(generator, n, Nx, Ny);
         divide_and_conquer(g);
     }
     CurvilinearMPIGrid2d( const CylindricalMPIGrid3d<LocalContainer>& g):
-        dg::MPIGrid2d( g.global().x0(), g.global().x1(), g.global().y0(), g.global().y1(), g.global().n(), g.global().Nx(), g.global().Ny(), g.global().bcx(), g.global().bcy(), get_reduced_comm( g.communicator() )),
+        dg::aMPIGeometry2d( g.global().x0(), g.global().x1(), g.global().y0(), g.global().y1(), g.global().n(), g.global().Nx(), g.global().Ny(), g.global().bcx(), g.global().bcy(), get_reduced_comm( g.communicator() )),
         xr_(dg::evaluate( dg::one, *this)), xz_(xr_), yr_(xr_), yz_(xr_), 
         g_xx_(xr_), g_xy_(g_xx_), g_yy_(g_xx_), vol2d_(g_xx_),
         handle_(g.generator())
@@ -270,10 +244,9 @@ struct CurvilinearMPIGrid2d : public dg::aMPIGeometry2d
         dg::blas1::transfer( tempvol, vol2d_.data());
     }
 
-    thrust::host_vector<double> r_,z_;
-    dg::MPI_Vector<thrust::host_vector<double> > xr_, xz_, yr_, yz_; //2d vector
-    dg::MPIContainer g_xx_, g_xy_, g_yy_, vol2d_;
-    dg::Handle<dg::geo::aGenerator> handle_;
+    dg::SparseTensor<host_vector > jac_, metric_;
+    std::vector<host_vector > map_;
+    dg::Handle<dg::geo::aGenerator2d> handle_;
 };
 ///@}
 }//namespace dg
-- 
GitLab


From fd80b9624f86743b481d86d6963d654050356063 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Wed, 9 Aug 2017 02:48:06 -0700
Subject: [PATCH 138/453] made TopologyX2d and removed GridX3d

---
 inc/dg/backend/derivativesX.h     | 215 +-------------
 inc/dg/backend/derivativesX_t.cu  |  45 +--
 inc/dg/backend/evaluationX.cuh    |  28 +-
 inc/dg/backend/grid.h             |  15 +-
 inc/dg/backend/gridX.h            | 452 ++++++------------------------
 inc/dg/backend/interpolationX.cuh |  43 +--
 inc/dg/backend/weightsX.cuh       |  23 +-
 inc/dg/geometry/cartesianX.h      |  15 -
 inc/dg/geometry/curvilinear.h     |   2 +-
 9 files changed, 111 insertions(+), 727 deletions(-)

diff --git a/inc/dg/backend/derivativesX.h b/inc/dg/backend/derivativesX.h
index d45e64f1e..63c78adc5 100644
--- a/inc/dg/backend/derivativesX.h
+++ b/inc/dg/backend/derivativesX.h
@@ -81,7 +81,7 @@ namespace create{
  *
  * @return A host matrix 
  */
-Composite<EllSparseBlockMat<double> > dx( const GridX2d& g, bc bcx, direction dir = centered)
+Composite<EllSparseBlockMat<double> > dx( const aTopologyX2d& g, bc bcx, direction dir = centered)
 {
     EllSparseBlockMat<double>  dx;
     dx = dx_normed( g.n(), g.Nx(), g.hx(), bcx, dir);
@@ -98,7 +98,7 @@ Composite<EllSparseBlockMat<double> > dx( const GridX2d& g, bc bcx, direction di
  *
  * @return A host matrix
  */
-Composite<EllSparseBlockMat<double> > dx( const GridX2d& g, direction dir = centered) { return dx( g, g.bcx(), dir);}
+Composite<EllSparseBlockMat<double> > dx( const aTopologyX2d& g, direction dir = centered) { return dx( g, g.bcx(), dir);}
 
 /**
  * @brief Create 2d derivative in y-direction
@@ -109,7 +109,7 @@ Composite<EllSparseBlockMat<double> > dx( const GridX2d& g, direction dir = cent
  *
  * @return A host matrix
  */
-Composite<EllSparseBlockMat<double> > dy( const GridX2d& g, bc bcy, direction dir = centered)
+Composite<EllSparseBlockMat<double> > dy( const aTopologyX2d& g, bc bcy, direction dir = centered)
 {
     EllSparseBlockMat<double>  dy_inner, dy_outer;
     GridX1d g1d_inner( g.y0(), g.y1(), g.fy(), g.n(), g.Ny(), bcy);
@@ -135,7 +135,7 @@ Composite<EllSparseBlockMat<double> > dy( const GridX2d& g, bc bcy, direction di
  *
  * @return A host matrix 
  */
-Composite<EllSparseBlockMat<double> > dy( const GridX2d& g, direction dir = centered){ return dy( g, g.bcy(), dir);}
+Composite<EllSparseBlockMat<double> > dy( const aTopologyX2d& g, direction dir = centered){ return dy( g, g.bcy(), dir);}
 
 /**
  * @brief Matrix that contains 2d jump terms in X direction
@@ -145,7 +145,7 @@ Composite<EllSparseBlockMat<double> > dy( const GridX2d& g, direction dir = cent
  *
  * @return A host matrix 
  */
-Composite<EllSparseBlockMat<double> > jumpX( const GridX2d& g, bc bcx)
+Composite<EllSparseBlockMat<double> > jumpX( const aTopologyX2d& g, bc bcx)
 {
     EllSparseBlockMat<double>  jx;
     jx = jump( g.n(), g.Nx(), g.hx(), bcx);
@@ -162,7 +162,7 @@ Composite<EllSparseBlockMat<double> > jumpX( const GridX2d& g, bc bcx)
  *
  * @return A host matrix 
  */
-Composite<EllSparseBlockMat<double> > jumpY( const GridX2d& g, bc bcy)
+Composite<EllSparseBlockMat<double> > jumpY( const aTopologyX2d& g, bc bcy)
 {
     EllSparseBlockMat<double>  jy_inner, jy_outer;
     GridX1d g1d_inner( g.y0(), g.y1(), g.fy(), g.n(), g.Ny(), bcy);
@@ -187,7 +187,7 @@ Composite<EllSparseBlockMat<double> > jumpY( const GridX2d& g, bc bcy)
  *
  * @return A host matrix 
  */
-Composite<EllSparseBlockMat<double> > jumpX( const GridX2d& g)
+Composite<EllSparseBlockMat<double> > jumpX( const aTopologyX2d& g)
 {
     return jumpX( g, g.bcx());
 }
@@ -199,210 +199,11 @@ Composite<EllSparseBlockMat<double> > jumpX( const GridX2d& g)
  *
  * @return A host matrix 
  */
-Composite<EllSparseBlockMat<double> > jumpY( const GridX2d& g)
+Composite<EllSparseBlockMat<double> > jumpY( const aTopologyX2d& g)
 {
     return jumpY( g, g.bcy());
 }
 
-///////////////////////////////////////////3D VERSIONS//////////////////////
-//jumpX, jumpY, jumpZ, dx, dy, dz
-/**
- * @brief Matrix that contains jump terms in X direction in 3D
- *
- * @param g The 3D grid
- * @param bcx boundary condition in x
- *
- * @return A host matrix 
- */
-Composite<EllSparseBlockMat<double> > jumpX( const GridX3d& g, bc bcx)
-{
-    EllSparseBlockMat<double>  jx;
-    jx = jump( g.n(), g.Nx(), g.hx(), bcx);
-    jx.left_size = g.n()*g.Ny()*g.Nz();
-    jx.set_default_range();
-    return jx;
-}
-
-/**
- * @brief Matrix that contains jump terms in Y direction in 3D
- *
- * @param g The 3D grid
- * @param bcy boundary condition in y
- *
- * @return A host matrix 
- */
-Composite<EllSparseBlockMat<double> > jumpY( const GridX3d& g, bc bcy)
-{
-    EllSparseBlockMat<double>  jy_inner, jy_outer;
-    GridX1d g1d_inner( g.y0(), g.y1(), g.fy(), g.n(), g.Ny(), bcy);
-    Grid1d g1d_outer( g.y0(), g.y1(), g.n(), g.Ny(), bcy);
-    jy_inner = jump( g1d_inner, bcy);
-    jy_outer = jump( g1d_outer, bcy);
-    jy_inner.right_size = g.n()*g.Nx();
-    jy_inner.right_range[0] = 0;
-    jy_inner.right_range[1] = g.n()*g.inner_Nx();
-    jy_outer.right_range[0] = g.n()*g.inner_Nx();
-    jy_outer.right_range[1] = g.n()*g.Nx();
-    jy_outer.right_size = g.n()*g.Nx();
-    jy_inner.left_size = g.Nz();
-    jy_outer.left_size = g.Nz();
-
-    Composite<EllSparseBlockMat<double> > c( jy_inner, jy_outer);
-    return c;
-}
-
-/**
- * @brief Matrix that contains jump terms in Z direction in 3D
- *
- * @param g The 3D grid
- * @param bcz boundary condition in z
- *
- * @return A host matrix 
- */
-Composite<EllSparseBlockMat<double> > jumpZ( const GridX3d& g, bc bcz)
-{
-    EllSparseBlockMat<double>  jz;
-    jz = jump( 1, g.Nz(), g.hz(), bcz);
-    jz.right_size = g.n()*g.Nx()*g.n()*g.Ny();
-    jz.set_default_range();
-    return jz;
-}
-
-/**
- * @brief Matrix that contains 3d jump terms in X direction taking boundary conditions from the grid
- *
- * @param g grid
- *
- * @return A host matrix
- */
-Composite<EllSparseBlockMat<double> > jumpX( const GridX3d& g)
-{
-    return jumpX( g, g.bcx());
-}
-
-/**
- * @brief Matrix that contains 3d jump terms in Y direction taking boundary conditions from the grid
- *
- * @param g grid
- *
- * @return A host matrix
- */
-Composite<EllSparseBlockMat<double> > jumpY( const GridX3d& g)
-{
-    return jumpY( g, g.bcy());
-}
-
-/**
- * @brief Matrix that contains 3d jump terms in Z direction taking boundary conditions from the grid
- *
- * @param g grid
- *
- * @return A host matrix
- */
-Composite<EllSparseBlockMat<double> > jumpZ( const GridX3d& g)
-{
-    return jumpZ( g, g.bcz());
-}
-
-
-/**
- * @brief Create 3d derivative in x-direction
- *
- * @param g The grid on which to create dx
- * @param bcx The boundary condition
- * @param dir The direction of the first derivative
- *
- * @return A host matrix 
- */
-Composite<EllSparseBlockMat<double> > dx( const GridX3d& g, bc bcx, direction dir = centered)
-{
-    EllSparseBlockMat<double>  dx;
-    dx = dx_normed( g.n(), g.Nx(), g.hx(), bcx, dir);
-    dx.left_size = g.n()*g.Ny()*g.Nz();
-    dx.set_default_range();
-    return dx;
-}
-
-/**
- * @brief Create 3d derivative in x-direction
- *
- * @param g The grid on which to create dx (boundary condition is taken from here)
- * @param dir The direction of the first derivative
- *
- * @return A host matrix 
- */
-Composite<EllSparseBlockMat<double> > dx( const GridX3d& g, direction dir = centered) { return dx( g, g.bcx(), dir);}
-
-/**
- * @brief Create 3d derivative in y-direction
- *
- * @param g The grid on which to create dy
- * @param bcy The boundary condition
- * @param dir The direction of the first derivative
- *
- * @return A host matrix 
- */
-Composite<EllSparseBlockMat<double> > dy( const GridX3d& g, bc bcy, direction dir = centered)
-{
-    EllSparseBlockMat<double>  dy_inner, dy_outer;
-    GridX1d g1d_inner( g.y0(), g.y1(), g.fy(), g.n(), g.Ny(), bcy);
-    Grid1d g1d_outer( g.y0(), g.y1(), g.n(), g.Ny(), bcy);
-    dy_inner = dx( g1d_inner, bcy, dir);
-    dy_outer = dx( g1d_outer, bcy, dir);
-    dy_inner.right_size = g.n()*g.Nx();
-    dy_inner.right_range[0] = 0;
-    dy_inner.right_range[1] = g.n()*g.inner_Nx();
-    dy_outer.right_range[0] = g.n()*g.inner_Nx();
-    dy_outer.right_range[1] = g.n()*g.Nx();
-    dy_outer.right_size = g.n()*g.Nx();
-    dy_inner.left_size = g.Nz();
-    dy_outer.left_size = g.Nz();
-
-    Composite<EllSparseBlockMat<double> > c( dy_inner, dy_outer);
-    return c;
-}
-
-/**
- * @brief Create 3d derivative in y-direction
- *
- * @param g The grid on which to create dy (boundary condition is taken from here)
- * @param dir The direction of the first derivative
- *
- * @return A host matrix 
- */
-Composite<EllSparseBlockMat<double> > dy( const GridX3d& g, direction dir = centered){ return dy( g, g.bcy(), dir);}
-
-/**
- * @brief Create 3d derivative in z-direction
- *
- * @param g The grid on which to create dz
- * @param bcz The boundary condition
- * @param dir The direction of the stencil
- *
- * @return A host matrix 
- */
-Composite<EllSparseBlockMat<double> > dz( const GridX3d& g, bc bcz, direction dir = centered)
-{
-    EllSparseBlockMat<double>  dz;
-    dz = dx_normed( 1, g.Nz(), g.hz(), bcz, dir);
-    dz.right_size = g.n()*g.n()*g.Nx()*g.Ny();
-    dz.set_default_range();
-    return dz;
-
-}
-
-/**
- * @brief Create 3d derivative in z-direction
- *
- * @param g The grid on which to create dz (boundary condition is taken from here)
- * @param dir The direction of the stencil
- *
- * @return A host matrix 
- */
-Composite<EllSparseBlockMat<double> > dz( const GridX3d& g, direction dir = centered){ return dz( g, g.bcz(), dir);}
-
-
-
 ///@}
 
 } //namespace create
diff --git a/inc/dg/backend/derivativesX_t.cu b/inc/dg/backend/derivativesX_t.cu
index 7ceca2952..726e7fb6d 100644
--- a/inc/dg/backend/derivativesX_t.cu
+++ b/inc/dg/backend/derivativesX_t.cu
@@ -34,10 +34,6 @@ double cosy( double x, double y) {
     }
     return sin(x)*cos(y);
 }
-double sin(  double x, double y, double z) { return sin(x,y)*sin(z);}
-double cosx( double x, double y, double z) { return cosx(x,y)*sin(z);}
-double cosy( double x, double y, double z) { return cosy(x,y)*sin(z);}
-double cosz( double x, double y, double z) { return sin(x,y)*cos(z);}
 
 
 typedef dg::DVec Vector;
@@ -47,12 +43,14 @@ typedef dg::Composite<dg::EllSparseBlockMatDevice<double> > Matrix;
 
 int main()
 {
-    unsigned n, Nx, Ny, Nz;
-    std::cout << "Type in n, Nx (1./5.) and Ny (1./4.) and Nz!\n";
-    std::cin >> n >> Nx >> Ny >> Nz;
+    unsigned n, Nx, Ny;
+    //std::cout << "Type in n, Nx (1./5.) and Ny (1./4.)!\n";
+    //std::cin >> n >> Nx >> Ny;
     //dg::bc bcx=dg::DIR_NEU, bcy=dg::DIR, bcz = dg::PER;
     //dg::GridX2d g2d( -2.*M_PI, M_PI/2., -M_PI, 2*M_PI+M_PI, 1./5., 1./4., n, Nx, Ny, bcx, bcy);
-    dg::bc bcx=dg::DIR, bcy=dg::NEU, bcz = dg::PER;
+    std::cout << "Type in n, Nx (1./3.) and Ny (1./6.)!\n";
+    std::cin >> n >> Nx >> Ny;
+    dg::bc bcx=dg::DIR, bcy=dg::NEU;
     dg::GridX2d g2d( -2.*M_PI, M_PI, -M_PI/2., 2.*M_PI+M_PI/2., 1./3., 1./6., n, Nx, Ny, bcx, bcy);
     g2d.display(std::cout);
     //dg::GridX2d g2d( -2.*M_PI, M_PI/2., 0., 2*M_PI, 1./5., 0., n, Nx, Ny, bcx, bcy);
@@ -87,37 +85,6 @@ int main()
     dg::blas1::axpby( 1., tempX, 1., tempY, tempY);
     dg::blas1::axpby( 1., null2, -1., tempY);
     std::cout << "Distance to true solution: "<<sqrt(dg::blas2::dot(tempY, w2d, tempY))<<"\n";
-    //dg::GridX3d g3d( -2.*M_PI, M_PI/2., -M_PI, 2*M_PI+M_PI, 0., 2.*M_PI, 1./5., 1./4., n, Nx, Ny, Nz, bcx, bcy, bcz);
-    dg::GridX3d g3d( -2.*M_PI, M_PI/2., 0., 2*M_PI, 0., 2.*M_PI, 1./5., 0., n, Nx, Ny, Nz, bcx, bcy, bcz);
-    const Vector w3d = dg::create::weights( g3d);
-    Matrix dx3 = dg::create::dx( g3d, dg::forward);
-    Matrix dy3 = dg::create::dy( g3d, dg::backward);
-    Matrix dz3 = dg::create::dz( g3d, dg::centered);
-    Matrix jx3 = dg::create::jumpX( g3d);
-    Matrix jy3 = dg::create::jumpY( g3d);
-    Matrix jz3 = dg::create::jumpZ( g3d);
-    Matrix m3[] = {dx3, dy3, dz3, jx3, jy3, jz3};
-    const Vector f3d = dg::evaluate( sin, g3d);
-    const Vector dx3d = dg::evaluate( cosx, g3d);
-    const Vector dy3d = dg::evaluate( cosy, g3d);
-    const Vector dz3d = dg::evaluate( cosz, g3d);
-    const Vector null3 = dg::evaluate( zero, g3d);
-    Vector sol3[] = {dx3d, dy3d, dz3d, null3, null3, null3};
-
-    std::cout << "TEST 3D: DX, DY, DZ, JX, JY, JZ, JXY\n";
-    for( unsigned i=0; i<6; i++)
-    {
-        Vector error = f3d;
-        dg::blas2::symv( m3[i], f3d, error);
-        dg::blas1::axpby( 1., sol3[i], -1., error);
-        std::cout << "Distance to true solution: "<<sqrt(dg::blas2::dot(error, w3d, error))<<"\n";
-    }
-    Vector tX = f3d, tY(tX);
-    dg::blas2::symv( m3[3], f3d, tX);
-    dg::blas2::symv( m3[4], f3d, tY);
-    dg::blas1::axpby( 1., tX, 1., tY, tY);
-    dg::blas1::axpby( 1., null3, -1., tY);
-    std::cout << "Distance to true solution: "<<sqrt(dg::blas2::dot(tY, w3d, tY))<<"\n";
     //for periodic bc | dirichlet bc
     //n = 1 -> p = 2      2
     //n = 2 -> p = 1      1
diff --git a/inc/dg/backend/evaluationX.cuh b/inc/dg/backend/evaluationX.cuh
index de9d59dc2..2c7b7774e 100644
--- a/inc/dg/backend/evaluationX.cuh
+++ b/inc/dg/backend/evaluationX.cuh
@@ -50,36 +50,12 @@ thrust::host_vector<double> evaluate( double (f)(double), const GridX1d& g)
             may be constructed during function call.
  */
 template< class BinaryOp>
-thrust::host_vector<double> evaluate( BinaryOp f, const GridX2d& g)
+thrust::host_vector<double> evaluate( BinaryOp f, const aTopologyX2d& g)
 {
     return evaluate( f, g.grid());
 };
 ///@cond
-thrust::host_vector<double> evaluate( double(f)(double, double), const GridX2d& g)
-{
-    return evaluate( f, g.grid());
-};
-///@endcond
-
-/**
- * @brief Evaluate a function on gaussian abscissas
- *
- * Evaluates f(x,y,z) on the given grid
- * @tparam TernaryOp Model of Ternary Function
- * @param f The function to evaluate: f = f(x,y,z)
- * @param g The 3d grid on which to evaluate f
- *
- * @return  A dG Host Vector with values
- * @note Copies the ternary Operator. This function is meant for small function objects, that
-            may be constructed during function call.
- */
-template< class TernaryOp>
-thrust::host_vector<double> evaluate( TernaryOp f, const GridX3d& g)
-{
-    return evaluate( f, g.grid());
-};
-///@cond
-thrust::host_vector<double> evaluate( double(f)(double, double, double), const GridX3d& g)
+thrust::host_vector<double> evaluate( double(f)(double, double), const aTopologyX2d& g)
 {
     return evaluate( f, g.grid());
 };
diff --git a/inc/dg/backend/grid.h b/inc/dg/backend/grid.h
index e5eb98ec6..aa44a30cb 100644
--- a/inc/dg/backend/grid.h
+++ b/inc/dg/backend/grid.h
@@ -32,11 +32,11 @@ struct Grid1d
     /**
      * @brief 1D grid
      * 
-     @param x0 left boundary
-     @param x1 right boundary
-     @param n # of polynomial coefficients
-     @param N # of cells
-     @param bcx boundary conditions
+     * @param x0 left boundary
+     * @param x1 right boundary
+     * @param n # of polynomial coefficients
+     * @param N # of cells
+     * @param bcx boundary conditions
      */
     Grid1d( double x0, double x1, unsigned n, unsigned N, bc bcx = PER)
     {
@@ -411,10 +411,7 @@ struct aTopology2d
 
 
 /**
- * @brief A 3D grid class  for cartesian coordinates
- *
- * In the third dimension only 1 polynomial coefficient is used,
- * not n.
+ * @brief An abstract base class for three-dimensional grids
  * @note although it is abstract objects are not meant to be hold on the heap via a base class pointer ( we protected the destructor)
  */
 struct aTopology3d
diff --git a/inc/dg/backend/gridX.h b/inc/dg/backend/gridX.h
index e3d7cf1dd..fcb4d12da 100644
--- a/inc/dg/backend/gridX.h
+++ b/inc/dg/backend/gridX.h
@@ -7,7 +7,7 @@
 
 /*! @file 
   
-  Grid objects
+  GridX objects
   */
 
 
@@ -32,14 +32,14 @@ struct GridX1d
     typedef SharedTag memory_category;
     typedef OneDimensionalTag dimensionality;
     /**
-     * @brief 1D grid
+     * @brief 1D X-point grid
      * 
-     @param x0 left boundary
-     @param x1 right boundary
-     @param f factor 0<f<0.5 divides the domain
-     @param n # of polynomial coefficients
-     @param N # of cells
-     @param bcx boundary conditions
+     * @param x0 left boundary
+     * @param x1 right boundary
+     * @param f factor 0<f<0.5 divides the domain
+     * @param n # of polynomial coefficients
+     * @param N # of cells
+     * @param bcx boundary conditions
      */
     GridX1d( double x0, double x1, double f, unsigned n, unsigned N, bc bcx = NEU):
         x0_(x0), x1_(x1), f_(f),
@@ -51,8 +51,6 @@ struct GridX1d
         assert( N > 0  );
         assert( n != 0 );
         assert( bcx != PER);
-        lx_ = (x1-x0);
-        hx_ = lx_/(double)Nx_;
     }
     /**
      * @brief left boundary
@@ -77,13 +75,13 @@ struct GridX1d
      *
      * @return 
      */
-    double lx() const {return lx_;}
+    double lx() const {return x1_-x0_;}
     /**
      * @brief cell size
      *
      * @return 
      */
-    double h() const {return hx_;}
+    double h() const {return lx()/(double)Nx_;}
     /**
      * @brief number of cells
      *
@@ -95,7 +93,7 @@ struct GridX1d
      *
      * @return 
      */
-    unsigned outer_N() const {return (unsigned)(floor(f_*(double)Nx_+0.5));}
+    unsigned outer_N() const {return (unsigned)(round(f_*(double)Nx_));}
     /**
      * @brief number of cells in the inner region
      *
@@ -137,10 +135,10 @@ struct GridX1d
             <<"    N  = "<<Nx_<<"\n"
             <<"    inner N = "<<inner_N()<<"\n"
             <<"    outer N = "<<outer_N()<<"\n"
-            <<"    h  = "<<hx_<<"\n"
+            <<"    h  = "<<h()<<"\n"
             <<"    x0 = "<<x0_<<"\n"
             <<"    x1 = "<<x1_<<"\n"
-            <<"    lx = "<<lx_<<"\n"
+            <<"    lx = "<<lx()<<"\n"
             <<"Boundary conditions in x are: \n"
             <<"    "<<bc2str(bcx_)<<"\n";
     }
@@ -159,15 +157,15 @@ struct GridX1d
     {
         assert( contains(x0));
         double deltaX;
-        double xleft = x0_ + f_*lx_;
-        double xright = x1_ - f_*lx_;
+        double xleft = x0_ + f_*lx();
+        double xright = x1_ - f_*lx();
         if( x0 >= xleft && x0<xright)
         {
             if( x1 > xleft) deltaX = (x1 -xleft);
             else deltaX = xright - x1;
             unsigned N = floor(deltaX/(xright-xleft));
-            if( x1  > xright ) x1 -= N*lx_;
-            if( x1  < xleft ) x1 += N*lx_;
+            if( x1  > xright ) x1 -= N*lx();
+            if( x1  < xleft ) x1 += N*lx();
         }
         else if( x0 < xleft && x1 >=xleft)
             x1 += (xright-xleft);
@@ -191,14 +189,11 @@ struct GridX1d
     }
   private:
     double x0_, x1_, f_;
-    double lx_;
     unsigned n_, Nx_;
-    double hx_;
     bc bcx_;
     DLT<double> dlt_;
 };
 
-struct GridX3d; //forward declare 3d version
 
 /**
  * @brief A 2D grid class with X-point topology
@@ -216,49 +211,10 @@ struct GridX3d; //forward declare 3d version
  *
  * @tparam double scalar value type 
  */
-struct GridX2d
+struct aTopologyX2d
 {
-    typedef SharedTag memory_category;
+    typedef SharedTag memory_category; //!< tag for choosing default host vector type
     typedef TwoDimensionalTag dimensionality;
-    /**
-     * @brief Construct a 2D grid
-     *
-     * @param x0 left boundary in x
-     * @param x1 right boundary in x 
-     * @param y0 lower boundary in y
-     * @param y1 upper boundary in y 
-     * @param fx factor for x-direction (fx*Nx must be a natural number)
-     * @param fy factor for y-direction (fy*Ny must be a natural number)
-     * @param n  # of polynomial coefficients per dimension
-     * @param Nx # of points in x 
-     * @param Ny # of points in y
-     * @param bcx boundary condition in x
-     * @param bcy boundary condition in y
-     */
-    GridX2d( double x0, double x1, double y0, double y1, double fx, double fy, unsigned n, unsigned Nx, unsigned Ny, bc bcx = PER, bc bcy = NEU):
-        x0_(x0), x1_(x1), y0_(y0), y1_(y1), fx_(fx), fy_(fy),
-        n_(n), Nx_(Nx), Ny_(Ny), bcx_(bcx), bcy_( bcy), dlt_(n)
-    {
-        assert( (fy_ >= 0.) && (fy_ < 0.5) );
-        assert( (fx_ >= 0.) && (fx_ < 1.) );
-        assert( fabs(outer_Nx() - fx_*(double)Nx) < 1e-15); 
-        assert( fabs(outer_Ny() - fy_*(double)Ny) < 1e-15); 
-        assert( n != 0);
-        assert( x1 > x0 && y1 > y0);
-        assert( Nx_ > 0  && Ny > 0 );
-        assert( bcy != PER);
-        lx_ = (x1_-x0_), ly_ = (y1_-y0_);
-        hx_ = lx_/(double)Nx_, hy_ = ly_/(double)Ny_;
-    }
-    /**
-     * @brief Reduce from a 3d grid
-     *
-     * This takes the x and y dimension from the 3d grid.
-     * Note that this is possible because all our grids are product space grids  and only the first two dimensions are dG discretized.
-     *
-     * @param g The 3d counterpart
-     */
-    GridX2d( const GridX3d& g);
 
     /**
      * @brief Left boundary in x
@@ -289,25 +245,25 @@ struct GridX2d
      *
      * @return 
      */
-    double lx() const {return lx_;}
+    double lx() const {return x1_-x0_;}
     /**
      * @brief length of y
      *
      * @return 
      */
-    double ly() const {return ly_;}
+    double ly() const {return y1_-y0_;}
     /**
      * @brief cell size in x 
      *
      * @return 
      */
-    double hx() const {return hx_;}
+    double hx() const {return lx()/(double)Nx_;}
     /**
      * @brief cell size in y
      *
      * @return 
      */
-    double hy() const {return hy_;}
+    double hy() const {return ly()/(double)Ny_;}
     /**
      * @brief Factor
      *
@@ -343,7 +299,7 @@ struct GridX2d
      *
      * @return 
      */
-    unsigned outer_Nx() const {return (unsigned)floor(fx_*(double)Nx_+0.5);}
+    unsigned outer_Nx() const {return (unsigned)round(fx_*(double)Nx_);}
     /**
      * @brief number of cells in y
      *
@@ -361,7 +317,7 @@ struct GridX2d
      *
      * @return 
      */
-    unsigned outer_Ny() const {return (unsigned)floor(fy_*(double)Ny_+0.5);}
+    unsigned outer_Ny() const {return (unsigned)round(fy_*(double)Ny_);}
     /**
      * @brief boundary conditions in x
      *
@@ -379,7 +335,7 @@ struct GridX2d
      *
      * @return 
      */
-    GridX2d local_grid() const {return *this;}
+    aTopologyX2d local_grid() const {return *this;}
     /**
      * @brief Return a copy without topology
      *
@@ -413,14 +369,14 @@ struct GridX2d
             <<"    Ny = "<<Ny_<<"\n"
             <<"    inner Ny = "<<inner_Ny()<<"\n"
             <<"    outer Ny = "<<outer_Ny()<<"\n"
-            <<"    hx = "<<hx_<<"\n"
-            <<"    hy = "<<hy_<<"\n"
+            <<"    hx = "<<hx()<<"\n"
+            <<"    hy = "<<hy()<<"\n"
             <<"    x0 = "<<x0_<<"\n"
             <<"    x1 = "<<x1_<<"\n"
             <<"    y0 = "<<y0_<<"\n"
             <<"    y1 = "<<y1_<<"\n"
-            <<"    lx = "<<lx_<<"\n"
-            <<"    ly = "<<ly_<<"\n"
+            <<"    lx = "<<lx()<<"\n"
+            <<"    ly = "<<ly()<<"\n"
             <<"Boundary conditions in x are: \n"
             <<"    "<<bc2str(bcx_)<<"\n"
             <<"Boundary conditions in y are: \n"
@@ -443,22 +399,22 @@ struct GridX2d
         double deltaX;
         if( x1 > x0_) deltaX = (x1 -x0_);
         else deltaX = x1_ - x1;
-        unsigned N = floor(deltaX/lx_);
-        if( x1  > x1_ && bcx_ == dg::PER) x1 -= N*lx_;
-        if( x1  < x0_ && bcx_ == dg::PER) x1 += N*lx_;
+        unsigned N = floor(deltaX/lx());
+        if( x1  > x1_ && bcx_ == dg::PER) x1 -= N*lx();
+        if( x1  < x0_ && bcx_ == dg::PER) x1 += N*lx();
 
         if( x0 < x1_ - fx_*(x1_-x0_) ) //if x0 is  one of the inner points
         {
             double deltaY;
-            double yleft = y0_ + fy_*ly_;
-            double yright = y1_ - fy_*ly_;
+            double yleft = y0_ + fy_*ly();
+            double yright = y1_ - fy_*ly();
             if( y0 >= yleft && y0<yright)
             {
                 if( y1 > yleft) deltaY = (y1 -yleft);
                 else deltaY = yright - y1;
                 unsigned N = floor(deltaY/(yright-yleft));
-                if( y1  > yright ) y1 -= N*ly_;
-                if( y1  < yleft ) y1 += N*ly_;
+                if( y1  > yright ) y1 -= N*ly();
+                if( y1  < yleft ) y1 += N*ly();
             }
             else if( y0 < yleft && y1 >=yleft)
                 y1 += (yright-yleft);
@@ -482,323 +438,79 @@ struct GridX2d
         if( (x>=x0_ && x <= x1_) && (y>=y0_ && y <= y1_)) return true; 
         return false;
     }
-  protected:
-    virtual void init_X_boundaries( double x0, double x1)
-    {
-        x0_ = x0, x1_ = x1;
-        assert( x1 > x0 );
-        lx_ = (x1_-x0_);
-        hx_ = lx_/(double)Nx_;
+    void init_X_boundaries( double x0, double x1) {
+        do_init_X_boundaries(x0,x1);
     }
-  private:
-    double x0_, x1_, y0_, y1_;
-    double fx_, fy_;
-    double lx_, ly_;
-    unsigned n_, Nx_, Ny_;
-    double hx_, hy_;
-    bc bcx_, bcy_;
-    DLT<double> dlt_;
-};
-
-/**
- * @brief A 3D grid class  for cartesian coordinates
- *
- * In the third dimension only 1 polynomial coefficient is used,
- * not n. In 2d it looks like
- @code
- | -----> y 
- |  |---x----------x---|
- |  |---x----------x---|
- v  |--- ---------- ---|
- x  |--- ---------- ---| fx*Lx
-    |--- ---------- ---|
-    fy*Ly
- @endcode
- * @tparam double scalar value type 
- */
-struct GridX3d
-{
-    typedef SharedTag memory_category;
-    typedef ThreeDimensionalTag dimensionality;
+  protected:
+    ///disallow destruction through base class pointer
+    ~aTopologyX2d(){}
     /**
-     * @brief Construct a 3D grid
+     * @brief Construct a 2D grid
      *
      * @param x0 left boundary in x
      * @param x1 right boundary in x 
      * @param y0 lower boundary in y
      * @param y1 upper boundary in y 
-     * @param z0 lower boundary in z
-     * @param z1 upper boundary in z 
-     * @param fx factor for x-direction
-     * @param fy factor for y-direction
-     * @param n  # of polynomial coefficients per (x-,y-) dimension
+     * @param fx factor for x-direction (fx*Nx must be a natural number)
+     * @param fy factor for y-direction (fy*Ny must be a natural number)
+     * @param n  # of polynomial coefficients per dimension
      * @param Nx # of points in x 
      * @param Ny # of points in y
-     * @param Nz # of points in z
      * @param bcx boundary condition in x
      * @param bcy boundary condition in y
-     * @param bcz boundary condition in z
-     * @attention # of polynomial coefficients in z direction is always 1
      */
-    GridX3d( double x0, double x1, double y0, double y1, double z0, double z1, double fx, double fy, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = PER, bc bcy = NEU, bc bcz = PER):
-        x0_(x0), x1_(x1), y0_(y0), y1_(y1), z0_(z0), z1_(z1), fx_(fx), fy_(fy),
-        n_(n), Nx_(Nx), Ny_(Ny), Nz_(Nz), bcx_(bcx), bcy_( bcy), bcz_( bcz), dlt_(n)
+    aTopologyX2d( double x0, double x1, double y0, double y1, double fx, double fy, unsigned n, unsigned Nx, unsigned Ny, bc bcx, bc bcy):
+        x0_(x0), x1_(x1), y0_(y0), y1_(y1), fx_(fx), fy_(fy),
+        n_(n), Nx_(Nx), Ny_(Ny), bcx_(bcx), bcy_( bcy), dlt_(n)
     {
         assert( (fy_ >= 0.) && (fy_ < 0.5) );
         assert( (fx_ >= 0.) && (fx_ < 1.) );
         assert( fabs(outer_Nx() - fx_*(double)Nx) < 1e-15); 
         assert( fabs(outer_Ny() - fy_*(double)Ny) < 1e-15); 
         assert( n != 0);
-        assert( x1 > x0 && y1 > y0 ); assert( z1 > z0 );         
-        assert( Nx_ > 0  && Ny > 0); assert( Nz > 0);
-        lx_ = (x1-x0), ly_ = (y1-y0), lz_ = (z1-z0);
-        hx_ = lx_/(double)Nx_, hy_ = ly_/(double)Ny_, hz_ = lz_/(double)Nz_;
+        assert( x1 > x0 && y1 > y0);
+        assert( Nx_ > 0  && Ny > 0 );
+        assert( bcy != PER);
     }
-    /**
-     * @brief left boundary in x
-     *
-     * @return 
-     */
-    double x0() const {return x0_;}
-    /**
-     * @brief right boundary in x
-     *
-     * @return 
-     */
-    double x1() const {return x1_;}
-
-    /**
-     * @brief left boundary in y 
-     *
-     * @return 
-     */
-    double y0() const {return y0_;}
-    /**
-     * @brief right boundary in y
-     *
-     * @return 
-     */
-    double y1() const {return y1_;}
-
-    /**
-     * @brief left boundary in z
-     *
-     * @return 
-     */
-    double z0() const {return z0_;}
-    /**
-     * @brief right boundary in z
-     *
-     * @return 
-     */
-    double z1() const {return z1_;}
-
-    /**
-     * @brief length in x
-     *
-     * @return 
-     */
-    double lx() const {return lx_;}
-    /**
-     * @brief length in y
-     *
-     * @return 
-     */
-    double ly() const {return ly_;}
-    /**
-     * @brief length in z
-     *
-     * @return 
-     */
-    double lz() const {return lz_;}
-    
-    /**
-     * @brief cell size in x
-     *
-     * @return 
-     */
-    double hx() const {return hx_;}
-    /**
-     * @brief cell size in y
-     *
-     * @return 
-     */
-    double hy() const {return hy_;}
-    /**
-     * @brief cell size in z
-     *
-     * @return 
-     */
-    double hz() const {return hz_;}
-    /**
-     * @brief Factor
-     *
-     * @return 
-     */
-    double fx() const {return fx_;}
-    /**
-     * @brief Factor
-     *
-     * @return 
-     */
-    double fy() const {return fy_;}
-    /**
-     * @brief number of polynomial coefficients in x and y
-     *
-     * @return 
-     */
-    unsigned n() const {return n_;}
-    /**
-     * @brief number of points in x
-     *
-     * @return 
-     */
-    unsigned Nx() const {return Nx_;}
-    /**
-     * @brief number of topological cells in x
-     *
-     * @return 
-     */
-    unsigned inner_Nx() const {return Nx_ - outer_Nx();}
-    /**
-     * @brief number of smooth rows in x
-     *
-     * @return 
-     */
-    unsigned outer_Nx() const {return (unsigned)floor(fx_*(double)Nx_+0.5);}
-    /**
-     * @brief number of cells in y
-     *
-     * @return 
-     */
-    unsigned Ny() const {return Ny_;}
-    /**
-     * @brief number of cells in the inner region of y
-     *
-     * @return 
-     */
-    unsigned inner_Ny() const {return Ny_-2*outer_Ny();}
-    /**
-     * @brief number of cells in one of the outer regions of y
-     *
-     * @return 
-     */
-    unsigned outer_Ny() const {return (unsigned)floor(fy_*(double)Ny_+0.5);}
-    /**
-     * @brief number of points in z
-     *
-     * @return 
-     */
-    unsigned Nz() const {return Nz_;}
-    /**
-     * @brief boundary conditions in x 
-     *
-     * @return 
-     */
-    bc bcx() const {return bcx_;}
-    /**
-     * @brief boundary conditions in y
-     *
-     * @return 
-     */
-    bc bcy() const {return bcy_;}
-    /**
-     * @brief boundary conditions in z 
-     *
-     * @return 
-     */
-    bc bcz() const {return bcz_;}
-    /**
-     * @brief Return a copy without topology
-     *
-     * @return 
-     */
-    Grid3d grid() const {return Grid3d( x0_,x1_,y0_,y1_,z0_,z1_,n_,Nx_,Ny_,Nz_,bcx_,bcy_,bcz_);}
-    /**
-     * @brief discrete legendre transformation
-     *
-     * @return 
-     */
-    const DLT<double>& dlt() const{return dlt_;}
-    /**
-     * @brief doublehe total number of points
-     *
-     * @return n*n*Nx*Ny*Nz
-     */
-    unsigned size() const { return n_*n_*Nx_*Ny_*Nz_;}
-    /**
-     * @brief Display 
-     *
-     * @param os output stream
-     */
-    void display( std::ostream& os = std::cout) const
-    {
-        os << "Grid parameters are: \n"
-            <<"    n  = "<<n_<<"\n"
-            <<"    Nx = "<<Nx_<<"\n"
-            <<"    inner Nx = "<<inner_Nx()<<"\n"
-            <<"    outer Nx = "<<outer_Nx()<<"\n"
-            <<"    Ny = "<<Ny_<<"\n"
-            <<"    inner Ny = "<<inner_Ny()<<"\n"
-            <<"    outer Ny = "<<outer_Ny()<<"\n"
-            <<"    Nz = "<<Nz_<<"\n"
-            <<"    hx = "<<hx_<<"\n"
-            <<"    hy = "<<hy_<<"\n"
-            <<"    hz = "<<hz_<<"\n"
-            <<"    x0 = "<<x0_<<"\n"
-            <<"    x1 = "<<x1_<<"\n"
-            <<"    y0 = "<<y0_<<"\n"
-            <<"    y1 = "<<y1_<<"\n"
-            <<"    z0 = "<<z0_<<"\n"
-            <<"    z1 = "<<z1_<<"\n"
-            <<"    lx = "<<lx_<<"\n"
-            <<"    ly = "<<ly_<<"\n"
-            <<"    lz = "<<lz_<<"\n"
-            <<"Boundary conditions in x are: \n"
-            <<"    "<<bc2str(bcx_)<<"\n"
-            <<"Boundary conditions in y are: \n"
-            <<"    "<<bc2str(bcy_)<<"\n"
-            <<"Boundary conditions in z are: \n"
-            <<"    "<<bc2str(bcz_)<<"\n";
+    aTopologyX2d(const aTopologyX2d& src){
+        x0_=src.x0_, x1_=src.x1_;
+        y0_=src.y0_, y1_=src.y1_;
+        fx_=src.fx_, fy_=src.fy_;
+        n_=src.n_, Nx_=src.Nx_, Ny_=src.Ny_, bcx_=src.bcx_, bcy_=src.bcy_;
+        dlt_=src.dlt_;
     }
-    /**
-     * @brief Check if the grid contains a point
-     *
-     * @note doesn't check periodicity!!
-     * @param x x-point to check
-     * @param y y-point to check
-     * @param z z-point to check
-     *
-     * @return true if x is between x0 and x1, false else
-     */
-    bool contains( double x, double y, double z)const
-    {
-        if( (x>=x0_ && x <= x1_) && (y>=y0_ && y <= y1_) && (z>=z0_ && z<=z1_)) 
-            return true; 
-        return false;
+    aTopologyX2d& operator=(const aTopologyX2d& src){
+        x0_=src.x0_, x1_=src.x1_;
+        y0_=src.y0_, y1_=src.y1_;
+        fx_=src.fx_, fy_=src.fy_;
+        n_=src.n_, Nx_=src.Nx_, Ny_=src.Ny_, bcx_=src.bcx_, bcy_=src.bcy_;
+        dlt_=src.dlt_;
+        return *this;
     }
-  protected:
-    virtual void init_X_boundaries( double x0, double x1)
+    virtual void do_init_X_boundaries( double x0, double x1)
     {
         x0_ = x0, x1_ = x1;
         assert( x1 > x0 );
-        lx_ = (x1_-x0_);
-        hx_ = lx_/(double)Nx_;
     }
   private:
-    double x0_, x1_, y0_, y1_, z0_, z1_;
-    double fx_,fy_;
-    double lx_, ly_, lz_;
-    unsigned n_, Nx_, Ny_, Nz_;
-    double hx_, hy_, hz_;
-    bc bcx_, bcy_, bcz_;
+    double x0_, x1_, y0_, y1_;
+    double fx_, fy_;
+    unsigned n_, Nx_, Ny_;
+    bc bcx_, bcy_;
     DLT<double> dlt_;
 };
-///@}
+/**
+ * @brief The simplest implementation of aTopologyX2d
+ */
+struct GridX2d : public aTopologyX2d
+{
+    ///@copydoc aTopologyX2d::aTopologyX2d()
+    GridX2d( double x0, double x1, double y0, double y1, double fx, double fy, unsigned n, unsigned Nx, unsigned Ny, bc bcx=PER, bc bcy=PER):
+        aTopologyX2d(x0,x1,y0,y1,fx,fy,n,Nx,Ny,bcx,bcy) { }
+    ///allow explicit type conversion from any other topology
+    explicit GridX2d( const aTopologyX2d& src): aTopologyX2d(src){}
+};
 
-///@cond
-GridX2d::GridX2d( const GridX3d& g) : x0_(g.x0()), x1_(g.x1()), y0_(g.y0()), y1_(g.y1()), fx_(g.fx()), fy_(g.fy()), n_(g.n()), Nx_(g.Nx()), Ny_(g.Ny()), bcx_(g.bcx()), bcy_(g.bcy()), dlt_(g.n())
-{}
-///@endcond
+///@}
 
 }// namespace dg
diff --git a/inc/dg/backend/interpolationX.cuh b/inc/dg/backend/interpolationX.cuh
index 7c49150ff..74ab2042f 100644
--- a/inc/dg/backend/interpolationX.cuh
+++ b/inc/dg/backend/interpolationX.cuh
@@ -38,31 +38,12 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const thrust::ho
  *
  * @return interpolation matrix
  */
-cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const thrust::host_vector<double>& x, const thrust::host_vector<double>& y, const GridX2d& g , dg::bc globalbcz = dg::NEU)
+cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const thrust::host_vector<double>& x, const thrust::host_vector<double>& y, const aTopologyX2d& g , dg::bc globalbcz = dg::NEU)
 {
     return interpolation( x,y, g.grid(), globalbcz);
 }
 
 
-
-/**
- * @brief Create interpolation matrix
- *
- * The matrix, when applied to a vector, interpolates its values to the given coordinates. In z-direction only a nearest neighbor interpolation is used
- * @param x X-coordinates of interpolation points
- * @param y Y-coordinates of interpolation points
- * @param z Z-coordinates of interpolation points
- * @param g The Grid on which to operate
- * @param globalbcz determines what to do if values lie exactly on the boundary
- *
- * @return interpolation matrix
- * @note The values of x, y and z must lie within the boundaries of g
- */
-cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const thrust::host_vector<double>& x, const thrust::host_vector<double>& y, const thrust::host_vector<double>& z, const GridX3d& g, dg::bc globalbcz= dg::NEU)
-{
-    return interpolation( x,y,z, g.grid(), globalbcz);
-}
-
 /**
  * @brief Create interpolation between two grids
  *
@@ -91,27 +72,11 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const GridX1d& g
  * @return Interpolation matrix
  * @note The boundaries of the old grid must lie within the boundaries of the new grid
  */
-cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const GridX2d& g_new, const GridX2d& g_old)
+cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const aTopologyX2d& g_new, const aTopologyX2d& g_old)
 {
     return interpolation( g_new.grid(), g_old.grid());
 }
 
-/**
- * @brief Create interpolation between two grids
- *
- * This matrix can be applied to vectors defined on the old grid to obtain
- * its values on the new grid.
- * 
- * @param g_new The new points 
- * @param g_old The old grid
- *
- * @return Interpolation matrix
- * @note The boundaries of the old grid must lie within the boundaries of the new grid
- */
-cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const GridX3d& g_new, const GridX3d& g_old)
-{
-    return interpolation( g_new.grid(), g_old.grid());
-}
 ///@}
 
 /**
@@ -122,7 +87,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const GridX3d& g
  *
  * @return the vector in LSPACE
  */
-thrust::host_vector<double> forward_transform( const thrust::host_vector<double>& in, const GridX2d& g)
+thrust::host_vector<double> forward_transform( const thrust::host_vector<double>& in, const aTopologyX2d& g)
 {
     return forward_transform( in, g.grid());
 }
@@ -138,7 +103,7 @@ thrust::host_vector<double> forward_transform( const thrust::host_vector<double>
  *
  * @return interpolated point
  */
-double interpolate( double x, double y,  const thrust::host_vector<double>& v, const GridX2d& g )
+double interpolate( double x, double y,  const thrust::host_vector<double>& v, const aTopologyX2d& g )
 {
     return interpolate( x,y,v,g.grid());
 }
diff --git a/inc/dg/backend/weightsX.cuh b/inc/dg/backend/weightsX.cuh
index 79884a227..505f38b64 100644
--- a/inc/dg/backend/weightsX.cuh
+++ b/inc/dg/backend/weightsX.cuh
@@ -39,7 +39,7 @@ thrust::host_vector<double> inv_weights( const GridX1d& g) { return inv_weights(
 *
 * @return Host Vector
 */
-thrust::host_vector<double> weights( const GridX2d& g) { return weights( g.grid()); }
+thrust::host_vector<double> weights( const aTopologyX2d& g) { return weights( g.grid()); }
 /**
 * @brief create host_vector containing 2d X-space inverse weight coefficients
 *
@@ -47,26 +47,7 @@ thrust::host_vector<double> weights( const GridX2d& g) { return weights( g.grid(
 *
 * @return Host Vector
 */
-thrust::host_vector<double> inv_weights( const GridX2d& g) { return inv_weights( g.grid()); }
-
-/**
-* @brief create host_vector containing 3d X-space weight coefficients for integration
-*
-* @param g The grid 
-*
-* @return Host Vector
-*/
-thrust::host_vector<double> weights( const GridX3d& g) { return weights(g.grid()); }
-
-/**
-* @brief create host_vector containing 3d X-space inverse weight coefficients
-*
-* @tparam T value type
-* @param g The grid 
-*
-* @return Host Vector
-*/
-thrust::host_vector<double> inv_weights( const GridX3d& g) { return inv_weights(g.grid()); }
+thrust::host_vector<double> inv_weights( const aTopologyX2d& g) { return inv_weights( g.grid()); }
 
 ///@}
 }//namespace create
diff --git a/inc/dg/geometry/cartesianX.h b/inc/dg/geometry/cartesianX.h
index de1b6001d..79737a101 100644
--- a/inc/dg/geometry/cartesianX.h
+++ b/inc/dg/geometry/cartesianX.h
@@ -9,21 +9,6 @@ namespace dg
 ///@addtogroup basicgrids
 ///@{
 //
-/**
- * @brief one-dimensional Grid with Cartesian metric
- */
-struct CartesianGridX1d: public dg::GridX1d
-{
-    typedef OrthonormalTag metric_category; 
-    ///@copydoc GridX1d::GridX1d()
-    CartesianGridX1d( double x0, double x1, double f, unsigned n, unsigned N, bc bcx = NEU):
-        dg::GridX1d(x0,x1,f,n,N,bcx){}
-    /**
-     * @brief Construct from existing topology
-     * @param grid existing grid class
-     */
-    CartesianGridX1d( const dg::GridX1d& grid):dg::GridX1d(grid){}
-};
 
 /**
  * @brief two-dimensional Grid with Cartesian metric
diff --git a/inc/dg/geometry/curvilinear.h b/inc/dg/geometry/curvilinear.h
index 467dba94c..ec060e3a4 100644
--- a/inc/dg/geometry/curvilinear.h
+++ b/inc/dg/geometry/curvilinear.h
@@ -137,7 +137,7 @@ struct CurvilinearGrid2d : public dg::aGeometry2d
         construct( n,Nx,Ny);
     }
     CurvilinearGrid2d( CylindricalProductGrid3d<container> g):
-        dg::aGeometry2d( g.x0(), g.x1(), g.y0(), g.y1(), g.n(), g.Nx(), g.Ny(), g.bcx(), g.bcy()), handle_(g.generator())
+        dg::aGeometry2d( g.x0(), g.x1(), g.y0(), g.y1(), g.n(), g.Nx(), g.Ny(), g.bcx(), g.bcy() ), handle_(g.generator())
     {
         g.set( n(), Nx(), Ny(), 1); //shouldn't trigger 2d grid generator
         map_=g.map();
-- 
GitLab


From 3e0923b095209b3cb0f69434e76298580d267e28 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Wed, 9 Aug 2017 07:04:45 -0700
Subject: [PATCH 139/453] work on mpi curvilinear grids

---
 inc/dg/backend/mpi_evaluation.h   |  54 +++++-
 inc/dg/dg_doc.h                   |   2 +-
 inc/dg/geometry/cartesianX.h      |  15 --
 inc/dg/geometry/curvilinear.h     |  15 +-
 inc/dg/geometry/mpi_curvilinear.h | 289 ++++++++++++------------------
 5 files changed, 179 insertions(+), 196 deletions(-)

diff --git a/inc/dg/backend/mpi_evaluation.h b/inc/dg/backend/mpi_evaluation.h
index d174a1292..c10060a62 100644
--- a/inc/dg/backend/mpi_evaluation.h
+++ b/inc/dg/backend/mpi_evaluation.h
@@ -68,8 +68,60 @@ MPI_Vector<thrust::host_vector<double> > evaluate( double(f)(double, double, dou
     return evaluate<double(double, double, double)>( f, g);
 };
 ///@endcond
+//
+///@}
 
+/**
+ * @brief Take the relevant local part of a global vector
+ *
+ * @param global a vector the size of the global grid 
+ * @param g the assumed topology
+ * @return an MPI_Vector that is the distributed version of the global vector
+ * @ingroup scatter
+ */
+MPI_Vector<thrust::host_vector<double> > global2local( const thrust::host_vector<double>& global, const aMPITopology3d& g)
+{
+    assert( in.size() == g.global().size());
+    thrust::host_vector<double> temp(g.size());
+    int dims[3], periods[3], coords[3];
+    MPI_Cart_get( g.communicator(), 3, dims, periods, coords);
+    for( unsigned s=0; s<g.Nz(); s++)
+        //for( unsigned py=0; py<dims[1]; py++)
+            for( unsigned i=0; i<g.n()*g.Ny(); i++)
+                //for( unsigned px=0; px<dims[0]; px++)
+                    for( unsigned j=0; j<g.n()*g.Nx(); j++)
+                    {
+                        unsigned idx1 = (s*g.n()*g.Ny()+i)*g.n()*g.Nx() + j;
+                        unsigned idx2 = (((s*dims[1]+coords[1])*g.n()*g.Ny()+i)*dims[0] + coords[0])*g.n()*g.Nx() + j;
+                        temp[idx1] = global[idx2];
+                    }
+    return MPI_Vector<thrust::host_vector<double> >(temp, g.communicator()); 
+}
+/**
+ * @brief Take the relevant local part of a global vector
+ *
+ * @param global a vector the size of the global grid 
+ * @param g the assumed topology
+ * @return an MPI_Vector that is the distributed version of the global vector
+ * @ingroup scatter
+ */
+MPI_Vector<thrust::host_vector<double> > global2local( const thrust::host_vector<double>& global, const aMPITopology2d& g)
+{
+    assert( in.size() == g.global().size());
+    thrust::host_vector<double> temp(g.size());
+    int dims[2], periods[2], coords[2];
+    MPI_Cart_get( g.communicator(), 2, dims, periods, coords);
+    //for( unsigned py=0; py<dims[1]; py++)
+        for( unsigned i=0; i<g.n()*g.Ny(); i++)
+            //for( unsigned px=0; px<dims[0]; px++)
+                for( unsigned j=0; j<g.n()*g.Nx(); j++)
+                {
+                    unsigned idx1 = (i*g.n()*g.Nx() + j;
+                    unsigned idx2 = ((coords[1]*g.n()*g.Ny()+i)*dims[0] + coords[0])*g.n()*g.Nx() + j;
+                    temp[idx1] = global[idx2];
+                }
+    return MPI_Vector<thrust::host_vector<double> >(temp, g.communicator()); 
+}
 
-///@}
 }//namespace dg
 
diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index b4417a1eb..24f7fc8a8 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -61,7 +61,7 @@
  *
  *             High level matrix creation functions
            @defgroup interpolation Interpolation and projection
- *         @defgroup scatter Scatter
+ *         @defgroup scatter Scatter and Gather
  *     @}
  *     @defgroup geometry Geometric grids and operations
  *
diff --git a/inc/dg/geometry/cartesianX.h b/inc/dg/geometry/cartesianX.h
index 79737a101..f39839809 100644
--- a/inc/dg/geometry/cartesianX.h
+++ b/inc/dg/geometry/cartesianX.h
@@ -25,21 +25,6 @@ struct CartesianGridX2d: public dg::GridX2d
     CartesianGridX2d( const dg::GridX2d& grid):dg::GridX2d(grid){}
 };
 
-/**
- * @brief three-dimensional Grid with Cartesian metric
- */
-struct CartesianGridX3d: public dg::GridX3d
-{
-    typedef OrthonormalTag metric_category; 
-    ///@copydoc GridX3d::GridX3d()
-    CartesianGridX3d( double x0, double x1, double y0, double y1, double z0, double z1, double fx, double fy, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = PER, bc bcy = NEU, bc bcz = PER): dg::GridX3d(x0,x1,y0,y1,z0,z1,fx,fy,n,Nx,Ny,Nz,bcx,bcy,bcz){}
-    /**
-     * @brief Construct from existing topology
-     * @param grid existing grid class
-     */
-    CartesianGridX3d( const dg::GridX3d& grid):dg::GridX3d(grid){}
-};
-
 ///@}
 
 } //namespace dg
diff --git a/inc/dg/geometry/curvilinear.h b/inc/dg/geometry/curvilinear.h
index ec060e3a4..81e3d2bbf 100644
--- a/inc/dg/geometry/curvilinear.h
+++ b/inc/dg/geometry/curvilinear.h
@@ -88,10 +88,10 @@ struct CylindricalProductGrid3d : public dg::aGeometry3d
         handle_.get().generate( x_vec, y_vec, map_[0], map_[1], jac_.value(0), jac_.value(1), jac_.value(2), jac_.value(3));
         jac_.idx(0,0) = 0, jac_.idx(0,1) = 1, jac_.idx(1,0)=2, jac_.idx(1,1) = 3;
     }
-    virtual SparseTensor<thrust::host_vector> do_compute_jacobian( ) const {
+    virtual SparseTensor<thrust::host_vector<double> > do_compute_jacobian( ) const {
         return jac_;
     }
-    virtual SparseTensor<thrust::host_vector> do_compute_metric( ) const
+    virtual SparseTensor<thrust::host_vector<double> > do_compute_metric( ) const
     {
         thrust::host_vector<double> tempxx( size()), tempxy(size()), tempyy(size()), temppp(size());
         for( unsigned i=0; i<size(); i++)
@@ -110,8 +110,9 @@ struct CylindricalProductGrid3d : public dg::aGeometry3d
             metric.idx(0,1) = metric.idx(1,0) = 3; 
             metric.value(3) = tempxy;
         }
+        return metric;
     }
-    virtual std::vector<thrurst::host_vector<double> > do_compute_map()const{return map_;}
+    virtual std::vector<thrust::host_vector<double> > do_compute_map()const{return map_;}
     std::vector<thrust::host_vector<double> > map_;
     SparseTensor<thrust::host_vector<double> > jac_;
     dg::Handle<aGenerator> handle_;
@@ -136,7 +137,7 @@ struct CurvilinearGrid2d : public dg::aGeometry2d
     {
         construct( n,Nx,Ny);
     }
-    CurvilinearGrid2d( CylindricalProductGrid3d<container> g):
+    explicit CurvilinearGrid2d( CylindricalProductGrid3d<container> g):
         dg::aGeometry2d( g.x0(), g.x1(), g.y0(), g.y1(), g.n(), g.Nx(), g.Ny(), g.bcx(), g.bcy() ), handle_(g.generator())
     {
         g.set( n(), Nx(), Ny(), 1); //shouldn't trigger 2d grid generator
@@ -161,13 +162,13 @@ struct CurvilinearGrid2d : public dg::aGeometry2d
         map_=g.map();
         metric_=g.metric();
     }
-    virtual SparseTensor<thrust::host_vector> do_compute_jacobian( ) const {
+    virtual SparseTensor<thrust::host_vector<double> > do_compute_jacobian( ) const {
         return jac_;
     }
-    virtual SparseTensor<thrust::host_vector> do_compute_metric( ) const {
+    virtual SparseTensor<thrust::host_vector<double> > do_compute_metric( ) const {
         return metric_;
     }
-    virtual std::vector<thrurst::host_vector<double> > do_compute_map()const{return map_;}
+    virtual std::vector<thrust::host_vector<double> > do_compute_map()const{return map_;}
     dg::SparseTensor<thrust::host_vector<double> > jac_, metric_;
     std::vector<thrust::host_vector<double> > map_;
     dg::Handle<aGenerator2d> handle_;
diff --git a/inc/dg/geometry/mpi_curvilinear.h b/inc/dg/geometry/mpi_curvilinear.h
index 7f5362d57..5231a7b53 100644
--- a/inc/dg/geometry/mpi_curvilinear.h
+++ b/inc/dg/geometry/mpi_curvilinear.h
@@ -21,184 +21,138 @@ struct CurvilinearMPIGrid2d;
 ///@{
 
 /**
- * This is s 2x1 product space grid
- * @tparam MPIContainer Vector class that holds metric coefficients
+ * This is s 2x1 product space MPI grid
  */
 struct CylindricalProductMPIGrid3d : public dg::aMPIGeometry3d
 {
-    typedef dg::CurvilinearMPIGrid2d<LocalContainer> perpendicular_grid; //!< the two-dimensional grid
+    typedef dg::CurvilinearMPIGrid2d perpendicular_grid; //!< the two-dimensional grid
     typedef typename MPIContainer::container_type LocalContainer; //!< the local container type
-    /**
-     * @copydoc Grid3d::Grid3d()
-     * @param comm a three-dimensional Cartesian communicator
-     * @note the paramateres given in the constructor are global parameters 
-     */
-    CylindricalMPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, MPI_Comm comm): 
-        dg::MPIGrid3d( 0, x1-x0, 0, y1-y0, z0, z1, n, Nx, Ny, Nz, comm),
-        handle_( new ShiftedIdentityGenerator(x0,x1,y0,y1), n,Nx,Ny,local().Nz(), bcx,bcy,bcz)
-     {
-        CylindricalGrid3d<LocalContainer> g(handle_.get(),n,Nx,Ny,this->Nz());
-        divide_and_conquer();
-     }
-
-    /**
-     * @copydoc Grid3d::Grid3d()
-     * @param comm a three-dimensional Cartesian communicator
-     * @note the paramateres given in the constructor are global parameters 
-     */
-    CylindricalMPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz, MPI_Comm comm):
-        dg::MPIGrid3d( 0, x1-x0, 0, y1-y0, z0, z1, n, Nx, Ny, Nz, bcx, bcy, bcz, comm),
-        handle_( new ShiftedIdentityGenerator(x0,x1,y0,y1))
-     {
-        CylindricalGrid3d<LocalContainer> g(handle_.get(),n,Nx,Ny,this->Nz());
-        divide_and_conquer(g);
-     }
-
-    CylindricalMPIGrid3d( const geo::aGenerator& generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx, MPI_Comm comm): 
-        dg::MPIGrid3d( 0, generator.width(), 0., generator.height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, dg::PER, dg::PER, comm),
+    CylindricalMPIGrid3d( const geo::aGenerator& generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz, MPI_Comm comm): 
+        dg::aMPITopology3d( 0, generator.width(), 0., generator.height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, bcy, bcz, comm),
         handle_( generator)
     {
-        CylindricalGrid3d<LocalContainer> g(generator,n,Nx,Ny,this->Nz());
-        divide_and_conquer(g);
+        map_.resize(3);
+        CurvilinearMPIGrid2d g(generator,n,Nx,Ny, bcx, bcy, get_reduced_comm(comm));
+        constructPerp( g);
+        constructParallel(this->Nz());
     }
 
     perpendicular_grid perp_grid() const { return perpendicular_grid(*this);}
 
-    template<class TernaryOp>
-    dg::MPI_Vector<thrust::host_vector<double> > doPullback(TernaryOp f)const{
-        thrust::host_vector<double> vec( g.size());
-        unsigned size2d = g.n()*g.n()*g.Nx()*g.Ny();
-        for( unsigned k=0; k<g.Nz(); k++)
-            for( unsigned i=0; i<size2d; i++)
-                vec[k*size2d+i] = f( g.r()[i], g.z()[i], g.phi()[k]);
-        MPI_Vector<thrust::host_vector<double> > v( vec, g.communicator());
-        return v;
-    }
-    const dg::MPI_Vector<thrust::host_vector<double> >& xr()const{return xr_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& yr()const{return yr_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& xz()const{return xz_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& yz()const{return yz_;}
-    const MPIContainer& g_xx()const{return g_xx_;}
-    const MPIContainer& g_yy()const{return g_yy_;}
-    const MPIContainer& g_xy()const{return g_xy_;}
-    const MPIContainer& g_pp()const{return g_pp_;}
-    const MPIContainer& vol()const{return vol_;}
-    const MPIContainer& perpVol()const{return vol2d_;}
-    const geo::aGenerator& generator() const{return g.generator();}
-    bool isOrthonormal() const { return g.isOrthonormal();}
-    bool isOrthogonal() const { return g.isOrthogonal();}
-    bool isConformal() const { return g.isConformal();}
+    const aGenerator2d& generator() const{return handle_.get();}
     private:
+    MPI_Comm get_reduced_comm( MPI_Comm src)
+    {
+        MPI_Comm planeComm;
+        int remain_dims[] = {true,true,false}; //true true false
+        MPI_Cart_sub( src, remain_dims, &planeComm);
+        return planeComm;
+    }
     virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz)
     {
-        dg::MPIGrid3d::do_set(new_n, new_Nx, new_Ny, new_Nz);
-        dg::CylindricalGrid3d<thrust::host_vector> g( handle_.get(), new_n, new_Nx,new_Ny,Nz());
-        divide_and_conquer(g);//distribute to processes
+        dg::aMPITopology3d::do_set(new_n, new_Nx, new_Ny, new_Nz);
+        if( !( new_n == n() && new_Nx == Nx() && new_Ny == Ny() ) )
+        {
+            CurvilinearMPIGrid2d g(handle.get(),new_n,new_Nx,new_Ny, this->bcx(), this->bcy(), get_reduced_comm(communicator()));
+            constructPerp( g);
+        }
+        constructParallel(this->Nz());
     }
-    void divide_and_conquer( const dg::CylindricalGrid3d<thrust::host_vector<double> & g )
+    void constructPerp( CurvilinearMPIGrid2d& g2d)
     {
-        r_.resize( this->n()*this->n()*this->Nx()*this->Ny());
-        z_ = r_;
-        phi_.resize( this->Nz());
-        xr_=dg::evaluate( dg::one, *this), xz_=xr_, yr_=xr_, yz_=xr_;
-        dg::blas1::transfer( xr_, g_xx_);
-        g_xy_=g_xx_, g_yy_=g_xx_, g_pp_=g_xx_, vol_=g_xx_, vol2d_=g_xx_;
-        thrust::host_vector<double> tempxx(size()), tempxy(size()),tempyy(size()),temppp(size()),tempvol2d(size()),tempvol(size());
-        //divide and conquer
-        int dims[3], periods[3], coords[3];
-        MPI_Cart_get( comm, 3, dims, periods, coords);
-        init_X_boundaries( g.x0(), g.x1());
-        for( unsigned s=0; s<this->Nz(); s++)
-            //for( unsigned py=0; py<dims[1]; py++)
-                for( unsigned i=0; i<this->n()*this->Ny(); i++)
-                    //for( unsigned px=0; px<dims[0]; px++)
-                        for( unsigned j=0; j<this->n()*this->Nx(); j++)
-                        {
-                            unsigned idx2D1 = i*this->n()*this->Nx() + j;
-                            unsigned idx2D2 = ((coords[1]*this->n()*this->Ny()+i)*dims[0] + coords[0])*this->n()*this->Nx() + j;
-                            r_[idx2D1] = g.r()[idx2D2];
-                            z_[idx2D1] = g.z()[idx2D2];
-                            phi_[s] = g.phi()[coords[2]*this->Nz()+s];
-                            unsigned idx1 = (s*this->n()*this->Ny()+i)*this->n()*this->Nx() + j;
-                            unsigned idx2 = (((s*dims[1]+coords[1])*this->n()*this->Ny()+i)*dims[0] + coords[0])*this->n()*this->Nx() + j;
-                            xr_.data()[idx1] = g.xr()[idx2];
-                            xz_.data()[idx1] = g.xz()[idx2];
-                            yr_.data()[idx1] = g.yr()[idx2];
-                            yz_.data()[idx1] = g.yz()[idx2];
-
-                            tempxx[idx1] = g.g_xx()[idx2];
-                            tempxy[idx1] = g.g_xy()[idx2];
-                            tempyy[idx1] = g.g_yy()[idx2];
-                            temppp[idx1] = g.g_pp()[idx2];
-                            tempvol[idx1] = g.vol()[idx2];
-                            tempvol2d[idx1] = g.perpVol()[idx2];
-                        }
-        dg::blas1::transfer( tempxx, g_xx_.data());
-        dg::blas1::transfer( tempxy, g_xy_.data());
-        dg::blas1::transfer( tempyy, g_yy_.data());
-        dg::blas1::transfer( temppp, g_pp_.data());
-        dg::blas1::transfer( tempvol, vol_.data());
-        dg::blas1::transfer( tempvol2d, vol2d_.data());
+        jac_=g.jacobian();
+        map_=g.map();
     }
-    thrust::host_vector<double> r_,z_,phi_;
-    dg::MPI_Vector<thrust::host_vector<double> > xr_, xz_, yr_, yz_; //3d vector
-    MPIContainer g_xx_, g_xy_, g_yy_, g_pp_, vol_, vol2d_;
+    void constructParallel( unsigned localNz )
+    {
+        map_[2]=dg::evaluate(dg::cooZ3d, *this);
+        unsigned size = this->size();
+        unsigned size2d = this->n()*this->n()*this->Nx()*this->Ny();
+        //resize for 3d values
+        for( unsigned r=0; r<4;r++)
+        {
+            jac_.value(r).data().resize(size);
+            jac_.communicator() = communicator();
+        }
+        map_[0].data().resize(size); 
+        map_[0].communicator() = communicator();
+        map_[1].data().resize(size);
+        map_[1].communicator() = communicator();
+        //lift to 3D grid
+        for( unsigned k=1; k<localNz; k++)
+            for( unsigned i=0; i<size2d; i++)
+            {
+                for(unsigned r=0; r<4; r++)
+                    jac_.value(r).data()[k*size2d+i] = jac_.value(r).data()[(k-1)*size2d+i];
+                map_[0].data()[k*size2d+i] = map_[0].data()[(k-1)*size2d+i];
+                map_[1].data()[k*size2d+i] = map_[1].data()[(k-1)*size2d+i];
+            }
+    }
+    virtual SparseTensor<host_vector> do_compute_jacobian( ) const {
+        return jac_;
+    }
+    virtual SparseTensor<host_vector> do_compute_metric( ) const {
+        thrust::host_vector<double> tempxx( size()), tempxy(size()), tempyy(size()), temppp(size());
+        for( unsigned i=0; i<size(); i++)
+        {
+            tempxx[i] = (jac_.value(0,0).data()[i]*jac_.value(0,0).data()[i]+jac_.value(0,1).data()[i]*jac_.value(0,1).data()[i]);
+            tempxy[i] = (jac_.value(0,0).data()[i]*jac_.value(1,0).data()[i]+jac_.value(0,1).data()[i]*jac_.value(1,1).data()[i]);
+            tempyy[i] = (jac_.value(1,0).data()[i]*jac_.value(1,0).data()[i]+jac_.value(1,1).data()[i]*jac_.value(1,1).data()[i]);
+            temppp[i] = 1./map_[2][i]/map_[2][i]; //1/R^2
+        }
+        SparseTensor<thrust::host_vector<double> > metric;
+        metric.idx(0,0) = 0; metric.value(0) = host_vector(tempxx, communicator());
+        metric.idx(1,1) = 1; metric.value(1) = host_vector(tempyy, communicator());
+        metric.idx(2,2) = 2; metric.value(2) = host_vector(temppp, communicator());
+        if( !handle_.get().isOrthogonal())
+        {
+            metric.idx(0,1) = metric.idx(1,0) = 3; 
+            metric.value(3) = host_vector(tempxy, communicator());
+        }
+        return metric;
+    }
+    virtual std::vector<host_vector > do_compute_map()const{return map_;}
+    dg::SparseTensor<host_vector > jac_;
+    std::vector<host_vector > map_;
     Handle<dg::geo::aGenerator> handle_;
 };
 
 /**
- * @tparam MPIContainer Vector class that holds metric coefficients
+ * @brief A two-dimensional MPI grid based on curvilinear coordinates
  */
 struct CurvilinearMPIGrid2d : public dg::aMPIGeometry2d
 {
     typedef typename MPIContainer::container_type LocalContainer; //!< the local container type
-
-    CurvilinearMPIGrid2d( const geo::aGenerator& generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx, MPI_Comm comm2d): 
-        dg::aMPIGeometry2d( 0, generator.width(), 0., generator.height(), n, Nx, Ny, bcx, dg::PER, comm2d),handle_(generator)
+    CurvilinearMPIGrid2d( const aGenerator2d& generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx, dg::bc bcy, MPI_Comm comm2d): 
+        dg::aMPIGeometry2d( 0, generator.width(), 0., generator.height(), n, Nx, Ny, bcx, bcy, comm2d), handle_(generator)
     {
+        //generate global 2d grid and then reduce to local 
         dg::CurvilinearGrid2d<thrust::host_vector<double> > g(generator, n, Nx, Ny);
         divide_and_conquer(g);
     }
-    CurvilinearMPIGrid2d( const CylindricalMPIGrid3d<LocalContainer>& g):
+    explicit CurvilinearMPIGrid2d( const CylindricalMPIGrid3d<LocalContainer>& g):
         dg::aMPIGeometry2d( g.global().x0(), g.global().x1(), g.global().y0(), g.global().y1(), g.global().n(), g.global().Nx(), g.global().Ny(), g.global().bcx(), g.global().bcy(), get_reduced_comm( g.communicator() )),
-        xr_(dg::evaluate( dg::one, *this)), xz_(xr_), yr_(xr_), yz_(xr_), 
-        g_xx_(xr_), g_xy_(g_xx_), g_yy_(g_xx_), vol2d_(g_xx_),
         handle_(g.generator())
     {
-        r_=g.r(); 
-        z_=g.z(); 
+        map_=g.map();
+        jac_=g.jacobian();
+        metric_=g.metric();
+        //now resize to 2d
         unsigned s = this->size();
-        for( unsigned i=0; i<s; i++)
-        {
-            xr_.data()[i]=g.xr().data()[i]; 
-            xz_.data()[i]=g.xz().data()[i]; 
-            yr_.data()[i]=g.yr().data()[i]; 
-            yz_.data()[i]=g.yz().data()[i];
-        }
-        thrust::copy( g.g_xx().data().begin(), g.g_xx().data().begin()+s, g_xx_.data().begin());
-        thrust::copy( g.g_xy().data().begin(), g.g_xy().data().begin()+s, g_xy_.data().begin());
-        thrust::copy( g.g_yy().data().begin(), g.g_yy().data().begin()+s, g_yy_.data().begin());
-        thrust::copy( g.perpVol().data().begin(), g.perpVol().data().begin()+s, vol2d_.data().begin());
+        for( unsigned i=0; i<jac_.values().size(); i++)
+            jac_.value(i).resize(s);
+        for( unsigned i=0; i<metric_.values().size(); i++)
+            metric_.value(i).resize(s);
+        for( unsigned i=0; i<map_.size(); i++)
+            map_[i].resize(s);
     }
 
-    const thrust::host_vector<double>& r()const{return r_;}
-    const thrust::host_vector<double>& z()const{return z_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& xr()const{return xr_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& yr()const{return yr_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& xz()const{return xz_;}
-    const dg::MPI_Vector<thrust::host_vector<double> >& yz()const{return yz_;}
-    const MPIContainer& g_xx()const{return g_xx_;}
-    const MPIContainer& g_yy()const{return g_yy_;}
-    const MPIContainer& g_xy()const{return g_xy_;}
-    const MPIContainer& vol()const{return vol2d_;}
-    const geo::aGenerator& generator() const{return g.generator();}
-    bool isOrthonormal() const { return g.isOrthonormal();}
-    bool isOrthogonal() const { return g.isOrthogonal();}
-    bool isConformal() const { return g.isConformal();}
+    const aGenerator2d& generator() const{return g.generator();}
+    virtual CurvilinearMPIGrid2d* clone()const{return new CurvilinearMPIGrid2d(*this);}
     private:
     virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny)
     {
-        dg::MPIGrid3d::do_set(new_n, new_Nx, new_Ny);
+        dg::aMPITopology2d::do_set(new_n, new_Nx, new_Ny);
         dg::CurvilinearGrid2d<thrust::host_vector<double> > g( handle_.get(), new_n, new_Nx, new_Ny);
         divide_and_conquer(g);//distribute to processes
     }
@@ -211,39 +165,30 @@ struct CurvilinearMPIGrid2d : public dg::aMPIGeometry2d
     }
     void divide_and_conquer(const dg::CurvilinearGrid2d<thrust::host_vector<double> >& g_)
     {
-        r_.resize( size());
-        z_=r_;
-        xr_=dg::evaluate( dg::one, *this), xz_=xr_, yr_=xr_, yz_=xr_;
-        thrust::host_vector<double> tempxx(size()), tempxy(size()),tempyy(size()), tempvol(size());
-        dg::blas1::transfer( xr_, g_xx_);
-        g_xy_=g_xx_, g_yy_=g_xx_, vol2d_=g_xx_;
-        //divide and conquer
-        int dims[2], periods[2], coords[2];
-        MPI_Cart_get( communicator(), 2, dims, periods, coords);
-        //for( unsigned py=0; py<dims[1]; py++)
-            for( unsigned i=0; i<this->n()*this->Ny(); i++)
-                //for( unsigned px=0; px<dims[0]; px++)
-                    for( unsigned j=0; j<this->n()*this->Nx(); j++)
-                    {
-                        unsigned idx1 = i*this->n()*this->Nx() + j;
-                        unsigned idx2 = ((coords[1]*this->n()*this->Ny()+i)*dims[0] + coords[0])*this->n()*this->Nx() + j;
-                        r_[idx1] = g_.r()[idx2];
-                        z_[idx1] = g_.z()[idx2];
-                        xr_.data()[idx1] = g_.xr()[idx2];
-                        xz_.data()[idx1] = g_.xz()[idx2];
-                        yr_.data()[idx1] = g_.yr()[idx2];
-                        yz_.data()[idx1] = g_.yz()[idx2];
-                        tempxx[idx1] = g_.g_xx()[idx2];
-                        tempxy[idx1] = g_.g_xy()[idx2];
-                        tempyy[idx1] = g_.g_yy()[idx2];
-                        tempvol2d[idx1] = g_.perpVol()[idx2];
-                    }
-        dg::blas1::transfer( tempxx, g_xx_.data());
-        dg::blas1::transfer( tempxy, g_xy_.data());
-        dg::blas1::transfer( tempyy, g_yy_.data());
-        dg::blas1::transfer( tempvol, vol2d_.data());
+        dg::SparseTensor<host_vector > jacobian=g_.jacobian(); 
+        dg::SparseTensor<host_vector > metric=g_.metric(); 
+        std::vector<host_vector > map = g_.map();
+        for( unsigned i=0; i<3; i++)
+            for( unsigned j=0; j<3; j++)
+            {
+                metric_(i,j) = metric(i,j)
+                jac_(i,j) = jacobian(i,j)
+            }
+        for( unsigned i=0; i<jacobian.values().size(); i++)
+            jac_.value(i) = global2local( jacobian.value(i), *this);
+        for( unsigned i=0; i<metric.values().size(); i++)
+            metric_.value(i) = global2local( metric.value(i), *this);
+        for( unsigned i=0; i<map.size(); i++)
+            map_[i] = global2local( map[i]);
     }
 
+    virtual SparseTensor<host_vector> do_compute_jacobian( ) const {
+        return jac_;
+    }
+    virtual SparseTensor<host_vector> do_compute_metric( ) const {
+        return metric_;
+    }
+    virtual std::vector<host_vector > do_compute_map()const{return map_;}
     dg::SparseTensor<host_vector > jac_, metric_;
     std::vector<host_vector > map_;
     dg::Handle<dg::geo::aGenerator2d> handle_;
-- 
GitLab


From 08db805c1afcafce51c67ee47c182296ac2157a3 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Wed, 9 Aug 2017 07:35:23 -0700
Subject: [PATCH 140/453] work on documentation

---
 inc/dg/backend/cusp_matrix_blas.cuh |  3 +-
 inc/dg/backend/grid.h               |  8 ++--
 inc/dg/backend/gridX.h              |  7 ++--
 inc/dg/backend/mpi_grid.h           | 16 +++++---
 inc/dg/backend/selfmade_blas.cuh    |  5 ++-
 inc/dg/dg_doc.h                     | 61 +++++++++++++++--------------
 inc/dg/geometry/base_geometry.h     |  4 +-
 inc/dg/geometry/mpi_base.h          |  4 +-
 8 files changed, 59 insertions(+), 49 deletions(-)

diff --git a/inc/dg/backend/cusp_matrix_blas.cuh b/inc/dg/backend/cusp_matrix_blas.cuh
index 5178be2f1..01d328707 100644
--- a/inc/dg/backend/cusp_matrix_blas.cuh
+++ b/inc/dg/backend/cusp_matrix_blas.cuh
@@ -11,7 +11,7 @@
 
 #include "matrix_categories.h"
 
-
+///@cond
 namespace dg{
 
 namespace blas2
@@ -93,5 +93,6 @@ inline void doGemv( Matrix& m,
 } //namespace detail
 } //namespace blas2
 } //namespace dg
+///@endcond
 
 #endif //_DG_BLAS_LAPLACE_CUH
diff --git a/inc/dg/backend/grid.h b/inc/dg/backend/grid.h
index aa44a30cb..c2b72a17f 100644
--- a/inc/dg/backend/grid.h
+++ b/inc/dg/backend/grid.h
@@ -14,10 +14,9 @@
 
 namespace dg{
 
-///@addtogroup grid
-///@{
 /**
 * @brief 1D grid
+* @ingroup grid
 *
 */
 struct Grid1d
@@ -193,6 +192,7 @@ struct Grid1d
 /**
  * @brief An abstract base class for two-dimensional grids
  * @note although it is abstract objects are not meant to be hold on the heap via a base class pointer ( we protected the destructor)
+ * @ingroup basictopology
  */
 struct aTopology2d
 {
@@ -413,6 +413,7 @@ struct aTopology2d
 /**
  * @brief An abstract base class for three-dimensional grids
  * @note although it is abstract objects are not meant to be hold on the heap via a base class pointer ( we protected the destructor)
+ * @ingroup basictopology
  */
 struct aTopology3d
 {
@@ -681,6 +682,7 @@ struct aTopology3d
 
 /**
  * @brief The simplest implementation of aTopology2d
+ * @ingroup grid
  */
 struct Grid2d : public aTopology2d
 {
@@ -701,6 +703,7 @@ struct Grid2d : public aTopology2d
 
 /**
  * @brief The simplest implementation of aTopology3d
+ * @ingroup grid
  */
 struct Grid3d : public aTopology3d
 {
@@ -716,7 +719,6 @@ struct Grid3d : public aTopology3d
         aTopology3d::do_set(n,Nx,Ny,Nz);
     }
 };
-///@}
 //
 ///@cond
 void aTopology2d::do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny)
diff --git a/inc/dg/backend/gridX.h b/inc/dg/backend/gridX.h
index fcb4d12da..e40d7fc5a 100644
--- a/inc/dg/backend/gridX.h
+++ b/inc/dg/backend/gridX.h
@@ -13,8 +13,6 @@
 
 namespace dg{
 
-///@addtogroup grid
-///@{
 /**
 * @brief 1D grid for X-point topology
 *
@@ -26,6 +24,7 @@ namespace dg{
 * The left boundary is x0 and the right x1, the inner boundaries lie at
 * x0 + f*Lx and x1-f*Lx
 * therefore f must be smaller than 0.5
+* @ingroup grid
 */
 struct GridX1d
 {
@@ -209,7 +208,7 @@ struct GridX1d
     fy*Ly
  @endcode
  *
- * @tparam double scalar value type 
+ * @ingroup basictopology
  */
 struct aTopologyX2d
 {
@@ -501,6 +500,7 @@ struct aTopologyX2d
 };
 /**
  * @brief The simplest implementation of aTopologyX2d
+ * @ingroup grid
  */
 struct GridX2d : public aTopologyX2d
 {
@@ -511,6 +511,5 @@ struct GridX2d : public aTopologyX2d
     explicit GridX2d( const aTopologyX2d& src): aTopologyX2d(src){}
 };
 
-///@}
 
 }// namespace dg
diff --git a/inc/dg/backend/mpi_grid.h b/inc/dg/backend/mpi_grid.h
index 4e437b538..2b1aef09b 100644
--- a/inc/dg/backend/mpi_grid.h
+++ b/inc/dg/backend/mpi_grid.h
@@ -10,8 +10,6 @@
 
 namespace dg
 {
-///@addtogroup grid
-///@{
 
 
 /**
@@ -23,7 +21,7 @@ namespace dg
  * @note although it is abstract objects are not meant to be hold on the heap via a base class pointer ( we protected the destructor)
  * @attention
  * The boundaries in the constructors are global boundaries, the boundaries returned by the access functions are local boundaries, this is because the grid represents the information given to one process
- *
+ * @ingroup basictopology
  */
 struct aMPITopology2d
 {
@@ -338,6 +336,7 @@ struct aMPITopology2d
  *
  * @note Note that a single cell is never divided across processes.
  * @note although it is abstract objects are not meant to be hold on the heap via a base class pointer ( we protected the destructor)
+ * @ingroup basictopology
  */
 struct aMPITopology3d
 {
@@ -695,6 +694,10 @@ void aMPITopology3d::do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, u
 }
 ///@endcond
 
+/**
+ * @brief The simplest implementation of aMPITopology2d
+ * @ingroup grid
+ */
 struct MPIGrid2d: public aMPITopology2d
 {
     /**
@@ -720,8 +723,10 @@ struct MPIGrid2d: public aMPITopology2d
     }
 };
 
-
-
+/**
+ * @brief The simplest implementation of aMPITopology3d
+ * @ingroup grid
+ */
 struct MPIGrid3d : public aMPITopology3d
 {
     MPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, MPI_Comm comm):
@@ -743,5 +748,4 @@ struct MPIGrid3d : public aMPITopology3d
 };
 
 
-///@}
 }//namespace dg
diff --git a/inc/dg/backend/selfmade_blas.cuh b/inc/dg/backend/selfmade_blas.cuh
index 45d973951..142d0b5aa 100644
--- a/inc/dg/backend/selfmade_blas.cuh
+++ b/inc/dg/backend/selfmade_blas.cuh
@@ -1,6 +1,7 @@
 #ifndef _DG_BLAS_SELFMADE_
 #define _DG_BLAS_SELFMADE_
-
+//
+///@cond
 namespace dg{
 namespace blas2{
 namespace detail{
@@ -52,4 +53,6 @@ inline void doSymv(
 } //namespace detail
 } //namespace blas2
 } //namespace dg
+///@endcond
+//
 #endif//_DG_BLAS_SELFMADE_
diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index 24f7fc8a8..62ad0d64e 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -44,7 +44,8 @@
  *     Objects that store topological information (which point is neighbour of which other point) 
  *     about the grid. 
  *     @{
- *         @defgroup evaluation evaluate functions
+ *         @defgroup basictopology Topology base classes
+ *         @defgroup evaluation evaluate
  *             
  *             The function discretisation routines compute the DG discretisation
  *             of analytic functions on a given grid. In 1D the discretisation
@@ -60,20 +61,20 @@
  *         @defgroup creation create derivatives 
  *
  *             High level matrix creation functions
-           @defgroup interpolation Interpolation and projection
+ *         @defgroup interpolation Interpolation and projection
  *         @defgroup scatter Scatter and Gather
  *     @}
  *     @defgroup geometry Geometric grids and operations
  *
-          These routines form the heart of our geometry free numerical algorithms. 
-          They are called by our geometric operators like the Poisson bracket. 
-      @{
-          @defgroup pullback pullback and pushforward
-          @defgroup metric create metric
-          @defgroup basicgrids basic grids
+ *        These routines form the heart of our geometry free numerical algorithms. 
+ *        They are called by our geometric operators like the Poisson bracket. 
+ *    @{
+ *        @defgroup basicgeometry Geometry base classes
+ *        @defgroup pullback pullback and pushforward
+ *        @defgroup metric create volume
  *        @defgroup utilities Fieldalignment and Averaging
  *            The classes to perform field line integration for DS and averaging classes
-      @}
+ *    @}
  * @}
  * @defgroup numerical1 Level 4: Advanced numerical schemes
  *
@@ -84,37 +85,37 @@
  *     @defgroup fieldaligned Fieldaligned derivatives
  * @}
  * @defgroup templates Level 99: Template models
-   Documentation for template models
+ * Documentation for template models
  * @defgroup misc Level 00: Miscellaneous additions
-   @{
+ * @{
+ *     @defgroup timer Timer class
  *     @defgroup functions Functions and Functors
  * 
  *         The functions are useful mainly in the constructor of Operator objects. 
  *         The functors are useful for either vector transformations or
  *         as init functions in the evaluate routines.
-       @defgroup timer Timer class
  *     @defgroup lowlevel Lowlevel helper functions and classes
  *         Low level helper routines.
-   @}
+ * @}
  * 
  */
-/*! @mainpage
- * Welcome to the DG library. 
- *
- * @par Design principles
- *
- * The DG library is built on top of the <a href="https://thrust.github.io/">thrust</a> and <a href="http://cusplibrary.github.io/index.html">cusp</a> libraries. 
- * Its intention is to provide easy to use
- * functions and objects needed for the integration of 2D and 3D partial differential equations discretized with a
- * discontinuous galerkin method.  
- * Since it is built on top of <a href="https://thrust.github.io/">thrust</a> and <a href="http://cusplibrary.github.io/index.html">cusp</a>, code can run on a CPU as well as a GPU by simply 
- * switching between thrust's host_vector and device_vector. 
- * The DG library uses a design pattern also employed in the cusp library and other modern C++ codes. 
- * It might be referred to as <a href="http://dx.doi.org/10.1063/1.168674">container-free numerical algorithms</a>. 
- *
- *
- *
- */
+//* @mainpage
+// * Welcome to the DG library. 
+// *
+// * @par Design principles
+// *
+// * The DG library is built on top of the <a href="https://thrust.github.io/">thrust</a> and <a href="http://cusplibrary.github.io/index.html">cusp</a> libraries. 
+// * Its intention is to provide easy to use
+// * functions and objects needed for the integration of 2D and 3D partial differential equations discretized with a
+// * discontinuous galerkin method.  
+// * Since it is built on top of <a href="https://thrust.github.io/">thrust</a> and <a href="http://cusplibrary.github.io/index.html">cusp</a>, code can run on a CPU as well as a GPU by simply 
+// * switching between thrust's host_vector and device_vector. 
+// * The DG library uses a design pattern also employed in the cusp library and other modern C++ codes. 
+// * It might be referred to as <a href="http://dx.doi.org/10.1063/1.168674">container-free numerical algorithms</a>. 
+// *
+// *
+// *
+// */
 
 /**
  * @brief Struct that performs collective scatter and gather operations across processes
diff --git a/inc/dg/geometry/base_geometry.h b/inc/dg/geometry/base_geometry.h
index 643c211da..caf33b70f 100644
--- a/inc/dg/geometry/base_geometry.h
+++ b/inc/dg/geometry/base_geometry.h
@@ -6,7 +6,7 @@
 namespace dg
 {
 
-///@addtogroup geometry
+///@addtogroup basicgeometry
 ///@{
 
 /**
@@ -102,7 +102,7 @@ struct aGeometry3d : public aTopology3d
 
 ///@}
 
-///@addtogroup basicgrids
+///@addtogroup geometry
 ///@{
 
 /**
diff --git a/inc/dg/geometry/mpi_base.h b/inc/dg/geometry/mpi_base.h
index 026168133..965a1d27d 100644
--- a/inc/dg/geometry/mpi_base.h
+++ b/inc/dg/geometry/mpi_base.h
@@ -6,7 +6,7 @@
 namespace dg
 {
 
-///@addtogroup geometry
+///@addtogroup basicgeometry
 ///@{
 
 /**
@@ -92,7 +92,7 @@ struct aMPIGeometry3d : public aMPITopology3d
 
 ///@}
 
-///@addtogroup basicgrids
+///@addtogroup geometry
 ///@{
 
 /**
-- 
GitLab


From 45c3621f4cc549df92c088e767a9a1dbf44e96c8 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Wed, 9 Aug 2017 08:06:08 -0700
Subject: [PATCH 141/453] more on documentation

---
 inc/dg/Doxyfile                 | 128 ++++++++++++++++----------------
 inc/dg/{dg_doc.h => dg_doc.dox} |   5 ++
 inc/dg/geometry/multiply.h      |  40 +++++-----
 3 files changed, 89 insertions(+), 84 deletions(-)
 rename inc/dg/{dg_doc.h => dg_doc.dox} (98%)

diff --git a/inc/dg/Doxyfile b/inc/dg/Doxyfile
index d162b2f7e..8d8e11d65 100644
--- a/inc/dg/Doxyfile
+++ b/inc/dg/Doxyfile
@@ -38,7 +38,7 @@ PROJECT_NAME           = "Discontinuous Galerkin Library"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = 
+PROJECT_NUMBER         =
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
@@ -51,7 +51,7 @@ PROJECT_BRIEF          = "Discontinuous Galerkin numerical methods and container
 # pixels and the maximum width should not exceed 200 pixels. Doxygen will copy
 # the logo to the output directory.
 
-PROJECT_LOGO           = 
+PROJECT_LOGO           =
 
 # The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path
 # into which the generated documentation will be written. If a relative path is
@@ -162,7 +162,7 @@ FULL_PATH_NAMES        = YES
 # will be relative from the directory where doxygen is started.
 # This tag requires that the tag FULL_PATH_NAMES is set to YES.
 
-STRIP_FROM_PATH        = 
+STRIP_FROM_PATH        =
 
 # The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the
 # path mentioned in the documentation of a class, which tells the reader which
@@ -171,7 +171,7 @@ STRIP_FROM_PATH        =
 # specify the list of include paths that are normally passed to the compiler
 # using the -I flag.
 
-STRIP_FROM_INC_PATH    = 
+STRIP_FROM_INC_PATH    =
 
 # If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but
 # less readable) file names. This can be useful is your file systems doesn't
@@ -238,13 +238,13 @@ TAB_SIZE               = 8
 # "Side Effects:". You can put \n's in the value part of an alias to insert
 # newlines.
 
-ALIASES                = 
+ALIASES                =
 
 # This tag can be used to specify a number of word-keyword mappings (TCL only).
 # A mapping has the form "name=value". For example adding "class=itcl::class"
 # will allow you to use the command class in the itcl::class meaning.
 
-TCL_SUBST              = 
+TCL_SUBST              =
 
 # Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources
 # only. Doxygen will then generate output that is more tailored for C. For
@@ -639,7 +639,7 @@ GENERATE_DEPRECATEDLIST= YES
 # sections, marked by \if <section_label> ... \endif and \cond <section_label>
 # ... \endcond blocks.
 
-ENABLED_SECTIONS       = 
+ENABLED_SECTIONS       =
 
 # The MAX_INITIALIZER_LINES tag determines the maximum number of lines that the
 # initial value of a variable or macro / define can have for it to appear in the
@@ -681,7 +681,7 @@ SHOW_NAMESPACES        = YES
 # by doxygen. Whatever the program writes to standard output is used as the file
 # version. For an example see the documentation.
 
-FILE_VERSION_FILTER    = 
+FILE_VERSION_FILTER    =
 
 # The LAYOUT_FILE tag can be used to specify a layout file which will be parsed
 # by doxygen. The layout file controls the global structure of the generated
@@ -694,7 +694,7 @@ FILE_VERSION_FILTER    =
 # DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE
 # tag is left empty.
 
-LAYOUT_FILE            = 
+LAYOUT_FILE            =
 
 # The CITE_BIB_FILES tag can be used to specify one or more bib files containing
 # the reference definitions. This must be a list of .bib files. The .bib
@@ -704,7 +704,7 @@ LAYOUT_FILE            =
 # LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the
 # search path. See also \cite for info how to create references.
 
-CITE_BIB_FILES         = 
+CITE_BIB_FILES         =
 
 #---------------------------------------------------------------------------
 # Configuration options related to warning and progress messages
@@ -769,7 +769,7 @@ WARN_FORMAT            = "$file:$line: $text"
 # messages should be written. If left blank the output is written to standard
 # error (stderr).
 
-WARN_LOGFILE           = 
+WARN_LOGFILE           =
 
 #---------------------------------------------------------------------------
 # Configuration options related to the input files
@@ -824,7 +824,7 @@ RECURSIVE              = YES
 # Note that relative paths are relative to the directory from which doxygen is
 # run.
 
-EXCLUDE                = 
+EXCLUDE                =
 
 # The EXCLUDE_SYMLINKS tag can be used to select whether or not files or
 # directories that are symbolic links (a Unix file system feature) are excluded
@@ -840,7 +840,7 @@ EXCLUDE_SYMLINKS       = NO
 # Note that the wildcards are matched against the file with absolute path, so to
 # exclude all test directories for example use the pattern */test/*
 
-EXCLUDE_PATTERNS       = 
+EXCLUDE_PATTERNS       =
 
 # The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names
 # (namespaces, classes, functions, etc.) that should be excluded from the
@@ -851,13 +851,13 @@ EXCLUDE_PATTERNS       =
 # Note that the wildcards are matched against the file with absolute path, so to
 # exclude all test directories use the pattern */test/*
 
-EXCLUDE_SYMBOLS        = 
+EXCLUDE_SYMBOLS        = hide_*
 
 # The EXAMPLE_PATH tag can be used to specify one or more files or directories
 # that contain example code fragments that are included (see the \include
 # command).
 
-EXAMPLE_PATH           = 
+EXAMPLE_PATH           =
 
 # If the value of the EXAMPLE_PATH tag contains directories, you can use the
 # EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and
@@ -877,7 +877,7 @@ EXAMPLE_RECURSIVE      = NO
 # that contain images that are to be included in the documentation (see the
 # \image command).
 
-IMAGE_PATH             = 
+IMAGE_PATH             =
 
 # The INPUT_FILTER tag can be used to specify a program that doxygen should
 # invoke to filter for each input file. Doxygen will invoke the filter program
@@ -898,7 +898,7 @@ IMAGE_PATH             =
 # need to set EXTENSION_MAPPING for the extension otherwise the files are not
 # properly processed by doxygen.
 
-INPUT_FILTER           = 
+INPUT_FILTER           =
 
 # The FILTER_PATTERNS tag can be used to specify filters on a per file pattern
 # basis. Doxygen will compare the file name with each pattern and apply the
@@ -911,7 +911,7 @@ INPUT_FILTER           =
 # need to set EXTENSION_MAPPING for the extension otherwise the files are not
 # properly processed by doxygen.
 
-FILTER_PATTERNS        = 
+FILTER_PATTERNS        =
 
 # If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using
 # INPUT_FILTER) will also be used to filter the input files that are used for
@@ -926,14 +926,14 @@ FILTER_SOURCE_FILES    = NO
 # *.ext= (so without naming a filter).
 # This tag requires that the tag FILTER_SOURCE_FILES is set to YES.
 
-FILTER_SOURCE_PATTERNS = 
+FILTER_SOURCE_PATTERNS =
 
 # If the USE_MDFILE_AS_MAINPAGE tag refers to the name of a markdown file that
 # is part of the input, its contents will be placed on the main page
 # (index.html). This can be useful if you have a project on for instance GitHub
 # and want to reuse the introduction page also for the doxygen output.
 
-USE_MDFILE_AS_MAINPAGE = 
+USE_MDFILE_AS_MAINPAGE =
 
 #---------------------------------------------------------------------------
 # Configuration options related to source browsing
@@ -1038,7 +1038,7 @@ CLANG_ASSISTED_PARSING = NO
 # specified with INPUT and INCLUDE_PATH.
 # This tag requires that the tag CLANG_ASSISTED_PARSING is set to YES.
 
-CLANG_OPTIONS          = 
+CLANG_OPTIONS          =
 
 #---------------------------------------------------------------------------
 # Configuration options related to the alphabetical class index
@@ -1064,7 +1064,7 @@ COLS_IN_ALPHA_INDEX    = 5
 # while generating the index headers.
 # This tag requires that the tag ALPHABETICAL_INDEX is set to YES.
 
-IGNORE_PREFIX          = 
+IGNORE_PREFIX          =
 
 #---------------------------------------------------------------------------
 # Configuration options related to the HTML output
@@ -1108,7 +1108,7 @@ HTML_FILE_EXTENSION    = .html
 # of the possible markers and block names see the documentation.
 # This tag requires that the tag GENERATE_HTML is set to YES.
 
-HTML_HEADER            = 
+HTML_HEADER            =
 
 # The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each
 # generated HTML page. If the tag is left blank doxygen will generate a standard
@@ -1118,7 +1118,7 @@ HTML_HEADER            =
 # that doxygen normally uses.
 # This tag requires that the tag GENERATE_HTML is set to YES.
 
-HTML_FOOTER            = 
+HTML_FOOTER            =
 
 # The HTML_STYLESHEET tag can be used to specify a user-defined cascading style
 # sheet that is used by each HTML page. It can be used to fine-tune the look of
@@ -1130,7 +1130,7 @@ HTML_FOOTER            =
 # obsolete.
 # This tag requires that the tag GENERATE_HTML is set to YES.
 
-HTML_STYLESHEET        = 
+HTML_STYLESHEET        =
 
 # The HTML_EXTRA_STYLESHEET tag can be used to specify additional user-defined
 # cascading style sheets that are included after the standard style sheets
@@ -1143,7 +1143,7 @@ HTML_STYLESHEET        =
 # list). For an example see the documentation.
 # This tag requires that the tag GENERATE_HTML is set to YES.
 
-HTML_EXTRA_STYLESHEET  = 
+HTML_EXTRA_STYLESHEET  =
 
 # The HTML_EXTRA_FILES tag can be used to specify one or more extra images or
 # other source files which should be copied to the HTML output directory. Note
@@ -1153,7 +1153,7 @@ HTML_EXTRA_STYLESHEET  =
 # files will be copied as-is; there are no commands or markers available.
 # This tag requires that the tag GENERATE_HTML is set to YES.
 
-HTML_EXTRA_FILES       = 
+HTML_EXTRA_FILES       =
 
 # The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen
 # will adjust the colors in the style sheet and background images according to
@@ -1282,7 +1282,7 @@ GENERATE_HTMLHELP      = NO
 # written to the html output directory.
 # This tag requires that the tag GENERATE_HTMLHELP is set to YES.
 
-CHM_FILE               = 
+CHM_FILE               =
 
 # The HHC_LOCATION tag can be used to specify the location (absolute path
 # including file name) of the HTML help compiler (hhc.exe). If non-empty,
@@ -1290,7 +1290,7 @@ CHM_FILE               =
 # The file has to be specified with full path.
 # This tag requires that the tag GENERATE_HTMLHELP is set to YES.
 
-HHC_LOCATION           = 
+HHC_LOCATION           =
 
 # The GENERATE_CHI flag controls if a separate .chi index file is generated
 # (YES) or that it should be included in the master .chm file (NO).
@@ -1303,7 +1303,7 @@ GENERATE_CHI           = NO
 # and project file content.
 # This tag requires that the tag GENERATE_HTMLHELP is set to YES.
 
-CHM_INDEX_ENCODING     = 
+CHM_INDEX_ENCODING     =
 
 # The BINARY_TOC flag controls whether a binary table of contents is generated
 # (YES) or a normal table of contents (NO) in the .chm file. Furthermore it
@@ -1334,7 +1334,7 @@ GENERATE_QHP           = NO
 # the HTML output folder.
 # This tag requires that the tag GENERATE_QHP is set to YES.
 
-QCH_FILE               = 
+QCH_FILE               =
 
 # The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help
 # Project output. For more information please see Qt Help Project / Namespace
@@ -1359,7 +1359,7 @@ QHP_VIRTUAL_FOLDER     = doc
 # filters).
 # This tag requires that the tag GENERATE_QHP is set to YES.
 
-QHP_CUST_FILTER_NAME   = 
+QHP_CUST_FILTER_NAME   =
 
 # The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the
 # custom filter to add. For more information please see Qt Help Project / Custom
@@ -1367,21 +1367,21 @@ QHP_CUST_FILTER_NAME   =
 # filters).
 # This tag requires that the tag GENERATE_QHP is set to YES.
 
-QHP_CUST_FILTER_ATTRS  = 
+QHP_CUST_FILTER_ATTRS  =
 
 # The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this
 # project's filter section matches. Qt Help Project / Filter Attributes (see:
 # http://qt-project.org/doc/qt-4.8/qthelpproject.html#filter-attributes).
 # This tag requires that the tag GENERATE_QHP is set to YES.
 
-QHP_SECT_FILTER_ATTRS  = 
+QHP_SECT_FILTER_ATTRS  =
 
 # The QHG_LOCATION tag can be used to specify the location of Qt's
 # qhelpgenerator. If non-empty doxygen will try to run qhelpgenerator on the
 # generated .qhp file.
 # This tag requires that the tag GENERATE_QHP is set to YES.
 
-QHG_LOCATION           = 
+QHG_LOCATION           =
 
 # If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files will be
 # generated, together with the HTML files, they form an Eclipse help plugin. To
@@ -1514,7 +1514,7 @@ MATHJAX_RELPATH        = /usr/share/javascript/mathjax
 # MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols
 # This tag requires that the tag USE_MATHJAX is set to YES.
 
-MATHJAX_EXTENSIONS     = 
+MATHJAX_EXTENSIONS     =
 
 # The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces
 # of code that will be used on startup of the MathJax code. See the MathJax site
@@ -1522,7 +1522,7 @@ MATHJAX_EXTENSIONS     =
 # example see the documentation.
 # This tag requires that the tag USE_MATHJAX is set to YES.
 
-MATHJAX_CODEFILE       = 
+MATHJAX_CODEFILE       =
 
 # When the SEARCHENGINE tag is enabled doxygen will generate a search box for
 # the HTML output. The underlying search engine uses javascript and DHTML and
@@ -1582,7 +1582,7 @@ EXTERNAL_SEARCH        = NO
 # Searching" for details.
 # This tag requires that the tag SEARCHENGINE is set to YES.
 
-SEARCHENGINE_URL       = 
+SEARCHENGINE_URL       =
 
 # When SERVER_BASED_SEARCH and EXTERNAL_SEARCH are both enabled the unindexed
 # search data is written to a file for indexing by an external tool. With the
@@ -1598,7 +1598,7 @@ SEARCHDATA_FILE        = searchdata.xml
 # projects and redirect the results back to the right project.
 # This tag requires that the tag SEARCHENGINE is set to YES.
 
-EXTERNAL_SEARCH_ID     = 
+EXTERNAL_SEARCH_ID     =
 
 # The EXTRA_SEARCH_MAPPINGS tag can be used to enable searching through doxygen
 # projects other than the one defined by this configuration file, but that are
@@ -1608,7 +1608,7 @@ EXTERNAL_SEARCH_ID     =
 # EXTRA_SEARCH_MAPPINGS = tagname1=loc1 tagname2=loc2 ...
 # This tag requires that the tag SEARCHENGINE is set to YES.
 
-EXTRA_SEARCH_MAPPINGS  = 
+EXTRA_SEARCH_MAPPINGS  =
 
 #---------------------------------------------------------------------------
 # Configuration options related to the LaTeX output
@@ -1672,7 +1672,7 @@ PAPER_TYPE             = a4
 # If left blank no extra packages will be included.
 # This tag requires that the tag GENERATE_LATEX is set to YES.
 
-EXTRA_PACKAGES         = 
+EXTRA_PACKAGES         =
 
 # The LATEX_HEADER tag can be used to specify a personal LaTeX header for the
 # generated LaTeX document. The header should contain everything until the first
@@ -1688,7 +1688,7 @@ EXTRA_PACKAGES         =
 # to HTML_HEADER.
 # This tag requires that the tag GENERATE_LATEX is set to YES.
 
-LATEX_HEADER           = 
+LATEX_HEADER           =
 
 # The LATEX_FOOTER tag can be used to specify a personal LaTeX footer for the
 # generated LaTeX document. The footer should contain everything after the last
@@ -1699,7 +1699,7 @@ LATEX_HEADER           =
 # Note: Only use a user-defined footer if you know what you are doing!
 # This tag requires that the tag GENERATE_LATEX is set to YES.
 
-LATEX_FOOTER           = 
+LATEX_FOOTER           =
 
 # The LATEX_EXTRA_STYLESHEET tag can be used to specify additional user-defined
 # LaTeX style sheets that are included after the standard style sheets created
@@ -1710,7 +1710,7 @@ LATEX_FOOTER           =
 # list).
 # This tag requires that the tag GENERATE_LATEX is set to YES.
 
-LATEX_EXTRA_STYLESHEET = 
+LATEX_EXTRA_STYLESHEET =
 
 # The LATEX_EXTRA_FILES tag can be used to specify one or more extra images or
 # other source files which should be copied to the LATEX_OUTPUT output
@@ -1718,7 +1718,7 @@ LATEX_EXTRA_STYLESHEET =
 # markers available.
 # This tag requires that the tag GENERATE_LATEX is set to YES.
 
-LATEX_EXTRA_FILES      = 
+LATEX_EXTRA_FILES      =
 
 # If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated is
 # prepared for conversion to PDF (using ps2pdf or pdflatex). The PDF file will
@@ -1826,14 +1826,14 @@ RTF_HYPERLINKS         = NO
 # default style sheet that doxygen normally uses.
 # This tag requires that the tag GENERATE_RTF is set to YES.
 
-RTF_STYLESHEET_FILE    = 
+RTF_STYLESHEET_FILE    =
 
 # Set optional variables used in the generation of an RTF document. Syntax is
 # similar to doxygen's config file. A template extensions file can be generated
 # using doxygen -e rtf extensionFile.
 # This tag requires that the tag GENERATE_RTF is set to YES.
 
-RTF_EXTENSIONS_FILE    = 
+RTF_EXTENSIONS_FILE    =
 
 # If the RTF_SOURCE_CODE tag is set to YES then doxygen will include source code
 # with syntax highlighting in the RTF output.
@@ -1878,7 +1878,7 @@ MAN_EXTENSION          = .3
 # MAN_EXTENSION with the initial . removed.
 # This tag requires that the tag GENERATE_MAN is set to YES.
 
-MAN_SUBDIR             = 
+MAN_SUBDIR             =
 
 # If the MAN_LINKS tag is set to YES and doxygen generates man output, then it
 # will generate one additional man file for each entity documented in the real
@@ -1991,7 +1991,7 @@ PERLMOD_PRETTY         = YES
 # overwrite each other's variables.
 # This tag requires that the tag GENERATE_PERLMOD is set to YES.
 
-PERLMOD_MAKEVAR_PREFIX = 
+PERLMOD_MAKEVAR_PREFIX =
 
 #---------------------------------------------------------------------------
 # Configuration options related to the preprocessor
@@ -2032,7 +2032,7 @@ SEARCH_INCLUDES        = YES
 # preprocessor.
 # This tag requires that the tag SEARCH_INCLUDES is set to YES.
 
-INCLUDE_PATH           = 
+INCLUDE_PATH           =
 
 # You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard
 # patterns (like *.h and *.hpp) to filter out the header-files in the
@@ -2040,7 +2040,7 @@ INCLUDE_PATH           =
 # used.
 # This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
 
-INCLUDE_FILE_PATTERNS  = 
+INCLUDE_FILE_PATTERNS  =
 
 # The PREDEFINED tag can be used to specify one or more macro names that are
 # defined before the preprocessor is started (similar to the -D option of e.g.
@@ -2050,7 +2050,7 @@ INCLUDE_FILE_PATTERNS  =
 # recursively expanded use the := operator instead of the = operator.
 # This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
 
-PREDEFINED             = 
+PREDEFINED             =
 
 # If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this
 # tag can be used to specify a list of macro names that should be expanded. The
@@ -2059,7 +2059,7 @@ PREDEFINED             =
 # definition found in the source code.
 # This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
 
-EXPAND_AS_DEFINED      = 
+EXPAND_AS_DEFINED      =
 
 # If the SKIP_FUNCTION_MACROS tag is set to YES then doxygen's preprocessor will
 # remove all references to function-like macros that are alone on a line, have
@@ -2088,13 +2088,13 @@ SKIP_FUNCTION_MACROS   = YES
 # the path). If a tag file is not located in the directory in which doxygen is
 # run, you must also specify the path to the tagfile here.
 
-TAGFILES               = 
+TAGFILES               =
 
 # When a file name is specified after GENERATE_TAGFILE, doxygen will create a
 # tag file that is based on the input files it reads. See section "Linking to
 # external documentation" for more information about the usage of tag files.
 
-GENERATE_TAGFILE       = 
+GENERATE_TAGFILE       =
 
 # If the ALLEXTERNALS tag is set to YES, all external class will be listed in
 # the class index. If set to NO, only the inherited external classes will be
@@ -2143,14 +2143,14 @@ CLASS_DIAGRAMS         = YES
 # the mscgen tool resides. If left empty the tool is assumed to be found in the
 # default search path.
 
-MSCGEN_PATH            = 
+MSCGEN_PATH            =
 
 # You can include diagrams made with dia in doxygen documentation. Doxygen will
 # then run dia to produce the diagram and insert it in the documentation. The
 # DIA_PATH tag allows you to specify the directory where the dia binary resides.
 # If left empty dia is assumed to be found in the default search path.
 
-DIA_PATH               = 
+DIA_PATH               =
 
 # If set to YES the inheritance and collaboration graphs will hide inheritance
 # and usage relations if the target is undocumented or is not a class.
@@ -2199,7 +2199,7 @@ DOT_FONTSIZE           = 10
 # the path where dot can find it using this tag.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
-DOT_FONTPATH           = 
+DOT_FONTPATH           =
 
 # If the CLASS_GRAPH tag is set to YES then doxygen will generate a graph for
 # each documented class showing the direct and indirect inheritance relations.
@@ -2345,26 +2345,26 @@ INTERACTIVE_SVG        = NO
 # found. If left blank, it is assumed the dot tool can be found in the path.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
-DOT_PATH               = 
+DOT_PATH               =
 
 # The DOTFILE_DIRS tag can be used to specify one or more directories that
 # contain dot files that are included in the documentation (see the \dotfile
 # command).
 # This tag requires that the tag HAVE_DOT is set to YES.
 
-DOTFILE_DIRS           = 
+DOTFILE_DIRS           =
 
 # The MSCFILE_DIRS tag can be used to specify one or more directories that
 # contain msc files that are included in the documentation (see the \mscfile
 # command).
 
-MSCFILE_DIRS           = 
+MSCFILE_DIRS           =
 
 # The DIAFILE_DIRS tag can be used to specify one or more directories that
 # contain dia files that are included in the documentation (see the \diafile
 # command).
 
-DIAFILE_DIRS           = 
+DIAFILE_DIRS           =
 
 # When using plantuml, the PLANTUML_JAR_PATH tag should be used to specify the
 # path where java can find the plantuml.jar file. If left blank, it is assumed
@@ -2372,12 +2372,12 @@ DIAFILE_DIRS           =
 # generate a warning when it encounters a \startuml command in this case and
 # will not generate output for the diagram.
 
-PLANTUML_JAR_PATH      = 
+PLANTUML_JAR_PATH      =
 
 # When using plantuml, the specified paths are searched for files specified by
 # the !include statement in a plantuml block.
 
-PLANTUML_INCLUDE_PATH  = 
+PLANTUML_INCLUDE_PATH  =
 
 # The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of nodes
 # that will be shown in the graph. If the number of nodes in a graph becomes
diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.dox
similarity index 98%
rename from inc/dg/dg_doc.h
rename to inc/dg/dg_doc.dox
index 62ad0d64e..c640a78cb 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.dox
@@ -117,6 +117,11 @@
 // *
 // */
 
+ /**
+  * @class hide_container
+  * @tparam container A container class that can be used in blas functions
+  */
+
 /**
  * @brief Struct that performs collective scatter and gather operations across processes
  * on distributed vectors using mpi
diff --git a/inc/dg/geometry/multiply.h b/inc/dg/geometry/multiply.h
index 2dcdc672f..da940dd32 100644
--- a/inc/dg/geometry/multiply.h
+++ b/inc/dg/geometry/multiply.h
@@ -14,7 +14,7 @@ namespace tensor
 ///@{
 /**
  * @brief calls sqrt transform function on value
- * @tparam container container class 
+ * @copydoc container
  * @param mu if empty, stays empty, else contains sqrt of input
  */
 template<class container>
@@ -76,7 +76,7 @@ void scal( SparseTensor<container>& t, const SparseElement<container>& mu)
  * @brief Multiply container with form
  *
  * @tparam container container class 
- * @param mu if mu.isEmpty() then nothing happens, else the input is pointwise multiplied with the value in mu
+ * @param mu if mu.isEmpty() then out=in, else the input is pointwise multiplied with the value in mu
  * @param in input vector
  * @param out output vector (may alias in)
  */
@@ -94,7 +94,7 @@ void pointwiseDot( const SparseElement<container>& mu, const container& in, cont
  *
  * @tparam container container class 
  * @param in input vector
- * @param mu if mu.isEmpty() then nothing happens, else the input is pointwise divided with the value in mu
+ * @param mu if mu.isEmpty() then out=in, else the input is pointwise divided with the value in mu
  * @param out output vector (may alias in)
  */
 template<class container>
@@ -112,10 +112,10 @@ void pointwiseDivide( const container& in, const SparseElement<container>& mu, c
  * Compute \f$ w^i = t^{ij}v_j\f$ for \f$ i,j\in \{1,2\}\f$ in the first two dimensions (ignores the 3rd dimension in t)
  * @tparam container the container class
  * @param t input Tensor
- * @param in0 (input) covariant first component 
- * @param in1 (input) covariant second component
- * @param out0 (output) contravariant first component 
- * @param out1 (output) contravariant second component 
+ * @param in0 (input) first component 
+ * @param in1 (input) second component
+ * @param out0 (output) first component 
+ * @param out1 (output) second component 
  * @note this version keeps the input intact 
  * @attention aliasing only allowed if tensor is either lower, or upper triangular 
  */
@@ -153,8 +153,8 @@ void multiply( const SparseTensor<container>& t, const container& in0, const con
  * Compute \f$ v^i = t^{ij}v_j\f$ for \f$ i,j\in \{1,2\}\f$ in the first two dimensions (ignores the 3rd dimension in t)
  * @tparam container the container class
  * @param t input Tensor
- * @param inout0 (input/output) covariant first component 
- * @param inout1 (input/output) covariant second component
+ * @param inout0 (input/output) first component 
+ * @param inout1 (input/output) second component
  * @param workspace (write) optional workspace
  * @note this version overwrites the input and the workspace
  * @attention aliasing not allowed
@@ -185,15 +185,15 @@ void multiply_inplace( const SparseTensor<container>& t, container& inout0, cont
 /**
  * @brief Multiply a tensor with a vector in 2d
  *
- * Compute \f$ w^i = t^{ij}v_j\f$ for \f$ i,j\in \{1,2\}\f$ in the first two dimensions (ignores the 3rd dimension in t)
+ * Compute \f$ w^i = t^{ij}v_j\f$ for \f$ i,j\in \{1,2,3\}\f$
  * @tparam container the container class
  * @param t input Tensor
- * @param in0 (input) covariant first component 
- * @param in1 (input) covariant second component
- * @param in2 (input) covariant third component
- * @param out0 (output) contravariant first component 
- * @param out1 (output) contravariant second component 
- * @param out2 (output) contravariant third component 
+ * @param in0 (input)  first component 
+ * @param in1 (input)  second component
+ * @param in2 (input)  third component
+ * @param out0 (output)  first component 
+ * @param out1 (output)  second component 
+ * @param out2 (output)  third component 
  * @note this version keeps the input intact 
  * @attention aliasing only allowed if tensor is either lower, or upper triangular 
  */
@@ -243,12 +243,12 @@ void multiply( const SparseTensor<container>& t, const container& in0, const con
 /**
  * @brief Multiply a tensor with a vector in 2d inplace
  *
- * Compute \f$ v^i = t^{ij}v_j\f$ for \f$ i,j\in \{1,2\}\f$ in the first two dimensions (ignores the 3rd dimension in t)
+ * Compute \f$ v^i = t^{ij}v_j\f$ for \f$ i,j\in \{1,2,3\}\f$
  * @tparam container the container class
  * @param t input Tensor
- * @param inout0 (input/output) covariant first component 
- * @param inout1 (input/output) covariant second component
- * @param inout2 (input/output) covariant second component
+ * @param inout0 (input/output) first component 
+ * @param inout1 (input/output) second component
+ * @param inout2 (input/output) second component
  * @param workspace0 (write) optional workspace
  * @param workspace1 (write) optional workspace
  * @note this version overwrites the input and may or may not write into the workspace
-- 
GitLab


From 95f6a5f64f43f3188ad5f25ddf46d9606ade7442 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Wed, 9 Aug 2017 09:08:44 -0700
Subject: [PATCH 142/453] arakawa_t works with new interface

---
 inc/dg/Doxyfile                 |  2 +-
 inc/dg/arakawa.h                | 30 +++++++-----
 inc/dg/backend/typedefs.cuh     |  2 +-
 inc/dg/{dg_doc.dox => dg_doc.h} | 16 ++++++-
 inc/dg/geometry/base_geometry.h | 25 +++++-----
 inc/dg/geometry/geometry.h      |  6 +--
 inc/dg/geometry/mpi_base.h      |  2 +-
 inc/dg/geometry/multiply.h      | 46 +++++++++---------
 inc/dg/geometry/multiply_t.cu   | 12 ++---
 inc/dg/geometry/tensor.h        | 82 ++++++++++++++++++---------------
 inc/dg/geometry/transform.h     | 33 +++++++------
 11 files changed, 143 insertions(+), 113 deletions(-)
 rename inc/dg/{dg_doc.dox => dg_doc.h} (93%)

diff --git a/inc/dg/Doxyfile b/inc/dg/Doxyfile
index 8d8e11d65..18ce2c8ba 100644
--- a/inc/dg/Doxyfile
+++ b/inc/dg/Doxyfile
@@ -783,7 +783,7 @@ WARN_LOGFILE           =
 
 INPUT                  = . \
                          ./backend \
-                         ./geometry
+                         ./geometry 
 
 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
diff --git a/inc/dg/arakawa.h b/inc/dg/arakawa.h
index 281fbf3b8..e5a827d30 100644
--- a/inc/dg/arakawa.h
+++ b/inc/dg/arakawa.h
@@ -2,7 +2,7 @@
 #define _DG_ARAKAWA_CUH
 
 #include "blas.h"
-#include "geometry.h"
+#include "geometry/geometry.h"
 #include "enums.h"
 #include "backend/evaluation.cuh"
 #include "backend/derivatives.h"
@@ -21,9 +21,8 @@ namespace dg
 /**
  * @brief X-space generalized version of Arakawa's scheme
  *
+ * @copydoc hide_matrix_container
  * @ingroup arakawa
- * @tparam Matrix The Matrix class to use
- * @tparam container The vector class on which to operate on. The blas2 function symv( m, x, y) must be callable and may not change x. 
  */
 template< class Geometry, class Matrix, class container >
 struct ArakawaX
@@ -85,9 +84,7 @@ struct ArakawaX
     {
         blas2::symv( bdxf, phi, dxrhs);
         blas2::symv( bdyf, phi, dyrhs);
-        blas1::copy( dxrhs, dxlhs);//save results
-        blas1::copy( dyrhs, dylhs);
-        geo::raisePerpIndex( dxlhs, dylhs, varphi, helper_, grid); //input gets destroyed
+        tensor::multiply2d( metric_, dxrhs, dyrhs, varphi, helper_);
         blas1::pointwiseDot( varphi, dxrhs, varphi);
         blas1::pointwiseDot( 1., helper_, dyrhs,1., varphi );
     }
@@ -95,21 +92,30 @@ struct ArakawaX
   private:
     container dxlhs, dxrhs, dylhs, dyrhs, helper_;
     Matrix bdxf, bdyf;
-    Geometry grid;
+    SparseElement<container> perp_vol_inv_;
+    SparseTensor<container> metric_;
 };
 
 template<class Geometry, class Matrix, class container>
 ArakawaX<Geometry, Matrix, container>::ArakawaX( Geometry g ): 
     dxlhs( dg::evaluate( one, g) ), dxrhs(dxlhs), dylhs(dxlhs), dyrhs( dxlhs), helper_( dxlhs), 
     bdxf( dg::create::dx( g, g.bcx())),
-    bdyf( dg::create::dy( g, g.bcy())), grid( g)
-{ }
+    bdyf( dg::create::dy( g, g.bcy()))
+{
+    metric_=g.metric().perp();
+    perp_vol_inv_ = dg::tensor::determinant(metric_);
+    dg::tensor::sqrt(perp_vol_inv_);
+}
 template<class Geometry, class Matrix, class container>
 ArakawaX<Geometry, Matrix, container>::ArakawaX( Geometry g, bc bcx, bc bcy): 
     dxlhs( dg::evaluate( one, g) ), dxrhs(dxlhs), dylhs(dxlhs), dyrhs( dxlhs), helper_( dxlhs),
     bdxf(dg::create::dx( g, bcx)),
-    bdyf(dg::create::dy( g, bcy)), grid(g)
-{ }
+    bdyf(dg::create::dy( g, bcy))
+{ 
+    metric_=g.metric().perp();
+    perp_vol_inv_ = dg::tensor::determinant(metric_);
+    dg::tensor::sqrt(perp_vol_inv_);
+}
 
 template< class Geometry, class Matrix, class container>
 void ArakawaX< Geometry, Matrix, container>::operator()( const container& lhs, const container& rhs, container& result)
@@ -149,7 +155,7 @@ void ArakawaX< Geometry, Matrix, container>::operator()( const container& lhs, c
     //now sum everything up
     blas1::axpby( 1., dxlhs, 1., result); //result + dxlhs -> result
     blas1::axpby( 1., dxrhs, 1., result); //result + dyrhs -> result
-    geo::dividePerpVolume( result, grid);
+    tensor::pointwiseDot( perp_vol_inv_, result, result);
 }
 
 }//namespace dg
diff --git a/inc/dg/backend/typedefs.cuh b/inc/dg/backend/typedefs.cuh
index d14198a31..1bdbf8b83 100644
--- a/inc/dg/backend/typedefs.cuh
+++ b/inc/dg/backend/typedefs.cuh
@@ -28,7 +28,7 @@ typedef EllSparseBlockMat<double> HMatrix; //!< Host Matrix for derivatives
 #ifdef MPI_VERSION
 //typedef MPI_Vector<thrust::device_vector<double> >  MDVec; //!< MPI Device Vector s.a. dg::DVec
 typedef MPI_Vector<DVec >  MDVec; //!< MPI Device Vector s.a. dg::DVec
-typedef MPI_Vector<thrust::host_vector<double>  >   MHVec; //!< MPI Host Vector
+typedef MPI_Vector<thrust::host_vector<double>  >   MHVec; //!< MPI Host Vector s.a. dg::HVec
 
 typedef NearestNeighborComm<thrust::host_vector<int>, thrust::host_vector<double> > NNCH; //!< host Communicator for the use in an mpi matrix for derivatives
 //typedef NearestNeighborComm<thrust::device_vector<int>, thrust::device_vector<double> > NNCD; //!< device Communicator for the use in an mpi matrix for derivatives
diff --git a/inc/dg/dg_doc.dox b/inc/dg/dg_doc.h
similarity index 93%
rename from inc/dg/dg_doc.dox
rename to inc/dg/dg_doc.h
index c640a78cb..7bdd9320c 100644
--- a/inc/dg/dg_doc.dox
+++ b/inc/dg/dg_doc.h
@@ -118,8 +118,20 @@
 // */
 
  /**
-  * @class hide_container
-  * @tparam container A container class that can be used in blas functions
+  * @class hide_container_lvl1
+  * @tparam container A data container class for which the blas1 functionality is overloaded. Currently this is one of 
+  *   dg::HVec, dg::DVec, dg::MHVec or dg::MDVec
+  */
+ /**
+  * @class hide_matrix_container
+  * @tparam Matrix A class for which the blas2 functions are callable in connection with the container class
+  *  - dg::HMatrix with dg::HVec
+  *  - dg::DMatrix with dg::DVec
+  *  - dg::MHMatrix with dg::MHVec
+  *  - dg::MDMatrix with dg::MDVec
+  *
+  * @tparam container A data container class for which the blas1 functionality is overloaded. Currently this is one of 
+  *   dg::HVec, dg::DVec, dg::MHVec or dg::MDVec
   */
 
 /**
diff --git a/inc/dg/geometry/base_geometry.h b/inc/dg/geometry/base_geometry.h
index caf33b70f..aa4d7b769 100644
--- a/inc/dg/geometry/base_geometry.h
+++ b/inc/dg/geometry/base_geometry.h
@@ -20,7 +20,7 @@ struct aGeometry2d : public aTopology2d
     SparseTensor<thrust::host_vector<double> > metric()const { 
         return do_compute_metric(); 
     }
-    std::vector<thrust::host_vector<double> > map(){
+    std::vector<thrust::host_vector<double> > map()const{
         return do_compute_map();
     }
     ///Geometries are cloneable
@@ -40,10 +40,10 @@ struct aGeometry2d : public aTopology2d
     }
     private:
     virtual SparseTensor<thrust::host_vector<double> > do_compute_metric()const {
-        return SharedContainer<thrust::host_vector<double> >();
+        return SparseTensor<thrust::host_vector<double> >();
     }
     virtual SparseTensor<thrust::host_vector<double> > do_compute_jacobian()const {
-        return SharedContainer<thrust::host_vector<double> >();
+        return SparseTensor<thrust::host_vector<double> >();
     }
     virtual std::vector<thrust::host_vector<double> > do_compute_map()const{
         std::vector<thrust::host_vector<double> > map(2);
@@ -66,7 +66,7 @@ struct aGeometry3d : public aTopology3d
     SparseTensor<thrust::host_vector<double> > metric()const { 
         return do_compute_metric(); 
     }
-    std::vector<thrust::host_vector<double> > map(){
+    std::vector<thrust::host_vector<double> > map()const{
         return do_compute_map();
     }
     ///Geometries are cloneable
@@ -86,10 +86,10 @@ struct aGeometry3d : public aTopology3d
     }
     private:
     virtual SparseTensor<thrust::host_vector<double> > do_compute_metric()const {
-        return SharedContainer<thrust::host_vector<double> >();
+        return SparseTensor<thrust::host_vector<double> >();
     }
     virtual SparseTensor<thrust::host_vector<double> > do_compute_jacobian()const {
-        return SharedContainer<thrust::host_vector<double> >();
+        return SparseTensor<thrust::host_vector<double> >();
     }
     virtual std::vector<thrust::host_vector<double> > do_compute_map()const{
         std::vector<thrust::host_vector<double> > map(3);
@@ -114,10 +114,9 @@ struct CartesianGrid2d: public dg::aGeometry2d
     CartesianGrid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx = PER, bc bcy = PER): dg::aGeometry2d(x0,x1,y0,y1,n,Nx,Ny,bcx,bcy){}
     /**
      * @brief Construct from existing topology
-     *
-     * @param grid existing grid class
+     * @param g existing grid class
      */
-    CartesianGrid2d( const dg::Grid2d& grid):dg::Grid2d(grid){}
+    CartesianGrid2d( const dg::Grid2d& g):dg::aGeometry2d(g.x0(),g.x1(),g.y0(),g.y1(),g.n(),g.Nx(),g.Ny(),g.bcx(),g.bcy()){}
     virtual CartesianGrid2d* clone()const{return new CartesianGrid2d(*this);}
     private:
     virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny){
@@ -134,9 +133,9 @@ struct CartesianGrid3d: public dg::aGeometry3d
     CartesianGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = PER, bc bcy = PER, bc bcz = PER): dg::aGeometry3d(x0,x1,y0,y1,z0,z1,n,Nx,Ny,Nz,bcx,bcy,bcz){}
     /**
      * @brief Implicit type conversion from Grid3d
-     * @param grid existing grid class
+     * @param g existing grid class
      */
-    CartesianGrid3d( const dg::Grid3d& grid):dg::Grid3d(grid){}
+    CartesianGrid3d( const dg::Grid3d& g):dg::aGeometry3d(g.x0(), g.x1(), g.y0(), g.y1(), g.z0(), g.z1(),g.n(),g.Nx(),g.Ny(),g.Nz(),g.bcx(),g.bcy(),g.bcz()){}
     virtual CartesianGrid3d* clone()const{return new CartesianGrid3d(*this);}
     private:
     virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz){
@@ -153,8 +152,8 @@ struct CylindricalGrid3d: public dg::aGeometry3d
     virtual CylindricalGrid3d* clone()const{return new CylindricalGrid3d(*this);}
     private:
     virtual SparseTensor<thrust::host_vector<double> > do_compute_metric()const{
-        SparseTensor<thrust::host_vector<double> metric(1);
-        thrust::host_vector<double> R = dg::evaluate(dg::coo1, *this);
+        SparseTensor<thrust::host_vector<double> > metric(1);
+        thrust::host_vector<double> R = dg::evaluate(dg::cooX3d, *this);
         for( unsigned i = 0; i<size(); i++)
             R[i] = 1./R[i]/R[i];
         metric.idx(2,2)=0;
diff --git a/inc/dg/geometry/geometry.h b/inc/dg/geometry/geometry.h
index f236dd095..ffc7465dd 100644
--- a/inc/dg/geometry/geometry.h
+++ b/inc/dg/geometry/geometry.h
@@ -9,7 +9,7 @@
 #include "../backend/mpi_evaluation.h"
 #include "../backend/mpi_precon.h"
 #endif//MPI_VERSION
-#include "base.h"
+#include "base_geometry.h"
 //#include "curvilinear.h"
 //#include "cartesianX.h"
 #ifdef MPI_VERSION
@@ -44,7 +44,7 @@ namespace create{
 template< class Geometry>
 typename GeometryTraits<Geometry>::host_vector volume( const Geometry& g)
 {
-    typedef typename GeometryTraits< Geometry>::host_vector host_vec;
+    typedef typename GeometryTraits< Geometry>::host_vector host_vector;
     SparseElement<host_vector> vol = dg::tensor::determinant(g.metric());
     host_vector temp = dg::create::weights( g);
     dg::tensor::pointwiseDot( vol, temp, temp);
@@ -63,7 +63,7 @@ typename GeometryTraits<Geometry>::host_vector volume( const Geometry& g)
 template< class Geometry>
 typename GeometryTraits<Geometry>::host_vector inv_volume( const Geometry& g)
 {
-    typedef typename GeometryTraits< Geometry>::host_vector host_vec;
+    typedef typename GeometryTraits< Geometry>::host_vector host_vector;
     host_vector temp = volume(g);
     dg::blas1::transform(temp,temp,dg::INVERT<double>());
     return temp;
diff --git a/inc/dg/geometry/mpi_base.h b/inc/dg/geometry/mpi_base.h
index 965a1d27d..3d1716997 100644
--- a/inc/dg/geometry/mpi_base.h
+++ b/inc/dg/geometry/mpi_base.h
@@ -61,7 +61,7 @@ struct aMPIGeometry3d : public aMPITopology3d
     SparseTensor<host_vector > metric()const { 
         return do_compute_metric(); 
     }
-    std::vector<host_vector > map(){
+    std::vector<host_vector > map()const{
         return do_compute_map();
     }
     ///Geometries are cloneable
diff --git a/inc/dg/geometry/multiply.h b/inc/dg/geometry/multiply.h
index da940dd32..818b79d88 100644
--- a/inc/dg/geometry/multiply.h
+++ b/inc/dg/geometry/multiply.h
@@ -14,7 +14,7 @@ namespace tensor
 ///@{
 /**
  * @brief calls sqrt transform function on value
- * @copydoc container
+ * @copydoc hide_container_lvl1
  * @param mu if empty, stays empty, else contains sqrt of input
  */
 template<class container>
@@ -25,7 +25,7 @@ void sqrt( SparseElement<container>& mu){
 
 /**
  * @brief calls invert transform function on value
- * @tparam container container class 
+ * @copydoc hide_container_lvl1
  * @param mu if empty, stays empty, else contains inverse of input
  */
 template<class container>
@@ -38,7 +38,7 @@ void invert(SparseElement<container>& mu){
  * @brief Scale tensor with a container
  *
  * Computes \f$ t^{ij} = \mu t^{ij}\f$ 
- * @tparam container container class 
+ * @copydoc hide_container_lvl1
  * @param t input (contains result on output)
  * @param mu all elements in t are scaled with mu
  */
@@ -61,7 +61,7 @@ void scal( SparseTensor<container>& t, const container& mu)
  * @brief Scale tensor with a form
  *
  * Computes \f$ t^{ij} = \mu t^{ij}\f$ 
- * @tparam container container class 
+ * @copydoc hide_container_lvl1
  * @param t input (contains result on output)
  * @param mu if mu.isEmpty() then nothing happens, else all elements in t are scaled with its value
  */
@@ -75,7 +75,7 @@ void scal( SparseTensor<container>& t, const SparseElement<container>& mu)
 /**
  * @brief Multiply container with form
  *
- * @tparam container container class 
+ * @copydoc hide_container_lvl1
  * @param mu if mu.isEmpty() then out=in, else the input is pointwise multiplied with the value in mu
  * @param in input vector
  * @param out output vector (may alias in)
@@ -92,7 +92,7 @@ void pointwiseDot( const SparseElement<container>& mu, const container& in, cont
 /**
  * @brief Divide container with form
  *
- * @tparam container container class 
+ * @copydoc hide_container_lvl1
  * @param in input vector
  * @param mu if mu.isEmpty() then out=in, else the input is pointwise divided with the value in mu
  * @param out output vector (may alias in)
@@ -110,7 +110,7 @@ void pointwiseDivide( const container& in, const SparseElement<container>& mu, c
  * @brief Multiply a tensor with a vector in 2d
  *
  * Compute \f$ w^i = t^{ij}v_j\f$ for \f$ i,j\in \{1,2\}\f$ in the first two dimensions (ignores the 3rd dimension in t)
- * @tparam container the container class
+ * @copydoc hide_container_lvl1
  * @param t input Tensor
  * @param in0 (input) first component 
  * @param in1 (input) second component
@@ -120,7 +120,7 @@ void pointwiseDivide( const container& in, const SparseElement<container>& mu, c
  * @attention aliasing only allowed if tensor is either lower, or upper triangular 
  */
 template<class container>
-void multiply( const SparseTensor<container>& t, const container& in0, const container& in1, container& out0, container& out1)
+void multiply2d( const SparseTensor<container>& t, const container& in0, const container& in1, container& out0, container& out1)
 {
     if(!t.isSet(0,1))//lower triangular
     {
@@ -151,7 +151,7 @@ void multiply( const SparseTensor<container>& t, const container& in0, const con
  * @brief Multiply a tensor with a vector in 2d inplace
  *
  * Compute \f$ v^i = t^{ij}v_j\f$ for \f$ i,j\in \{1,2\}\f$ in the first two dimensions (ignores the 3rd dimension in t)
- * @tparam container the container class
+ * @copydoc hide_container_lvl1
  * @param t input Tensor
  * @param inout0 (input/output) first component 
  * @param inout1 (input/output) second component
@@ -160,7 +160,7 @@ void multiply( const SparseTensor<container>& t, const container& in0, const con
  * @attention aliasing not allowed
  */
 template<class container>
-void multiply_inplace( const SparseTensor<container>& t, container& inout0, container& inout1, container& workspace)
+void multiply2d_inplace( const SparseTensor<container>& t, container& inout0, container& inout1, container& workspace)
 {
     if( t.isSet(0,1) ) dg::blas1::pointwiseDot( t.value(0,1), inout1, workspace);
     //compute out1 inplace
@@ -186,7 +186,7 @@ void multiply_inplace( const SparseTensor<container>& t, container& inout0, cont
  * @brief Multiply a tensor with a vector in 2d
  *
  * Compute \f$ w^i = t^{ij}v_j\f$ for \f$ i,j\in \{1,2,3\}\f$
- * @tparam container the container class
+ * @copydoc hide_container_lvl1
  * @param t input Tensor
  * @param in0 (input)  first component 
  * @param in1 (input)  second component
@@ -198,7 +198,7 @@ void multiply_inplace( const SparseTensor<container>& t, container& inout0, cont
  * @attention aliasing only allowed if tensor is either lower, or upper triangular 
  */
 template<class container>
-void multiply( const SparseTensor<container>& t, const container& in0, const container& in1, const container& in2, container& out0, container& out1, container& out2)
+void multiply3d( const SparseTensor<container>& t, const container& in0, const container& in1, const container& in2, container& out0, container& out1, container& out2)
 {
     if( !t.isSet(0,1)&&!t.isSet(0,2)&&!t.isSet(1,2))
     {
@@ -244,7 +244,7 @@ void multiply( const SparseTensor<container>& t, const container& in0, const con
  * @brief Multiply a tensor with a vector in 2d inplace
  *
  * Compute \f$ v^i = t^{ij}v_j\f$ for \f$ i,j\in \{1,2,3\}\f$
- * @tparam container the container class
+ * @copydoc hide_container_lvl1
  * @param t input Tensor
  * @param inout0 (input/output) first component 
  * @param inout1 (input/output) second component
@@ -255,7 +255,7 @@ void multiply( const SparseTensor<container>& t, const container& in0, const con
  * @attention aliasing not allowed
  */
 template<class container>
-void multiply_inplace( const SparseTensor<container>& t, container& inout0, container& inout1, container& inout2, container& workspace0, container& workspace1)
+void multiply3d_inplace( const SparseTensor<container>& t, container& inout0, container& inout1, container& inout2, container& workspace0, container& workspace1)
 {
     //first: store off-diagonals of first two rows
     if( t.isSet(0,1) ) {
@@ -307,7 +307,7 @@ void multiply_inplace( const SparseTensor<container>& t, container& inout0, cont
 
 /**
 * @brief Compute the determinant of a tensor
-* @tparam container the container class
+* @copydoc hide_container_lvl1
 * @param t the input tensor 
 * @return the determinant of t as a SparseElement (unset if t is empty)
 */
@@ -340,18 +340,18 @@ SparseElement<container> determinant( const SparseTensor<container>& t)
 ///@cond
 //alias always allowed
 template<class container>
-void multiply( const CholeskyTensor<container>& ch, const container& in0, const container& in1, container& out0, container& out1)
+void multiply2d( const CholeskyTensor<container>& ch, const container& in0, const container& in1, container& out0, container& out1)
 {
-    multiply(ch.upper(),     in0,  in1,  out0, out1);
-    multiply(ch.diagonal(),  out0, out1, out0, out1);
-    multiply(ch.lower(),     out0, out1, out0, out1);
+    multiply2d(ch.upper(),     in0,  in1,  out0, out1);
+    multiply2d(ch.diagonal(),  out0, out1, out0, out1);
+    multiply2d(ch.lower(),     out0, out1, out0, out1);
 }
 template<class container>
-void multiply( const CholeskyTensor<container>& ch, const container& in0, const container& in1, const container& in2, container& out0, container& out1, container& out2)
+void multiply3d( const CholeskyTensor<container>& ch, const container& in0, const container& in1, const container& in2, container& out0, container& out1, container& out2)
 {
-    multiply(ch.upper(),    in0,  in1, in2,  out0, out1, out2);
-    multiply(ch.diagonal(), out0, out1,out2, out0, out1, out2);
-    multiply(ch.lower(),    out0, out1,out2, out0, out1, out2);
+    multiply3d(ch.upper(),    in0,  in1, in2,  out0, out1, out2);
+    multiply3d(ch.diagonal(), out0, out1,out2, out0, out1, out2);
+    multiply3d(ch.lower(),    out0, out1,out2, out0, out1, out2);
 }
 
 template<class container>
diff --git a/inc/dg/geometry/multiply_t.cu b/inc/dg/geometry/multiply_t.cu
index bbd97ab8f..7d137f644 100644
--- a/inc/dg/geometry/multiply_t.cu
+++ b/inc/dg/geometry/multiply_t.cu
@@ -62,16 +62,16 @@ int main()
     std::cout << "Restore 8 = "<<inout0[0]<<"\n";
     std::cout << "Test Tensor multiplies \n";
     std::cout << "Multiply T with [8,9]\n";
-    dg::tensor::multiply(t, inout0, inout1, work0, work1);
+    dg::tensor::multiply2d(t, inout0, inout1, work0, work1);
     std::cout << "Result         is ["<<work0[0]<<" "<<work1[0]<<"]\n";
-    dg::tensor::multiply_inplace(t, inout0, inout1, work1);
+    dg::tensor::multiply2d_inplace(t, inout0, inout1, work1);
     std::cout << "Result inplace is ["<<inout0[0]<<" "<<inout1[0]<<"]\n";
     t.idx(0,2) = 2; std::swap( t.idx(1,1), t.idx(2,1));  print(t);
     std::cout << "Multiply T with [8,9,2]\n";
-    dg::tensor::multiply(t, eight, nine,two, work0, work1, work2);
+    dg::tensor::multiply3d(t, eight, nine,two, work0, work1, work2);
     std::cout << "Result         is ["<<work0[0]<<" "<<work1[0]<<" "<<work2[0]<<"]\n";
     inout0=eight, inout1=nine, inout2=two;
-    dg::tensor::multiply_inplace(t, inout0, inout1,inout2, work1,work2);
+    dg::tensor::multiply3d_inplace(t, inout0, inout1,inout2, work1,work2);
     std::cout << "Result inplace is ["<<inout0[0]<<" "<<inout1[0]<<" "<<inout2[0]<<"]\n";
     std::cout << "Determinant of T: "<<dg::tensor::determinant(t).value()[0]<<" (320)\n";
     std::swap(t.idx(2,1), t.idx(2,0)); 
@@ -87,10 +87,10 @@ int main()
     std::cout << "upper \n"; print(ch.upper());
     std::cout << "Multiply T with [8,9]\n";
     inout0=eight, inout1=nine, inout2=two;
-    dg::tensor::multiply(ch, inout0, inout1, inout0, inout1);
+    dg::tensor::multiply2d(ch, inout0, inout1, inout0, inout1);
     std::cout << "Result         is ["<<inout0[0]<<", "<<inout1[0]<<"] ([94, 93])\n";
     std::cout << "Multiply T with [8,9,2]\n";
-    dg::tensor::multiply(ch, eight,nine,two, work0, work1, work2);
+    dg::tensor::multiply3d(ch, eight,nine,two, work0, work1, work2);
     std::cout << "Result         is ["<<work0[0]<<" "<<work1[0]<<" "<<work2[0]<<"] (110, 93, 74)\n";
     return 0;
 
diff --git a/inc/dg/geometry/tensor.h b/inc/dg/geometry/tensor.h
index d90328a41..f0ccaa600 100644
--- a/inc/dg/geometry/tensor.h
+++ b/inc/dg/geometry/tensor.h
@@ -11,10 +11,10 @@ namespace dg
 /**
  * @brief This is a sparse Tensor with only one element i.e. a Form
  *
- * @tparam container a container class
+ * @tparam T a T class
  * @ingroup misc
  */
-template<class container>
+template<class T>
 struct SparseElement
 {
     ///create empty object
@@ -23,9 +23,14 @@ struct SparseElement
      * @brief copy construct value
      * @param value a value
      */
-    SparseElement(const container& value):value_(1,value){ }
-    template<class OtherContainer>
-    SparseElement( const SparseElement<OtherContainer>& src)
+    SparseElement(const T& value):value_(1,value){ }
+    /**
+     * @brief Type conversion from other value types
+     * @tparam OtherT dg::blas1::transfer must be callable for T and OtherT
+     * @param src the source matrix to convert
+     */
+    template<class OtherT>
+    SparseElement( const SparseElement<OtherT>& src)
     {
         if(src.isSet())
         {
@@ -36,14 +41,14 @@ struct SparseElement
 
     ///@brief Read access
     ///@return read access to contained value
-    const container& value( )const { 
+    const T& value( )const { 
         return value_[0];
     }
     /**
-     * @brief write access, create a container if there isn't one already
-     * @return write access, always returns a container 
+     * @brief write access, create a T if there isn't one already
+     * @return write access, always returns a T 
      */
-    container& value() {
+    T& value() {
         if(!isSet()) value_.resize(1);
         return value_[0];
     }
@@ -59,48 +64,48 @@ struct SparseElement
     ///@brief Clear contained value
     void clear(){value_.clear();}
     private:
-    std::vector<container> value_;
+    std::vector<T> value_;
 };
 
 /**
 * @brief Class for 2x2 and 3x3 matrices sharing or implicitly assuming elements 
 *
-* This class enables shared access to stored containers 
-* or not store them at all since the storage of (and computation with) a container is expensive.
+* This class enables shared access to stored Ts 
+* or not store them at all since the storage of (and computation with) a T is expensive.
 
 * This class contains both a (dense) matrix of integers.
-* If positive or zero, the integer represents a gather index into the stored array of containers, 
-if negative the value of the container is assumed to be 1, except for the off-diagonal entries
+* If positive or zero, the integer represents a gather index into the stored array of Ts, 
+if negative the value of the T is assumed to be 1, except for the off-diagonal entries
     in the matrix where it is assumed to be 0.
-* We then only need to store non-trivial and non-repetitive containers.
-* @tparam container container class
+* We then only need to store non-trivial and non-repetitive Ts.
+* @tparam T must be default constructible and copyable
 * @ingroup misc
 */
-template<class container>
+template<class T>
 struct SparseTensor
 {
     ///no element is set
     SparseTensor( ):mat_idx_(3,-1.) {}
 
     /**
-     * @brief reserve space for containers in the values array
-     * @param value_size reserve space for this number of containers (default constructor) 
+     * @brief reserve space for Ts in the values array
+     * @param value_size reserve space for this number of Ts (default constructor) 
      */
     SparseTensor( unsigned value_size): mat_idx_(3,-1.), values_(value_size){}
 
     /**
-    * @brief pass array of containers
-    * @param values The contained containers must all have the same size
+    * @brief pass array of Ts
+    * @param values The contained Ts must all have the same size
     */
-    SparseTensor( const std::vector<container>& values ): mat_idx_(3,-1.), values_(values){}
+    SparseTensor( const std::vector<T>& values ): mat_idx_(3,-1.), values_(values){}
 
     /**
-     * @brief Type conversion from other container types
-     * @tparam OtherContainer calls dg::blas1::transfer to convert OtherContainer to container
+     * @brief Type conversion from other value types
+     * @tparam OtherT dg::blas1::transfer must be callable for T and OtherT
      * @param src the source matrix to convert
      */
-    template<class OtherContainer>
-    SparseTensor( const SparseTensor<OtherContainer>& src): mat_idx_(3,-1.), values_(src.values().size()){
+    template<class OtherT>
+    SparseTensor( const SparseTensor<OtherT>& src): mat_idx_(3,-1.), values_(src.values().size()){
         for(unsigned i=0; i<3; i++)
             for(unsigned j=0; j<3; j++)
                 mat_idx_(i,j)=src.idx(i,j);
@@ -113,7 +118,7 @@ struct SparseTensor
     * @brief check if a value is set at the given position or not
     * @param i row index 0<=i<3
     * @param j col index 0<=j<3
-    * @return true if container is non-empty, false if value is assumed implicitly
+    * @return true if T is non-empty, false if value is assumed implicitly
     */
     bool isSet(size_t i, size_t j)const{
         if( mat_idx_(i,j) <0) return false;
@@ -151,24 +156,24 @@ struct SparseTensor
       */
      void clear_unused_values();
 
-    /*!@brief Read access the underlying container
+    /*!@brief Read access the underlying T
      * @return if !isSet(i,j) the result is undefined, otherwise values[idx(i,j)] is returned. 
      * @param i row index 0<=i<3
      * @param j col index 0<=j<3
      * @note If the indices fall out of range of index the result is undefined
      */
-    const container& value(size_t i, size_t j)const{ 
+    const T& value(size_t i, size_t j)const{ 
         int k = mat_idx_(i,j);
         return values_[k];
     }
     //if you're looking for this function: YOU DON'T NEED IT!!ALIASING
-    //container& value(size_t i, size_t j);
+    //T& value(size_t i, size_t j);
     /**
-     * @brief Return the container at given position, create one if there isn't one already
+     * @brief Return the T at given position, create one if there isn't one already
      * @param i index into the values array
-     * @return  always returns a container 
+     * @return  always returns a T 
      */
-    container& value( size_t i)
+    T& value( size_t i)
     {
         if(i>=values_.size() ) values_.resize(i+1);
         return values_[i];
@@ -177,7 +182,7 @@ struct SparseTensor
      * @brief Return read access to the values array
      * @return read access to values array
      */
-    const std::vector<container>& values()const{return values_;}
+    const std::vector<T>& values()const{return values_;}
     ///clear all values, Tensor is empty after that
     void clear(){
         mat_idx_=dg::Operator<int>(3,-1);
@@ -265,7 +270,7 @@ struct SparseTensor
 
     private:
     dg::Operator<int> mat_idx_;
-    std::vector<container> values_;
+    std::vector<T> values_;
     void unique_insert(std::vector<int>& indices, int& idx);
 };
 
@@ -275,7 +280,7 @@ struct SparseTensor
  * @brief data structure to hold the LDL^T decomposition of a symmetric positive definite matrix
  *
  * LDL^T stands for a lower triangular matrix L,  a diagonal matrix D and the transpose L^T
- * @tparam container a valid container class
+ * @copydoc hide_container_lvl1
  * @attention the tensor in the Elliptic classes actually only need to be positive **semi-definite**
  * and unfortunately the decomposition is unstable for semi-definite matrices.
 * @ingroup misc
@@ -292,6 +297,11 @@ struct CholeskyTensor
     {
         decompose(in);
     }
+    /**
+     * @brief Type conversion from other value types
+     * @tparam OtherContainer dg::blas1::transfer must be callable for container and OtherContainer
+     * @param src the source matrix to convert
+     */
     template<class OtherContainer>
     CholeskyTensor( const CholeskyTensor<OtherContainer>& in):q_(in.lower()),diag_(in.diagonal()),upper_(in.upper())
     { 
diff --git a/inc/dg/geometry/transform.h b/inc/dg/geometry/transform.h
index 8baf8e0b8..e54d9055b 100644
--- a/inc/dg/geometry/transform.h
+++ b/inc/dg/geometry/transform.h
@@ -1,15 +1,11 @@
 #pragma once
+#include "../backend/topological_traits.h"
+#include "multiply.h"
 
 
 namespace dg
 {
 ///@cond
-template<class Geometry>
-class GeometryTraits
-{
-    typedef typename MemoryTraits< typename TopologyTraits<Geometry>::memory_category>::host_vector host_vector;
-
-};
 template<class MemoryTag>
 struct MemoryTraits {
 };
@@ -19,6 +15,13 @@ struct MemoryTraits< SharedTag>
     typedef thrust::host_vector<double> host_vector;
 };
 
+template<class Geometry>
+class GeometryTraits
+{
+    typedef typename MemoryTraits< typename TopologyTraits<Geometry>::memory_category>::host_vector host_vector;
+
+};
+
 ///@endcond
 /**
  * @brief This function pulls back a function defined in some basic coordinates to the curvilinear coordinate system
@@ -114,8 +117,8 @@ void pushForwardPerp( Functor1 vR, Functor2 vZ,
 {
     typedef typename GeometryTraits< Geometry>::host_vector host_vec;
     host_vec out1 = pullback( vR, g), temp1(out1);
-    host_vec out2 = pullback( vZ, g), temp2(out2);
-    dg::tensor::multiply(g.map(), out1, out2, temp1, temp2);
+    host_vec out2 = pullback( vZ, g);
+    dg::tensor::multiply2d_inplace(g.jacobian(), out1, out2, temp1);
     dg::blas1::transfer( out1, vx);
     dg::blas1::transfer( out2, vy);
 }
@@ -137,7 +140,7 @@ void pushForwardPerp( Functor1 vR, Functor2 vZ,
  * @param g The geometry object
  * @ingroup pullback
  */
-template<class Functor1, class Functor2, class Functor3 class container, class Geometry> 
+template<class Functor1, class Functor2, class Functor3, class container, class Geometry> 
 void pushForward( Functor1 vR, Functor2 vZ, Functor3 vPhi,
         container& vx, container& vy, container& vz,
         const Geometry& g)
@@ -145,8 +148,8 @@ void pushForward( Functor1 vR, Functor2 vZ, Functor3 vPhi,
     typedef typename GeometryTraits< Geometry>::host_vector host_vec;
     host_vec out1 = pullback( vR, g), temp1(out1);
     host_vec out2 = pullback( vZ, g), temp2(out2);
-    host_vec out3 = pullback( vPhi, g), temp3(out3);
-    dg::tensor::multiply(g.map(), out1, out2, out3, temp1, temp2, temp3);
+    host_vec out3 = pullback( vPhi, g);
+    dg::tensor::multiply3d_inplace(g.jacobian(), out1, out2, out3, temp1, temp2);
     dg::blas1::transfer( out1, vx);
     dg::blas1::transfer( out2, vy);
     dg::blas1::transfer( out3, vz);
@@ -197,12 +200,12 @@ void pushForwardPerp( FunctorRR chiRR, FunctorRZ chiRZ, FunctorZZ chiZZ,
     SparseTensor<container> d = jac.dense(); //now we have a dense tensor
     container tmp00(d.getValue(0,0)), tmp01(tmp00), tmp10(tmp00), tmp11(tmp00);
     // multiply Chi*t -> tmp
-    dg::tensor::const_multiply( chi, d.getValue(0,0), d.getValue(1,0), tmp00, tmp10);
-    dg::tensor::const_multiply( chi, d.getValue(0,1), d.getValue(1,1), tmp01, tmp11);
+    dg::tensor::multiply2d( chi, d.getValue(0,0), d.getValue(1,0), tmp00, tmp10);
+    dg::tensor::multiply2d( chi, d.getValue(0,1), d.getValue(1,1), tmp01, tmp11);
     // multiply tT * tmp -> Chi
     SparseTensor<container> transpose = jac.transpose();
-    dg::tensor::multiply( transpose, tmp00, tmp01, chixx, chixy);
-    dg::tensor::multiply( transpose, tmp10, tmp11, chixy, chiyy);
+    dg::tensor::multiply2d( transpose, tmp00, tmp01, chixx, chixy);
+    dg::tensor::multiply2d( transpose, tmp10, tmp11, chixy, chiyy);
 }
 
 } //namespace dg
-- 
GitLab


From a3f3d912f7a44f49df5f4c532d1b17c3a08d2e7f Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Wed, 9 Aug 2017 09:12:53 -0700
Subject: [PATCH 143/453] small corrections in transform.h

---
 inc/dg/geometry/transform.h | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/inc/dg/geometry/transform.h b/inc/dg/geometry/transform.h
index e54d9055b..0f8c5e6fa 100644
--- a/inc/dg/geometry/transform.h
+++ b/inc/dg/geometry/transform.h
@@ -27,7 +27,7 @@ class GeometryTraits
  * @brief This function pulls back a function defined in some basic coordinates to the curvilinear coordinate system
  *
  * e.g. F(x,y) = f(R(x,y), Z(x,y)) in 2d 
- * @tparam Functor The binary or ternary function object 
+ * @tparam Functor The binary or ternary function class
  * @param f The function defined in cartesian coordinates
  * @param g a two- or three dimensional Geometry
  * @note Template deduction for the Functor will fail if you overload functions with different 
@@ -184,24 +184,24 @@ void pushForwardPerp( FunctorRR chiRR, FunctorRZ chiRZ, FunctorZZ chiZZ,
     host_vec chiRZ_ = pullback( chiRZ, g);
     host_vec chiZZ_ = pullback( chiZZ, g);
     //transfer to device
-    if(g.map().isEmpty())
+    if(g.jacobian().isEmpty())
     {
         chiRR_.swap(chixx);
         chiRZ_.swap(chixy);
         chiZZ_.swap(chiyy);
         return;
     }
-    const dg::SparseTensor<container> jac = g.map();
+    const dg::SparseTensor<container> jac = g.jacobian();
     std::vector<container> values( 3); 
     values[0] = chiRR_, values[1] = chiRZ_, values[2] = chiZZ_;
     SparseTensor<container> chi(values);
     chi(0,0)=0, chi(0,1)=chi(1,0)=1, chi(1,1)=2;
 
     SparseTensor<container> d = jac.dense(); //now we have a dense tensor
-    container tmp00(d.getValue(0,0)), tmp01(tmp00), tmp10(tmp00), tmp11(tmp00);
+    container tmp00(d.value(0,0)), tmp01(tmp00), tmp10(tmp00), tmp11(tmp00);
     // multiply Chi*t -> tmp
-    dg::tensor::multiply2d( chi, d.getValue(0,0), d.getValue(1,0), tmp00, tmp10);
-    dg::tensor::multiply2d( chi, d.getValue(0,1), d.getValue(1,1), tmp01, tmp11);
+    dg::tensor::multiply2d( chi, d.value(0,0), d.value(1,0), tmp00, tmp10);
+    dg::tensor::multiply2d( chi, d.value(0,1), d.value(1,1), tmp01, tmp11);
     // multiply tT * tmp -> Chi
     SparseTensor<container> transpose = jac.transpose();
     dg::tensor::multiply2d( transpose, tmp00, tmp01, chixx, chixy);
-- 
GitLab


From 87a75dd8d18512871c8e9da9c796fe3e804c38c0 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Fri, 11 Aug 2017 02:39:28 -0700
Subject: [PATCH 144/453] reinstated the 3d X-Point grid

---
 inc/dg/backend/derivativesX.h     | 199 ++++++++++++++++++
 inc/dg/backend/derivativesX_t.cu  |  47 ++++-
 inc/dg/backend/evaluationX.cuh    |  24 +++
 inc/dg/backend/gridX.h            | 333 +++++++++++++++++++++++++++++-
 inc/dg/backend/interpolationX.cuh |  35 ++++
 inc/dg/backend/weightsX.cuh       |  19 ++
 inc/dg/geometry/cartesianX.h      |  21 +-
 7 files changed, 661 insertions(+), 17 deletions(-)

diff --git a/inc/dg/backend/derivativesX.h b/inc/dg/backend/derivativesX.h
index 63c78adc5..73b0c6b66 100644
--- a/inc/dg/backend/derivativesX.h
+++ b/inc/dg/backend/derivativesX.h
@@ -204,6 +204,205 @@ Composite<EllSparseBlockMat<double> > jumpY( const aTopologyX2d& g)
     return jumpY( g, g.bcy());
 }
 
+///////////////////////////////////////////3D VERSIONS//////////////////////
+//jumpX, jumpY, jumpZ, dx, dy, dz
+/**
+ * @brief Matrix that contains jump terms in X direction in 3D
+ *
+ * @param g The 3D grid
+ * @param bcx boundary condition in x
+ *
+ * @return A host matrix 
+ */
+Composite<EllSparseBlockMat<double> > jumpX( const aTopologyX3d& g, bc bcx)
+{
+    EllSparseBlockMat<double>  jx;
+    jx = jump( g.n(), g.Nx(), g.hx(), bcx);
+    jx.left_size = g.n()*g.Ny()*g.Nz();
+    jx.set_default_range();
+    return jx;
+}
+
+/**
+ * @brief Matrix that contains jump terms in Y direction in 3D
+ *
+ * @param g The 3D grid
+ * @param bcy boundary condition in y
+ *
+ * @return A host matrix 
+ */
+Composite<EllSparseBlockMat<double> > jumpY( const aTopologyX3d& g, bc bcy)
+{
+    EllSparseBlockMat<double>  jy_inner, jy_outer;
+    GridX1d g1d_inner( g.y0(), g.y1(), g.fy(), g.n(), g.Ny(), bcy);
+    Grid1d g1d_outer( g.y0(), g.y1(), g.n(), g.Ny(), bcy);
+    jy_inner = jump( g1d_inner, bcy);
+    jy_outer = jump( g1d_outer, bcy);
+    jy_inner.right_size = g.n()*g.Nx();
+    jy_inner.right_range[0] = 0;
+    jy_inner.right_range[1] = g.n()*g.inner_Nx();
+    jy_outer.right_range[0] = g.n()*g.inner_Nx();
+    jy_outer.right_range[1] = g.n()*g.Nx();
+    jy_outer.right_size = g.n()*g.Nx();
+    jy_inner.left_size = g.Nz();
+    jy_outer.left_size = g.Nz();
+
+    Composite<EllSparseBlockMat<double> > c( jy_inner, jy_outer);
+    return c;
+}
+
+/**
+ * @brief Matrix that contains jump terms in Z direction in 3D
+ *
+ * @param g The 3D grid
+ * @param bcz boundary condition in z
+ *
+ * @return A host matrix 
+ */
+Composite<EllSparseBlockMat<double> > jumpZ( const aTopologyX3d& g, bc bcz)
+{
+    EllSparseBlockMat<double>  jz;
+    jz = jump( 1, g.Nz(), g.hz(), bcz);
+    jz.right_size = g.n()*g.Nx()*g.n()*g.Ny();
+    jz.set_default_range();
+    return jz;
+}
+
+/**
+ * @brief Matrix that contains 3d jump terms in X direction taking boundary conditions from the grid
+ *
+ * @param g grid
+ *
+ * @return A host matrix
+ */
+Composite<EllSparseBlockMat<double> > jumpX( const aTopologyX3d& g)
+{
+    return jumpX( g, g.bcx());
+}
+
+/**
+ * @brief Matrix that contains 3d jump terms in Y direction taking boundary conditions from the grid
+ *
+ * @param g grid
+ *
+ * @return A host matrix
+ */
+Composite<EllSparseBlockMat<double> > jumpY( const aTopologyX3d& g)
+{
+    return jumpY( g, g.bcy());
+}
+
+/**
+ * @brief Matrix that contains 3d jump terms in Z direction taking boundary conditions from the grid
+ *
+ * @param g grid
+ *
+ * @return A host matrix
+ */
+Composite<EllSparseBlockMat<double> > jumpZ( const aTopologyX3d& g)
+{
+    return jumpZ( g, g.bcz());
+}
+
+
+/**
+ * @brief Create 3d derivative in x-direction
+ *
+ * @param g The grid on which to create dx
+ * @param bcx The boundary condition
+ * @param dir The direction of the first derivative
+ *
+ * @return A host matrix 
+ */
+Composite<EllSparseBlockMat<double> > dx( const aTopologyX3d& g, bc bcx, direction dir = centered)
+{
+    EllSparseBlockMat<double>  dx;
+    dx = dx_normed( g.n(), g.Nx(), g.hx(), bcx, dir);
+    dx.left_size = g.n()*g.Ny()*g.Nz();
+    dx.set_default_range();
+    return dx;
+}
+
+/**
+ * @brief Create 3d derivative in x-direction
+ *
+ * @param g The grid on which to create dx (boundary condition is taken from here)
+ * @param dir The direction of the first derivative
+ *
+ * @return A host matrix 
+ */
+Composite<EllSparseBlockMat<double> > dx( const aTopologyX3d& g, direction dir = centered) { return dx( g, g.bcx(), dir);}
+
+/**
+ * @brief Create 3d derivative in y-direction
+ *
+ * @param g The grid on which to create dy
+ * @param bcy The boundary condition
+ * @param dir The direction of the first derivative
+ *
+ * @return A host matrix 
+ */
+Composite<EllSparseBlockMat<double> > dy( const aTopologyX3d& g, bc bcy, direction dir = centered)
+{
+    EllSparseBlockMat<double>  dy_inner, dy_outer;
+    GridX1d g1d_inner( g.y0(), g.y1(), g.fy(), g.n(), g.Ny(), bcy);
+    Grid1d g1d_outer( g.y0(), g.y1(), g.n(), g.Ny(), bcy);
+    dy_inner = dx( g1d_inner, bcy, dir);
+    dy_outer = dx( g1d_outer, bcy, dir);
+    dy_inner.right_size = g.n()*g.Nx();
+    dy_inner.right_range[0] = 0;
+    dy_inner.right_range[1] = g.n()*g.inner_Nx();
+    dy_outer.right_range[0] = g.n()*g.inner_Nx();
+    dy_outer.right_range[1] = g.n()*g.Nx();
+    dy_outer.right_size = g.n()*g.Nx();
+    dy_inner.left_size = g.Nz();
+    dy_outer.left_size = g.Nz();
+
+    Composite<EllSparseBlockMat<double> > c( dy_inner, dy_outer);
+    return c;
+}
+
+/**
+ * @brief Create 3d derivative in y-direction
+ *
+ * @param g The grid on which to create dy (boundary condition is taken from here)
+ * @param dir The direction of the first derivative
+ *
+ * @return A host matrix 
+ */
+Composite<EllSparseBlockMat<double> > dy( const aTopologyX3d& g, direction dir = centered){ return dy( g, g.bcy(), dir);}
+
+/**
+ * @brief Create 3d derivative in z-direction
+ *
+ * @param g The grid on which to create dz
+ * @param bcz The boundary condition
+ * @param dir The direction of the stencil
+ *
+ * @return A host matrix 
+ */
+Composite<EllSparseBlockMat<double> > dz( const aTopologyX3d& g, bc bcz, direction dir = centered)
+{
+    EllSparseBlockMat<double>  dz;
+    dz = dx_normed( 1, g.Nz(), g.hz(), bcz, dir);
+    dz.right_size = g.n()*g.n()*g.Nx()*g.Ny();
+    dz.set_default_range();
+    return dz;
+
+}
+
+/**
+ * @brief Create 3d derivative in z-direction
+ *
+ * @param g The grid on which to create dz (boundary condition is taken from here)
+ * @param dir The direction of the stencil
+ *
+ * @return A host matrix 
+ */
+Composite<EllSparseBlockMat<double> > dz( const aTopologyX3d& g, direction dir = centered){ return dz( g, g.bcz(), dir);}
+
+
+
 ///@}
 
 } //namespace create
diff --git a/inc/dg/backend/derivativesX_t.cu b/inc/dg/backend/derivativesX_t.cu
index 726e7fb6d..c306e6c94 100644
--- a/inc/dg/backend/derivativesX_t.cu
+++ b/inc/dg/backend/derivativesX_t.cu
@@ -34,6 +34,10 @@ double cosy( double x, double y) {
     }
     return sin(x)*cos(y);
 }
+double sin(  double x, double y, double z) { return sin(x,y)*sin(z);}
+double cosx( double x, double y, double z) { return cosx(x,y)*sin(z);}
+double cosy( double x, double y, double z) { return cosy(x,y)*sin(z);}
+double cosz( double x, double y, double z) { return sin(x,y)*cos(z);}
 
 
 typedef dg::DVec Vector;
@@ -43,14 +47,14 @@ typedef dg::Composite<dg::EllSparseBlockMatDevice<double> > Matrix;
 
 int main()
 {
-    unsigned n, Nx, Ny;
-    //std::cout << "Type in n, Nx (1./5.) and Ny (1./4.)!\n";
-    //std::cin >> n >> Nx >> Ny;
+    unsigned n, Nx, Ny, Nz;
+    //std::cout << "Type in n, Nx (1./5.) and Ny (1./4.) and Nz!\n";
+    //std::cin >> n >> Nx >> Ny >> Nz;
     //dg::bc bcx=dg::DIR_NEU, bcy=dg::DIR, bcz = dg::PER;
     //dg::GridX2d g2d( -2.*M_PI, M_PI/2., -M_PI, 2*M_PI+M_PI, 1./5., 1./4., n, Nx, Ny, bcx, bcy);
-    std::cout << "Type in n, Nx (1./3.) and Ny (1./6.)!\n";
-    std::cin >> n >> Nx >> Ny;
-    dg::bc bcx=dg::DIR, bcy=dg::NEU;
+    std::cout << "Type in n, Nx (1./3.) and Ny (1./6.) and Nz!\n";
+    std::cin >> n >> Nx >> Ny >> Nz;
+    dg::bc bcx=dg::DIR, bcy=dg::NEU, bcz = dg::PER;
     dg::GridX2d g2d( -2.*M_PI, M_PI, -M_PI/2., 2.*M_PI+M_PI/2., 1./3., 1./6., n, Nx, Ny, bcx, bcy);
     g2d.display(std::cout);
     //dg::GridX2d g2d( -2.*M_PI, M_PI/2., 0., 2*M_PI, 1./5., 0., n, Nx, Ny, bcx, bcy);
@@ -85,6 +89,37 @@ int main()
     dg::blas1::axpby( 1., tempX, 1., tempY, tempY);
     dg::blas1::axpby( 1., null2, -1., tempY);
     std::cout << "Distance to true solution: "<<sqrt(dg::blas2::dot(tempY, w2d, tempY))<<"\n";
+    //dg::GridX3d g3d( -2.*M_PI, M_PI/2., -M_PI, 2*M_PI+M_PI, 0., 2.*M_PI, 1./5., 1./4., n, Nx, Ny, Nz, bcx, bcy, bcz);
+    dg::GridX3d g3d( -2.*M_PI, M_PI, 0., 2*M_PI, 0., 2.*M_PI, 1./3., 0., n, Nx, Ny, Nz, bcx, bcy, bcz);
+    const Vector w3d = dg::create::weights( g3d);
+    Matrix dx3 = dg::create::dx( g3d, dg::forward);
+    Matrix dy3 = dg::create::dy( g3d, dg::backward);
+    Matrix dz3 = dg::create::dz( g3d, dg::centered);
+    Matrix jx3 = dg::create::jumpX( g3d);
+    Matrix jy3 = dg::create::jumpY( g3d);
+    Matrix jz3 = dg::create::jumpZ( g3d);
+    Matrix m3[] = {dx3, dy3, dz3, jx3, jy3, jz3};
+    const Vector f3d = dg::evaluate( sin, g3d);
+    const Vector dx3d = dg::evaluate( cosx, g3d);
+    const Vector dy3d = dg::evaluate( cosy, g3d);
+    const Vector dz3d = dg::evaluate( cosz, g3d);
+    const Vector null3 = dg::evaluate( zero, g3d);
+    Vector sol3[] = {dx3d, dy3d, dz3d, null3, null3, null3};
+
+    std::cout << "TEST 3D: DX, DY, DZ, JX, JY, JZ, JXY\n";
+    for( unsigned i=0; i<6; i++)
+    {
+        Vector error = f3d;
+        dg::blas2::symv( m3[i], f3d, error);
+        dg::blas1::axpby( 1., sol3[i], -1., error);
+        std::cout << "Distance to true solution: "<<sqrt(dg::blas2::dot(error, w3d, error))<<"\n";
+    }
+    Vector tX = f3d, tY(tX);
+    dg::blas2::symv( m3[3], f3d, tX);
+    dg::blas2::symv( m3[4], f3d, tY);
+    dg::blas1::axpby( 1., tX, 1., tY, tY);
+    dg::blas1::axpby( 1., null3, -1., tY);
+    std::cout << "Distance to true solution: "<<sqrt(dg::blas2::dot(tY, w3d, tY))<<"\n";
     //for periodic bc | dirichlet bc
     //n = 1 -> p = 2      2
     //n = 2 -> p = 1      1
diff --git a/inc/dg/backend/evaluationX.cuh b/inc/dg/backend/evaluationX.cuh
index 2c7b7774e..fbded8a26 100644
--- a/inc/dg/backend/evaluationX.cuh
+++ b/inc/dg/backend/evaluationX.cuh
@@ -61,6 +61,30 @@ thrust::host_vector<double> evaluate( double(f)(double, double), const aTopology
 };
 ///@endcond
 
+/**
+ * @brief Evaluate a function on gaussian abscissas
+ *
+ * Evaluates f(x,y,z) on the given grid
+ * @tparam TernaryOp Model of Ternary Function
+ * @param f The function to evaluate: f = f(x,y,z)
+ * @param g The 3d grid on which to evaluate f
+ *
+ * @return  A dG Host Vector with values
+ * @note Copies the ternary Operator. This function is meant for small function objects, that
+            may be constructed during function call.
+ */
+template< class TernaryOp>
+thrust::host_vector<double> evaluate( TernaryOp f, const aTopologyX3d& g)
+{
+    return evaluate( f, g.grid());
+};
+///@cond
+thrust::host_vector<double> evaluate( double(f)(double, double, double), const aTopologyX3d& g)
+{
+    return evaluate( f, g.grid());
+};
+///@endcond
+
 ///@}
 }//namespace dg
 
diff --git a/inc/dg/backend/gridX.h b/inc/dg/backend/gridX.h
index e40d7fc5a..fa36c8c2d 100644
--- a/inc/dg/backend/gridX.h
+++ b/inc/dg/backend/gridX.h
@@ -193,6 +193,7 @@ struct GridX1d
     DLT<double> dlt_;
 };
 
+struct aTopologyX3d; //forward declare 3d version
 
 /**
  * @brief A 2D grid class with X-point topology
@@ -329,12 +330,6 @@ struct aTopologyX2d
      * @return 
      */
     bc bcy() const {return bcy_;}
-    /**
-     * @brief Return a copy
-     *
-     * @return 
-     */
-    aTopologyX2d local_grid() const {return *this;}
     /**
      * @brief Return a copy without topology
      *
@@ -505,11 +500,335 @@ struct aTopologyX2d
 struct GridX2d : public aTopologyX2d
 {
     ///@copydoc aTopologyX2d::aTopologyX2d()
-    GridX2d( double x0, double x1, double y0, double y1, double fx, double fy, unsigned n, unsigned Nx, unsigned Ny, bc bcx=PER, bc bcy=PER):
+    GridX2d( double x0, double x1, double y0, double y1, double fx, double fy, unsigned n, unsigned Nx, unsigned Ny, bc bcx=PER, bc bcy=NEU):
         aTopologyX2d(x0,x1,y0,y1,fx,fy,n,Nx,Ny,bcx,bcy) { }
     ///allow explicit type conversion from any other topology
     explicit GridX2d( const aTopologyX2d& src): aTopologyX2d(src){}
 };
 
+/**
+ * @brief A 3D grid class  for cartesian coordinates
+ *
+ * In the third dimension only 1 polynomial coefficient is used,
+ * not n. In 2d it looks like
+ @code
+ | -----> y 
+ |  |---x----------x---|
+ |  |---x----------x---|
+ v  |--- ---------- ---|
+ x  |--- ---------- ---| fx*Lx
+    |--- ---------- ---|
+    fy*Ly
+ @endcode
+ * @tparam double scalar value type 
+ */
+struct aTopologyX3d
+{
+    typedef SharedTag memory_category;
+    typedef ThreeDimensionalTag dimensionality;
+    /**
+     * @brief left boundary in x
+     *
+     * @return 
+     */
+    double x0() const {return x0_;}
+    /**
+     * @brief right boundary in x
+     *
+     * @return 
+     */
+    double x1() const {return x1_;}
+
+    /**
+     * @brief left boundary in y 
+     *
+     * @return 
+     */
+    double y0() const {return y0_;}
+    /**
+     * @brief right boundary in y
+     *
+     * @return 
+     */
+    double y1() const {return y1_;}
+
+    /**
+     * @brief left boundary in z
+     *
+     * @return 
+     */
+    double z0() const {return z0_;}
+    /**
+     * @brief right boundary in z
+     *
+     * @return 
+     */
+    double z1() const {return z1_;}
+
+    /**
+     * @brief length in x
+     *
+     * @return 
+     */
+    double lx() const {return x1_-x0_;}
+    /**
+     * @brief length in y
+     *
+     * @return 
+     */
+    double ly() const {return y1_-y0_;}
+    /**
+     * @brief length in z
+     *
+     * @return 
+     */
+    double lz() const {return z1_-z0_;}
+    
+    /**
+     * @brief cell size in x
+     *
+     * @return 
+     */
+    double hx() const {return lx()/(double)Nx_;}
+    /**
+     * @brief cell size in y
+     *
+     * @return 
+     */
+    double hy() const {return ly()/(double)Ny_;}
+    /**
+     * @brief cell size in z
+     *
+     * @return 
+     */
+    double hz() const {return lz()/(double)Nz_;}
+    /**
+     * @brief Factor
+     *
+     * @return 
+     */
+    double fx() const {return fx_;}
+    /**
+     * @brief Factor
+     *
+     * @return 
+     */
+    double fy() const {return fy_;}
+    /**
+     * @brief number of polynomial coefficients in x and y
+     *
+     * @return 
+     */
+    unsigned n() const {return n_;}
+    /**
+     * @brief number of points in x
+     *
+     * @return 
+     */
+    unsigned Nx() const {return Nx_;}
+    /**
+     * @brief number of topological cells in x
+     *
+     * @return 
+     */
+    unsigned inner_Nx() const {return Nx_ - outer_Nx();}
+    /**
+     * @brief number of smooth rows in x
+     *
+     * @return 
+     */
+    unsigned outer_Nx() const {return (unsigned)round(fx_*(double)Nx_);}
+    /**
+     * @brief number of cells in y
+     *
+     * @return 
+     */
+    unsigned Ny() const {return Ny_;}
+    /**
+     * @brief number of cells in the inner region of y
+     *
+     * @return 
+     */
+    unsigned inner_Ny() const {return Ny_-2*outer_Ny();}
+    /**
+     * @brief number of cells in one of the outer regions of y
+     *
+     * @return 
+     */
+    unsigned outer_Ny() const {return (unsigned)round(fy_*(double)Ny_);}
+    /**
+     * @brief number of points in z
+     *
+     * @return 
+     */
+    unsigned Nz() const {return Nz_;}
+    /**
+     * @brief boundary conditions in x 
+     *
+     * @return 
+     */
+    bc bcx() const {return bcx_;}
+    /**
+     * @brief boundary conditions in y
+     *
+     * @return 
+     */
+    bc bcy() const {return bcy_;}
+    /**
+     * @brief boundary conditions in z 
+     *
+     * @return 
+     */
+    bc bcz() const {return bcz_;}
+    /**
+     * @brief Return a copy without topology
+     *
+     * @return 
+     */
+    Grid3d grid() const {return Grid3d( x0_,x1_,y0_,y1_,z0_,z1_,n_,Nx_,Ny_,Nz_,bcx_,bcy_,bcz_);}
+    /**
+     * @brief discrete legendre transformation
+     *
+     * @return 
+     */
+    const DLT<double>& dlt() const{return dlt_;}
+    /**
+     * @brief doublehe total number of points
+     *
+     * @return n*n*Nx*Ny*Nz
+     */
+    unsigned size() const { return n_*n_*Nx_*Ny_*Nz_;}
+    /**
+     * @brief Display 
+     *
+     * @param os output stream
+     */
+    void display( std::ostream& os = std::cout) const
+    {
+        os << "Grid parameters are: \n"
+            <<"    n  = "<<n_<<"\n"
+            <<"    Nx = "<<Nx_<<"\n"
+            <<"    inner Nx = "<<inner_Nx()<<"\n"
+            <<"    outer Nx = "<<outer_Nx()<<"\n"
+            <<"    Ny = "<<Ny_<<"\n"
+            <<"    inner Ny = "<<inner_Ny()<<"\n"
+            <<"    outer Ny = "<<outer_Ny()<<"\n"
+            <<"    Nz = "<<Nz_<<"\n"
+            <<"    hx = "<<hx()<<"\n"
+            <<"    hy = "<<hy()<<"\n"
+            <<"    hz = "<<hz()<<"\n"
+            <<"    x0 = "<<x0_<<"\n"
+            <<"    x1 = "<<x1_<<"\n"
+            <<"    y0 = "<<y0_<<"\n"
+            <<"    y1 = "<<y1_<<"\n"
+            <<"    z0 = "<<z0_<<"\n"
+            <<"    z1 = "<<z1_<<"\n"
+            <<"    lx = "<<lx()<<"\n"
+            <<"    ly = "<<ly()<<"\n"
+            <<"    lz = "<<lz()<<"\n"
+            <<"Boundary conditions in x are: \n"
+            <<"    "<<bc2str(bcx_)<<"\n"
+            <<"Boundary conditions in y are: \n"
+            <<"    "<<bc2str(bcy_)<<"\n"
+            <<"Boundary conditions in z are: \n"
+            <<"    "<<bc2str(bcz_)<<"\n";
+    }
+    /**
+     * @brief Check if the grid contains a point
+     *
+     * @note doesn't check periodicity!!
+     * @param x x-point to check
+     * @param y y-point to check
+     * @param z z-point to check
+     *
+     * @return true if x is between x0 and x1, false else
+     */
+    bool contains( double x, double y, double z)const
+    {
+        if( (x>=x0_ && x <= x1_) && (y>=y0_ && y <= y1_) && (z>=z0_ && z<=z1_)) 
+            return true; 
+        return false;
+    }
+    void init_X_boundaries( double x0, double x1) {
+        do_init_X_boundaries(x0,x1);
+    }
+  protected:
+    ///disallow destruction through base class pointer
+    ~aTopologyX3d(){}
+    /**
+     * @brief Construct a 3D grid
+     *
+     * @param x0 left boundary in x
+     * @param x1 right boundary in x 
+     * @param y0 lower boundary in y
+     * @param y1 upper boundary in y 
+     * @param z0 lower boundary in z
+     * @param z1 upper boundary in z 
+     * @param fx factor for x-direction
+     * @param fy factor for y-direction
+     * @param n  # of polynomial coefficients per (x-,y-) dimension
+     * @param Nx # of points in x 
+     * @param Ny # of points in y
+     * @param Nz # of points in z
+     * @param bcx boundary condition in x
+     * @param bcy boundary condition in y
+     * @param bcz boundary condition in z
+     * @attention # of polynomial coefficients in z direction is always 1
+     */
+    aTopologyX3d( double x0, double x1, double y0, double y1, double z0, double z1, double fx, double fy, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz):
+        x0_(x0), x1_(x1), y0_(y0), y1_(y1), z0_(z0), z1_(z1), fx_(fx), fy_(fy),
+        n_(n), Nx_(Nx), Ny_(Ny), Nz_(Nz), bcx_(bcx), bcy_( bcy), bcz_( bcz), dlt_(n)
+    {
+        assert( (fy_ >= 0.) && (fy_ < 0.5) );
+        assert( (fx_ >= 0.) && (fx_ < 1.) );
+        assert( fabs(outer_Nx() - fx_*(double)Nx) < 1e-15); 
+        assert( fabs(outer_Ny() - fy_*(double)Ny) < 1e-15); 
+        assert( n != 0);
+        assert( x1 > x0 && y1 > y0 ); assert( z1 > z0 );         
+        assert( Nx_ > 0  && Ny > 0); assert( Nz > 0);
+    }
+    aTopologyX3d(const aTopologyX3d& src){
+        x0_=src.x0_, x1_=src.x1_;
+        y0_=src.y0_, y1_=src.y1_;
+        z0_=src.z0_, z1_=src.z1_;
+        fx_=src.fx_, fy_=src.fy_;
+        n_=src.n_, Nx_=src.Nx_, Ny_=src.Ny_, Nz_=src.Nz_,bcx_=src.bcx_, bcy_=src.bcy_, bcz_=src.bcz_;
+        dlt_=src.dlt_;
+    }
+    aTopologyX3d& operator=(const aTopologyX3d& src){
+        x0_=src.x0_, x1_=src.x1_;
+        y0_=src.y0_, y1_=src.y1_;
+        z0_=src.z0_, z1_=src.z1_;
+        fx_=src.fx_, fy_=src.fy_;
+        n_=src.n_, Nx_=src.Nx_, Ny_=src.Ny_, Nz_=src.Nz_,bcx_=src.bcx_, bcy_=src.bcy_, bcz_=src.bcz_;
+        dlt_=src.dlt_;
+        return *this;
+    }
+    virtual void do_init_X_boundaries( double x0, double x1)
+    {
+        x0_ = x0, x1_ = x1;
+        assert( x1 > x0 );
+    }
+  private:
+    double x0_, x1_, y0_, y1_, z0_, z1_;
+    double fx_,fy_;
+    unsigned n_, Nx_, Ny_, Nz_;
+    bc bcx_, bcy_, bcz_;
+    DLT<double> dlt_;
+};
+
+/**
+ * @brief The simplest implementation of aTopologyX3d
+ * @ingroup grid
+ */
+struct GridX3d : public aTopologyX3d
+{
+    ///@copydoc aTopologyX2d::aTopologyX2d()
+    GridX3d( double x0, double x1, double y0, double y1, double z0, double z1, double fx, double fy, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx=PER, bc bcy=NEU, bc bcz=PER):
+        aTopologyX3d(x0,x1,y0,y1,z0,z1,fx,fy,n,Nx,Ny,Nz,bcx,bcy,bcz) { }
+    ///allow explicit type conversion from any other topology
+    explicit GridX3d( const aTopologyX3d& src): aTopologyX3d(src){}
+};
+
 
 }// namespace dg
diff --git a/inc/dg/backend/interpolationX.cuh b/inc/dg/backend/interpolationX.cuh
index 74ab2042f..3678353ea 100644
--- a/inc/dg/backend/interpolationX.cuh
+++ b/inc/dg/backend/interpolationX.cuh
@@ -44,6 +44,25 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const thrust::ho
 }
 
 
+
+/**
+ * @brief Create interpolation matrix
+ *
+ * The matrix, when applied to a vector, interpolates its values to the given coordinates. In z-direction only a nearest neighbor interpolation is used
+ * @param x X-coordinates of interpolation points
+ * @param y Y-coordinates of interpolation points
+ * @param z Z-coordinates of interpolation points
+ * @param g The Grid on which to operate
+ * @param globalbcz determines what to do if values lie exactly on the boundary
+ *
+ * @return interpolation matrix
+ * @note The values of x, y and z must lie within the boundaries of g
+ */
+cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const thrust::host_vector<double>& x, const thrust::host_vector<double>& y, const thrust::host_vector<double>& z, const aTopologyX3d& g, dg::bc globalbcz= dg::NEU)
+{
+    return interpolation( x,y,z, g.grid(), globalbcz);
+}
+
 /**
  * @brief Create interpolation between two grids
  *
@@ -77,6 +96,22 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const aTopologyX
     return interpolation( g_new.grid(), g_old.grid());
 }
 
+/**
+ * @brief Create interpolation between two grids
+ *
+ * This matrix can be applied to vectors defined on the old grid to obtain
+ * its values on the new grid.
+ * 
+ * @param g_new The new points 
+ * @param g_old The old grid
+ *
+ * @return Interpolation matrix
+ * @note The boundaries of the old grid must lie within the boundaries of the new grid
+ */
+cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const aTopologyX3d& g_new, const aTopologyX3d& g_old)
+{
+    return interpolation( g_new.grid(), g_old.grid());
+}
 ///@}
 
 /**
diff --git a/inc/dg/backend/weightsX.cuh b/inc/dg/backend/weightsX.cuh
index 505f38b64..92e7a31e9 100644
--- a/inc/dg/backend/weightsX.cuh
+++ b/inc/dg/backend/weightsX.cuh
@@ -49,6 +49,25 @@ thrust::host_vector<double> weights( const aTopologyX2d& g) { return weights( g.
 */
 thrust::host_vector<double> inv_weights( const aTopologyX2d& g) { return inv_weights( g.grid()); }
 
+/**
+* @brief create host_vector containing 3d X-space weight coefficients for integration
+*
+* @param g The grid 
+*
+* @return Host Vector
+*/
+thrust::host_vector<double> weights( const aTopologyX3d& g) { return weights(g.grid()); }
+
+/**
+* @brief create host_vector containing 3d X-space inverse weight coefficients
+*
+* @tparam T value type
+* @param g The grid 
+*
+* @return Host Vector
+*/
+thrust::host_vector<double> inv_weights( const aTopologyX3d& g) { return inv_weights(g.grid()); }
+
 ///@}
 }//namespace create
 }//namespace dg
diff --git a/inc/dg/geometry/cartesianX.h b/inc/dg/geometry/cartesianX.h
index f39839809..d34a83480 100644
--- a/inc/dg/geometry/cartesianX.h
+++ b/inc/dg/geometry/cartesianX.h
@@ -13,16 +13,29 @@ namespace dg
 /**
  * @brief two-dimensional Grid with Cartesian metric
  */
-struct CartesianGridX2d: public dg::GridX2d
+struct CartesianGridX2d: public dg::aTopologyX2d
 {
-    typedef OrthonormalTag metric_category; 
     ///@copydoc GridX2d::GridX2d()
-    CartesianGridX2d( double x0, double x1, double y0, double y1, double fx, double fy, unsigned n, unsigned Nx, unsigned Ny, bc bcx = PER, bc bcy = NEU):dg::GridX2d(x0,x1,y0,y1,fx,fy,n,Nx,Ny,bcx,bcy){}
+    CartesianGridX2d( double x0, double x1, double y0, double y1, double fx, double fy, unsigned n, unsigned Nx, unsigned Ny, bc bcx = PER, bc bcy = NEU):dg::aTopologyX2d(x0,x1,y0,y1,fx,fy,n,Nx,Ny,bcx,bcy){}
     /**
      * @brief Construct from existing topology
      * @param grid existing grid class
      */
-    CartesianGridX2d( const dg::GridX2d& grid):dg::GridX2d(grid){}
+    explicit CartesianGridX2d( const dg::GridX2d& grid):dg::aTopologyX2d(grid){}
+};
+
+/**
+ * @brief three-dimensional Grid with Cartesian metric
+ */
+struct CartesianGridX3d: public dg::aTopologyX3d
+{
+    ///@copydoc GridX3d::GridX3d()
+    CartesianGridX3d( double x0, double x1, double y0, double y1, double z0, double z1, double fx, double fy, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = PER, bc bcy = NEU, bc bcz = PER): dg::aTopologyX3d(x0,x1,y0,y1,z0,z1,fx,fy,n,Nx,Ny,Nz,bcx,bcy,bcz){}
+    /**
+     * @brief Construct from existing topology
+     * @param grid existing grid class
+     */
+    explicit CartesianGridX3d( const dg::GridX3d& grid):dg::aTopologyX3d(grid){}
 };
 
 ///@}
-- 
GitLab


From cb8fd7532864db6ce147b8303c9d5b60d75b8f11 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Fri, 11 Aug 2017 05:32:50 -0700
Subject: [PATCH 145/453] trying to work on refined grid

---
 inc/dg/geometry/refined_grid.h | 221 +++++++++++++++------------------
 1 file changed, 99 insertions(+), 122 deletions(-)

diff --git a/inc/dg/geometry/refined_grid.h b/inc/dg/geometry/refined_grid.h
index baa913f18..3b5da7a93 100644
--- a/inc/dg/geometry/refined_grid.h
+++ b/inc/dg/geometry/refined_grid.h
@@ -6,8 +6,7 @@
 #include "dg/backend/interpolation.cuh"
 #include "dg/blas.h"
 
-#include "cartesian.h"
-#include "cylindrical.h"
+#include "base_geometry.h"
 
 
 namespace dg
@@ -252,15 +251,57 @@ int linear_ref( unsigned multiple_x, const Grid1d& g, thrust::host_vector<double
 ///@endcond
 }//namespace detail
 
-struct RefinedGrid3d;
 ///@endcond
 /**
- * @brief Refined grid 
+ * @brief aRefined grid 
  * @deprecated
  * @ingroup grid
  */
-struct RefinedGrid2d : public dg::aTopology2d
+struct aRefinedGrid2d : public dg::aGeometry2d
 {
+
+    /**
+     * @brief The grid that this object refines
+     *
+     * This function is vitual so that derived classes can also construct the associated grid 
+     * @return  2d grid
+     */
+    virtual const dg::Grid2d& associated()const {return g_assoc_;}
+    /**
+     * @brief Return the abscissas in X-direction 
+     *
+     * @return A 2d vector
+     */
+    const thrust::host_vector<double>& abscissasX() const {return absX_;} 
+    /**
+     * @brief Return the abscissas in Y-direction 
+     *
+     * @return A 2d vector
+     */
+    const thrust::host_vector<double>& abscissasY() const {return absY_;} 
+    /**
+     * @brief Return the weights in X-direction 
+     *
+     * @return A 2d vector
+     */
+    const thrust::host_vector<double>& weightsX() const {return wx_;} 
+    /**
+     * @brief Return the weights in Y-direction 
+     *
+     * @return A 2d vector
+     */
+    const thrust::host_vector<double>& weightsY() const {return wy_;} 
+
+    protected:
+    void init_X_boundaries( double x0, double x1)
+    {
+        double alpha = (x1-x0)/this->lx();
+        double beta = (x0*this->x1()-x1*this->x0())/this->lx();
+        //weights are invariant
+        for( unsigned i=0; i<absX_.size(); i++)
+            absX_[i]=alpha*absX_[i]+beta;
+        dg::Grid2d::init_X_boundaries( x0, x1);
+    }
     /**
      * @brief Refine a corner of a grid
      *
@@ -272,10 +313,10 @@ struct RefinedGrid2d : public dg::aTopology2d
      * @param howmanyY Add number of cells to the existing one
      * @copydetails Grid2d::Grid2d()
      */
-    RefinedGrid2d( unsigned node_x, unsigned node_y, unsigned add_x, unsigned add_y, 
+    aRefinedGrid2d( unsigned node_x, unsigned node_y, unsigned add_x, unsigned add_y, 
             unsigned howmanyX, unsigned howmanyY,
             double x0, double x1, double y0, double y1, 
-            unsigned n, unsigned Nx, unsigned Ny, bc bcx = dg::PER, bc bcy = dg::PER) : dg::aTopology2d( x0, x1, y0, y1, n, n_new(Nx, add_x*howmanyX, bcx), n_new(Ny, add_y*howmanyY, bcy), bcx, bcy), 
+            unsigned n, unsigned Nx, unsigned Ny, bc bcx = dg::PER, bc bcy = dg::PER) : dg::aGeometry2d( x0, x1, y0, y1, n, n_new(Nx, add_x*howmanyX, bcx), n_new(Ny, add_y*howmanyY, bcy), bcx, bcy), 
         wx_(size()), wy_(size()), absX_(size()), absY_(size()),
         g_assoc_( x0, x1, y0, y1, n, Nx, Ny, bcx, bcy)
     {
@@ -305,9 +346,9 @@ struct RefinedGrid2d : public dg::aTopology2d
      * @param n_old the polynomials in the old grid
      * @copydetails Grid2d::Grid2d()
      */
-    RefinedGrid2d( unsigned multiple_x, unsigned multiple_y,
+    aRefinedGrid2d( unsigned multiple_x, unsigned multiple_y,
             double x0, double x1, double y0, double y1, unsigned n,
-            unsigned n_old, unsigned Nx, unsigned Ny, bc bcx = dg::PER, bc bcy = dg::PER) : dg::aTopology2d( x0, x1, y0, y1, n, multiple_x*Nx, multiple_y*Ny, bcx, bcy), 
+            unsigned n_old, unsigned Nx, unsigned Ny, bc bcx = dg::PER, bc bcy = dg::PER) : dg::aGeometry2d( x0, x1, y0, y1, n, multiple_x*Nx, multiple_y*Ny, bcx, bcy), 
         wx_(size()), wy_(size()), absX_(size()), absY_(size()),
         g_assoc_( x0, x1, y0, y1, n_old, Nx, Ny, bcx, bcy)
     {
@@ -327,21 +368,31 @@ struct RefinedGrid2d : public dg::aTopology2d
             }
     }
 
-    /**
-     * @brief Reduce from a 3d grid 
-     *
-     * This is possible because all our grids are product space grids. 
-     *
-     * @param g The 3d grid
-     */
-    RefinedGrid2d( const dg::RefinedGrid3d& g);
+    private:
+    unsigned n_new( unsigned N, unsigned factor, dg::bc bc)
+    {
+        if( bc == dg::PER) return N + 2*factor; 
+        return N + factor;
+    }
+    thrust::host_vector<double> wx_, wy_; //weights
+    thrust::host_vector<double> absX_, absY_; //abscissas 
+    dg::Grid2d g_assoc_;
+};
+
+/**
+ * @brief aRefined grid 
+ * @deprecated
+ * @ingroup grid
+ */
+struct aRefinedGrid3d : public dg::aGeometry3d
+{
     /**
      * @brief The grid that this object refines
      *
      * This function is vitual so that derived classes can also construct the associated grid 
      * @return  2d grid
      */
-    virtual const dg::Grid2d& associated()const {return g_assoc_;}
+    virtual const dg::Grid3d& associated()const {return g_assoc_;}
     /**
      * @brief Return the abscissas in X-direction 
      *
@@ -366,36 +417,16 @@ struct RefinedGrid2d : public dg::aTopology2d
      * @return A 2d vector
      */
     const thrust::host_vector<double>& weightsY() const {return wy_;} 
-
     protected:
     void init_X_boundaries( double x0, double x1)
     {
         double alpha = (x1-x0)/this->lx();
         double beta = (x0*this->x1()-x1*this->x0())/this->lx();
-        //weights are invariant
+        //weights are invariant under linear transformation
         for( unsigned i=0; i<absX_.size(); i++)
             absX_[i]=alpha*absX_[i]+beta;
-        dg::Grid2d::init_X_boundaries( x0, x1);
-    }
-
-    private:
-    unsigned n_new( unsigned N, unsigned factor, dg::bc bc)
-    {
-        if( bc == dg::PER) return N + 2*factor; 
-        return N + factor;
+        dg::Grid3d::init_X_boundaries( x0, x1);
     }
-    thrust::host_vector<double> wx_, wy_; //weights
-    thrust::host_vector<double> absX_, absY_; //abscissas 
-    dg::Grid2d g_assoc_;
-};
-
-/**
- * @brief Refined grid 
- * @deprecated
- * @ingroup grid
- */
-struct RefinedGrid3d : public dg::aTopology3d
-{
     /**
      * @brief Refine a corner of a grid
      *
@@ -407,10 +438,10 @@ struct RefinedGrid3d : public dg::aTopology3d
      * @param howmanyY howmany cells should be refined in y
      * @copydetails Grid3d::Grid3d()
      */
-    RefinedGrid3d( unsigned node_x, unsigned node_y, unsigned add_x, unsigned add_y, 
+    aRefinedGrid3d( unsigned node_x, unsigned node_y, unsigned add_x, unsigned add_y, 
             unsigned howmanyX, unsigned howmanyY,
             double x0, double x1, double y0, double y1, double z0, double z1, 
-            unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = dg::PER, bc bcy = dg::PER, bc bcz = dg::PER) : dg::aTopology3d( x0, x1, y0, y1, z0, z1, n, n_new(Nx, add_x*howmanyX, bcx), n_new(Ny, add_y*howmanyY, bcy), Nz, bcx, bcy, bcz), 
+            unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = dg::PER, bc bcy = dg::PER, bc bcz = dg::PER) : dg::aGeometry3d( x0, x1, y0, y1, z0, z1, n, n_new(Nx, add_x*howmanyX, bcx), n_new(Ny, add_y*howmanyY, bcy), Nz, bcx, bcy, bcz), 
         wx_(size()), wy_(size()), absX_(size()), absY_(size()),
         g_assoc_( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, bcx, bcy, bcz)
     {
@@ -440,12 +471,12 @@ struct RefinedGrid3d : public dg::aTopology3d
      * @param multiple_y Multiply all cells in y - direction
      * @copydetails Grid3d::Grid3d()
      */
-    RefinedGrid3d( unsigned multiple_x, unsigned multiple_y,
+    aRefinedGrid3d( unsigned multiple_x, unsigned multiple_y,
             double x0, double x1, double y0, double y1, double z0, double z1, 
             unsigned n,
             unsigned n_old, unsigned Nx, unsigned Ny, unsigned Nz, 
             bc bcx = dg::PER, bc bcy = dg::PER, bc bcz = dg::PER) : 
-        dg::aTopology3d( x0, x1, y0, y1, z0, z1, n, multiple_x*Nx, multiple_y*Ny, Nz, bcx, bcy, bcz), 
+        dg::aGeometry3d( x0, x1, y0, y1, z0, z1, n, multiple_x*Nx, multiple_y*Ny, Nz, bcx, bcy, bcz), 
         wx_(size()), wy_(size()), absX_(size()), absY_(size()),
         g_assoc_( x0, x1, y0, y1, z0, z1, n_old, Nx, Ny, Nz, bcx, bcy, bcz)
     {
@@ -465,47 +496,6 @@ struct RefinedGrid3d : public dg::aTopology3d
                     absY_[(s*wy.size()+i)*wx.size()+j] = ay[i];
                 }
     }
-    /**
-     * @brief The grid that this object refines
-     *
-     * This function is vitual so that derived classes can also construct the associated grid 
-     * @return  2d grid
-     */
-    virtual const dg::Grid3d& associated()const {return g_assoc_;}
-    /**
-     * @brief Return the abscissas in X-direction 
-     *
-     * @return A 2d vector
-     */
-    const thrust::host_vector<double>& abscissasX() const {return absX_;} 
-    /**
-     * @brief Return the abscissas in Y-direction 
-     *
-     * @return A 2d vector
-     */
-    const thrust::host_vector<double>& abscissasY() const {return absY_;} 
-    /**
-     * @brief Return the weights in X-direction 
-     *
-     * @return A 2d vector
-     */
-    const thrust::host_vector<double>& weightsX() const {return wx_;} 
-    /**
-     * @brief Return the weights in Y-direction 
-     *
-     * @return A 2d vector
-     */
-    const thrust::host_vector<double>& weightsY() const {return wy_;} 
-    protected:
-    void init_X_boundaries( double x0, double x1)
-    {
-        double alpha = (x1-x0)/this->lx();
-        double beta = (x0*this->x1()-x1*this->x0())/this->lx();
-        //weights are invariant under linear transformation
-        for( unsigned i=0; i<absX_.size(); i++)
-            absX_[i]=alpha*absX_[i]+beta;
-        dg::Grid3d::init_X_boundaries( x0, x1);
-    }
 
     private:
     unsigned n_new( unsigned N, unsigned factor, dg::bc bc)
@@ -516,38 +506,32 @@ struct RefinedGrid3d : public dg::aTopology3d
     thrust::host_vector<double> wx_, wy_; //weights
     thrust::host_vector<double> absX_, absY_; //abscissas 
     dg::Grid3d g_assoc_;
+    virtual SparseTensor<thrust::host_vector<double> > do_compute_metric()const {
 
-};
-
-RefinedGrid2d::RefinedGrid2d( const dg::RefinedGrid3d& g) : 
-    dg::Grid2d( g.x0(), g.x1(), g.y0(), g.y1(), g.n(), g.Nx(), g.Ny(), g.bcx(), g.bcy()),
-    wx_( this->size()), wy_(this->size()), absX_(this->size()), absY_(this->size()),
-    g_assoc_( g.associated())
-{
-    for(unsigned i=0; i<this->size(); i++)
-    {
-        wx_[i] = g.weightsX()[i];
-        wy_[i] = g.weightsY()[i];
-        absX_[i] = g.abscissasX()[i];
-        absY_[i] = g.abscissasY()[i];
+        return SparseTensor<thrust::host_vector<double> >();
+    }
+    virtual SparseTensor<thrust::host_vector<double> > do_compute_jacobian()const {
+        return SparseTensor<thrust::host_vector<double> >();
+    }
+    virtual std::vector<thrust::host_vector<double> > do_compute_map()const{
+        std::vector<thrust::host_vector<double> > map(2);
+        map[0] = dg::evaluate(dg::cooX2d, *this);
+        map[1] = dg::evaluate(dg::cooY2d, *this);
+        return map;
     }
-}
-
-
 
+};
 
 /**
  * @brief A refined cartesian grid
  *
  * @ingroup basicgrids
  * @deprecated
- * @tparam container
  */
-template<class container>
-struct CartesianRefinedGrid2d : public dg::RefinedGrid2d
+struct CartesianRefinedGrid2d : public dg::aRefinedGrid2d
 {
     typedef CurvilinearPerpTag metric_category; 
-    CartesianRefinedGrid2d( unsigned multiple_x, unsigned multiple_y, double x0, double x1, double y0, double y1, unsigned n, unsigned n_old, unsigned Nx, unsigned Ny, bc bcx = PER, bc bcy = PER): dg::RefinedGrid2d(multiple_x, multiple_y,x0,x1,y0,y1,n,n_old,Nx,Ny,bcx,bcy), g_assoc_(x0,x1,y0,y1,n_old,Nx,Ny,bcx,bcy){ 
+    CartesianaRefinedGrid2d( unsigned multiple_x, unsigned multiple_y, double x0, double x1, double y0, double y1, unsigned n, unsigned n_old, unsigned Nx, unsigned Ny, bc bcx = PER, bc bcy = PER): dg::aRefinedGrid2d(multiple_x, multiple_y,x0,x1,y0,y1,n,n_old,Nx,Ny,bcx,bcy), g_assoc_(x0,x1,y0,y1,n_old,Nx,Ny,bcx,bcy){ 
         dg::blas1::transfer( weightsX(), g_xx_);
         dg::blas1::transfer( weightsY(), g_yy_);
         dg::blas1::transfer( weightsX(), vol2d_);
@@ -557,24 +541,17 @@ struct CartesianRefinedGrid2d : public dg::RefinedGrid2d
         dg::blas1::pointwiseDivide( vol2d_, g_xx_, vol2d_);
         dg::blas1::pointwiseDivide( vol2d_, g_yy_, vol2d_);
     }
-    const thrust::host_vector<double>& r()const{return this->abscissasX();}
-    const thrust::host_vector<double>& z()const{return this->abscissasY();}
+
     const dg::CartesianGrid2d& associated() const {return g_assoc_;}
-    const container& g_xx()const{return g_xx_;}
-    const container& g_yy()const{return g_yy_;}
-    const container& vol()const{return vol2d_;}
-    const container& perpVol()const{return vol2d_;}
-    bool isOrthogonal()const{return true;}
-    bool isConformal()const{return false;}
+    virtual CartesianRefinedGrid2d* clone()const{return new CartesianRefinedGrid2d(*this);}
     private:
-    container g_xx_, g_yy_, vol2d_;
     dg::CartesianGrid2d g_assoc_;
 };
 
 namespace create{
 
 ///@deprecated
-cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const dg::RefinedGrid2d& g_fine)
+cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const dg::aRefinedGrid2d& g_fine)
 {
     dg::Grid2d g = g_fine.associated();
     thrust::host_vector<double> x = g_fine.abscissasX();
@@ -583,7 +560,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const dg::Refine
 
 }
 ///@deprecated
-cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const dg::RefinedGrid2d& g_fine)
+cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const dg::aRefinedGrid2d& g_fine)
 {
     cusp::coo_matrix<int, double, cusp::host_memory> temp = interpolation( g_fine), A;
     cusp::transpose( temp, A);
@@ -591,7 +568,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const dg::Refin
 }
 
 ///@deprecated
-cusp::coo_matrix<int, double, cusp::host_memory> projection( const dg::RefinedGrid2d& g_fine)
+cusp::coo_matrix<int, double, cusp::host_memory> projection( const dg::aRefinedGrid2d& g_fine)
 {
     //form the adjoint
     thrust::host_vector<double> w_f = dg::create::weights( g_fine);
@@ -617,7 +594,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> projection( const dg::RefinedGr
 }
 
 ///@deprecated
-cusp::coo_matrix<int, double, cusp::host_memory> smoothing( const dg::RefinedGrid2d& g)
+cusp::coo_matrix<int, double, cusp::host_memory> smoothing( const dg::aRefinedGrid2d& g)
 {
     cusp::coo_matrix<int, double, cusp::host_memory> A = interpolation(g);
     cusp::coo_matrix<int, double, cusp::host_memory> B = projection(g);
@@ -628,7 +605,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> smoothing( const dg::RefinedGri
 }
 
 ///@deprecated
-cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const dg::RefinedGrid3d& g_fine)
+cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const dg::aRefinedGrid3d& g_fine)
 {
     dg::Grid3d g = g_fine.associated();
     thrust::host_vector<double> x = g_fine.abscissasX();
@@ -638,7 +615,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const dg::Refine
 }
 
 ///@deprecated
-cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const dg::RefinedGrid3d& g_fine)
+cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const dg::aRefinedGrid3d& g_fine)
 {
     cusp::coo_matrix<int, double, cusp::host_memory> temp = interpolation( g_fine), A;
     cusp::transpose( temp, A);
@@ -646,7 +623,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const dg::Refin
 }
 
 ///@deprecated
-cusp::coo_matrix<int, double, cusp::host_memory> projection( const dg::RefinedGrid3d& g_fine)
+cusp::coo_matrix<int, double, cusp::host_memory> projection( const dg::aRefinedGrid3d& g_fine)
 {
     //form the adjoint
     thrust::host_vector<double> w_f = dg::create::weights( g_fine);
@@ -672,7 +649,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> projection( const dg::RefinedGr
 }
 
 ///@deprecated
-cusp::coo_matrix<int, double, cusp::host_memory> smoothing( const dg::RefinedGrid3d& g)
+cusp::coo_matrix<int, double, cusp::host_memory> smoothing( const dg::aRefinedGrid3d& g)
 {
     cusp::coo_matrix<int, double, cusp::host_memory> A = interpolation(g);
     cusp::coo_matrix<int, double, cusp::host_memory> B = projection(g);
-- 
GitLab


From fce26ca8e50e64f5259d53c6735e811c684de9f8 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Fri, 11 Aug 2017 23:24:26 +0200
Subject: [PATCH 146/453] introduced 1d Refinement base class

---
 inc/dg/dg_doc.h                |  18 +-
 inc/dg/geometry/refined_grid.h | 582 +++++++++++++++++----------------
 2 files changed, 294 insertions(+), 306 deletions(-)

diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index 7bdd9320c..43dba4025 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -73,6 +73,7 @@
  *        @defgroup pullback pullback and pushforward
  *        @defgroup metric create volume
  *        @defgroup utilities Fieldalignment and Averaging
+ *        @defgroup generators Grid Generator classes
  *            The classes to perform field line integration for DS and averaging classes
  *    @}
  * @}
@@ -99,23 +100,6 @@
  * @}
  * 
  */
-//* @mainpage
-// * Welcome to the DG library. 
-// *
-// * @par Design principles
-// *
-// * The DG library is built on top of the <a href="https://thrust.github.io/">thrust</a> and <a href="http://cusplibrary.github.io/index.html">cusp</a> libraries. 
-// * Its intention is to provide easy to use
-// * functions and objects needed for the integration of 2D and 3D partial differential equations discretized with a
-// * discontinuous galerkin method.  
-// * Since it is built on top of <a href="https://thrust.github.io/">thrust</a> and <a href="http://cusplibrary.github.io/index.html">cusp</a>, code can run on a CPU as well as a GPU by simply 
-// * switching between thrust's host_vector and device_vector. 
-// * The DG library uses a design pattern also employed in the cusp library and other modern C++ codes. 
-// * It might be referred to as <a href="http://dx.doi.org/10.1063/1.168674">container-free numerical algorithms</a>. 
-// *
-// *
-// *
-// */
 
  /**
   * @class hide_container_lvl1
diff --git a/inc/dg/geometry/refined_grid.h b/inc/dg/geometry/refined_grid.h
index baa913f18..df025ad22 100644
--- a/inc/dg/geometry/refined_grid.h
+++ b/inc/dg/geometry/refined_grid.h
@@ -6,140 +6,16 @@
 #include "dg/backend/interpolation.cuh"
 #include "dg/blas.h"
 
-#include "cartesian.h"
-#include "cylindrical.h"
+#include "base_geometry.h"
 
 
 namespace dg
 {
-    //actually there shouldn't be a reason to manually refine a grid when we can use our generators instead
 
-///@cond
 namespace detail
 {
-    ///@cond
-
-/**
- * @brief Divide cells in an equally distributed number of new cells
- *
- * @param add_x the number of cells to add 
- * @param node the node around which to refine
- * @param n polynomials
- * @param N # of cells
- * @param bcx boundary condition
- * @param howmany number of cells to refine
- *
- * @return Weights
- */
-thrust::host_vector<double> equidist_ref( unsigned add_x, unsigned node, unsigned n, unsigned N, dg::bc bcx, unsigned howmany)
-{
-    assert( howmany <= N);
-    assert( node <= N);
-    if( node != 0 && node != N) assert( howmany <= node && howmany <= N-node);
-    if( add_x == 0 || howmany == 0)
-    {
-        thrust::host_vector<double> w_( n*N, 1);
-        return w_;
-    }
-    assert( node <= N);
-    //there are add_x+1 finer cells per refined cell ...
-    thrust::host_vector< double> left( n*N+n*add_x*howmany, 1), right(left);
-    for( unsigned i=0; i<(add_x+1)*howmany; i++)//the original cell and the additional ones
-        for( unsigned k=0; k<n; k++)
-            left[i*n+k] = add_x + 1;
-    //mirror left into right
-    for( unsigned i=0; i<right.size(); i++)
-        right[i] = left[ (left.size()-1)-i];
-    thrust::host_vector< double> both( n*N+2*n*add_x*howmany, 1);
-    for( unsigned i=0; i<left.size(); i++)
-        both[i] *= left[i];
-    for( unsigned i=0; i<right.size(); i++)
-        both[i+n*add_x*howmany] *= right[i];
-    if(      node == 0     && bcx != dg::PER) { return left; }
-    else if( node == N && bcx != dg::PER) { return right; }
-    else if((node == N || node == 0) && bcx == dg::PER) { return both; }
-    else 
-    {
-        thrust::host_vector<double> w_ = both;
-        //now shift indices so that refinement is around nodes
-        for( unsigned i=0; i<both.size(); i++)
-            w_[((howmany*add_x+node)*n+i)%both.size()] = both[i];
-        return w_;
-    }
-}
-
-/**
- * @brief Divide cells in an equally distributed number of new cells
- *
- * @param add_x the number of cells to add 
- * @param node the node around which to refine
- * @param n polynomials
- * @param N # of cells
- * @param bcx boundary condition
- *
- * @return Weights
- */
-thrust::host_vector<double> equidist_ref( unsigned add_x, unsigned node, unsigned n, unsigned N, dg::bc bcx)
-{
-    return equidist_ref(add_x, node, n, N, bcx, 1);
-}
-
-thrust::host_vector<double> exponential_ref( unsigned add_x, unsigned node, unsigned n, unsigned N, dg::bc bcx)
-{
-    if( add_x == 0)
-    {
-        thrust::host_vector<double> w_( n*N, 1);
-        return w_;
-    }
-    assert( node <= N);
-    //there are add_x+1 finer cells per refined cell ...
-    thrust::host_vector< double> left( n*N+n*add_x, 1), right(left);
-    for( unsigned k=0; k<n; k++)//the original cell and the additional ones
-        left[k] = pow( 2, add_x);
-    for( unsigned i=0; i<add_x; i++) 
-        for( unsigned k=0; k<n; k++)
-            left[(i+1)*n+k] = pow( 2, add_x-i);
-    //mirror left into right
-    for( unsigned i=0; i<right.size(); i++)
-        right[i] = left[ (left.size()-1)-i];
-    thrust::host_vector< double> both( n*N+2*n*add_x, 1);
-    for( unsigned i=0; i<left.size(); i++)
-        both[i] *= left[i];
-    for( unsigned i=0; i<right.size(); i++)
-        both[i+n*add_x] *= right[i];
-    if(      node == 0     && bcx != dg::PER) { return left; }
-    else if( node == N && bcx != dg::PER) { return right; }
-    else if((node == N || node == 0) && bcx == dg::PER) { return both; }
-    else 
-    {
-        thrust::host_vector<double> w_ = both;
-        //now shift indices so that refinement is around nodes
-        for( unsigned i=0; i<both.size(); i++)
-            w_[((add_x+node)*n+i)%both.size()] = both[i];
-        return w_;
-    }
-}
-
-/**
- * @brief Refine every cell in the grid by an integer number of new cells
- *
- * @param multiple_x
- * @param n
- * @param N
- * @param bcx
- *
- * @return 
- */
-thrust::host_vector<double> linear_ref( unsigned multiple_x, unsigned n, unsigned N, dg::bc bcx)
-{
-    assert( multiple_x >= 1);
-    //there are add_x+1 finer cells per refined cell ...
-    thrust::host_vector< double> left( n*N*multiple_x, 1);
-    for( unsigned k=0; k<left.size(); k++)
-        left[k] = (double)multiple_x;
-    return left;
-}
 
+///@cond
 /**
  * @brief Normalize the given weights and compute the abscissas of the grid
  *
@@ -168,99 +44,281 @@ thrust::host_vector<double> normalize_weights_and_compute_abscissas( const Grid1
     }
     return abs;
 }
+///@endcond
+
+}//namespace detail
+
+///@addtogroup generators
+///@{
 
 /**
- * @brief Create 1d refinement weights and abscissas for the exponential refinement around a node 
- *
- * There will be two refined cells at the end except if a corner node is 
- * given and the boundary condition is not periodic. We count nodes from
- * 0 (left corner) to N (right corner). 
- * @param add_x number of additional cells in the cells idx-1 and idx
- * @param node The cells node-1 and node will be refined
- * @param g The 1d grid to refine
- *
- * @param weights A 1d vector of size n*(Nx+add_x) for one-sided refinement and n*(Nx+2*add_x)) for two-sided refinement
- * @param abscissas A 1d vector of size n*(Nx+add_x) for one-sided refinement and n*(Nx+2*add_x)) for two-sided refinement
- * @return the new number of cells
- */
-int exponential_ref( unsigned add_x, unsigned node, const Grid1d& g, thrust::host_vector<double>& weights, thrust::host_vector<double>& abscissas)
+* @brief Abstract base class for 1d grid refinement that increases the number of grid cells of a fixed basis grid
+*/
+struct aRefinement1d
 {
-    if( add_x == 0)
+    /*! @brief Generate the grid transformation
+     *  
+     * @param g The 1d grid to refine
+     * @param weights A 1d vector of size N_new. These represent the Jacobian of the transformation \f[\frac{\partial \zeta}{\partial x} \f]. The new metric element has thus to be multiplied by weights^2 and the volume by 1/weights
+     * @param abscissas A 1d vector of size N_new. These are the new abscissas \f$ x(\zeta) \f$ of the grid. 
+    */
+    void generate( const Grid1d& g_old, thrust::host_vector<double>& weights, thrust::host_vector<double>& abscissas)
     {
-        thrust::host_vector<double> w_( g.size(), 1);
-        thrust::host_vector<double> abs_= dg::create::abscissas(g);
-        weights = w_; abscissas = abs_; 
-        return g.N();
+        weights.resize( new_size(g_old.N(), g_old.bcx()));
+        abscissas.resize( new_size(g_old.N(), g_old.bcx()));
+        do_generate(g_old,weights,abscissas);
     }
-    weights = exponential_ref( add_x, node, g.n(), g.N(), g.bcx());
-    unsigned Nx_new = weights.size()/g.n();
-    abscissas = normalize_weights_and_compute_abscissas( g, weights);
-    return Nx_new;
-}
+    /*! @brief the new number of cells
+     * @param N_old the old number of cells
+     * @param bcx the boundary condition of the grid
+     */
+    unsigned N_new( unsigned N_old, bc bcx)
+    {
+        do_N_new(N_old, bcx);
+    }
+    private:
+    virtual void do_generate( const Grid1d& g, thrust::host_vector<double>& weights, thrust::host_vector<double>& abscissas) const =0;
+    virtual unsigned new_size( unsigned N_old, bc bcx) const =0;
+};
 
 /**
- * @brief Create 1d refinement weights and abscissas for the equidist refinement around a node 
- *
- * There will be two*howmany refined cells at the end except if a corner node is 
- * given and the boundary condition is not periodic. We count nodes from
- * 0 (left corner) to N (right corner). 
- * @param add_x number of additional cells in the cells idx-1 and idx
- * @param node The cells node-1 and node will be refined
- * @param g The 1d grid to refine
- *
- * @param weights A 1d vector of size n*(Nx+add_x*howmany) for one-sided refinement and n*(Nx+2*add_x*howmany)) for two-sided refinement
- * @param abscissas A 1d vector of size n*(Nx+add_x*howmany) for one-sided refinement and n*(Nx+2*add_x*howmany)) for two-sided refinement
- * @param howmany  number of cells around a node to refine
- * @return the new number of cells
+ * @brief Linear refinement consists of multiplying every cell in the grid by a factor
  */
-int equidist_ref( unsigned add_x, unsigned node, const Grid1d& g, thrust::host_vector<double>& weights, thrust::host_vector<double>& abscissas, unsigned howmany)
+struct LinearRefinement : public aRefinement1d
 {
-    if( add_x == 0 || howmany == 0)
+    /**
+     * @brief Refine every cell in the grid by an integer number of new cells
+     * @param multiple multiply every cell
+     */
+    LinearRefinement( unsigned multiple): m_(multiple){}
+    private:
+    unsigned m_;
+    virtual void do_generate( const Grid1d& g, thrust::host_vector<double>& weights, thrust::host_vector<double>& abscissas) const 
     {
-        thrust::host_vector<double> w_( g.size(), 1);
-        thrust::host_vector<double> abs_= dg::create::abscissas(g);
-        weights = w_; abscissas = abs_; 
-        return g.N();
+        weights = linear_ref( m_, g.n(), g.N(), g.bcx());
+        abscissas = dg::detail::normalize_weights_and_compute_abscissas( g, weights);
     }
-    weights = equidist_ref( add_x, node, g.n(), g.N(), g.bcx(), howmany);
-    unsigned Nx_new = weights.size()/g.n();
-    abscissas = normalize_weights_and_compute_abscissas( g, weights);
-    return Nx_new;
-}
-int equidist_ref( unsigned add_x, unsigned node, const Grid1d& g, thrust::host_vector<double>& weights, thrust::host_vector<double>& abscissas)
+    virtual unsigned do_N_new( unsigned N_old, bc bcx) const
+    {
+        return N_old*m_;
+    }
+    thrust::host_vector<double> linear_ref( unsigned multiple_x, unsigned n, unsigned N, dg::bc bcx)
+    {
+        assert( multiple_x >= 1);
+        //there are add_x+1 finer cells per refined cell ...
+        thrust::host_vector< double> left( n*N*multiple_x, 1);
+        for( unsigned k=0; k<left.size(); k++)
+            left[k] = (double)multiple_x;
+        return left;
+    }
+
+};
+
+/**
+ * @brief Equidistant cell refinement around a given node 
+ */
+struct EquidistRefinement : public aRefinement1d
 {
-    return equidist_ref( add_x, node, g, weights, abscissas, 1);
-}
+    /**
+     * @brief Divide a number of cells left and right of a node into an equidistant number of new cells
+     *
+     * There will be 2*howmany refined cells (each containing 1+add_x equidistant cells) at the end except if a 
+     * corner node is given and the boundary condition is not periodic. 
+     * @param add_x number of additional cells in the cells around the node
+     * @param node the node around which to refine. We count nodes from 0 (left corner) to N (right corner). 
+     * @param howmany  number of cells around a node to refine
+     */
+    EquidistRefinement( unsigned add_x, unsigned node, unsigned howmany=1): add_x_(add_x), node_(node), howm_(howmany){ }
+    private:
+    unsigned add_x_, node_, howm_;
+    virtual void do_generate( const Grid1d& g, thrust::host_vector<double>& weights, thrust::host_vector<double>& abscissas) const 
+    {
+        if( add_x_ == 0 || howm_ == 0)
+        {
+            thrust::host_vector<double> w_( g.size(), 1);
+            abscissas = dg::create::abscissas(g);
+            weights = w_; 
+            return;
+        }
+        weights = equidist_ref( add_x_, node_, g.n(), g.N(), g.bcx(), howm_);
+        abscissas = normalize_weights_and_compute_abscissas( g, weights);
+    }
+    virtual unsigned do_N_new( unsigned N_old, bc bcx) const
+    {
+        if( bc == dg::PER) return N_old + 2*add_x_*howm_; 
+        return N_old + add_x_*howm_;
+    }
+    thrust::host_vector<double> equidist_ref( unsigned add_x, unsigned node, unsigned n, unsigned N, dg::bc bcx, unsigned howmany)
+    {
+        assert( howm_ <= N);
+        assert( node_ <= N);
+        if( node_ != 0 && node_ != N) 
+            assert( howm_ <= node_ && howm_ <= N-node_);
+        if( add_x_ == 0 || howm_ == 0)
+        {
+            thrust::host_vector<double> w_( n*N, 1);
+            return w_;
+        }
+        //there are add_x+1 finer cells per refined cell ...
+        thrust::host_vector< double> left( n*N+n*add_x_*howm_, 1), right(left);
+        for( unsigned i=0; i<(add_x_+1)*howm_; i++)//the original cell and the additional ones
+            for( unsigned k=0; k<n; k++)
+                left[i*n+k] = add_x_ + 1;
+        //mirror left into right
+        for( unsigned i=0; i<right.size(); i++)
+            right[i] = left[ (left.size()-1)-i];
+        thrust::host_vector< double> both( n*N+2*n*add_x_*howm_, 1);
+        for( unsigned i=0; i<left.size(); i++)
+            both[i] *= left[i];
+        for( unsigned i=0; i<right.size(); i++)
+            both[i+n*add_x_*howm_] *= right[i];
+        if(      node_ == 0     && bcx != dg::PER) { return left; }
+        else if( node_ == N && bcx != dg::PER) { return right; }
+        else if((node_ == N || node_ == 0) && bcx == dg::PER) { return both; }
+        else 
+        {
+            thrust::host_vector<double> w_ = both;
+            //now shift indices so that refinement is around node_s
+            for( unsigned i=0; i<both.size(); i++)
+                w_[((howm_*add_x_+node_)*n+i)%both.size()] = both[i];
+            return w_;
+        }
+    }
+
+};
+
 /**
- * @brief Create 1d refinement weights and abscissas for the linear refinement 
- *
- * @param multiple_x number of additional cells in the cells 
- * @param g The 1d grid to refine
- *
- * @param weights A 1d vector of size n*(Nx+add_x) for one-sided refinement and n*(Nx+2*add_x)) for two-sided refinement
- * @param abscissas A 1d vector of size n*(Nx+add_x) for one-sided refinement and n*(Nx+2*add_x)) for two-sided refinement
- * @return the new number of cells
+ * @brief The exponential refinement around a node 
  */
-int linear_ref( unsigned multiple_x, const Grid1d& g, thrust::host_vector<double>& weights, thrust::host_vector<double>& abscissas)
+struct ExponentialRefinement : public aRefinement1d
 {
-    weights = linear_ref( multiple_x, g.n(), g.N(), g.bcx());
-    unsigned Nx_new = weights.size()/g.n();
-    abscissas = normalize_weights_and_compute_abscissas( g, weights);
-    return Nx_new;
-}
+    /**
+     * @brief Construct exponential refinement
+     *
+     * There will be two refined cells at the end except if a corner node is 
+     * given and the boundary condition is not periodic.  
+     * @param add_x number of additional cells in the cells node-1 and node. Every new additional cell divides the cell closest to the node in half.
+     * @param node The cells node-1 and node will be refined. We count nodes from
+     * 0 (left corner) to N (right corner).
+     */
+    ExponentialRefinement( unsigned add_x, unsigned node): add_x_(add_x), node_(node) {}
+    private:
+    unsigned add_x_, node_;
+    virtual void do_generate( const Grid1d& g, thrust::host_vector<double>& weights, thrust::host_vector<double>& abscissas) const 
+    {
+        if( add_x_ == 0)
+        {
+            thrust::host_vector<double> w_( g.size(), 1);
+            abscissas= dg::create::abscissas(g);
+            weights = w_; 
+            return;
+        }
+        weights = exponential_ref( add_x_, node_, g.n(), g.N(), g.bcx());
+        abscissas = normalize_weights_and_compute_abscissas( g, weights);
+    }
+    virtual unsigned do_N_new( unsigned N_old, bc bcx) const
+    {
+        if( bc == dg::PER) return N_old + 2*add_x_; 
+        return N_old + add_x_;
+    }
+    thrust::host_vector<double> exponential_ref( unsigned add_x, unsigned node, unsigned n, unsigned N, dg::bc bcx)
+    {
+        if( add_x_ == 0)
+        {
+            thrust::host_vector<double> w_( n*N, 1);
+            return w_;
+        }
+        assert( node_ <= N);
+        //there are add_x_+1 finer cells per refined cell ...
+        thrust::host_vector< double> left( n*N+n*add_x_, 1), right(left);
+        for( unsigned k=0; k<n; k++)//the original cell and the additional ones
+            left[k] = pow( 2, add_x_);
+        for( unsigned i=0; i<add_x_; i++) 
+            for( unsigned k=0; k<n; k++)
+                left[(i+1)*n+k] = pow( 2, add_x_-i);
+        //mirror left into right
+        for( unsigned i=0; i<right.size(); i++)
+            right[i] = left[ (left.size()-1)-i];
+        thrust::host_vector< double> both( n*N+2*n*add_x_, 1);
+        for( unsigned i=0; i<left.size(); i++)
+            both[i] *= left[i];
+        for( unsigned i=0; i<right.size(); i++)
+            both[i+n*add_x_] *= right[i];
+        if(      node_ == 0     && bcx != dg::PER) { return left; }
+        else if( node_ == N && bcx != dg::PER) { return right; }
+        else if((node_ == N || node_ == 0) && bcx == dg::PER) { return both; }
+        else 
+        {
+            thrust::host_vector<double> w_ = both;
+            //now shift indices so that refinement is around node_s
+            for( unsigned i=0; i<both.size(); i++)
+                w_[((add_x_+node_)*n+i)%both.size()] = both[i];
+            return w_;
+        }
+    }
+};
 
-///@endcond
-}//namespace detail
+///@}
 
-struct RefinedGrid3d;
-///@endcond
+struct aRefinedGrid3d;
 /**
- * @brief Refined grid 
+ * @brief aRefined grid 
  * @deprecated
  * @ingroup grid
  */
-struct RefinedGrid2d : public dg::aTopology2d
+struct aRefinedGrid2d : public dg::aGeometry2d
 {
+
+    /**
+     * @brief Reduce from a 3d grid 
+     *
+     * This is possible because all our grids are product space grids. 
+     *
+     * @param g The 3d grid
+     */
+    aRefinedGrid2d( const dg::aRefinedGrid3d& g);
+    /**
+     * @brief The grid that this object refines
+     *
+     * This function is vitual so that derived classes can also construct the associated grid 
+     * @return  2d grid
+     */
+    virtual const dg::Grid2d& associated()const {return g_assoc_;}
+    /**
+     * @brief Return the abscissas in X-direction 
+     *
+     * @return A 2d vector
+     */
+    const thrust::host_vector<double>& abscissasX() const {return absX_;} 
+    /**
+     * @brief Return the abscissas in Y-direction 
+     *
+     * @return A 2d vector
+     */
+    const thrust::host_vector<double>& abscissasY() const {return absY_;} 
+    /**
+     * @brief Return the weights in X-direction 
+     *
+     * @return A 2d vector
+     */
+    const thrust::host_vector<double>& weightsX() const {return wx_;} 
+    /**
+     * @brief Return the weights in Y-direction 
+     *
+     * @return A 2d vector
+     */
+    const thrust::host_vector<double>& weightsY() const {return wy_;} 
+
+    protected:
+    //void init_X_boundaries( double x0, double x1)
+    //{
+    //    double alpha = (x1-x0)/this->lx();
+    //    double beta = (x0*this->x1()-x1*this->x0())/this->lx();
+    //    //weights are invariant
+    //    for( unsigned i=0; i<absX_.size(); i++)
+    //        absX_[i]=alpha*absX_[i]+beta;
+    //    dg::Grid2d::init_X_boundaries( x0, x1);
+    //}
     /**
      * @brief Refine a corner of a grid
      *
@@ -272,12 +330,11 @@ struct RefinedGrid2d : public dg::aTopology2d
      * @param howmanyY Add number of cells to the existing one
      * @copydetails Grid2d::Grid2d()
      */
-    RefinedGrid2d( unsigned node_x, unsigned node_y, unsigned add_x, unsigned add_y, 
+    aRefinedGrid2d( unsigned node_x, unsigned node_y, unsigned add_x, unsigned add_y, 
             unsigned howmanyX, unsigned howmanyY,
             double x0, double x1, double y0, double y1, 
-            unsigned n, unsigned Nx, unsigned Ny, bc bcx = dg::PER, bc bcy = dg::PER) : dg::aTopology2d( x0, x1, y0, y1, n, n_new(Nx, add_x*howmanyX, bcx), n_new(Ny, add_y*howmanyY, bcy), bcx, bcy), 
+            unsigned n, unsigned Nx, unsigned Ny, bc bcx = dg::PER, bc bcy = dg::PER) : dg::aGeometry2d( x0, x1, y0, y1, n, n_new(Nx, add_x*howmanyX, bcx), n_new(Ny, add_y*howmanyY, bcy), bcx, bcy), 
         wx_(size()), wy_(size()), absX_(size()), absY_(size()),
-        g_assoc_( x0, x1, y0, y1, n, Nx, Ny, bcx, bcy)
     {
         //assert( howmanyX <= node_x && howmanyX <= Nx - node_x);
         //assert( howmanyY <= node_y && howmanyY <= Ny - node_y);
@@ -305,11 +362,10 @@ struct RefinedGrid2d : public dg::aTopology2d
      * @param n_old the polynomials in the old grid
      * @copydetails Grid2d::Grid2d()
      */
-    RefinedGrid2d( unsigned multiple_x, unsigned multiple_y,
+    aRefinedGrid2d( unsigned multiple_x, unsigned multiple_y,
             double x0, double x1, double y0, double y1, unsigned n,
-            unsigned n_old, unsigned Nx, unsigned Ny, bc bcx = dg::PER, bc bcy = dg::PER) : dg::aTopology2d( x0, x1, y0, y1, n, multiple_x*Nx, multiple_y*Ny, bcx, bcy), 
+            unsigned n_old, unsigned Nx, unsigned Ny, bc bcx = dg::PER, bc bcy = dg::PER) : dg::aGeometry2d( x0, x1, y0, y1, n, multiple_x*Nx, multiple_y*Ny, bcx, bcy), 
         wx_(size()), wy_(size()), absX_(size()), absY_(size()),
-        g_assoc_( x0, x1, y0, y1, n_old, Nx, Ny, bcx, bcy)
     {
         Grid1d gx( x0, x1, n, Nx, bcx);
         Grid1d gy( y0, y1, n, Ny, bcy);
@@ -327,57 +383,6 @@ struct RefinedGrid2d : public dg::aTopology2d
             }
     }
 
-    /**
-     * @brief Reduce from a 3d grid 
-     *
-     * This is possible because all our grids are product space grids. 
-     *
-     * @param g The 3d grid
-     */
-    RefinedGrid2d( const dg::RefinedGrid3d& g);
-    /**
-     * @brief The grid that this object refines
-     *
-     * This function is vitual so that derived classes can also construct the associated grid 
-     * @return  2d grid
-     */
-    virtual const dg::Grid2d& associated()const {return g_assoc_;}
-    /**
-     * @brief Return the abscissas in X-direction 
-     *
-     * @return A 2d vector
-     */
-    const thrust::host_vector<double>& abscissasX() const {return absX_;} 
-    /**
-     * @brief Return the abscissas in Y-direction 
-     *
-     * @return A 2d vector
-     */
-    const thrust::host_vector<double>& abscissasY() const {return absY_;} 
-    /**
-     * @brief Return the weights in X-direction 
-     *
-     * @return A 2d vector
-     */
-    const thrust::host_vector<double>& weightsX() const {return wx_;} 
-    /**
-     * @brief Return the weights in Y-direction 
-     *
-     * @return A 2d vector
-     */
-    const thrust::host_vector<double>& weightsY() const {return wy_;} 
-
-    protected:
-    void init_X_boundaries( double x0, double x1)
-    {
-        double alpha = (x1-x0)/this->lx();
-        double beta = (x0*this->x1()-x1*this->x0())/this->lx();
-        //weights are invariant
-        for( unsigned i=0; i<absX_.size(); i++)
-            absX_[i]=alpha*absX_[i]+beta;
-        dg::Grid2d::init_X_boundaries( x0, x1);
-    }
-
     private:
     unsigned n_new( unsigned N, unsigned factor, dg::bc bc)
     {
@@ -386,15 +391,14 @@ struct RefinedGrid2d : public dg::aTopology2d
     }
     thrust::host_vector<double> wx_, wy_; //weights
     thrust::host_vector<double> absX_, absY_; //abscissas 
-    dg::Grid2d g_assoc_;
 };
 
 /**
- * @brief Refined grid 
+ * @brief aRefined grid 
  * @deprecated
  * @ingroup grid
  */
-struct RefinedGrid3d : public dg::aTopology3d
+struct aRefinedGrid3d : public dg::aGeometry3d
 {
     /**
      * @brief Refine a corner of a grid
@@ -407,12 +411,11 @@ struct RefinedGrid3d : public dg::aTopology3d
      * @param howmanyY howmany cells should be refined in y
      * @copydetails Grid3d::Grid3d()
      */
-    RefinedGrid3d( unsigned node_x, unsigned node_y, unsigned add_x, unsigned add_y, 
+    aRefinedGrid3d( unsigned node_x, unsigned node_y, unsigned add_x, unsigned add_y, 
             unsigned howmanyX, unsigned howmanyY,
             double x0, double x1, double y0, double y1, double z0, double z1, 
-            unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = dg::PER, bc bcy = dg::PER, bc bcz = dg::PER) : dg::aTopology3d( x0, x1, y0, y1, z0, z1, n, n_new(Nx, add_x*howmanyX, bcx), n_new(Ny, add_y*howmanyY, bcy), Nz, bcx, bcy, bcz), 
+            unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = dg::PER, bc bcy = dg::PER, bc bcz = dg::PER) : dg::aGeometry3d( x0, x1, y0, y1, z0, z1, n, n_new(Nx, add_x*howmanyX, bcx), n_new(Ny, add_y*howmanyY, bcy), Nz, bcx, bcy, bcz), 
         wx_(size()), wy_(size()), absX_(size()), absY_(size()),
-        g_assoc_( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, bcx, bcy, bcz)
     {
         //assert( howmanyX <= node_x && howmanyX <= Nx - node_x);
         //assert( howmanyY <= node_y && howmanyY <= Ny - node_y);
@@ -440,14 +443,13 @@ struct RefinedGrid3d : public dg::aTopology3d
      * @param multiple_y Multiply all cells in y - direction
      * @copydetails Grid3d::Grid3d()
      */
-    RefinedGrid3d( unsigned multiple_x, unsigned multiple_y,
+    aRefinedGrid3d( unsigned multiple_x, unsigned multiple_y,
             double x0, double x1, double y0, double y1, double z0, double z1, 
             unsigned n,
             unsigned n_old, unsigned Nx, unsigned Ny, unsigned Nz, 
             bc bcx = dg::PER, bc bcy = dg::PER, bc bcz = dg::PER) : 
-        dg::aTopology3d( x0, x1, y0, y1, z0, z1, n, multiple_x*Nx, multiple_y*Ny, Nz, bcx, bcy, bcz), 
+        dg::aGeometry3d( x0, x1, y0, y1, z0, z1, n, multiple_x*Nx, multiple_y*Ny, Nz, bcx, bcy, bcz), 
         wx_(size()), wy_(size()), absX_(size()), absY_(size()),
-        g_assoc_( x0, x1, y0, y1, z0, z1, n_old, Nx, Ny, Nz, bcx, bcy, bcz)
     {
         Grid1d gx( x0, x1, n, Nx, bcx);
         Grid1d gy( y0, y1, n, Ny, bcy);
@@ -465,13 +467,14 @@ struct RefinedGrid3d : public dg::aTopology3d
                     absY_[(s*wy.size()+i)*wx.size()+j] = ay[i];
                 }
     }
+
     /**
      * @brief The grid that this object refines
      *
      * This function is vitual so that derived classes can also construct the associated grid 
      * @return  2d grid
      */
-    virtual const dg::Grid3d& associated()const {return g_assoc_;}
+    virtual const dg::aGeometry3d& associated()const=0;
     /**
      * @brief Return the abscissas in X-direction 
      *
@@ -515,14 +518,12 @@ struct RefinedGrid3d : public dg::aTopology3d
     }
     thrust::host_vector<double> wx_, wy_; //weights
     thrust::host_vector<double> absX_, absY_; //abscissas 
-    dg::Grid3d g_assoc_;
 
 };
 
-RefinedGrid2d::RefinedGrid2d( const dg::RefinedGrid3d& g) : 
+aRefinedGrid2d::aRefinedGrid2d( const dg::aRefinedGrid3d& g) : 
     dg::Grid2d( g.x0(), g.x1(), g.y0(), g.y1(), g.n(), g.Nx(), g.Ny(), g.bcx(), g.bcy()),
     wx_( this->size()), wy_(this->size()), absX_(this->size()), absY_(this->size()),
-    g_assoc_( g.associated())
 {
     for(unsigned i=0; i<this->size(); i++)
     {
@@ -534,8 +535,6 @@ RefinedGrid2d::RefinedGrid2d( const dg::RefinedGrid3d& g) :
 }
 
 
-
-
 /**
  * @brief A refined cartesian grid
  *
@@ -544,37 +543,42 @@ RefinedGrid2d::RefinedGrid2d( const dg::RefinedGrid3d& g) :
  * @tparam container
  */
 template<class container>
-struct CartesianRefinedGrid2d : public dg::RefinedGrid2d
+struct CartesianaRefinedGrid2d : public dg::aRefinedGrid2d
 {
     typedef CurvilinearPerpTag metric_category; 
-    CartesianRefinedGrid2d( unsigned multiple_x, unsigned multiple_y, double x0, double x1, double y0, double y1, unsigned n, unsigned n_old, unsigned Nx, unsigned Ny, bc bcx = PER, bc bcy = PER): dg::RefinedGrid2d(multiple_x, multiple_y,x0,x1,y0,y1,n,n_old,Nx,Ny,bcx,bcy), g_assoc_(x0,x1,y0,y1,n_old,Nx,Ny,bcx,bcy){ 
+    CartesianaRefinedGrid2d( unsigned multiple_x, unsigned multiple_y, double x0, double x1, double y0, double y1, unsigned n, unsigned n_old, unsigned Nx, unsigned Ny, bc bcx = PER, bc bcy = PER): dg::aRefinedGrid2d(multiple_x, multiple_y,x0,x1,y0,y1,n,n_old,Nx,Ny,bcx,bcy), g_assoc_(x0,x1,y0,y1,n_old,Nx,Ny,bcx,bcy){ 
         dg::blas1::transfer( weightsX(), g_xx_);
         dg::blas1::transfer( weightsY(), g_yy_);
-        dg::blas1::transfer( weightsX(), vol2d_);
         dg::blas1::pointwiseDot( g_xx_,g_yy_,vol2d_);
         dg::blas1::pointwiseDot( g_xx_,g_xx_,g_xx_);
         dg::blas1::pointwiseDot( g_yy_,g_yy_,g_yy_);
         dg::blas1::pointwiseDivide( vol2d_, g_xx_, vol2d_);
         dg::blas1::pointwiseDivide( vol2d_, g_yy_, vol2d_);
     }
-    const thrust::host_vector<double>& r()const{return this->abscissasX();}
-    const thrust::host_vector<double>& z()const{return this->abscissasY();}
     const dg::CartesianGrid2d& associated() const {return g_assoc_;}
-    const container& g_xx()const{return g_xx_;}
-    const container& g_yy()const{return g_yy_;}
-    const container& vol()const{return vol2d_;}
-    const container& perpVol()const{return vol2d_;}
-    bool isOrthogonal()const{return true;}
-    bool isConformal()const{return false;}
     private:
-    container g_xx_, g_yy_, vol2d_;
     dg::CartesianGrid2d g_assoc_;
+    virtual SparseTensor<thrust::host_vector<double> > do_compute_metric()const {
+        return SparseTensor<thrust::host_vector<double> >();
+    }
+    virtual SparseTensor<thrust::host_vector<double> > do_compute_jacobian()const {
+        return SparseTensor<thrust::host_vector<double> >();
+    }
+    virtual std::vector<thrust::host_vector<double> > do_compute_map()const{
+        std::vector<thrust::host_vector<double> > map(2);
+        map[0] = dg::evaluate(dg::cooX2d, *this);
+        map[1] = dg::evaluate(dg::cooY2d, *this);
+        return map;
+    }
+    virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny){
+        aTopology2d::do_set(new_n,new_Nx,new_Ny);
+    }
 };
 
 namespace create{
 
 ///@deprecated
-cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const dg::RefinedGrid2d& g_fine)
+cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const dg::aRefinedGrid2d& g_fine)
 {
     dg::Grid2d g = g_fine.associated();
     thrust::host_vector<double> x = g_fine.abscissasX();
@@ -583,7 +587,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const dg::Refine
 
 }
 ///@deprecated
-cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const dg::RefinedGrid2d& g_fine)
+cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const dg::aRefinedGrid2d& g_fine)
 {
     cusp::coo_matrix<int, double, cusp::host_memory> temp = interpolation( g_fine), A;
     cusp::transpose( temp, A);
@@ -591,7 +595,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const dg::Refin
 }
 
 ///@deprecated
-cusp::coo_matrix<int, double, cusp::host_memory> projection( const dg::RefinedGrid2d& g_fine)
+cusp::coo_matrix<int, double, cusp::host_memory> projection( const dg::aRefinedGrid2d& g_fine)
 {
     //form the adjoint
     thrust::host_vector<double> w_f = dg::create::weights( g_fine);
@@ -617,7 +621,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> projection( const dg::RefinedGr
 }
 
 ///@deprecated
-cusp::coo_matrix<int, double, cusp::host_memory> smoothing( const dg::RefinedGrid2d& g)
+cusp::coo_matrix<int, double, cusp::host_memory> smoothing( const dg::aRefinedGrid2d& g)
 {
     cusp::coo_matrix<int, double, cusp::host_memory> A = interpolation(g);
     cusp::coo_matrix<int, double, cusp::host_memory> B = projection(g);
@@ -628,7 +632,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> smoothing( const dg::RefinedGri
 }
 
 ///@deprecated
-cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const dg::RefinedGrid3d& g_fine)
+cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const dg::aRefinedGrid3d& g_fine)
 {
     dg::Grid3d g = g_fine.associated();
     thrust::host_vector<double> x = g_fine.abscissasX();
@@ -638,7 +642,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const dg::Refine
 }
 
 ///@deprecated
-cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const dg::RefinedGrid3d& g_fine)
+cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const dg::aRefinedGrid3d& g_fine)
 {
     cusp::coo_matrix<int, double, cusp::host_memory> temp = interpolation( g_fine), A;
     cusp::transpose( temp, A);
@@ -646,7 +650,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const dg::Refin
 }
 
 ///@deprecated
-cusp::coo_matrix<int, double, cusp::host_memory> projection( const dg::RefinedGrid3d& g_fine)
+cusp::coo_matrix<int, double, cusp::host_memory> projection( const dg::aRefinedGrid3d& g_fine)
 {
     //form the adjoint
     thrust::host_vector<double> w_f = dg::create::weights( g_fine);
@@ -672,7 +676,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> projection( const dg::RefinedGr
 }
 
 ///@deprecated
-cusp::coo_matrix<int, double, cusp::host_memory> smoothing( const dg::RefinedGrid3d& g)
+cusp::coo_matrix<int, double, cusp::host_memory> smoothing( const dg::aRefinedGrid3d& g)
 {
     cusp::coo_matrix<int, double, cusp::host_memory> A = interpolation(g);
     cusp::coo_matrix<int, double, cusp::host_memory> B = projection(g);
-- 
GitLab


From 2c8d5d38bffc03272281bd6ca059ee96d16ebf20 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sat, 12 Aug 2017 00:24:54 +0200
Subject: [PATCH 147/453] little corrections in documentation and sparse matrix
 matrix multiply

---
 inc/dg/geometry/base_geometry.h |  5 ++++-
 inc/dg/geometry/curvilinear.h   |  2 ++
 inc/dg/geometry/multiply.h      | 22 ++++++++++++++++++++++
 inc/dg/geometry/tensor.h        | 12 +++++++++---
 4 files changed, 37 insertions(+), 4 deletions(-)

diff --git a/inc/dg/geometry/base_geometry.h b/inc/dg/geometry/base_geometry.h
index aa4d7b769..48ff268a9 100644
--- a/inc/dg/geometry/base_geometry.h
+++ b/inc/dg/geometry/base_geometry.h
@@ -163,5 +163,8 @@ struct CylindricalGrid3d: public dg::aGeometry3d
     virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz){
         aTopology3d::do_set(new_n,new_Nx,new_Ny,new_Nz);
     }
-}; ///@}
+};
+
+///@}
+
 } //namespace dg
diff --git a/inc/dg/geometry/curvilinear.h b/inc/dg/geometry/curvilinear.h
index 81e3d2bbf..98dd75e6f 100644
--- a/inc/dg/geometry/curvilinear.h
+++ b/inc/dg/geometry/curvilinear.h
@@ -21,6 +21,8 @@ struct CurvilinearGrid2d;
 
 /**
  * @brief A three-dimensional grid based on curvilinear coordinates
+ * 
+ * The base coordinate system is the cylindrical coordinate system R,Z,phi
  */
 struct CylindricalProductGrid3d : public dg::aGeometry3d
 {
diff --git a/inc/dg/geometry/multiply.h b/inc/dg/geometry/multiply.h
index 818b79d88..0d7c09ead 100644
--- a/inc/dg/geometry/multiply.h
+++ b/inc/dg/geometry/multiply.h
@@ -337,6 +337,28 @@ SparseElement<container> determinant( const SparseTensor<container>& t)
     return SparseElement<container>(det);
 }
 
+/**
+* @brief Compute the product of a tensor with its transpose
+* @copydoc hide_container_lvl1
+* @param t the input tensor 
+* @return the product of t with its transpose
+*/
+template<class container>
+SparseTensor<container> square( const SparseTensor<container>& t)
+{
+    if(t.isEmpty()) return SparseTensor<container>();
+    SparseTensor<container> tt;
+    container tmp(t.value(0));
+    if(t.isSet(0,0)){
+        tt.idx(0,0) = 0;
+        dg::pointwiseDot( t.value(0,0), t.value(0,0), tmp);
+        tt.value(0) = tmp;
+    }
+    if( t.isSet(0,1) || t.isSet(0,2) ) 
+    {
+        if( tt.isSet(0,0)
+    }
+}
 ///@cond
 //alias always allowed
 template<class container>
diff --git a/inc/dg/geometry/tensor.h b/inc/dg/geometry/tensor.h
index f0ccaa600..046ad5435 100644
--- a/inc/dg/geometry/tensor.h
+++ b/inc/dg/geometry/tensor.h
@@ -78,7 +78,8 @@ struct SparseElement
 if negative the value of the T is assumed to be 1, except for the off-diagonal entries
     in the matrix where it is assumed to be 0.
 * We then only need to store non-trivial and non-repetitive Ts.
-* @tparam T must be default constructible and copyable
+* @tparam T must be default constructible and copyable. If dense is called the dg::blas1::transform function 
+* needs to be callable.
 * @ingroup misc
 */
 template<class T>
@@ -251,8 +252,13 @@ struct SparseTensor
      
      ///construct an empty Tensor
      SparseTensor empty()const{return SparseTensor();}
-     ///Construct a tensor with all unset values filled with explicit 0 or 1
-     /// @note undefined isEmpty() returns true
+     /**
+     * @brief Construct a tensor with all unset values filled with explicit 0 or 1
+     *
+     * T must support the dg::blas1::transform function 
+     * @return a dense tensor
+     * @note undefined if isEmpty() returns true
+     */
      SparseTensor dense()const;
      ///copy and erase all values in the third dimension
      ///@note calls clear_unused_values() to get rid of the elements
-- 
GitLab


From a1506b29277ee003a8ae0575535e38e79947bafc Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sat, 12 Aug 2017 22:17:10 +0200
Subject: [PATCH 148/453] made Cartesian refined grid and give up idea of
 square fct

---
 inc/dg/backend/projection.cuh         |  57 +++
 inc/dg/geometry/multiply.h            |  22 --
 inc/dg/geometry/refined_curvilinear.h |  12 +-
 inc/dg/geometry/refined_grid.h        | 481 +++++---------------------
 4 files changed, 153 insertions(+), 419 deletions(-)

diff --git a/inc/dg/backend/projection.cuh b/inc/dg/backend/projection.cuh
index a2261ce20..e7a582289 100644
--- a/inc/dg/backend/projection.cuh
+++ b/inc/dg/backend/projection.cuh
@@ -247,6 +247,63 @@ cusp::coo_matrix< int, double, cusp::host_memory> transformation( const Grid1d&
 }
 ///@}
 
+/*
+///@deprecated
+cusp::coo_matrix<int, double, cusp::host_memory> projection( const dg::aRefinedGrid2d& g_fine)
+{
+    //form the adjoint
+    thrust::host_vector<double> w_f = dg::create::weights( g_fine);
+    thrust::host_vector<double> v_c = dg::create::inv_weights( g_fine.associated() );
+    cusp::coo_matrix<int, double, cusp::host_memory> Wf( w_f.size(), w_f.size(), w_f.size());
+    cusp::coo_matrix<int, double, cusp::host_memory> Vc( v_c.size(), v_c.size(), v_c.size());
+    for( int i =0; i<(int)w_f.size(); i++)
+    {
+        Wf.row_indices[i] = Wf.column_indices[i] = i;
+        Wf.values[i] = w_f[i]/g_fine.weightsX()[i]/g_fine.weightsY()[i];
+    }
+    for( int i =0; i<(int)v_c.size(); i++)
+    {
+        Vc.row_indices[i] = Vc.column_indices[i] = i;
+        Vc.values[i] = v_c[i];
+    }
+    cusp::coo_matrix<int, double, cusp::host_memory> temp = interpolation( g_fine), A;
+    cusp::transpose( temp, A);
+    cusp::multiply( A, Wf, temp);
+    cusp::multiply( Vc, temp, A);
+    A.sort_by_row_and_column();
+    return A;
+}
+
+
+
+///@deprecated
+cusp::coo_matrix<int, double, cusp::host_memory> projection( const dg::aRefinedGrid3d& g_fine)
+{
+    //form the adjoint
+    thrust::host_vector<double> w_f = dg::create::weights( g_fine);
+    thrust::host_vector<double> v_c = dg::create::inv_weights( g_fine.associated() );
+    cusp::coo_matrix<int, double, cusp::host_memory> Wf( w_f.size(), w_f.size(), w_f.size());
+    cusp::coo_matrix<int, double, cusp::host_memory> Vc( v_c.size(), v_c.size(), v_c.size());
+    for( int i =0; i<(int)w_f.size(); i++)
+    {
+        Wf.row_indices[i] = Wf.column_indices[i] = i;
+        Wf.values[i] = w_f[i]/g_fine.weightsX()[i]/g_fine.weightsY()[i];
+    }
+    for( int i =0; i<(int)v_c.size(); i++)
+    {
+        Vc.row_indices[i] = Vc.column_indices[i] = i;
+        Vc.values[i] = v_c[i];
+    }
+    cusp::coo_matrix<int, double, cusp::host_memory> temp = interpolation( g_fine), A;
+    cusp::transpose( temp, A);
+    cusp::multiply( A, Wf, temp);
+    cusp::multiply( Vc, temp, A);
+    A.sort_by_row_and_column();
+    return A;
+}
+*/
+
+
 }//namespace create
 
 
diff --git a/inc/dg/geometry/multiply.h b/inc/dg/geometry/multiply.h
index 0d7c09ead..818b79d88 100644
--- a/inc/dg/geometry/multiply.h
+++ b/inc/dg/geometry/multiply.h
@@ -337,28 +337,6 @@ SparseElement<container> determinant( const SparseTensor<container>& t)
     return SparseElement<container>(det);
 }
 
-/**
-* @brief Compute the product of a tensor with its transpose
-* @copydoc hide_container_lvl1
-* @param t the input tensor 
-* @return the product of t with its transpose
-*/
-template<class container>
-SparseTensor<container> square( const SparseTensor<container>& t)
-{
-    if(t.isEmpty()) return SparseTensor<container>();
-    SparseTensor<container> tt;
-    container tmp(t.value(0));
-    if(t.isSet(0,0)){
-        tt.idx(0,0) = 0;
-        dg::pointwiseDot( t.value(0,0), t.value(0,0), tmp);
-        tt.value(0) = tmp;
-    }
-    if( t.isSet(0,1) || t.isSet(0,2) ) 
-    {
-        if( tt.isSet(0,0)
-    }
-}
 ///@cond
 //alias always allowed
 template<class container>
diff --git a/inc/dg/geometry/refined_curvilinear.h b/inc/dg/geometry/refined_curvilinear.h
index 20e4d6af3..15570aa71 100644
--- a/inc/dg/geometry/refined_curvilinear.h
+++ b/inc/dg/geometry/refined_curvilinear.h
@@ -7,7 +7,6 @@ namespace dg
 {
 
 ///@cond
-template< class container>
 struct CurvilinearRefinedGrid2d; 
 ///@endcond
 
@@ -16,16 +15,13 @@ struct CurvilinearRefinedGrid2d;
 
 /**
  * @brief A curvilinear refined grid
- * @deprecated
  */
-template< class container>
-struct CurvilinearRefinedGrid3d : public dg::RefinedGrid3d
+struct CurvilinearRefinedGrid3d : public dg::aGeometry3d
 {
-    typedef dg::CurvilinearCylindricalTag metric_category; //!< metric tag
-    typedef CurvilinearRefinedGrid2d<container> perpendicular_grid; //!< the two-dimensional grid type
+    typedef CurvilinearRefinedGrid2d perpendicular_grid; //!< the two-dimensional grid type
 
-    CurvilinearRefinedGrid3d( unsigned multiple_x, unsigned multiple_y, const geo::aGenerator* generator, unsigned n, unsigned n_old, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx): 
-        dg::RefinedGrid3d( multiple_x, multiple_y, 0, 1, 0., 2.*M_PI, 0., 2.*M_PI, n, n_old, Nx, Ny, Nz, bcx, dg::PER, dg::PER),
+    CurvilinearRefinedGrid3d( const aRefinement1d& refX, const aRefinement1d& refY, const aRefinement1d& refZ, const aGenerator2d& generator, unsigned n, unsigned n_old, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx): 
+        dg::aGeometry3d( 0, generator.width(), 0., 2.*M_PI, 0., 2.*M_PI, n, n_old, Nx, Ny, Nz, bcx, dg::PER, dg::PER),
         g_assoc_( generator, n_old, Nx, Ny, Nz, bcx)
     { 
         construct( generator);
diff --git a/inc/dg/geometry/refined_grid.h b/inc/dg/geometry/refined_grid.h
index df025ad22..29be1cd46 100644
--- a/inc/dg/geometry/refined_grid.h
+++ b/inc/dg/geometry/refined_grid.h
@@ -1,6 +1,7 @@
 #pragma once
 
 #include "cusp/transpose.h"
+#include "dg/backend/manage.h"
 #include "dg/backend/grid.h"
 #include "dg/backend/weights.cuh"
 #include "dg/backend/interpolation.cuh"
@@ -81,6 +82,21 @@ struct aRefinement1d
     virtual unsigned new_size( unsigned N_old, bc bcx) const =0;
 };
 
+/**
+* @brief No refinement
+*/
+struct IdentityRefinement : public aRefinementd
+{
+    private:
+    virtual void do_generate( const Grid1d& g, thrust::host_vector<double>& weights, thrust::host_vector<double>& abscissas) const {
+        weights=dg::create::weights(g);
+        abscissas=dg::create::abscissas(g);
+    }
+    virtual unsigned new_size( unsigned N_old, bc bcx) const {
+        return N_old;
+    }
+};
+
 /**
  * @brief Linear refinement consists of multiplying every cell in the grid by a factor
  */
@@ -260,432 +276,119 @@ struct ExponentialRefinement : public aRefinement1d
 
 ///@}
 
-struct aRefinedGrid3d;
 /**
- * @brief aRefined grid 
- * @deprecated
+ * @brief Refined Cartesian grid 
  * @ingroup grid
  */
-struct aRefinedGrid2d : public dg::aGeometry2d
+struct CartesianRefinedGrid2d : public dg::aGeometry2d
 {
-
-    /**
-     * @brief Reduce from a 3d grid 
-     *
-     * This is possible because all our grids are product space grids. 
-     *
-     * @param g The 3d grid
-     */
-    aRefinedGrid2d( const dg::aRefinedGrid3d& g);
-    /**
-     * @brief The grid that this object refines
-     *
-     * This function is vitual so that derived classes can also construct the associated grid 
-     * @return  2d grid
-     */
-    virtual const dg::Grid2d& associated()const {return g_assoc_;}
-    /**
-     * @brief Return the abscissas in X-direction 
-     *
-     * @return A 2d vector
-     */
-    const thrust::host_vector<double>& abscissasX() const {return absX_;} 
-    /**
-     * @brief Return the abscissas in Y-direction 
-     *
-     * @return A 2d vector
-     */
-    const thrust::host_vector<double>& abscissasY() const {return absY_;} 
-    /**
-     * @brief Return the weights in X-direction 
-     *
-     * @return A 2d vector
-     */
-    const thrust::host_vector<double>& weightsX() const {return wx_;} 
-    /**
-     * @brief Return the weights in Y-direction 
-     *
-     * @return A 2d vector
-     */
-    const thrust::host_vector<double>& weightsY() const {return wy_;} 
-
-    protected:
-    //void init_X_boundaries( double x0, double x1)
-    //{
-    //    double alpha = (x1-x0)/this->lx();
-    //    double beta = (x0*this->x1()-x1*this->x0())/this->lx();
-    //    //weights are invariant
-    //    for( unsigned i=0; i<absX_.size(); i++)
-    //        absX_[i]=alpha*absX_[i]+beta;
-    //    dg::Grid2d::init_X_boundaries( x0, x1);
-    //}
-    /**
-     * @brief Refine a corner of a grid
-     *
-     * @param node_x
-     * @param node_y
-     * @param add_x Add number of cells to the existing one
-     * @param add_y Add number of cells to the existing one
-     * @param howmanyX Add number of cells to the existing one
-     * @param howmanyY Add number of cells to the existing one
-     * @copydetails Grid2d::Grid2d()
-     */
-    aRefinedGrid2d( unsigned node_x, unsigned node_y, unsigned add_x, unsigned add_y, 
-            unsigned howmanyX, unsigned howmanyY,
-            double x0, double x1, double y0, double y1, 
-            unsigned n, unsigned Nx, unsigned Ny, bc bcx = dg::PER, bc bcy = dg::PER) : dg::aGeometry2d( x0, x1, y0, y1, n, n_new(Nx, add_x*howmanyX, bcx), n_new(Ny, add_y*howmanyY, bcy), bcx, bcy), 
-        wx_(size()), wy_(size()), absX_(size()), absY_(size()),
+    CartesianRefinedGrid2d( const aRefinement1d& refX, const aRefinement1d& refY, double x0, double x1, double y0, double y1, 
+            unsigned n, unsigned Nx, unsigned Ny, bc bcx = dg::PER, bc bcy = dg::PER) : dg::aGeometry2d( x0, x1, y0, y1, n, refX.N_new(Nx, bcx), refY.N_new(Ny,bcy), bcx, bcy), refX_(refX), refY_(refY), w_(2), a_(2)
     {
-        //assert( howmanyX <= node_x && howmanyX <= Nx - node_x);
-        //assert( howmanyY <= node_y && howmanyY <= Ny - node_y);
-        Grid1d gx( x0, x1, n, Nx, bcx);
-        Grid1d gy( y0, y1, n, Ny, bcy);
-        thrust::host_vector<double> wx, ax, wy, ay;
-        detail::equidist_ref( add_x, node_x, gx, wx, ax, howmanyX);
-        detail::equidist_ref( add_y, node_y, gy, wy, ay, howmanyY);
-        //now make product space
-        for( unsigned i=0; i<wy.size(); i++)
-            for( unsigned j=0; j<wx.size(); j++)
-            {
-                wx_[i*wx.size()+j] = wx[j];
-                wy_[i*wx.size()+j] = wy[i];
-                absX_[i*wx.size()+j] = ax[j];
-                absY_[i*wx.size()+j] = ay[i];
-            }
+        construct_weights_and_abscissas();
     }
 
-    /**
-     * @brief Refine a all cells of a grid
-     *
-     * @param multiple_x refine all cells in x 
-     * @param multiple_y refine all cells in y
-     * @param n_old the polynomials in the old grid
-     * @copydetails Grid2d::Grid2d()
-     */
-    aRefinedGrid2d( unsigned multiple_x, unsigned multiple_y,
-            double x0, double x1, double y0, double y1, unsigned n,
-            unsigned n_old, unsigned Nx, unsigned Ny, bc bcx = dg::PER, bc bcy = dg::PER) : dg::aGeometry2d( x0, x1, y0, y1, n, multiple_x*Nx, multiple_y*Ny, bcx, bcy), 
-        wx_(size()), wy_(size()), absX_(size()), absY_(size()),
-    {
+    private:
+    Handle<aRefinement1d> refX_, refY_;
+    std::vector<thrust::host_vector<double> > w_, a_;
+    void construct_weights_and_abscissas()
+    { 
         Grid1d gx( x0, x1, n, Nx, bcx);
         Grid1d gy( y0, y1, n, Ny, bcy);
         thrust::host_vector<double> wx, ax, wy, ay;
-        detail::linear_ref( multiple_x, gx, wx, ax);
-        detail::linear_ref( multiple_y, gy, wy, ay);
+        refX_.get().generate( gx, wx, ax);
+        refY_.get().generate( gy, wy, ay);
+        w_[0].resize(size()), w_[1].resize(size()); 
+        a_[0].resize(size()), a_[1].resize(size());
         //now make product space
         for( unsigned i=0; i<wy.size(); i++)
             for( unsigned j=0; j<wx.size(); j++)
             {
-                wx_[i*wx.size()+j] = wx[j];
-                wy_[i*wx.size()+j] = wy[i];
-                absX_[i*wx.size()+j] = ax[j];
-                absY_[i*wx.size()+j] = ay[i];
+                w_[0][i*wx.size()+j] = wx[j];
+                w_[1][i*wx.size()+j] = wy[i];
+                a_[0][i*wx.size()+j] = ax[j];
+                a_[1][i*wx.size()+j] = ay[i];
             }
     }
-
-    private:
-    unsigned n_new( unsigned N, unsigned factor, dg::bc bc)
-    {
-        if( bc == dg::PER) return N + 2*factor; 
-        return N + factor;
+    virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny){
+        aTopology2d::do_set(new_n,new_Nx,new_Ny);
+        construct_weights_and_abscissas();
+    }
+    virtual SparseTensor<thrust::host_vector<double> > do_compute_metric()const {
+        SparseTensor<thrust::host_vector<double> > t(w_);
+        dg::blas1::pointwiseDot( w_[0], w_[0], t.value(0));
+        dg::blas1::pointwiseDot( w_[1], w_[1], t.value(1));
+        t.idx(0,0)=0, t.idx(1,1)=1; 
+        return t;
+    }
+    virtual SparseTensor<thrust::host_vector<double> > do_compute_jacobian()const {
+        SparseTensor<thrust::host_vector<double> > t(w_);
+        t.idx(0,0)=0, t.idx(1,1)=1; 
+        return t;
+    }
+    virtual std::vector<thrust::host_vector<double> > do_compute_map()const{
+        return a_;
     }
-    thrust::host_vector<double> wx_, wy_; //weights
-    thrust::host_vector<double> absX_, absY_; //abscissas 
 };
 
 /**
- * @brief aRefined grid 
- * @deprecated
+ * @brief Refined Cartesian grid 
  * @ingroup grid
  */
-struct aRefinedGrid3d : public dg::aGeometry3d
+struct CartesianRefinedGrid3d : public dg::aGeometry3d
 {
-    /**
-     * @brief Refine a corner of a grid
-     *
-     * @param node_x index of X-point
-     * @param node_y index of X-point
-     * @param add_x Add number of cells to the existing one
-     * @param add_y Add number of cells to the existing one
-     * @param howmanyX howmany cells should be refined in x
-     * @param howmanyY howmany cells should be refined in y
-     * @copydetails Grid3d::Grid3d()
-     */
-    aRefinedGrid3d( unsigned node_x, unsigned node_y, unsigned add_x, unsigned add_y, 
-            unsigned howmanyX, unsigned howmanyY,
-            double x0, double x1, double y0, double y1, double z0, double z1, 
-            unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = dg::PER, bc bcy = dg::PER, bc bcz = dg::PER) : dg::aGeometry3d( x0, x1, y0, y1, z0, z1, n, n_new(Nx, add_x*howmanyX, bcx), n_new(Ny, add_y*howmanyY, bcy), Nz, bcx, bcy, bcz), 
-        wx_(size()), wy_(size()), absX_(size()), absY_(size()),
+    CartesianRefinedGrid3d( const aRefinement1d& refX, const aRefinement1d& refY, aRefinement1d& refZ, double x0, double x1, double y0, double y1, double z0, double z1,
+            unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = dg::PER, bc bcy = dg::PER, bc bcz=dg::PER) : dg::aGeometry3d( x0, x1, y0, y1,z0,z1, n, refX.N_new(Nx, bcx), refY.N_new(Ny,bcy), refZ.N_new(Nz,bcz), bcx, bcy, bcz), refX_(refX), refY_(refY), refZ_(refZ), w_(3), a_(3)
     {
-        //assert( howmanyX <= node_x && howmanyX <= Nx - node_x);
-        //assert( howmanyY <= node_y && howmanyY <= Ny - node_y);
-        Grid1d gx( x0, x1, n, Nx, bcx);
-        Grid1d gy( y0, y1, n, Ny, bcy);
-        thrust::host_vector<double> wx, ax, wy, ay;
-        detail::equidist_ref( add_x, node_x, gx, wx, ax, howmanyX);
-        detail::equidist_ref( add_y, node_y, gy, wy, ay, howmanyY);
-        //now make product space
-        for( unsigned s=0; s<Nz; s++)
-            for( unsigned i=0; i<wy.size(); i++)
-                for( unsigned j=0; j<wx.size(); j++)
-                {
-                    wx_[(s*wy.size()+i)*wx.size()+j] = wx[j];
-                    wy_[(s*wy.size()+i)*wx.size()+j] = wy[i];
-                    absX_[(s*wy.size()+i)*wx.size()+j] = ax[j];
-                    absY_[(s*wy.size()+i)*wx.size()+j] = ay[i];
-                }
+        construct_weights_and_abscissas();
     }
 
-    /**
-     * @brief Refine all cells of a grid
-     *
-     * @param multiple_x Multiply all cells in x - direction
-     * @param multiple_y Multiply all cells in y - direction
-     * @copydetails Grid3d::Grid3d()
-     */
-    aRefinedGrid3d( unsigned multiple_x, unsigned multiple_y,
-            double x0, double x1, double y0, double y1, double z0, double z1, 
-            unsigned n,
-            unsigned n_old, unsigned Nx, unsigned Ny, unsigned Nz, 
-            bc bcx = dg::PER, bc bcy = dg::PER, bc bcz = dg::PER) : 
-        dg::aGeometry3d( x0, x1, y0, y1, z0, z1, n, multiple_x*Nx, multiple_y*Ny, Nz, bcx, bcy, bcz), 
-        wx_(size()), wy_(size()), absX_(size()), absY_(size()),
-    {
+    private:
+    Handle<aRefinement1d> refX_, refY_, refZ_;
+    std::vector<thrust::host_vector<double> > w_, a_;
+    void construct_weights_and_abscissas()
+    { 
         Grid1d gx( x0, x1, n, Nx, bcx);
         Grid1d gy( y0, y1, n, Ny, bcy);
-        thrust::host_vector<double> wx, ax, wy, ay;
-        detail::linear_ref( multiple_x, gx, wx, ax);
-        detail::linear_ref( multiple_y, gy, wy, ay);
+        Grid1d gz( y0, y1, 1, Nz, bcz);
+        thrust::host_vector<double> wx, ax, wy, ay, wz, az;
+        refX_.get().generate( gx, wx, ax);
+        refY_.get().generate( gy, wy, ay);
+        refZ_.get().generate( gz, wz, az);
+        w_[0].resize(size()), w_[1].resize(size()), w_[2].resize(size()); 
+        a_[0].resize(size()), a_[1].resize(size()), a_[2].resize(size()); 
         //now make product space
-        for( unsigned s=0; s<Nz; s++)
-            for( unsigned i=0; i<wy.size(); i++)
-                for( unsigned j=0; j<wx.size(); j++)
-                {
-                    wx_[(s*wy.size()+i)*wx.size()+j] = wx[j];
-                    wy_[(s*wy.size()+i)*wx.size()+j] = wy[i];
-                    absX_[(s*wy.size()+i)*wx.size()+j] = ax[j];
-                    absY_[(s*wy.size()+i)*wx.size()+j] = ay[i];
-                }
-    }
-
-    /**
-     * @brief The grid that this object refines
-     *
-     * This function is vitual so that derived classes can also construct the associated grid 
-     * @return  2d grid
-     */
-    virtual const dg::aGeometry3d& associated()const=0;
-    /**
-     * @brief Return the abscissas in X-direction 
-     *
-     * @return A 2d vector
-     */
-    const thrust::host_vector<double>& abscissasX() const {return absX_;} 
-    /**
-     * @brief Return the abscissas in Y-direction 
-     *
-     * @return A 2d vector
-     */
-    const thrust::host_vector<double>& abscissasY() const {return absY_;} 
-    /**
-     * @brief Return the weights in X-direction 
-     *
-     * @return A 2d vector
-     */
-    const thrust::host_vector<double>& weightsX() const {return wx_;} 
-    /**
-     * @brief Return the weights in Y-direction 
-     *
-     * @return A 2d vector
-     */
-    const thrust::host_vector<double>& weightsY() const {return wy_;} 
-    protected:
-    void init_X_boundaries( double x0, double x1)
-    {
-        double alpha = (x1-x0)/this->lx();
-        double beta = (x0*this->x1()-x1*this->x0())/this->lx();
-        //weights are invariant under linear transformation
-        for( unsigned i=0; i<absX_.size(); i++)
-            absX_[i]=alpha*absX_[i]+beta;
-        dg::Grid3d::init_X_boundaries( x0, x1);
-    }
-
-    private:
-    unsigned n_new( unsigned N, unsigned factor, dg::bc bc)
-    {
-        if( bc == dg::PER) return N + 2*factor; 
-        return N + factor;
-    }
-    thrust::host_vector<double> wx_, wy_; //weights
-    thrust::host_vector<double> absX_, absY_; //abscissas 
-
-};
-
-aRefinedGrid2d::aRefinedGrid2d( const dg::aRefinedGrid3d& g) : 
-    dg::Grid2d( g.x0(), g.x1(), g.y0(), g.y1(), g.n(), g.Nx(), g.Ny(), g.bcx(), g.bcy()),
-    wx_( this->size()), wy_(this->size()), absX_(this->size()), absY_(this->size()),
-{
-    for(unsigned i=0; i<this->size(); i++)
-    {
-        wx_[i] = g.weightsX()[i];
-        wy_[i] = g.weightsY()[i];
-        absX_[i] = g.abscissasX()[i];
-        absY_[i] = g.abscissasY()[i];
+        for( unsigned i=0; i<wy.size(); i++)
+            for( unsigned j=0; j<wx.size(); j++)
+            {
+                w_[0][i*wx.size()+j] = wx[j];
+                w_[1][i*wx.size()+j] = wy[i];
+                w_[2][i*wx.size()+j] = wz[i];
+                a_[0][i*wx.size()+j] = ax[j];
+                a_[1][i*wx.size()+j] = ay[i];
+                a_[2][i*wx.size()+j] = az[i];
+            }
     }
-}
-
-
-/**
- * @brief A refined cartesian grid
- *
- * @ingroup basicgrids
- * @deprecated
- * @tparam container
- */
-template<class container>
-struct CartesianaRefinedGrid2d : public dg::aRefinedGrid2d
-{
-    typedef CurvilinearPerpTag metric_category; 
-    CartesianaRefinedGrid2d( unsigned multiple_x, unsigned multiple_y, double x0, double x1, double y0, double y1, unsigned n, unsigned n_old, unsigned Nx, unsigned Ny, bc bcx = PER, bc bcy = PER): dg::aRefinedGrid2d(multiple_x, multiple_y,x0,x1,y0,y1,n,n_old,Nx,Ny,bcx,bcy), g_assoc_(x0,x1,y0,y1,n_old,Nx,Ny,bcx,bcy){ 
-        dg::blas1::transfer( weightsX(), g_xx_);
-        dg::blas1::transfer( weightsY(), g_yy_);
-        dg::blas1::pointwiseDot( g_xx_,g_yy_,vol2d_);
-        dg::blas1::pointwiseDot( g_xx_,g_xx_,g_xx_);
-        dg::blas1::pointwiseDot( g_yy_,g_yy_,g_yy_);
-        dg::blas1::pointwiseDivide( vol2d_, g_xx_, vol2d_);
-        dg::blas1::pointwiseDivide( vol2d_, g_yy_, vol2d_);
+    virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny){
+        aTopology3d::do_set(new_n,new_Nx,new_Ny);
+        construct_weights_and_abscissas();
     }
-    const dg::CartesianGrid2d& associated() const {return g_assoc_;}
-    private:
-    dg::CartesianGrid2d g_assoc_;
     virtual SparseTensor<thrust::host_vector<double> > do_compute_metric()const {
-        return SparseTensor<thrust::host_vector<double> >();
+        SparseTensor<thrust::host_vector<double> > t(w_);
+        dg::blas1::pointwiseDot( w_[0], w_[0], t.value(0));
+        dg::blas1::pointwiseDot( w_[1], w_[1], t.value(1));
+        dg::blas1::pointwiseDot( w_[2], w_[2], t.value(2));
+        t.idx(0,0)=0, t.idx(1,1)=1, t.idx(2,2)=2; 
+        return t;
     }
     virtual SparseTensor<thrust::host_vector<double> > do_compute_jacobian()const {
-        return SparseTensor<thrust::host_vector<double> >();
+        SparseTensor<thrust::host_vector<double> > t(w_);
+        t.idx(0,0)=0, t.idx(1,1)=1, t.idx(2,2)=2; 
+        return t;
     }
     virtual std::vector<thrust::host_vector<double> > do_compute_map()const{
-        std::vector<thrust::host_vector<double> > map(2);
-        map[0] = dg::evaluate(dg::cooX2d, *this);
-        map[1] = dg::evaluate(dg::cooY2d, *this);
-        return map;
-    }
-    virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny){
-        aTopology2d::do_set(new_n,new_Nx,new_Ny);
+        return a_;
     }
 };
 
-namespace create{
-
-///@deprecated
-cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const dg::aRefinedGrid2d& g_fine)
-{
-    dg::Grid2d g = g_fine.associated();
-    thrust::host_vector<double> x = g_fine.abscissasX();
-    thrust::host_vector<double> y = g_fine.abscissasY();
-    return dg::create::interpolation( x,y, g);
-
-}
-///@deprecated
-cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const dg::aRefinedGrid2d& g_fine)
-{
-    cusp::coo_matrix<int, double, cusp::host_memory> temp = interpolation( g_fine), A;
-    cusp::transpose( temp, A);
-    return A;
-}
-
-///@deprecated
-cusp::coo_matrix<int, double, cusp::host_memory> projection( const dg::aRefinedGrid2d& g_fine)
-{
-    //form the adjoint
-    thrust::host_vector<double> w_f = dg::create::weights( g_fine);
-    thrust::host_vector<double> v_c = dg::create::inv_weights( g_fine.associated() );
-    cusp::coo_matrix<int, double, cusp::host_memory> Wf( w_f.size(), w_f.size(), w_f.size());
-    cusp::coo_matrix<int, double, cusp::host_memory> Vc( v_c.size(), v_c.size(), v_c.size());
-    for( int i =0; i<(int)w_f.size(); i++)
-    {
-        Wf.row_indices[i] = Wf.column_indices[i] = i;
-        Wf.values[i] = w_f[i]/g_fine.weightsX()[i]/g_fine.weightsY()[i];
-    }
-    for( int i =0; i<(int)v_c.size(); i++)
-    {
-        Vc.row_indices[i] = Vc.column_indices[i] = i;
-        Vc.values[i] = v_c[i];
-    }
-    cusp::coo_matrix<int, double, cusp::host_memory> temp = interpolation( g_fine), A;
-    cusp::transpose( temp, A);
-    cusp::multiply( A, Wf, temp);
-    cusp::multiply( Vc, temp, A);
-    A.sort_by_row_and_column();
-    return A;
-}
-
-///@deprecated
-cusp::coo_matrix<int, double, cusp::host_memory> smoothing( const dg::aRefinedGrid2d& g)
-{
-    cusp::coo_matrix<int, double, cusp::host_memory> A = interpolation(g);
-    cusp::coo_matrix<int, double, cusp::host_memory> B = projection(g);
-    cusp::coo_matrix<int, double, cusp::host_memory> C;
-    cusp::multiply( A, B, C);
-    C.sort_by_row_and_column();
-    return C; 
-}
-
-///@deprecated
-cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const dg::aRefinedGrid3d& g_fine)
-{
-    dg::Grid3d g = g_fine.associated();
-    thrust::host_vector<double> x = g_fine.abscissasX();
-    thrust::host_vector<double> y = g_fine.abscissasY();
-    thrust::host_vector<double> z = dg::evaluate( dg::cooZ3d, g_fine);
-    return dg::create::interpolation( x,y,z, g);
-}
-
-///@deprecated
-cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const dg::aRefinedGrid3d& g_fine)
-{
-    cusp::coo_matrix<int, double, cusp::host_memory> temp = interpolation( g_fine), A;
-    cusp::transpose( temp, A);
-    return A;
-}
-
-///@deprecated
-cusp::coo_matrix<int, double, cusp::host_memory> projection( const dg::aRefinedGrid3d& g_fine)
-{
-    //form the adjoint
-    thrust::host_vector<double> w_f = dg::create::weights( g_fine);
-    thrust::host_vector<double> v_c = dg::create::inv_weights( g_fine.associated() );
-    cusp::coo_matrix<int, double, cusp::host_memory> Wf( w_f.size(), w_f.size(), w_f.size());
-    cusp::coo_matrix<int, double, cusp::host_memory> Vc( v_c.size(), v_c.size(), v_c.size());
-    for( int i =0; i<(int)w_f.size(); i++)
-    {
-        Wf.row_indices[i] = Wf.column_indices[i] = i;
-        Wf.values[i] = w_f[i]/g_fine.weightsX()[i]/g_fine.weightsY()[i];
-    }
-    for( int i =0; i<(int)v_c.size(); i++)
-    {
-        Vc.row_indices[i] = Vc.column_indices[i] = i;
-        Vc.values[i] = v_c[i];
-    }
-    cusp::coo_matrix<int, double, cusp::host_memory> temp = interpolation( g_fine), A;
-    cusp::transpose( temp, A);
-    cusp::multiply( A, Wf, temp);
-    cusp::multiply( Vc, temp, A);
-    A.sort_by_row_and_column();
-    return A;
-}
-
-///@deprecated
-cusp::coo_matrix<int, double, cusp::host_memory> smoothing( const dg::aRefinedGrid3d& g)
-{
-    cusp::coo_matrix<int, double, cusp::host_memory> A = interpolation(g);
-    cusp::coo_matrix<int, double, cusp::host_memory> B = projection(g);
-    cusp::coo_matrix<int, double, cusp::host_memory> C;
-    cusp::multiply( A, B, C);
-    C.sort_by_row_and_column();
-    return C; 
-}
-
-}//namespace create
-
 }//namespace dg
-- 
GitLab


From 784e4c1d279070d14b86a0b6bb3cfe3f8c686b30 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sat, 12 Aug 2017 22:17:32 +0200
Subject: [PATCH 149/453] remove CurvilinearRefinedGrid

---
 inc/dg/geometry/refined_curvilinear.h | 156 --------------------------
 1 file changed, 156 deletions(-)
 delete mode 100644 inc/dg/geometry/refined_curvilinear.h

diff --git a/inc/dg/geometry/refined_curvilinear.h b/inc/dg/geometry/refined_curvilinear.h
deleted file mode 100644
index 15570aa71..000000000
--- a/inc/dg/geometry/refined_curvilinear.h
+++ /dev/null
@@ -1,156 +0,0 @@
-#pragma once
-
-#include "dg/geometry/refined_grid.h"
-#include "curvilinear.h"
-
-namespace dg
-{
-
-///@cond
-struct CurvilinearRefinedGrid2d; 
-///@endcond
-
-///@addtogroup grids
-///@{
-
-/**
- * @brief A curvilinear refined grid
- */
-struct CurvilinearRefinedGrid3d : public dg::aGeometry3d
-{
-    typedef CurvilinearRefinedGrid2d perpendicular_grid; //!< the two-dimensional grid type
-
-    CurvilinearRefinedGrid3d( const aRefinement1d& refX, const aRefinement1d& refY, const aRefinement1d& refZ, const aGenerator2d& generator, unsigned n, unsigned n_old, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx): 
-        dg::aGeometry3d( 0, generator.width(), 0., 2.*M_PI, 0., 2.*M_PI, n, n_old, Nx, Ny, Nz, bcx, dg::PER, dg::PER),
-        g_assoc_( generator, n_old, Nx, Ny, Nz, bcx)
-    { 
-        construct( generator);
-    }
-    perpendicular_grid perp_grid() const { return perpendicular_grid(*this);}
-    const dg::CurvilinearGrid3d<container>& associated() const{ return g_assoc_;}
-
-    const thrust::host_vector<double>& r()const{return r_;}
-    const thrust::host_vector<double>& z()const{return z_;}
-    const thrust::host_vector<double>& xr()const{return xr_;}
-    const thrust::host_vector<double>& yr()const{return yr_;}
-    const thrust::host_vector<double>& xz()const{return xz_;}
-    const thrust::host_vector<double>& yz()const{return yz_;}
-    const container& g_xx()const{return g_xx_;}
-    const container& g_yy()const{return g_yy_;}
-    const container& g_xy()const{return g_xy_;}
-    const container& g_pp()const{return g_pp_;}
-    const container& vol()const{return vol_;}
-    const container& perpVol()const{return vol2d_;}
-    private:
-    void construct( const geo::aGenerator* generator)
-    {
-        init_X_boundaries( 0., generator->width());
-        unsigned sizeY = this->n()*this->Ny();
-        unsigned sizeX = this->n()*this->Nx();
-        thrust::host_vector<double> y_vec(sizeY), x_vec(sizeX);
-        for(unsigned i=0; i<sizeY; i++) y_vec[i] = this->abscissasY()[i*sizeX];
-        for(unsigned i=0; i<sizeX; i++) x_vec[i] = this->abscissasX()[i];
-        (*generator)( x_vec, y_vec, r_, z_, xr_, xz_, yr_, yz_);
-        lift3d( ); //lift to 3D grid
-        construct_metric();
-    }
-    void lift3d( )
-    {
-        //lift to 3D grid
-        unsigned size = this->size();
-        r_.resize( size), z_.resize(size), xr_.resize(size), yr_.resize( size), xz_.resize( size), yz_.resize(size);
-        unsigned Nx = this->n()*this->Nx(), Ny = this->n()*this->Ny();
-        thrust::host_vector<double> wx = this->weightsX();
-        thrust::host_vector<double> wy = this->weightsY();
-        for( unsigned k=1; k<this->Nz(); k++)
-            for( unsigned i=0; i<Nx*Ny; i++)
-            {
-                r_[k*Nx*Ny+i] = r_[(k-1)*Nx*Ny+i];
-                z_[k*Nx*Ny+i] = z_[(k-1)*Nx*Ny+i];
-                yr_[k*Nx*Ny+i] = yr_[(k-1)*Nx*Ny+i]*wy[k*Nx*Ny+i];
-                yz_[k*Nx*Ny+i] = yz_[(k-1)*Nx*Ny+i]*wy[k*Nx*Ny+i];
-                xr_[k*Nx*Ny+i] = xr_[(k-1)*Nx*Ny+i]*wx[k*Nx*Ny+i];
-                xz_[k*Nx*Ny+i] = xz_[(k-1)*Nx*Ny+i]*wx[k*Nx*Ny+i];
-            }
-    }
-    void construct_metric()
-    {
-        thrust::host_vector<double> tempxx( r_), tempxy(r_), tempyy(r_), tempvol(r_);
-        for( unsigned idx=0; idx<this->size(); idx++)
-        {
-            tempxx[idx] = (xr_[idx]*xr_[idx]+xz_[idx]*xz_[idx]);
-            tempxy[idx] = (yr_[idx]*xr_[idx]+yz_[idx]*xz_[idx]);
-            tempyy[idx] = (yr_[idx]*yr_[idx]+yz_[idx]*yz_[idx]);
-            tempvol[idx] = r_[idx]/sqrt( tempxx[idx]*tempyy[idx] - tempxy[idx]*tempxy[idx] );
-        }
-        g_xx_=tempxx, g_xy_=tempxy, g_yy_=tempyy, vol_=tempvol;
-        dg::blas1::pointwiseDivide( tempvol, r_, tempvol);
-        vol2d_ = tempvol;
-        thrust::host_vector<double> ones = dg::evaluate( dg::one, *this);
-        dg::blas1::pointwiseDivide( ones, r_, tempxx);
-        dg::blas1::pointwiseDivide( tempxx, r_, tempxx); //1/R^2
-        g_pp_=tempxx;
-    }
-    thrust::host_vector<double> r_, z_, xr_, xz_, yr_, yz_; 
-    container g_xx_, g_xy_, g_yy_, g_pp_, vol_, vol2d_;
-    dg::CurvilinearGrid3d<container> g_assoc_;
-    
-};
-
-/**
- * @brief A curvilinear refined grid
- * @deprecated
- */
-template< class container>
-struct CurvilinearRefinedGrid2d : public dg::RefinedGrid2d
-{
-    typedef dg::CurvilinearCylindricalTag metric_category;
-
-    CurvilinearRefinedGrid2d( unsigned multiple_x, unsigned multiple_y, const geo::aGenerator* generator, unsigned n, unsigned n_old, unsigned Nx, unsigned Ny, dg::bc bcx):
-        dg::RefinedGrid2d( multiple_x, multiple_y, 0, 1., 0., 2*M_PI, n,n_old,Nx,Ny, bcx, dg::PER),
-        g_assoc_( generator, n_old, Nx, Ny, bcx) 
-    {
-        dg::CurvilinearRefinedGrid3d<container> g( multiple_x, multiple_y, generator, n,n_old,Nx,Ny,1,bcx);
-        init_X_boundaries( g.x0(), g.x1());
-        r_=g.r(), z_=g.z(), xr_=g.xr(), xz_=g.xz(), yr_=g.yr(), yz_=g.yz();
-        g_xx_=g.g_xx(), g_xy_=g.g_xy(), g_yy_=g.g_yy();
-        vol2d_=g.perpVol();
-    }
-
-    CurvilinearRefinedGrid2d( const CurvilinearRefinedGrid3d<container>& g):
-        dg::RefinedGrid2d( g), g_assoc_( g.associated())
-    {
-        unsigned s = this->size();
-        r_.resize( s), z_.resize(s), xr_.resize(s), xz_.resize(s), yr_.resize(s), yz_.resize(s);
-        g_xx_.resize( s), g_xy_.resize(s), g_yy_.resize(s), vol2d_.resize(s);
-        for( unsigned i=0; i<s; i++)
-        {r_[i]=g.r()[i], z_[i]=g.z()[i], xr_[i]=g.xr()[i], xz_[i]=g.xz()[i], yr_[i]=g.yr()[i], yz_[i]=g.yz()[i];}
-        thrust::copy( g.g_xx().begin(), g.g_xx().begin()+s, g_xx_.begin());
-        thrust::copy( g.g_xy().begin(), g.g_xy().begin()+s, g_xy_.begin());
-        thrust::copy( g.g_yy().begin(), g.g_yy().begin()+s, g_yy_.begin());
-        thrust::copy( g.perpVol().begin(), g.perpVol().begin()+s, vol2d_.begin());
-    }
-
-    const dg::CurvilinearGrid2d<container>& associated()const{return g_assoc_;}
-
-    const thrust::host_vector<double>& r()const{return r_;}
-    const thrust::host_vector<double>& z()const{return z_;}
-    const thrust::host_vector<double>& xr()const{return xr_;}
-    const thrust::host_vector<double>& yr()const{return yr_;}
-    const thrust::host_vector<double>& xz()const{return xz_;}
-    const thrust::host_vector<double>& yz()const{return yz_;}
-    const container& g_xx()const{return g_xx_;}
-    const container& g_yy()const{return g_yy_;}
-    const container& g_xy()const{return g_xy_;}
-    const container& vol()const{return vol2d_;}
-    const container& perpVol()const{return vol2d_;}
-    bool isOrthogonal() const{return false;}
-    bool isConformal() const{return false;}
-    private:
-    thrust::host_vector<double> r_, z_, xr_, xz_, yr_, yz_; 
-    container g_xx_, g_xy_, g_yy_, vol2d_;
-    dg::CurvilinearGrid2d<container> g_assoc_;
-};
-
-///@}
-}//namespace dg
-- 
GitLab


From d891f5616dc79b17f22de73926ff575cb0b4008f Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sun, 13 Aug 2017 00:43:26 +0200
Subject: [PATCH 150/453] debugged volume, refined_grid, moved geometry_t to
 geometry

---
 inc/dg/backend/operator_tensor.cuh     |  8 +--
 inc/dg/backend/operator_tensor_t.cu    |  8 +--
 inc/dg/geometry/curvilinear.h          | 22 +++---
 inc/dg/geometry/generator.h            | 11 +--
 inc/dg/geometry/geometry.h             | 57 +--------------
 inc/dg/{ => geometry}/geometry_mpit.cu |  0
 inc/dg/{ => geometry}/geometry_t.cu    | 15 ++--
 inc/dg/geometry/refined_grid.h         | 97 +++++++++++++++-----------
 inc/dg/geometry/refined_grid_t.cu      | 45 ++++++------
 inc/dg/geometry/transform.h            | 49 ++++++++++++-
 10 files changed, 166 insertions(+), 146 deletions(-)
 rename inc/dg/{ => geometry}/geometry_mpit.cu (100%)
 rename inc/dg/{ => geometry}/geometry_t.cu (77%)

diff --git a/inc/dg/backend/operator_tensor.cuh b/inc/dg/backend/operator_tensor.cuh
index 858fd5be1..ba57c4598 100644
--- a/inc/dg/backend/operator_tensor.cuh
+++ b/inc/dg/backend/operator_tensor.cuh
@@ -25,7 +25,7 @@ namespace dg
 * @return The  tensor product
 */
 template< class T>
-Operator<T> tensor( const Operator<T>& op1, const Operator<T>& op2)
+Operator<T> tensorproduct( const Operator<T>& op1, const Operator<T>& op2)
 {
 #ifdef DG_DEBUG
     assert( op1.size() == op2.size());
@@ -53,7 +53,7 @@ Operator<T> tensor( const Operator<T>& op1, const Operator<T>& op2)
 * @return A newly allocated cusp matrix
 */
 template< class T>
-cusp::coo_matrix<int,T, cusp::host_memory> tensor( unsigned N, const Operator<T>& op)
+cusp::coo_matrix<int,T, cusp::host_memory> tensorproduct( unsigned N, const Operator<T>& op)
 {
     assert( N>0);
     unsigned n = op.size();
@@ -98,8 +98,8 @@ cusp::coo_matrix<int, T, cusp::host_memory> sandwich( const Operator<T>& left,
     typedef cusp::coo_matrix<int, T, cusp::host_memory> Matrix;
     unsigned n = left.size();
     unsigned N = m.num_rows/n;
-    Matrix r = tensor( N, right);
-    Matrix l = tensor( N, left);
+    Matrix r = tensorproduc( N, right);
+    Matrix l = tensorproduc( N, left);
     Matrix mr(m ), lmr(m);
 
     cusp::multiply( m, r, mr);
diff --git a/inc/dg/backend/operator_tensor_t.cu b/inc/dg/backend/operator_tensor_t.cu
index 5ee3da428..f33c05b46 100644
--- a/inc/dg/backend/operator_tensor_t.cu
+++ b/inc/dg/backend/operator_tensor_t.cu
@@ -17,10 +17,10 @@ int main()
     dg::Operator<double> Op2 = dlt.forward( );
     std::cout << "1st Operator is:\n" << Op1<<"\n";
     std::cout << "2nd Operator is:\n" << Op2<<"\n";
-    std::cout << "Tensor Product is: \n" << dg::tensor( Op1, Op2);
-    cusp::coo_matrix<int, double, cusp::host_memory> test1 = dg::tensor( 2, dg::create::delta(3));
-    cusp::coo_matrix<int, double, cusp::host_memory> test2 = dg::tensor( 2, Op2);
-    cusp::coo_matrix<int, double, cusp::host_memory> test3 = dg::tensor( 2, dg::tensor( Op1, Op2));
+    std::cout << "Tensor Product is: \n" << dg::tensorproduct( Op1, Op2);
+    cusp::coo_matrix<int, double, cusp::host_memory> test1 = dg::tensorproduct( 2, dg::create::delta(3));
+    cusp::coo_matrix<int, double, cusp::host_memory> test2 = dg::tensorproduct( 2, Op2);
+    cusp::coo_matrix<int, double, cusp::host_memory> test3 = dg::tensorproduct( 2, dg::tensorproduct( Op1, Op2));
     cusp::print(test1);
     cusp::print(test2);
     cusp::print(test3);
diff --git a/inc/dg/geometry/curvilinear.h b/inc/dg/geometry/curvilinear.h
index 98dd75e6f..7393f62d5 100644
--- a/inc/dg/geometry/curvilinear.h
+++ b/inc/dg/geometry/curvilinear.h
@@ -2,8 +2,7 @@
 
 #include "dg/backend/manage.h"
 #include "dg/blas1.h"
-#include "geometry.h"
-#include "geometry_traits.h"
+#include "base_geometry.h"
 #include "generator.h"
 
 namespace dg
@@ -26,7 +25,7 @@ struct CurvilinearGrid2d;
  */
 struct CylindricalProductGrid3d : public dg::aGeometry3d
 {
-    typedef CurvilinearGrid2d<container> perpendicular_grid;
+    typedef CurvilinearGrid2d perpendicular_grid;
 
     /*!@brief Constructor
     
@@ -40,7 +39,7 @@ struct CylindricalProductGrid3d : public dg::aGeometry3d
      * @param bcy boundary condition in y
      * @param bcz boundary condition in z
      */
-    CylindricalProductGrid3d( const aGenerator& generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx=dg::DIR, bc bcy=dg::PER, bc bcz=dg::PER):
+    CylindricalProductGrid3d( const aGenerator2d& generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx=dg::DIR, bc bcy=dg::PER, bc bcz=dg::PER):
         dg::aGeometry3d( 0, generator.width(), 0., generator.height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, bcy, bcz)
     { 
         map_.resize(3);
@@ -49,9 +48,10 @@ struct CylindricalProductGrid3d : public dg::aGeometry3d
         constructParallel(Nz);
     }
 
-    perpendicular_grid perp_grid() const { return perpendicular_grid(*this);}
+    perpendicular_grid perp_grid() const;// { return perpendicular_grid(*this);}
 
-    const aGenerator2d & generator() const{return handle_.get()}
+    const aGenerator2d & generator() const{return handle_.get();}
+    virtual CylindricalProductGrid3d* clone()const{return new CylindricalProductGrid3d(*this);}
     private:
     virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny,unsigned new_Nz){
         dg::aTopology3d::do_set( new_n, new_Nx, new_Ny,new_Nz);
@@ -117,7 +117,7 @@ struct CylindricalProductGrid3d : public dg::aGeometry3d
     virtual std::vector<thrust::host_vector<double> > do_compute_map()const{return map_;}
     std::vector<thrust::host_vector<double> > map_;
     SparseTensor<thrust::host_vector<double> > jac_;
-    dg::Handle<aGenerator> handle_;
+    dg::Handle<aGenerator2d> handle_;
 };
 
 /**
@@ -139,7 +139,7 @@ struct CurvilinearGrid2d : public dg::aGeometry2d
     {
         construct( n,Nx,Ny);
     }
-    explicit CurvilinearGrid2d( CylindricalProductGrid3d<container> g):
+    explicit CurvilinearGrid2d( CylindricalProductGrid3d g):
         dg::aGeometry2d( g.x0(), g.x1(), g.y0(), g.y1(), g.n(), g.Nx(), g.Ny(), g.bcx(), g.bcy() ), handle_(g.generator())
     {
         g.set( n(), Nx(), Ny(), 1); //shouldn't trigger 2d grid generator
@@ -159,7 +159,7 @@ struct CurvilinearGrid2d : public dg::aGeometry2d
     }
     void construct( unsigned n, unsigned Nx, unsigned Ny)
     {
-        CurvilinearGrid3d<container> g( handle_.get(), n,Nx,Ny,1,bcx());
+        CylindricalProductGrid3d g( handle_.get(), n,Nx,Ny,1,bcx());
         jac_=g.jacobian();
         map_=g.map();
         metric_=g.metric();
@@ -177,4 +177,8 @@ struct CurvilinearGrid2d : public dg::aGeometry2d
 };
 
 ///@}
+///@cond
+CylindricalProductGrid3d::perpendicular_grid CylindricalProductGrid3d::perp_grid() const { return CylindricalProductGrid3d::perpendicular_grid(*this);}
+///@endcond
+
 }//namespace dg
diff --git a/inc/dg/geometry/generator.h b/inc/dg/geometry/generator.h
index efaef6c96..2ca67b251 100644
--- a/inc/dg/geometry/generator.h
+++ b/inc/dg/geometry/generator.h
@@ -56,12 +56,15 @@ struct aGenerator2d
     *
     * @return a copy of *this on the heap
     */
-    virtual aGridGenerator* clone() const=0;
-    virtual ~aGridGenerator(){}
+    virtual aGenerator2d* clone() const=0;
+    virtual ~aGenerator2d(){}
 
     protected:
-    aGridGenerator(const aGridGenerator& src){}
-    aGridGenerator& operator=(const aGridGenerator& src){}
+    aGenerator2d(){}
+    aGenerator2d(const aGenerator2d& src){}
+    aGenerator2d& operator=(const aGenerator2d& src){
+        return *this;
+    }
     private:
     virtual void do_generate(
          const thrust::host_vector<double>& zeta1d, 
diff --git a/inc/dg/geometry/geometry.h b/inc/dg/geometry/geometry.h
index ffc7465dd..1d6ced142 100644
--- a/inc/dg/geometry/geometry.h
+++ b/inc/dg/geometry/geometry.h
@@ -10,65 +10,12 @@
 #include "../backend/mpi_precon.h"
 #endif//MPI_VERSION
 #include "base_geometry.h"
-//#include "curvilinear.h"
+#include "curvilinear.h"
 //#include "cartesianX.h"
 #ifdef MPI_VERSION
 #include "mpi_base.h"
-//#include "mpi_curvilinear.h"
+#include "mpi_curvilinear.h"
 #endif//MPI_VERSION
 #include "tensor.h"
 #include "transform.h"
 #include "multiply.h"
-
-
-/*!@file 
- *
- * geometry functions
- */
-
-namespace dg{
-
-namespace create{
-///@addtogroup metric
-///@{
-
-/**
- * @brief Create the volume element on the grid (including weights!!)
- *
- * This is the same as the weights multiplied by the volume form \f$ \sqrt{g}\f$
- * @tparam Geometry any Geometry class
- * @param g Geometry object
- *
- * @return  The volume form
- */
-template< class Geometry>
-typename GeometryTraits<Geometry>::host_vector volume( const Geometry& g)
-{
-    typedef typename GeometryTraits< Geometry>::host_vector host_vector;
-    SparseElement<host_vector> vol = dg::tensor::determinant(g.metric());
-    host_vector temp = dg::create::weights( g);
-    dg::tensor::pointwiseDot( vol, temp, temp);
-    return temp;
-}
-
-/**
- * @brief Create the inverse volume element on the grid (including weights!!)
- *
- * This is the same as the inv_weights divided by the volume form \f$ \sqrt{g}\f$
- * @tparam Geometry any Geometry class
- * @param g Geometry object
- *
- * @return  The inverse volume form
- */
-template< class Geometry>
-typename GeometryTraits<Geometry>::host_vector inv_volume( const Geometry& g)
-{
-    typedef typename GeometryTraits< Geometry>::host_vector host_vector;
-    host_vector temp = volume(g);
-    dg::blas1::transform(temp,temp,dg::INVERT<double>());
-    return temp;
-}
-
-///@}
-}//namespace create
-}//namespace dg
diff --git a/inc/dg/geometry_mpit.cu b/inc/dg/geometry/geometry_mpit.cu
similarity index 100%
rename from inc/dg/geometry_mpit.cu
rename to inc/dg/geometry/geometry_mpit.cu
diff --git a/inc/dg/geometry_t.cu b/inc/dg/geometry/geometry_t.cu
similarity index 77%
rename from inc/dg/geometry_t.cu
rename to inc/dg/geometry/geometry_t.cu
index 5d83cea22..1640cde5c 100644
--- a/inc/dg/geometry_t.cu
+++ b/inc/dg/geometry/geometry_t.cu
@@ -3,16 +3,16 @@
 #include <cusp/print.h>
 
 #include "geometry.h"
-#include "backend/evaluation.cuh"
+#include "dg/backend/evaluation.cuh"
 
-#include "blas2.h"
+#include "../blas2.h"
 
 double R_0 = 4.*M_PI;
 
-
 double sine( double R, double Z,double phi){ return sin(R-R_0)*sin(Z)*sin(phi)/sqrt(R);}
 
 namespace dg { typedef thrust::device_vector<double> DVec; }
+namespace dg { typedef thrust::host_vector<double> HVec; }
 
 //TEST geometry.h for every container and geometry that you want to use
 int main()
@@ -21,7 +21,10 @@ int main()
     //unsigned n, Nx, Ny, Nz;
     //std::cin >> n>> Nx>>Ny>>Nz;
     //dg::CylindricalGrid3d<dg::DVec> grid( R_0 , R_0+ 2.*M_PI, 0.,2.*M_PI, 0., 2.*M_PI,  n, Nx, Ny, Nz, dg::DIR, dg::DIR, dg::PER);
-    dg::CylindricalGrid3d<dg::DVec> grid( R_0 , R_0+ 2.*M_PI, 0.,2.*M_PI, 0., 2.*M_PI,  3,32,24,16, dg::DIR, dg::DIR, dg::PER);
+    dg::CylindricalGrid3d grid( R_0 , R_0+ 2.*M_PI, 0.,2.*M_PI, 0., 2.*M_PI,  3,32,24,16, dg::DIR, dg::DIR, dg::PER);
+    dg::SparseElement<dg::DVec> vol = dg::tensor::determinant(grid.metric());
+    dg::tensor::sqrt(vol);
+    dg::tensor::invert(vol);
 
     dg::DVec b = dg::evaluate( sine, grid);
     dg::DVec vol3d = dg::create::volume( grid);
@@ -30,7 +33,7 @@ int main()
     std::cout << "Test of volume:         "<<test<< " sol = " << sol<< "\t";
     std::cout << "rel diff = " <<( test -  sol)/ sol<<"\n";
     dg::DVec temp = dg::create::weights( grid);
-    dg::geo::multiplyVolume( temp, grid);
+    dg::tensor::pointwiseDot( vol, temp, temp);
     test = dg::blas2::dot( b, temp, b);
     std::cout << "Test of multiplyVolume: "<<test<< " sol = " << sol<< "\t";
     std::cout << "rel diff = " <<( test -  sol)/ sol<<"\n";
@@ -41,7 +44,7 @@ int main()
     std::cout << "Test of inv_volume:     "<<test<< " sol = " << sol<< "\t";
     std::cout << "rel diff = " <<( test -  sol)/ sol<<"\n";
     temp = dg::create::inv_weights( grid);
-    dg::geo::divideVolume( temp, grid);
+    dg::tensor::pointwiseDivide(temp, vol, temp );
     test = dg::blas2::dot( b, temp, b);
     std::cout << "Test of divideVolume:   "<<test<< " sol = " << sol<< "\t";
     std::cout << "rel diff = " <<( test -  sol)/ sol<<"\n";
diff --git a/inc/dg/geometry/refined_grid.h b/inc/dg/geometry/refined_grid.h
index 29be1cd46..d2d3da28d 100644
--- a/inc/dg/geometry/refined_grid.h
+++ b/inc/dg/geometry/refined_grid.h
@@ -63,36 +63,45 @@ struct aRefinement1d
      * @param weights A 1d vector of size N_new. These represent the Jacobian of the transformation \f[\frac{\partial \zeta}{\partial x} \f]. The new metric element has thus to be multiplied by weights^2 and the volume by 1/weights
      * @param abscissas A 1d vector of size N_new. These are the new abscissas \f$ x(\zeta) \f$ of the grid. 
     */
-    void generate( const Grid1d& g_old, thrust::host_vector<double>& weights, thrust::host_vector<double>& abscissas)
+    void generate( const Grid1d& g_old, thrust::host_vector<double>& weights, thrust::host_vector<double>& abscissas) const
     {
-        weights.resize( new_size(g_old.N(), g_old.bcx()));
-        abscissas.resize( new_size(g_old.N(), g_old.bcx()));
+        weights.resize( N_new(g_old.N(), g_old.bcx()));
+        abscissas.resize( N_new(g_old.N(), g_old.bcx()));
         do_generate(g_old,weights,abscissas);
     }
     /*! @brief the new number of cells
      * @param N_old the old number of cells
      * @param bcx the boundary condition of the grid
      */
-    unsigned N_new( unsigned N_old, bc bcx)
+    unsigned N_new( unsigned N_old, bc bcx) const
     {
-        do_N_new(N_old, bcx);
+        return do_N_new(N_old, bcx);
+    }
+    virtual aRefinement1d* clone()const=0;
+    virtual ~aRefinement1d(){}
+    protected:
+    aRefinement1d(){}
+    aRefinement1d(const aRefinement1d& src){}
+    aRefinement1d& operator=(const aRefinement1d& src){
+        return *this;
     }
     private:
     virtual void do_generate( const Grid1d& g, thrust::host_vector<double>& weights, thrust::host_vector<double>& abscissas) const =0;
-    virtual unsigned new_size( unsigned N_old, bc bcx) const =0;
+    virtual unsigned do_N_new( unsigned N_old, bc bcx) const =0;
 };
 
 /**
 * @brief No refinement
 */
-struct IdentityRefinement : public aRefinementd
+struct IdentityRefinement : public aRefinement1d
 {
+    IdentityRefinement* clone()const{return new IdentityRefinement();}
     private:
     virtual void do_generate( const Grid1d& g, thrust::host_vector<double>& weights, thrust::host_vector<double>& abscissas) const {
         weights=dg::create::weights(g);
         abscissas=dg::create::abscissas(g);
     }
-    virtual unsigned new_size( unsigned N_old, bc bcx) const {
+    virtual unsigned do_N_new( unsigned N_old, bc bcx) const {
         return N_old;
     }
 };
@@ -107,6 +116,7 @@ struct LinearRefinement : public aRefinement1d
      * @param multiple multiply every cell
      */
     LinearRefinement( unsigned multiple): m_(multiple){}
+    LinearRefinement* clone()const{return new LinearRefinement(*this);}
     private:
     unsigned m_;
     virtual void do_generate( const Grid1d& g, thrust::host_vector<double>& weights, thrust::host_vector<double>& abscissas) const 
@@ -118,7 +128,7 @@ struct LinearRefinement : public aRefinement1d
     {
         return N_old*m_;
     }
-    thrust::host_vector<double> linear_ref( unsigned multiple_x, unsigned n, unsigned N, dg::bc bcx)
+    thrust::host_vector<double> linear_ref( unsigned multiple_x, unsigned n, unsigned N, dg::bc bcx) const
     {
         assert( multiple_x >= 1);
         //there are add_x+1 finer cells per refined cell ...
@@ -145,6 +155,7 @@ struct EquidistRefinement : public aRefinement1d
      * @param howmany  number of cells around a node to refine
      */
     EquidistRefinement( unsigned add_x, unsigned node, unsigned howmany=1): add_x_(add_x), node_(node), howm_(howmany){ }
+    EquidistRefinement* clone()const{return new EquidistRefinement(*this);}
     private:
     unsigned add_x_, node_, howm_;
     virtual void do_generate( const Grid1d& g, thrust::host_vector<double>& weights, thrust::host_vector<double>& abscissas) const 
@@ -157,14 +168,14 @@ struct EquidistRefinement : public aRefinement1d
             return;
         }
         weights = equidist_ref( add_x_, node_, g.n(), g.N(), g.bcx(), howm_);
-        abscissas = normalize_weights_and_compute_abscissas( g, weights);
+        abscissas = detail::normalize_weights_and_compute_abscissas( g, weights);
     }
     virtual unsigned do_N_new( unsigned N_old, bc bcx) const
     {
-        if( bc == dg::PER) return N_old + 2*add_x_*howm_; 
+        if( bcx == dg::PER) return N_old + 2*add_x_*howm_; 
         return N_old + add_x_*howm_;
     }
-    thrust::host_vector<double> equidist_ref( unsigned add_x, unsigned node, unsigned n, unsigned N, dg::bc bcx, unsigned howmany)
+    thrust::host_vector<double> equidist_ref( unsigned add_x, unsigned node, unsigned n, unsigned N, dg::bc bcx, unsigned howmany) const
     {
         assert( howm_ <= N);
         assert( node_ <= N);
@@ -218,6 +229,7 @@ struct ExponentialRefinement : public aRefinement1d
      * 0 (left corner) to N (right corner).
      */
     ExponentialRefinement( unsigned add_x, unsigned node): add_x_(add_x), node_(node) {}
+    ExponentialRefinement* clone()const{return new ExponentialRefinement(*this);}
     private:
     unsigned add_x_, node_;
     virtual void do_generate( const Grid1d& g, thrust::host_vector<double>& weights, thrust::host_vector<double>& abscissas) const 
@@ -230,14 +242,14 @@ struct ExponentialRefinement : public aRefinement1d
             return;
         }
         weights = exponential_ref( add_x_, node_, g.n(), g.N(), g.bcx());
-        abscissas = normalize_weights_and_compute_abscissas( g, weights);
+        abscissas = detail::normalize_weights_and_compute_abscissas( g, weights);
     }
     virtual unsigned do_N_new( unsigned N_old, bc bcx) const
     {
-        if( bc == dg::PER) return N_old + 2*add_x_; 
+        if( bcx == dg::PER) return N_old + 2*add_x_; 
         return N_old + add_x_;
     }
-    thrust::host_vector<double> exponential_ref( unsigned add_x, unsigned node, unsigned n, unsigned N, dg::bc bcx)
+    thrust::host_vector<double> exponential_ref( unsigned add_x, unsigned node, unsigned n, unsigned N, dg::bc bcx) const
     {
         if( add_x_ == 0)
         {
@@ -285,16 +297,17 @@ struct CartesianRefinedGrid2d : public dg::aGeometry2d
     CartesianRefinedGrid2d( const aRefinement1d& refX, const aRefinement1d& refY, double x0, double x1, double y0, double y1, 
             unsigned n, unsigned Nx, unsigned Ny, bc bcx = dg::PER, bc bcy = dg::PER) : dg::aGeometry2d( x0, x1, y0, y1, n, refX.N_new(Nx, bcx), refY.N_new(Ny,bcy), bcx, bcy), refX_(refX), refY_(refY), w_(2), a_(2)
     {
-        construct_weights_and_abscissas();
+        construct_weights_and_abscissas(n,Nx,Ny);
     }
 
+    CartesianRefinedGrid2d* clone()const{return new CartesianRefinedGrid2d(*this);}
     private:
     Handle<aRefinement1d> refX_, refY_;
     std::vector<thrust::host_vector<double> > w_, a_;
-    void construct_weights_and_abscissas()
+    void construct_weights_and_abscissas(unsigned n, unsigned Nx, unsigned Ny)
     { 
-        Grid1d gx( x0, x1, n, Nx, bcx);
-        Grid1d gy( y0, y1, n, Ny, bcy);
+        Grid1d gx( x0(), x1(), n, Nx, bcx());
+        Grid1d gy( y0(), y1(), n, Ny, bcy());
         thrust::host_vector<double> wx, ax, wy, ay;
         refX_.get().generate( gx, wx, ax);
         refY_.get().generate( gy, wy, ay);
@@ -311,8 +324,8 @@ struct CartesianRefinedGrid2d : public dg::aGeometry2d
             }
     }
     virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny){
-        aTopology2d::do_set(new_n,new_Nx,new_Ny);
-        construct_weights_and_abscissas();
+        aTopology2d::do_set(new_n,refX_.get().N_new(new_Nx,bcx()),refY_.get().N_new(new_Ny, bcy()));
+        construct_weights_and_abscissas(new_n,new_Nx,new_Ny);
     }
     virtual SparseTensor<thrust::host_vector<double> > do_compute_metric()const {
         SparseTensor<thrust::host_vector<double> > t(w_);
@@ -340,38 +353,40 @@ struct CartesianRefinedGrid3d : public dg::aGeometry3d
     CartesianRefinedGrid3d( const aRefinement1d& refX, const aRefinement1d& refY, aRefinement1d& refZ, double x0, double x1, double y0, double y1, double z0, double z1,
             unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = dg::PER, bc bcy = dg::PER, bc bcz=dg::PER) : dg::aGeometry3d( x0, x1, y0, y1,z0,z1, n, refX.N_new(Nx, bcx), refY.N_new(Ny,bcy), refZ.N_new(Nz,bcz), bcx, bcy, bcz), refX_(refX), refY_(refY), refZ_(refZ), w_(3), a_(3)
     {
-        construct_weights_and_abscissas();
+        construct_weights_and_abscissas(n, Nx, Ny,Nz);
     }
 
+    CartesianRefinedGrid3d* clone()const{return new CartesianRefinedGrid3d(*this);}
     private:
     Handle<aRefinement1d> refX_, refY_, refZ_;
     std::vector<thrust::host_vector<double> > w_, a_;
-    void construct_weights_and_abscissas()
+    void construct_weights_and_abscissas(unsigned n, unsigned Nx, unsigned Ny,unsigned Nz)
     { 
-        Grid1d gx( x0, x1, n, Nx, bcx);
-        Grid1d gy( y0, y1, n, Ny, bcy);
-        Grid1d gz( y0, y1, 1, Nz, bcz);
-        thrust::host_vector<double> wx, ax, wy, ay, wz, az;
-        refX_.get().generate( gx, wx, ax);
-        refY_.get().generate( gy, wy, ay);
-        refZ_.get().generate( gz, wz, az);
+        Grid1d gx( x0(), x1(), n, Nx, bcx());
+        Grid1d gy( y0(), y1(), n, Ny, bcy());
+        Grid1d gz( y0(), y1(), 1, Nz, bcz());
+        thrust::host_vector<double> w[3], a[3];
+        refX_.get().generate( gx, w[0], a[0]);
+        refY_.get().generate( gy, w[1], a[1]);
+        refZ_.get().generate( gz, w[2], a[2]);
         w_[0].resize(size()), w_[1].resize(size()), w_[2].resize(size()); 
         a_[0].resize(size()), a_[1].resize(size()), a_[2].resize(size()); 
         //now make product space
-        for( unsigned i=0; i<wy.size(); i++)
-            for( unsigned j=0; j<wx.size(); j++)
+    for( unsigned s=0; s<w[2].size(); s++)
+        for( unsigned i=0; i<w[1].size(); i++)
+            for( unsigned j=0; j<w[0].size(); j++)
             {
-                w_[0][i*wx.size()+j] = wx[j];
-                w_[1][i*wx.size()+j] = wy[i];
-                w_[2][i*wx.size()+j] = wz[i];
-                a_[0][i*wx.size()+j] = ax[j];
-                a_[1][i*wx.size()+j] = ay[i];
-                a_[2][i*wx.size()+j] = az[i];
+                w_[0][(s*w[1].size()+i)*w[0].size()+j] = w[0][j];
+                w_[1][(s*w[1].size()+i)*w[0].size()+j] = w[1][i];
+                w_[2][(s*w[1].size()+i)*w[0].size()+j] = w[2][s];
+                a_[0][(s*w[1].size()+i)*w[0].size()+j] = a[0][j];
+                a_[1][(s*w[1].size()+i)*w[0].size()+j] = a[1][i];
+                a_[2][(s*w[1].size()+i)*w[0].size()+j] = a[1][s];
             }
     }
-    virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny){
-        aTopology3d::do_set(new_n,new_Nx,new_Ny);
-        construct_weights_and_abscissas();
+    virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz){
+        aTopology3d::do_set(new_n,refX_.get().N_new(new_Nx, bcx()),refY_.get().N_new(new_Ny,bcy()), refZ_.get().N_new(new_Nz,bcz()));
+        construct_weights_and_abscissas(new_n, new_Nx, new_Ny, new_Nz);
     }
     virtual SparseTensor<thrust::host_vector<double> > do_compute_metric()const {
         SparseTensor<thrust::host_vector<double> > t(w_);
diff --git a/inc/dg/geometry/refined_grid_t.cu b/inc/dg/geometry/refined_grid_t.cu
index 4cf1a7b33..2f522b22e 100644
--- a/inc/dg/geometry/refined_grid_t.cu
+++ b/inc/dg/geometry/refined_grid_t.cu
@@ -1,8 +1,11 @@
 #include <iostream>
 
-#include "refined_grid.h"
 #include "../blas.h"
 #include "dg/backend/derivatives.h"
+#include "dg/backend/interpolation.cuh"
+#include "dg/backend/projection.cuh"
+#include "transform.h"
+#include "refined_grid.h"
 
 
 double function( double x, double y){return sin(x)*cos(y);}
@@ -18,7 +21,9 @@ int main ()
     int node;
     std::cout<< "Type node to refine 0,..,4!\n";
     std::cin >> node;
-    int new_N = dg::detail::equidist_ref( 3, node, g, both, both_abs, 1);
+    dg::EquidistRefinement equi( 3, node, 1);
+    int new_N = equi.N_new(g.N(), g.bcx());
+    equi.generate( g, both, both_abs);
     double sum = 0;
     for( unsigned i=0; i<new_N*g.n(); i++)
     {
@@ -28,7 +33,9 @@ int main ()
     std::cout << "SUM IS: "<<sum<<" ("<<new_N<<")\n";
     std::cout<< "LEFT SIDE:\n";
     dg::Grid1d gl( 0,1, 2,5, dg::DIR);
-    new_N = dg::detail::equidist_ref( 2, 0, gl, left, left_abs,2 );
+    dg::EquidistRefinement equi2( 2, 0, 2);
+    new_N = equi2.N_new(gl.N(), gl.bcx());
+    equi2.generate( gl, left, left_abs);
     sum = 0;
     for( unsigned i=0; i<new_N*gl.n(); i++)
     {
@@ -38,7 +45,9 @@ int main ()
     std::cout << "SUM IS: "<<sum<<" ("<<new_N<<")\n";
     std::cout<< "RIGHT SIDE:\n";
     dg::Grid1d gr( 0,1, 1, 5, dg::DIR);
-    new_N = dg::detail::equidist_ref( 5, gr.N(), gr, right, right_abs, 2);
+    dg::EquidistRefinement equi3( 5, gr.N(), 2);
+    new_N = equi2.N_new(gr.N(), gl.bcx());
+    equi2.generate( gr, right, right_abs);
     sum =0;
     for( unsigned i=0; i<new_N*gr.n(); i++)
     {
@@ -47,24 +56,19 @@ int main ()
     }
     std::cout << "SUM IS: "<<sum<<" ("<<new_N<<")\n";
 
-    //dg::RefinedGrid2d g2d_f( 0,0, 3,3, 2,2, 0., 2*M_PI, 0., 2*M_PI, 3, 20, 20);
-    dg::CartesianRefinedGrid2d<dg::HVec> g2d_f( 3,3, 0., 2*M_PI, 0., 2*M_PI, 5, 3, 20, 20);
-    dg::Grid2d g2d_c = g2d_f.associated();
+    dg::LinearRefinement lin(3);
+    dg::CartesianRefinedGrid2d g2d_f( lin,lin, 0., 2*M_PI, 0., 2*M_PI, 5, 20, 20);
+    dg::CartesianGrid2d g2d_c( 0., 2*M_PI, 0., 2*M_PI, 3, 20, 20);
     dg::HVec w2d_c = dg::create::weights( g2d_c);
     dg::HVec vec_c = dg::evaluate( function, g2d_c);
-    dg::HVec vec( g2d_f.size());
-    dg::HVec w2d = dg::create::weights( g2d_f);
-    dg::blas1::pointwiseDivide( w2d, g2d_f.weightsX(), w2d);
-    dg::blas1::pointwiseDivide( w2d, g2d_f.weightsY(), w2d);
-    for( unsigned i=0; i<g2d_f.size(); i++)
-        vec[i] = function( g2d_f.abscissasX()[i], g2d_f.abscissasY()[i]);
+    dg::HVec vec = dg::pullback( function, g2d_f);
+    dg::HVec w2d = dg::create::volume( g2d_f);
     double integral = dg::blas2::dot( vec, w2d, vec);
     std::cout << "error of fine integral is "<<integral-M_PI*M_PI<<std::endl;
     integral = dg::blas2::dot( vec_c, w2d_c, vec_c);
     std::cout << "error of coarse integral is "<<integral-M_PI*M_PI<<std::endl;
-    dg::IHMatrix Q = dg::create::interpolation( g2d_f);
-    dg::IHMatrix P = dg::create::projection( g2d_f);
-    dg::IHMatrix S = dg::create::smoothing( g2d_f);
+    dg::IHMatrix Q = dg::create::interpolation( g2d_f, g2d_c);
+    dg::IHMatrix P = dg::create::projection( g2d_c, g2d_f);
 
     dg::HVec vec_cf(vec);
     dg::blas2::gemv( Q, vec_c, vec_cf);//here gemv instead of symv is important
@@ -74,10 +78,6 @@ int main ()
     integral = dg::blas2::dot( vec_c, w2d_c, vec_c);
     std::cout << "error of projected integral is "<<integral-M_PI*M_PI<<std::endl;
 
-    dg::HVec smoothed(vec);
-    dg::blas2::symv( S, vec, smoothed);
-    integral = dg::blas2::dot( smoothed, w2d, smoothed);
-    std::cout << "error of smoothed integral is "<<integral-M_PI*M_PI<<std::endl;
     //also test P D^f_x Q f_c = D^c_x f 
     std::cout << "TEST OF P D_x^f Q = D_x^c\n";
     const dg::HMatrix dx_f = dg::create::dx( g2d_f);
@@ -87,13 +87,14 @@ int main ()
     dg::blas2::symv( dx_c, vec_c, deri_num);
     dg::blas2::gemv( Q, vec_c, vec_cf);
     dg::blas2::symv( dx_f, vec_cf, vec);
-    dg::blas1::pointwiseDot( vec, g2d_f.weightsX(), vec);
+    dg::SparseTensor<thrust::host_vector<double> > jac=g2d_f.jacobian();
+    dg::blas1::pointwiseDot( vec, jac.value(0,0), vec);
     dg::blas2::gemv( P, vec, vec_c);
     dg::blas1::axpby( 1., vec_c, -1., deri_num, vec_c);
     double error = dg::blas2::dot( vec_c, w2d_c, vec_c);
     std::cout << "error of derivative is "<<error<<std::endl;
 
-    dg::RefinedGrid3d g3d_f( 3,3, 0., 2*M_PI, 0., 2*M_PI, 0., 2*M_PI, 5, 3, 20, 20, 20);
+    dg::CartesianRefinedGrid3d g3d_f( lin,lin,lin, 0., 2*M_PI, 0., 2*M_PI, 0., 2*M_PI, 5, 20, 20, 20);
     g3d_f.display();
 
     return 0;
diff --git a/inc/dg/geometry/transform.h b/inc/dg/geometry/transform.h
index 0f8c5e6fa..e59f9239a 100644
--- a/inc/dg/geometry/transform.h
+++ b/inc/dg/geometry/transform.h
@@ -1,6 +1,7 @@
 #pragma once
 #include "../backend/topological_traits.h"
 #include "multiply.h"
+#include "base_geometry.h"
 
 
 namespace dg
@@ -16,7 +17,7 @@ struct MemoryTraits< SharedTag>
 };
 
 template<class Geometry>
-class GeometryTraits
+struct GeometryTraits
 {
     typedef typename MemoryTraits< typename TopologyTraits<Geometry>::memory_category>::host_vector host_vector;
 
@@ -208,4 +209,50 @@ void pushForwardPerp( FunctorRR chiRR, FunctorRZ chiRZ, FunctorZZ chiZZ,
     dg::tensor::multiply2d( transpose, tmp10, tmp11, chixy, chiyy);
 }
 
+namespace create{
+///@addtogroup metric
+///@{
+
+
+/**
+ * @brief Create the inverse volume element on the grid (including weights!!)
+ *
+ * This is the same as the inv_weights divided by the volume form \f$ \sqrt{g}\f$
+ * @tparam Geometry any Geometry class
+ * @param g Geometry object
+ *
+ * @return  The inverse volume form
+ */
+template< class Geometry>
+typename GeometryTraits<Geometry>::host_vector inv_volume( const Geometry& g)
+{
+    typedef typename GeometryTraits< Geometry>::host_vector host_vector;
+    SparseElement<host_vector> inv_vol = dg::tensor::determinant(g.metric());
+    dg::tensor::sqrt(inv_vol);
+    host_vector temp = dg::create::inv_weights( g);
+    dg::tensor::pointwiseDot( inv_vol,temp, temp);
+    return temp;
+}
+
+/**
+ * @brief Create the volume element on the grid (including weights!!)
+ *
+ * This is the same as the weights multiplied by the volume form \f$ \sqrt{g}\f$
+ * @tparam Geometry any Geometry class
+ * @param g Geometry object
+ *
+ * @return  The volume form
+ */
+template< class Geometry>
+typename GeometryTraits<Geometry>::host_vector volume( const Geometry& g)
+{
+    typedef typename GeometryTraits< Geometry>::host_vector host_vector;
+    host_vector temp = inv_volume(g);
+    dg::blas1::transform(temp,temp,dg::INVERT<double>());
+    return temp;
+}
+
+///@}
+}//namespace create
+
 } //namespace dg
-- 
GitLab


From 4850f5e0277c12dedbb401106dee8f92ca719f71 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sun, 13 Aug 2017 19:04:25 +0200
Subject: [PATCH 151/453] added base X-point geometry

---
 inc/dg/backend/gridX.h           |   4 +-
 inc/dg/geometry/base_geometryX.h | 148 +++++++++++++++++++++++++++++++
 inc/dg/geometry/cartesianX.h     |  43 ---------
 3 files changed, 150 insertions(+), 45 deletions(-)
 create mode 100644 inc/dg/geometry/base_geometryX.h
 delete mode 100644 inc/dg/geometry/cartesianX.h

diff --git a/inc/dg/backend/gridX.h b/inc/dg/backend/gridX.h
index fa36c8c2d..c0c6e525f 100644
--- a/inc/dg/backend/gridX.h
+++ b/inc/dg/backend/gridX.h
@@ -507,7 +507,7 @@ struct GridX2d : public aTopologyX2d
 };
 
 /**
- * @brief A 3D grid class  for cartesian coordinates
+ * @brief A 3D grid class with X-point topology
  *
  * In the third dimension only 1 polynomial coefficient is used,
  * not n. In 2d it looks like
@@ -520,7 +520,7 @@ struct GridX2d : public aTopologyX2d
     |--- ---------- ---|
     fy*Ly
  @endcode
- * @tparam double scalar value type 
+ * @ingroup basictopology
  */
 struct aTopologyX3d
 {
diff --git a/inc/dg/geometry/base_geometryX.h b/inc/dg/geometry/base_geometryX.h
new file mode 100644
index 000000000..82ed83b09
--- /dev/null
+++ b/inc/dg/geometry/base_geometryX.h
@@ -0,0 +1,148 @@
+#pragma once
+
+#include "../backend/gridX.h"
+#include "tensor.h"
+
+namespace dg
+{
+
+///@addtogroup basicgeometry
+///@{
+
+/**
+ * @brief This is the abstract interface class for a two-dimensional GeometryX
+ */
+struct aGeometryX2d : public aTopologyX2d
+{
+    SparseTensor<thrust::host_vector<double> > jacobian()const{
+        return do_compute_jacobian();
+    }
+    SparseTensor<thrust::host_vector<double> > metric()const { 
+        return do_compute_metric(); 
+    }
+    std::vector<thrust::host_vector<double> > map()const{
+        return do_compute_map();
+    }
+    ///Geometries are cloneable
+    virtual aGeometryX2d* clone()const=0;
+    ///allow deletion through base class pointer
+    virtual ~aGeometryX2d(){}
+    protected:
+    /*!
+     * @copydoc aTopologyX2d::aTopologyX2d()
+     * @note the default coordinate map will be the identity 
+     */
+    aGeometryX2d( double x0, double x1, double y0, double y1, double fx, double fy, unsigned n, unsigned Nx, unsigned Ny, bc bcx, bc bcy):aTopologyX2d( x0,x1,y0,y1,fx,fy,n,Nx,Ny,bcx,bcy){}
+    aGeometryX2d( const aGeometryX2d& src):aTopologyX2d(src){}
+    aGeometryX2d& operator=( const aGeometryX2d& src){
+        aTopologyX2d::operator=(src);
+        return *this;
+    }
+    private:
+    virtual SparseTensor<thrust::host_vector<double> > do_compute_metric()const {
+        return SparseTensor<thrust::host_vector<double> >();
+    }
+    virtual SparseTensor<thrust::host_vector<double> > do_compute_jacobian()const {
+        return SparseTensor<thrust::host_vector<double> >();
+    }
+    virtual std::vector<thrust::host_vector<double> > do_compute_map()const{
+        std::vector<thrust::host_vector<double> > map(2);
+        map[0] = dg::evaluate(dg::cooX2d, *this);
+        map[1] = dg::evaluate(dg::cooY2d, *this);
+        return map;
+    }
+
+
+};
+
+/**
+ * @brief This is the abstract interface class for a three-dimensional GeometryX
+ */
+struct aGeometryX3d : public aTopologyX3d
+{
+    SparseTensor<thrust::host_vector<double> > jacobian()const{
+        return do_compute_jacobian();
+    }
+    SparseTensor<thrust::host_vector<double> > metric()const { 
+        return do_compute_metric(); 
+    }
+    std::vector<thrust::host_vector<double> > map()const{
+        return do_compute_map();
+    }
+    ///Geometries are cloneable
+    virtual aGeometryX3d* clone()const=0;
+    ///allow deletion through base class pointer
+    virtual ~aGeometryX3d(){}
+    protected:
+    /*!
+     * @copydoc aTopologyX3d::aTopologyX3d()
+     * @note the default coordinate map will be the identity 
+     */
+    aGeometryX3d( double x0, double x1, double y0, double y1, double z0, double z1, double fx, double fy, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz): aTopologyX3d(x0,x1,y0,y1,z0,z1,fx,fy,n,Nx,Ny,Nz,bcx,bcy,bcz){}
+    aGeometryX3d( const aGeometryX3d& src):aTopologyX3d(src){}
+    aGeometryX3d& operator=( const aGeometryX3d& src){
+        aTopologyX3d::operator=(src);
+        return *this;
+    }
+    private:
+    virtual SparseTensor<thrust::host_vector<double> > do_compute_metric()const {
+        return SparseTensor<thrust::host_vector<double> >();
+    }
+    virtual SparseTensor<thrust::host_vector<double> > do_compute_jacobian()const {
+        return SparseTensor<thrust::host_vector<double> >();
+    }
+    virtual std::vector<thrust::host_vector<double> > do_compute_map()const{
+        std::vector<thrust::host_vector<double> > map(3);
+        map[0] = dg::evaluate(dg::cooX3d, *this);
+        map[1] = dg::evaluate(dg::cooY3d, *this);
+        map[2] = dg::evaluate(dg::cooZ3d, *this);
+        return map;
+    }
+};
+
+///@}
+
+///@addtogroup geometry
+///@{
+
+/**
+ * @brief two-dimensional GridX with Cartesian metric
+ */
+struct CartesianGridX2d: public dg::aGeometryX2d
+{
+    ///@copydoc GridX2d::GridX2d()
+    CartesianGridX2d( double x0, double x1, double y0, double y1, double fx, double fy, unsigned n, unsigned Nx, unsigned Ny, bc bcx = PER, bc bcy = PER): dg::aGeometryX2d(x0,x1,y0,y1,fx,fy,n,Nx,Ny,bcx,bcy){}
+    /**
+     * @brief Construct from existing topology
+     * @param g existing grid class
+     */
+    CartesianGridX2d( const dg::GridX2d& g):dg::aGeometryX2d(g.x0(),g.x1(),g.y0(),g.y1(),g.fx(),g.fy(),g.n(),g.Nx(),g.Ny(),g.bcx(),g.bcy()){}
+    virtual CartesianGridX2d* clone()const{return new CartesianGridX2d(*this);}
+    private:
+    virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny){
+        aTopologyX2d::do_set(new_n,new_Nx,new_Ny);
+    }
+};
+
+/**
+ * @brief three-dimensional GridX with Cartesian metric
+ */
+struct CartesianGridX3d: public dg::aGeometryX3d
+{
+    ///@copydoc GridX3d::GridX3d()
+    CartesianGridX3d( double x0, double x1, double y0, double y1, double z0, double z1, double fx, double fy, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = PER, bc bcy = PER, bc bcz = PER): dg::aGeometryX3d(x0,x1,y0,y1,z0,z1,fx,fy,n,Nx,Ny,Nz,bcx,bcy,bcz){}
+    /**
+     * @brief Implicit type conversion from GridX3d
+     * @param g existing grid class
+     */
+    CartesianGridX3d( const dg::GridX3d& g):dg::aGeometryX3d(g.x0(), g.x1(), g.y0(), g.y1(), g.z0(), g.z1(),g.fx(),g.fy(),g.n(),g.Nx(),g.Ny(),g.Nz(),g.bcx(),g.bcy(),g.bcz()){}
+    virtual CartesianGridX3d* clone()const{return new CartesianGridX3d(*this);}
+    private:
+    virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz){
+        aTopologyX3d::do_set(new_n,new_Nx,new_Ny,new_Nz);
+    }
+};
+
+///@}
+
+} //namespace dg
diff --git a/inc/dg/geometry/cartesianX.h b/inc/dg/geometry/cartesianX.h
deleted file mode 100644
index d34a83480..000000000
--- a/inc/dg/geometry/cartesianX.h
+++ /dev/null
@@ -1,43 +0,0 @@
-#pragma once
-
-#include "../backend/gridX.h"
-#include "geometry_traits.h"
-
-namespace dg
-{
-
-///@addtogroup basicgrids
-///@{
-//
-
-/**
- * @brief two-dimensional Grid with Cartesian metric
- */
-struct CartesianGridX2d: public dg::aTopologyX2d
-{
-    ///@copydoc GridX2d::GridX2d()
-    CartesianGridX2d( double x0, double x1, double y0, double y1, double fx, double fy, unsigned n, unsigned Nx, unsigned Ny, bc bcx = PER, bc bcy = NEU):dg::aTopologyX2d(x0,x1,y0,y1,fx,fy,n,Nx,Ny,bcx,bcy){}
-    /**
-     * @brief Construct from existing topology
-     * @param grid existing grid class
-     */
-    explicit CartesianGridX2d( const dg::GridX2d& grid):dg::aTopologyX2d(grid){}
-};
-
-/**
- * @brief three-dimensional Grid with Cartesian metric
- */
-struct CartesianGridX3d: public dg::aTopologyX3d
-{
-    ///@copydoc GridX3d::GridX3d()
-    CartesianGridX3d( double x0, double x1, double y0, double y1, double z0, double z1, double fx, double fy, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = PER, bc bcy = NEU, bc bcz = PER): dg::aTopologyX3d(x0,x1,y0,y1,z0,z1,fx,fy,n,Nx,Ny,Nz,bcx,bcy,bcz){}
-    /**
-     * @brief Construct from existing topology
-     * @param grid existing grid class
-     */
-    explicit CartesianGridX3d( const dg::GridX3d& grid):dg::aTopologyX3d(grid){}
-};
-
-///@}
-
-} //namespace dg
-- 
GitLab


From 8aa44952206251fd24ea7bb7acf97d5b49dfde1d Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sun, 13 Aug 2017 19:53:17 +0200
Subject: [PATCH 152/453] placed curvilinearX.h in geometry

---
 inc/dg/backend/gridX.h                        |  16 --
 inc/dg/geometry/curvilinear.h                 |  14 +-
 inc/dg/geometry/curvilinearX.h                | 159 ++++++++++++++++++
 inc/dg/geometry/generatorX.h                  |  91 ++++++++++
 inc/dg/geometry/mpi_curvilinear.h             |   6 +-
 inc/geometries/curvilinearX.h                 | 148 ----------------
 .../{curvilinearX_t.cu => ribeiroX_t.cu}      |   0
 inc/geometries/separatrix_orthogonal.h        |   9 +-
 8 files changed, 267 insertions(+), 176 deletions(-)
 create mode 100644 inc/dg/geometry/curvilinearX.h
 create mode 100644 inc/dg/geometry/generatorX.h
 delete mode 100644 inc/geometries/curvilinearX.h
 rename inc/geometries/{curvilinearX_t.cu => ribeiroX_t.cu} (100%)

diff --git a/inc/dg/backend/gridX.h b/inc/dg/backend/gridX.h
index c0c6e525f..da927d121 100644
--- a/inc/dg/backend/gridX.h
+++ b/inc/dg/backend/gridX.h
@@ -432,9 +432,6 @@ struct aTopologyX2d
         if( (x>=x0_ && x <= x1_) && (y>=y0_ && y <= y1_)) return true; 
         return false;
     }
-    void init_X_boundaries( double x0, double x1) {
-        do_init_X_boundaries(x0,x1);
-    }
   protected:
     ///disallow destruction through base class pointer
     ~aTopologyX2d(){}
@@ -481,11 +478,6 @@ struct aTopologyX2d
         dlt_=src.dlt_;
         return *this;
     }
-    virtual void do_init_X_boundaries( double x0, double x1)
-    {
-        x0_ = x0, x1_ = x1;
-        assert( x1 > x0 );
-    }
   private:
     double x0_, x1_, y0_, y1_;
     double fx_, fy_;
@@ -749,9 +741,6 @@ struct aTopologyX3d
             return true; 
         return false;
     }
-    void init_X_boundaries( double x0, double x1) {
-        do_init_X_boundaries(x0,x1);
-    }
   protected:
     ///disallow destruction through base class pointer
     ~aTopologyX3d(){}
@@ -804,11 +793,6 @@ struct aTopologyX3d
         dlt_=src.dlt_;
         return *this;
     }
-    virtual void do_init_X_boundaries( double x0, double x1)
-    {
-        x0_ = x0, x1_ = x1;
-        assert( x1 > x0 );
-    }
   private:
     double x0_, x1_, y0_, y1_, z0_, z1_;
     double fx_,fy_;
diff --git a/inc/dg/geometry/curvilinear.h b/inc/dg/geometry/curvilinear.h
index 7393f62d5..0bb5746c8 100644
--- a/inc/dg/geometry/curvilinear.h
+++ b/inc/dg/geometry/curvilinear.h
@@ -15,7 +15,7 @@ struct CurvilinearGrid2d;
 ///@endcond
 
 //when we make a 3d grid with eta and phi swapped the metric structure and the transformation changes 
-//In practise it can only be orthogonal due to the projection matrix in the elliptic operator
+//In practise it can only be orthogonal due to the projection tensor in the elliptic operator
 
 
 /**
@@ -23,7 +23,7 @@ struct CurvilinearGrid2d;
  * 
  * The base coordinate system is the cylindrical coordinate system R,Z,phi
  */
-struct CylindricalProductGrid3d : public dg::aGeometry3d
+struct CurvilinearProductGrid3d : public dg::aGeometry3d
 {
     typedef CurvilinearGrid2d perpendicular_grid;
 
@@ -39,7 +39,7 @@ struct CylindricalProductGrid3d : public dg::aGeometry3d
      * @param bcy boundary condition in y
      * @param bcz boundary condition in z
      */
-    CylindricalProductGrid3d( const aGenerator2d& generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx=dg::DIR, bc bcy=dg::PER, bc bcz=dg::PER):
+    CurvilinearProductGrid3d( const aGenerator2d& generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx=dg::DIR, bc bcy=dg::PER, bc bcz=dg::PER):
         dg::aGeometry3d( 0, generator.width(), 0., generator.height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, bcy, bcz)
     { 
         map_.resize(3);
@@ -51,7 +51,7 @@ struct CylindricalProductGrid3d : public dg::aGeometry3d
     perpendicular_grid perp_grid() const;// { return perpendicular_grid(*this);}
 
     const aGenerator2d & generator() const{return handle_.get();}
-    virtual CylindricalProductGrid3d* clone()const{return new CylindricalProductGrid3d(*this);}
+    virtual CurvilinearProductGrid3d* clone()const{return new CurvilinearProductGrid3d(*this);}
     private:
     virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny,unsigned new_Nz){
         dg::aTopology3d::do_set( new_n, new_Nx, new_Ny,new_Nz);
@@ -139,7 +139,7 @@ struct CurvilinearGrid2d : public dg::aGeometry2d
     {
         construct( n,Nx,Ny);
     }
-    explicit CurvilinearGrid2d( CylindricalProductGrid3d g):
+    explicit CurvilinearGrid2d( CurvilinearProductGrid3d g):
         dg::aGeometry2d( g.x0(), g.x1(), g.y0(), g.y1(), g.n(), g.Nx(), g.Ny(), g.bcx(), g.bcy() ), handle_(g.generator())
     {
         g.set( n(), Nx(), Ny(), 1); //shouldn't trigger 2d grid generator
@@ -159,7 +159,7 @@ struct CurvilinearGrid2d : public dg::aGeometry2d
     }
     void construct( unsigned n, unsigned Nx, unsigned Ny)
     {
-        CylindricalProductGrid3d g( handle_.get(), n,Nx,Ny,1,bcx());
+        CurvilinearProductGrid3d g( handle_.get(), n,Nx,Ny,1,bcx());
         jac_=g.jacobian();
         map_=g.map();
         metric_=g.metric();
@@ -178,7 +178,7 @@ struct CurvilinearGrid2d : public dg::aGeometry2d
 
 ///@}
 ///@cond
-CylindricalProductGrid3d::perpendicular_grid CylindricalProductGrid3d::perp_grid() const { return CylindricalProductGrid3d::perpendicular_grid(*this);}
+CurvilinearProductGrid3d::perpendicular_grid CurvilinearProductGrid3d::perp_grid() const { return CurvilinearProductGrid3d::perpendicular_grid(*this);}
 ///@endcond
 
 }//namespace dg
diff --git a/inc/dg/geometry/curvilinearX.h b/inc/dg/geometry/curvilinearX.h
new file mode 100644
index 000000000..63583fe2b
--- /dev/null
+++ b/inc/dg/geometry/curvilinearX.h
@@ -0,0 +1,159 @@
+#pragma once
+
+#include "dg/backend/gridX.h"
+#include "dg/backend/evaluationX.cuh"
+#include "dg/backend/functions.h"
+#include "dg/blas1.h"
+#include "dg/geometry/geometry_traits.h"
+
+namespace dg
+{
+///@addtogroup basicgrids
+///@{
+
+///@cond
+struct CurvilinearGridX2d; 
+///@endcond
+
+/**
+ * @brief A three-dimensional grid based on curvilinear coordinates
+ * 
+ * The base coordinate system is the cylindrical coordinate system R,Z,phi
+ */
+struct CurvilinearProductGridX3d : public dg::aGeometryX3d
+{
+    /*!@brief Constructor
+    
+     * the coordinates of the computational space are called x,y,z
+     * @param generator must generate a grid
+     * @param n number of %Gaussian nodes in x and y
+     * @param fx
+     * @param fy
+     * @param Nx number of cells in x
+     * @param Ny number of cells in y 
+     * @param Nz  number of cells z
+     * @param bcx boundary condition in x
+     * @param bcy boundary condition in y
+     * @param bcz boundary condition in z
+     */
+    CurvilinearProductGridX3d( const aGeneratorX2d& generator, 
+        double fx, double fy, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, double fx, double fy, bc bcx=dg::DIR, bc bcy=dg::PER, bc bcz=dg::PER):
+        dg::aGeometryX3d( generator.zeta0(), generator.zeta1(), generator.eta0(), generator.eta1(), 0., 2.*M_PI, fx,fy,n, Nx, Ny, Nz, bcx, bcy, bcz)
+    { 
+        map_.resize(3);
+        handle_ = generator;
+        constructPerp( n, Nx, Ny);
+        constructParallel(Nz);
+    }
+
+    const aGeneratorX2d & generator() const{return handle_.get();}
+    virtual CurvilinearProductGridX3d* clone()const{return new CurvilinearProductGridX3d(*this);}
+    private:
+    //construct phi and lift rest to 3d
+    void constructParallel(unsigned Nz)
+    {
+        map_[2]=dg::evaluate(dg::cooZ3d, *this);
+        unsigned size = this->size();
+        unsigned size2d = this->n()*this->n()*this->Nx()*this->Ny();
+        //resize for 3d values
+        for( unsigned r=0; r<4;r++)
+            jac_.value(r).resize(size);
+        map_[0].resize(size);
+        map_[1].resize(size);
+        //lift to 3D grid
+        for( unsigned k=1; k<Nz; k++)
+            for( unsigned i=0; i<size2d; i++)
+            {
+                for(unsigned r=0; r<4; r++)
+                    jac_.value(r)[k*size2d+i] = jac_.value(r)[(k-1)*size2d+i];
+                map_[0][k*size2d+i] = map_[0][(k-1)*size2d+i];
+                map_[1][k*size2d+i] = map_[1][(k-1)*size2d+i];
+            }
+    }
+    //construct 2d plane
+    void constructPerp( unsigned n, unsigned Nx, unsigned Ny)
+    {
+        dg::Grid1d gX1d( x0(), x1(), n, Nx);
+        dg::GridX1d gY1d( y0(), y1(), fy(), n, Ny);
+        thrust::host_vector<double> x_vec = dg::evaluate( dg::cooX1d, gX1d);
+        thrust::host_vector<double> y_vec = dg::evaluate( dg::cooX1d, gY1d);
+        handle_.get().generate( x_vec, y_vec, gY1d.n()*gY1d.outer_N(), gY1d.n()*(gY1d.inner_N()+gY1d.outer_N()), map_[0], map_[1], jac_.value(0), jac_.value(1), jac_.value(2), jac_.value(3));
+        jac_.idx(0,0) = 0, jac_.idx(0,1) = 1, jac_.idx(1,0)=2, jac_.idx(1,1) = 3;
+    }
+    virtual SparseTensor<thrust::host_vector<double> > do_compute_jacobian( ) const {
+        return jac_;
+    }
+    virtual SparseTensor<thrust::host_vector<double> > do_compute_metric( ) const
+    {
+        thrust::host_vector<double> tempxx( size()), tempxy(size()), tempyy(size()), temppp(size());
+        for( unsigned i=0; i<size(); i++)
+        {
+            tempxx[i] = (jac_.value(0,0)[i]*jac_.value(0,0)[i]+jac_.value(0,1)[i]*jac_.value(0,1)[i]);
+            tempxy[i] = (jac_.value(0,0)[i]*jac_.value(1,0)[i]+jac_.value(0,1)[i]*jac_.value(1,1)[i]);
+            tempyy[i] = (jac_.value(1,0)[i]*jac_.value(1,0)[i]+jac_.value(1,1)[i]*jac_.value(1,1)[i]);
+            temppp[i] = 1./map_[2][i]/map_[2][i]; //1/R^2
+        }
+        SparseTensor<thrust::host_vector<double> > metric;
+        metric.idx(0,0) = 0; metric.value(0) = tempxx;
+        metric.idx(1,1) = 1; metric.value(1) = tempyy;
+        metric.idx(2,2) = 2; metric.value(2) = temppp;
+        if( !handle_.get().isOrthogonal())
+        {
+            metric.idx(0,1) = metric.idx(1,0) = 3; 
+            metric.value(3) = tempxy;
+        }
+        return metric;
+    }
+    virtual std::vector<thrust::host_vector<double> > do_compute_map()const{return map_;}
+    std::vector<thrust::host_vector<double> > map_;
+    SparseTensor<thrust::host_vector<double> > jac_;
+    dg::Handle<aGeneratorX2d> handle_;
+};
+
+/**
+ * @brief A two-dimensional grid based on curvilinear coordinates
+ */
+struct CurvilinearGridX2d : public dg::aGeometryX2d
+{
+    /*!@brief Constructor
+    
+     * @param generator must generate an orthogonal grid (class takes ownership of the pointer)
+     * @param fx
+     * @param fy
+     * @param n number of polynomial coefficients
+     * @param Nx number of cells in first coordinate
+     * @param Ny number of cells in second coordinate
+     * @param bcx boundary condition in first coordinate
+     * @param bcy boundary condition in second coordinate
+     */
+    CurvilinearGridX2d( const aGeneratorX2d& generator, double fx, double fy, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx=dg::DIR, bc bcy=dg::PER):
+        dg::aGeometryX2d( generator.zeta0(), generator.zeta1(), generator.eta0(), generator.eta1(),fx,fy, n, Nx, Ny, bcx, bcy), handle_(generator)
+    {
+        construct( n,Nx,Ny);
+    }
+
+    const aGeneratorX2d& generator() const{return handle_.get();}
+    virtual CurvilinearGridX2d* clone()const{return new CurvilinearGridX2d(*this);}
+    private:
+    void construct( unsigned n, unsigned Nx, unsigned Ny)
+    {
+        CurvilinearProductGridX3d g( handle_.get(), n,Nx,Ny,1,bcx());
+        jac_=g.jacobian();
+        map_=g.map();
+        metric_=g.metric();
+    }
+    virtual SparseTensor<thrust::host_vector<double> > do_compute_jacobian( ) const {
+        return jac_;
+    }
+    virtual SparseTensor<thrust::host_vector<double> > do_compute_metric( ) const {
+        return metric_;
+    }
+    virtual std::vector<thrust::host_vector<double> > do_compute_map()const{return map_;}
+    dg::SparseTensor<thrust::host_vector<double> > jac_, metric_;
+    std::vector<thrust::host_vector<double> > map_;
+    dg::Handle<aGeneratorX2d> handle_;
+};
+
+///@}
+
+} //namespace dg
diff --git a/inc/dg/geometry/generatorX.h b/inc/dg/geometry/generatorX.h
new file mode 100644
index 000000000..638c0bc78
--- /dev/null
+++ b/inc/dg/geometry/generatorX.h
@@ -0,0 +1,91 @@
+#pragma once
+
+namespace dg
+{
+
+/**
+* @brief The abstract generator base class 
+
+A generator is there to construct coordinate transformations from physical coordinates
+\f$ x,y\f$ to the computational domain \f$\zeta, \eta\f$, which
+is a product space. 
+ @note the origin of the computational space is assumed to be (0,0)
+ @ingroup generators
+*/
+struct aGeneratorX2d
+{
+    double zeta0() const{return do_zeta0();}
+    double zeta1() const{return do_zeta1();}
+    double eta0() const{return do_eta0();}
+    double eta1() const{return do_eta1();}
+    ///@brief sparsity pattern for metric
+    bool isOrthogonal() const { return doIsOrthogonal(); }
+
+    /**
+    * @brief Generate grid points and elements of the Jacobian 
+    *
+    * @param zeta1d (input) a list of \f$ N_\zeta\f$ points \f$ 0<\zeta_i<\f$width() 
+    * @param eta1d (input) a list of \f$ N_\eta\f$ points \f$ 0<\eta_j<\f$height() 
+    * @param nodeX0 is the index of the first point in eta1d  after the first jump in topology in \f$ \eta\f$
+    * @param nodeX1 is the index of the first point in eta1d  after the second jump in topology in \f$ \eta\f$
+    * @param x (output) the list of \f$ N_\eta N_\zeta\f$ coordinates \f$ x(\zeta_i, \eta_j)\f$ 
+    * @param y (output) the list of \f$ N_\eta N_\zeta\f$ coordinates \f$ y(\zeta_i, \eta_j)\f$ 
+    * @param zetaX (output) the list of \f$ N_\eta N_\zeta\f$ elements \f$ \partial\zeta/\partial x (\zeta_i, \eta_j)\f$ 
+    * @param zetaY (output) the list of \f$ N_\eta N_\zeta\f$ elements \f$ \partial\zeta/\partial y (\zeta_i, \eta_j)\f$ 
+    * @param etaX (output) the list of \f$ N_\eta N_\zeta\f$ elements \f$ \partial\eta/\partial x (\zeta_i, \eta_j)\f$ 
+    * @param etaY (output) the list of \f$ N_\eta N_\zeta\f$ elements \f$ \partial\eta/\partial y (\zeta_i, \eta_j)\f$ 
+    * @note the first (\f$ \zeta\f$) coordinate shall be constructed contiguously in memory, i.e. the resuling lists are \f$ x_0=x(\zeta_0, \eta_0),\ x_1=x(\zeta_1, \eta_0)\ x_2=x(\zeta_2, \eta_0)\dots x_{NM-1}=x(\zeta_{N-1} \eta_{M-1})\f$
+    * @note All the resulting vectors are write-only and get properly resized
+    */
+    void generate( 
+         const thrust::host_vector<double>& zeta1d, 
+         const thrust::host_vector<double>& eta1d, 
+         const unsigned nodeX0, const unsigned nodeX1, 
+         thrust::host_vector<double>& x, 
+         thrust::host_vector<double>& y, 
+         thrust::host_vector<double>& zetaX, 
+         thrust::host_vector<double>& zetaY, 
+         thrust::host_vector<double>& etaX, 
+         thrust::host_vector<double>& etaY) const
+    {
+        unsigned size = zeta1d.size()*eta1d.size();
+        x.resize(size), y.resize(size);
+        zetaX = zetaY = etaX = etaY =x ;
+        do_generate( zeta1d, eta1d, x,y,zetaX,zetaY,etaX,etaY);
+    }
+
+    /**
+    * @brief Abstract clone method that returns a copy on the heap
+    *
+    * @return a copy of *this on the heap
+    */
+    virtual aGeneratorX2d* clone() const=0;
+    virtual ~aGeneratorX2d(){}
+
+    protected:
+    aGeneratorX2d(){}
+    aGeneratorX2d(const aGeneratorX2d& src){}
+    aGeneratorX2d& operator=(const aGeneratorX2d& src){
+        return *this;
+    }
+    private:
+    virtual void do_generate(
+         const thrust::host_vector<double>& zeta1d, 
+         const thrust::host_vector<double>& eta1d, 
+         const unsigned nodeX0, const unsigned nodeX1, 
+         thrust::host_vector<double>& x, 
+         thrust::host_vector<double>& y, 
+         thrust::host_vector<double>& zetaX, 
+         thrust::host_vector<double>& zetaY, 
+         thrust::host_vector<double>& etaX, 
+         thrust::host_vector<double>& etaY) const = 0;
+    virtual bool doIsOrthogonal()const{return false;}
+    virtual double do_zeta0() const=0;
+    virtual double do_zeta1() const=0;
+    virtual double do_eta0() const=0;
+    virtual double do_eta1() const=0;
+
+
+};
+
+}//namespace dg
diff --git a/inc/dg/geometry/mpi_curvilinear.h b/inc/dg/geometry/mpi_curvilinear.h
index 5231a7b53..50fdbacbf 100644
--- a/inc/dg/geometry/mpi_curvilinear.h
+++ b/inc/dg/geometry/mpi_curvilinear.h
@@ -23,11 +23,11 @@ struct CurvilinearMPIGrid2d;
 /**
  * This is s 2x1 product space MPI grid
  */
-struct CylindricalProductMPIGrid3d : public dg::aMPIGeometry3d
+struct CurvilinearProductMPIGrid3d : public dg::aMPIGeometry3d
 {
     typedef dg::CurvilinearMPIGrid2d perpendicular_grid; //!< the two-dimensional grid
     typedef typename MPIContainer::container_type LocalContainer; //!< the local container type
-    CylindricalMPIGrid3d( const geo::aGenerator& generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz, MPI_Comm comm): 
+    CurvilinearMPIGrid3d( const geo::aGenerator& generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz, MPI_Comm comm): 
         dg::aMPITopology3d( 0, generator.width(), 0., generator.height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, bcy, bcz, comm),
         handle_( generator)
     {
@@ -130,7 +130,7 @@ struct CurvilinearMPIGrid2d : public dg::aMPIGeometry2d
         dg::CurvilinearGrid2d<thrust::host_vector<double> > g(generator, n, Nx, Ny);
         divide_and_conquer(g);
     }
-    explicit CurvilinearMPIGrid2d( const CylindricalMPIGrid3d<LocalContainer>& g):
+    explicit CurvilinearMPIGrid2d( const CurvilinearMPIGrid3d<LocalContainer>& g):
         dg::aMPIGeometry2d( g.global().x0(), g.global().x1(), g.global().y0(), g.global().y1(), g.global().n(), g.global().Nx(), g.global().Ny(), g.global().bcx(), g.global().bcy(), get_reduced_comm( g.communicator() )),
         handle_(g.generator())
     {
diff --git a/inc/geometries/curvilinearX.h b/inc/geometries/curvilinearX.h
deleted file mode 100644
index a64caf241..000000000
--- a/inc/geometries/curvilinearX.h
+++ /dev/null
@@ -1,148 +0,0 @@
-#pragma once
-
-#include "dg/backend/gridX.h"
-#include "dg/backend/evaluationX.cuh"
-#include "dg/backend/functions.h"
-#include "dg/blas1.h"
-#include "dg/geometry/geometry_traits.h"
-
-namespace dg
-{
-
-///@cond
-template< class container>
-struct CurvilinearGridX2d; 
-///@endcond
-
-/**
- * A curvilinear grid for X-point geometry in three dimensions
- * @ingroup grids
- */
-template< class container>
-struct CurvilinearGridX3d : public dg::GridX3d
-{
-    typedef dg::CurvilinearCylindricalTag metric_category;
-    typedef CurvilinearGridX2d<container> perpendicular_grid;
-
-    template< class Generator>
-    CurvilinearGridX3d( Generator generator, double psi_0, double fx, double fy, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx, dg::bc bcy):
-        dg::GridX3d( 0,1, -2.*M_PI*fy/(1.-2.*fy), 2.*M_PI*(1.+fy/(1.-2.*fy)), 0., 2*M_PI, fx, fy, n, Nx, Ny, Nz, bcx, bcy, dg::PER),
-        r_(this->size()), z_(r_), xr_(r_), xz_(r_), yr_(r_), yz_(r_)
-    {
-        construct( generator, psi_0, fx, n, Nx, Ny);
-    }
-
-    const thrust::host_vector<double>& r()const{return r_;}
-    const thrust::host_vector<double>& z()const{return z_;}
-    const thrust::host_vector<double>& xr()const{return xr_;}
-    const thrust::host_vector<double>& yr()const{return yr_;}
-    const thrust::host_vector<double>& xz()const{return xz_;}
-    const thrust::host_vector<double>& yz()const{return yz_;}
-    const container& g_xx()const{return g_xx_;}
-    const container& g_yy()const{return g_yy_;}
-    const container& g_xy()const{return g_xy_;}
-    const container& g_pp()const{return g_pp_;}
-    const container& vol()const{return vol_;}
-    const container& perpVol()const{return vol2d_;}
-    perpendicular_grid perp_grid() const { return CurvilinearGridX2d<container>(*this);}
-    private:
-    template<class Generator>
-    void construct( Generator generator, double psi_0, double fx, unsigned n, unsigned Nx, unsigned Ny )
-    {
-        const double x_0 = generator.f0()*psi_0;
-        const double x_1 = -fx/(1.-fx)*x_0;
-        init_X_boundaries( x_0, x_1);
-        dg::Grid1d gX1d( this->x0(), this->x1(), n, Nx, dg::DIR);
-        thrust::host_vector<double> x_vec = dg::evaluate( dg::cooX1d, gX1d);
-        dg::GridX1d gY1d( -this->fy()*2.*M_PI/(1.-2.*this->fy()), 2*M_PI+this->fy()*2.*M_PI/(1.-2.*this->fy()), this->fy(), this->n(), this->Ny(), dg::DIR);
-        thrust::host_vector<double> y_vec = dg::evaluate( dg::cooX1d, gY1d);
-        thrust::host_vector<double> rvec, zvec, yrvec, yzvec, xrvec, xzvec;
-        generator( x_vec, y_vec, gY1d.n()*gY1d.outer_N(), gY1d.n()*(gY1d.inner_N()+gY1d.outer_N()), rvec, zvec, xrvec, xzvec, yrvec, yzvec);
-
-        unsigned Mx = this->n()*this->Nx(), My = this->n()*this->Ny();
-        //now lift to 3D grid
-        for( unsigned k=0; k<this->Nz(); k++)
-            for( unsigned i=0; i<Mx*My; i++)
-            {
-                r_[k*Mx*My+i] = rvec[i];
-                z_[k*Mx*My+i] = zvec[i];
-                yr_[k*Mx*My+i] = yrvec[i];
-                yz_[k*Mx*My+i] = yzvec[i];
-                xr_[k*Mx*My+i] = xrvec[i];
-                xz_[k*Mx*My+i] = xzvec[i];
-            }
-        construct_metric();
-    }
-    //compute metric elements from xr, xz, yr, yz, r and z
-    void construct_metric()
-    {
-        //std::cout << "CONSTRUCTING METRIC\n";
-        thrust::host_vector<double> tempxx( r_), tempxy(r_), tempyy(r_), tempvol(r_);
-        for( unsigned idx=0; idx<this->size(); idx++)
-        {
-            tempxx[idx] = (xr_[idx]*xr_[idx]+xz_[idx]*xz_[idx]);
-            tempxy[idx] = (yr_[idx]*xr_[idx]+yz_[idx]*xz_[idx]);
-            tempyy[idx] = (yr_[idx]*yr_[idx]+yz_[idx]*yz_[idx]);
-            tempvol[idx] = r_[idx]/sqrt(tempxx[idx]*tempyy[idx]-tempxy[idx]*tempxy[idx]);
-        }
-        g_xx_=tempxx, g_xy_=tempxy, g_yy_=tempyy, vol_=tempvol;
-        dg::blas1::pointwiseDivide( tempvol, r_, tempvol);
-        vol2d_ = tempvol;
-        thrust::host_vector<double> ones = dg::evaluate( dg::one, *this);
-        dg::blas1::pointwiseDivide( ones, r_, tempxx);
-        dg::blas1::pointwiseDivide( tempxx, r_, tempxx); //1/R^2
-        g_pp_=tempxx;
-    }
-    thrust::host_vector<double> r_, z_, xr_, xz_, yr_, yz_;
-    container g_xx_, g_xy_, g_yy_, g_pp_, vol_, vol2d_;
-};
-
-/**
- * A curvilinear grid for X-point geometry in two dimensions
- * @ingroup grids
- */
-template< class container>
-struct CurvilinearGridX2d : public dg::GridX2d
-{
-    typedef dg::CurvilinearTag metric_category;
-    template<class Generator>
-    CurvilinearGridX2d(Generator generator, double psi_0, double fx, double fy, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx, dg::bc bcy):
-        dg::GridX2d( 0, 1,-fy*2.*M_PI/(1.-2.*fy), 2*M_PI+fy*2.*M_PI/(1.-2.*fy), fx, fy, n, Nx, Ny, bcx, bcy)
-    {
-        CurvilinearGridX3d<container> g( generator, psi_0, fx,fy, n,Nx,Ny,1,bcx,bcy);
-        init_X_boundaries( g.x0(),g.x1());
-        r_=g.r(), z_=g.z(), xr_=g.xr(), xz_=g.xz(), yr_=g.yr(), yz_=g.yz();
-        g_xx_=g.g_xx(), g_xy_=g.g_xy(), g_yy_=g.g_yy();
-        vol2d_=g.perpVol();
-    }
-    CurvilinearGridX2d( const CurvilinearGridX3d<container>& g):
-        dg::GridX2d( g.x0(), g.x1(), g.y0(), g.y1(), g.fx(), g.fy(), g.n(), g.Nx(), g.Ny(), g.bcx(), g.bcy())
-    {
-        unsigned s = this->size();
-        r_.resize( s), z_.resize(s), xr_.resize(s), xz_.resize(s), yr_.resize(s), yz_.resize(s);
-        g_xx_.resize( s), g_xy_.resize(s), g_yy_.resize(s), vol2d_.resize(s);
-        for( unsigned i=0; i<s; i++)
-        { r_[i]=g.r()[i], z_[i]=g.z()[i], xr_[i]=g.xr()[i], xz_[i]=g.xz()[i], yr_[i]=g.yr()[i], yz_[i]=g.yz()[i];}
-        thrust::copy( g.g_xx().begin(), g.g_xx().begin()+s, g_xx_.begin());
-        thrust::copy( g.g_xy().begin(), g.g_xy().begin()+s, g_xy_.begin());
-        thrust::copy( g.g_yy().begin(), g.g_yy().begin()+s, g_yy_.begin());
-        thrust::copy( g.perpVol().begin(), g.perpVol().begin()+s, vol2d_.begin());
-    }
-    const thrust::host_vector<double>& r()const{return r_;}
-    const thrust::host_vector<double>& z()const{return z_;}
-    const thrust::host_vector<double>& xr()const{return xr_;}
-    const thrust::host_vector<double>& yr()const{return yr_;}
-    const thrust::host_vector<double>& xz()const{return xz_;}
-    const thrust::host_vector<double>& yz()const{return yz_;}
-
-    const container& g_xx()const{return g_xx_;}
-    const container& g_yy()const{return g_yy_;}
-    const container& g_xy()const{return g_xy_;}
-    const container& vol()const{return vol2d_;}
-    const container& perpVol()const{return vol2d_;}
-    private:
-    thrust::host_vector<double> r_, z_, xr_, xz_, yr_, yz_;
-    container g_xx_, g_xy_, g_yy_, vol2d_;
-};
-
-} //namespace dg
diff --git a/inc/geometries/curvilinearX_t.cu b/inc/geometries/ribeiroX_t.cu
similarity index 100%
rename from inc/geometries/curvilinearX_t.cu
rename to inc/geometries/ribeiroX_t.cu
diff --git a/inc/geometries/separatrix_orthogonal.h b/inc/geometries/separatrix_orthogonal.h
index 0eac12977..a5faf4b72 100644
--- a/inc/geometries/separatrix_orthogonal.h
+++ b/inc/geometries/separatrix_orthogonal.h
@@ -4,6 +4,7 @@
 #include "dg/backend/interpolationX.cuh"
 #include "dg/backend/evaluationX.cuh"
 #include "dg/backend/weightsX.cuh"
+#include "dg/geometry/generatorX.h"
 #include "dg/runge_kutta.h"
 #include "utilitiesX.h"
 
@@ -123,7 +124,7 @@ void computeX_rzy( PsiX psiX, PsiY psiY,
  * @tparam LaplacePsi models aBinaryOperator
  */
 template< class Psi, class PsiX, class PsiY, class LaplacePsi>
-struct SimpleOrthogonalX
+struct SimpleOrthogonalX : public aGeneratorX2d
 {
     SimpleOrthogonalX(): f0_(1), firstline_(0){}
     SimpleOrthogonalX( Psi psi, PsiX psiX, PsiY psiY, LaplacePsi laplacePsi, double psi_0, //psi_0 must be the closed surface, 0 the separatrix
@@ -172,6 +173,10 @@ struct SimpleOrthogonalX
         }
     }
     double laplace(double x, double y) {return f0_*laplacePsi_(x,y);}
+        y_0= -2.*M_PI*fy/(1.-2.*fy); 
+        y_1= 2.*M_PI*(1.+fy/(1.-2.*fy));
+        const double x_0 = generator.f0()*psi_0;
+        const double x_1 = -fx/(1.-fx)*x_0;
     private:
     PsiX psiX_;
     PsiY psiY_;
@@ -188,7 +193,7 @@ struct SimpleOrthogonalX
  * @tparam Psi All the template parameters must model aBinaryOperator i.e. the bracket operator() must be callable with two arguments and return a double. 
  */
 template< class Psi, class PsiX, class PsiY, class LaplacePsi>
-struct SeparatrixOrthogonal
+struct SeparatrixOrthogonal : public aGeneratorX2d
 {
     typedef dg::OrthogonalTag metric_category;
     /**
-- 
GitLab


From f3b1970191ffabaed733b24509c877279c7236bf Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 14 Aug 2017 06:50:56 -0700
Subject: [PATCH 153/453] made refined_gridX.h

---
 .../geometry}/refined_curvilinearX.h          |   0
 inc/dg/geometry/refined_gridX.h               | 658 +++++++-----------
 2 files changed, 234 insertions(+), 424 deletions(-)
 rename inc/{geometries => dg/geometry}/refined_curvilinearX.h (100%)

diff --git a/inc/geometries/refined_curvilinearX.h b/inc/dg/geometry/refined_curvilinearX.h
similarity index 100%
rename from inc/geometries/refined_curvilinearX.h
rename to inc/dg/geometry/refined_curvilinearX.h
diff --git a/inc/dg/geometry/refined_gridX.h b/inc/dg/geometry/refined_gridX.h
index f5c58c792..c0843008a 100644
--- a/inc/dg/geometry/refined_gridX.h
+++ b/inc/dg/geometry/refined_gridX.h
@@ -9,484 +9,294 @@
 
 namespace dg
 {
-///@cond
-namespace detail
-{
-/**
- * @brief Create 1d refinement weights and abscissas for the X-point
- *
- * There will be four refined cells at the end except if the X-point is a corner node.
- * @param add_x number of additional cells in the cells around the X-point
- * @param g The 1d Xgrid to refine
- *
- * @param weights   A 1d vector of size n*(Nx+2*add_x) for one-sided refinement and n*(Nx+4*add_x)) for two-sided refinement
- * @param abscissas A 1d vector of size n*(Nx+2*add_x) for one-sided refinement and n*(Nx+4*add_x)) for two-sided refinement
- * @return the new number of cells
- */
-int equidist_Xref( unsigned add_x, const GridX1d& g, thrust::host_vector<double>& weights, thrust::host_vector<double>& abscissas, unsigned howmany)
-{
-    if( add_x == 0 || howmany == 0) { return equidist_ref( add_x, 0, g.grid(), weights, abscissas, howmany); }
-    if( g.f() == 0) { return equidist_ref( add_x, 0, Grid1d( g.x0(), g.x1(), g.n(), g.N(), dg::PER), weights, abscissas, howmany); }
-    thrust::host_vector<double> w1, w2, w3;
-    unsigned node1 = g.outer_N();
-    w1 = equidist_ref( add_x, node1, g.n(), g.outer_N(), dg::DIR, howmany); //left side
-    w2 = equidist_ref( add_x, 0, g.n(), g.inner_N(), dg::PER, howmany);//inner side
-    w3 = equidist_ref( add_x, 0, g.n(), g.outer_N(), dg::DIR, howmany);//right side
-    //now combine unnormalized weights
-    thrust::host_vector<double> wtot( w1.size() + w2.size() + w3.size());
-    for( unsigned i=0; i<w1.size() ; i++)
-        wtot[i] = w1[i];
-    for( unsigned i=0; i<w2.size(); i++)
-        wtot[w1.size()+i] = w2[i];
-    for( unsigned i=0; i<w3.size(); i++)
-        wtot[w1.size()+w2.size()+i] = w3[i];
-    weights = wtot;
-
-    unsigned Nx_new = weights.size()/g.n();
-    abscissas = normalize_weights_and_compute_abscissas( g.grid(), weights);
-    return Nx_new;
-}
-
-/**
- * @brief Create 1d refinement weights and abscissas for the X-point
- *
- * There will be four refined cells at the end except if the X-point is a corner node.
- * @param add_x number of additional cells in the cells around the X-point
- * @param g The 1d Xgrid to refine
- *
- * @param weights   A 1d vector of size n*(Nx+2*add_x) for one-sided refinement and n*(Nx+4*add_x)) for two-sided refinement
- * @param abscissas A 1d vector of size n*(Nx+2*add_x) for one-sided refinement and n*(Nx+4*add_x)) for two-sided refinement
- * @return the new number of cells
- */
-int exponential_Xref( unsigned add_x, const GridX1d& g, thrust::host_vector<double>& weights, thrust::host_vector<double>& abscissas)
-{
-    if( add_x == 0) { return exponential_ref( add_x, 0, g.grid(), weights, abscissas); }
-    if( g.f() == 0) { return exponential_ref( add_x, 0, Grid1d( g.x0(), g.x1(), g.n(), g.N(), dg::PER), weights, abscissas); }
-    thrust::host_vector<double> w1, w2, w3;
-    unsigned node1 = g.outer_N();
-    w1 = exponential_ref( add_x, node1, g.n(), g.outer_N(), dg::DIR); //left side
-    w2 = exponential_ref( add_x, 0, g.n(), g.inner_N(), dg::PER);//inner side
-    w3 = exponential_ref( add_x, 0, g.n(), g.outer_N(), dg::DIR);//right side
-    //now combine unnormalized weights
-    thrust::host_vector<double> wtot( g.size() + 4*g.n()*add_x);
-    for( unsigned i=0; i<w1.size() ; i++)
-        wtot[i] = w1[i];
-    for( unsigned i=0; i<w2.size(); i++)
-        wtot[w1.size()+i] = w2[i];
-    for( unsigned i=0; i<w3.size(); i++)
-        wtot[w1.size()+w2.size()+i] = w3[i];
-    weights = wtot;
-
-    unsigned Nx_new = weights.size()/g.n();
-    abscissas = normalize_weights_and_compute_abscissas( g.grid(), weights);
-    return Nx_new;
-}
-
-}//namespace detail
-
-
-struct RefinedGridX3d;
-///@endcond
 /**
- * @brief Refined grid 
- * @ingroup grid
- */
-struct RefinedGridX2d : public dg::GridX2d
+* @brief Abstract base class for 1d grid refinement that increases the number of grid cells of a fixed basis grid
+*/
+struct aRefinementX2d
 {
-    /**
-     * @brief Refine a corner of a grid
-     *
-     * @param add_x Add number of cells to the existing one
-     * @param add_y Add number of cells to the existing one
-     * @param howmanyX Add number of cells to the existing one
-     * @param howmanyY Add number of cells to the existing one
-     * @param n_ref The new number of polynomial coefficients
-     * @copydetails GridX2d::GridX2d()
-     */
-    RefinedGridX2d( unsigned add_x, unsigned add_y, unsigned howmanyX, unsigned howmanyY, 
-            double x0, double x1, double y0, double y1, 
-            double fx, double fy, 
-            unsigned n_ref, unsigned n, unsigned Nx, unsigned Ny, 
-            bc bcx = dg::PER, bc bcy = dg::PER) : dg::GridX2d( x0, x1, y0, y1, 
-                fx_new(Nx, add_x*howmanyX, fx), fy_new(Ny, add_y*howmanyY, fy), n_ref, nx_new(Nx, add_x*howmanyX, fx), ny_new(Ny, add_y*howmanyY, fy), bcx, bcy), 
-        wx_(size()), wy_(size()), absX_(size()), absY_(size()),
-        g_assoc_( x0, x1, y0, y1, fx, fy, n, Nx, Ny, bcx, bcy)
+    /*! @brief Generate the grid transformation
+     *  
+     * @param g The 1d grid to refine
+     * @param weights A 1d vector of size N_new. These represent the Jacobian of the transformation \f[\frac{\partial \zeta}{\partial x} \f]. The new metric element has thus to be multiplied by weights^2 and the volume by 1/weights
+     * @param abscissas A 1d vector of size N_new. These are the new abscissas \f$ x(\zeta) \f$ of the grid. 
+    */
+    void generate( const GridX2d& g_old, thrust::host_vector<double>& weightsX, thrust::host_vector<double>& weightsY, thrust::host_vector<double>& abscissasX, thrust::host_vector<double>& abscissasY) const
     {
-        Grid1d  gx( x0, x1, n_ref, Nx, bcx);
-        GridX1d         gy( y0, y1, fy, n_ref, Ny, bcy);
         thrust::host_vector<double> wx, ax, wy, ay;
-        detail::equidist_ref(  add_x, g_assoc_.inner_Nx(), gx, wx, ax, howmanyX);
-        detail::equidist_Xref( add_y, gy, wy, ay, howmanyY);
-        //detail::exponential_ref(  add_x, g_assoc_.inner_Nx(), gx, wx, ax);
-        //detail::exponential_Xref( add_y, gy, wy, ay);
+        Grid1d gx( g_old.x0(), g_old.x1(), g_old.n(), g_old.Nx(), g_old.bcx());
+        do_generateX(gx,g_old.inner_Nx(), wx,ax);
+        GridX1d gy( g_old.y0(), g_old.y1(), g_old.fy(), g_old.n(), g_old.Ny(), g_old.bcy());
+        do_generateY(gy,wy,ay);
+        weightsX.resize(size()), weightsY.resize(size()); 
+        abscissasX.resize(size()), abscissasY.resize(size());
         //now make product space
         for( unsigned i=0; i<wy.size(); i++)
             for( unsigned j=0; j<wx.size(); j++)
             {
-                wx_[i*wx.size()+j] = wx[j];
-                wy_[i*wx.size()+j] = wy[i];
-                absX_[i*wx.size()+j] = ax[j];
-                absY_[i*wx.size()+j] = ay[i];
+                weightsX[0][i*wx.size()+j] = wx[j];
+                weightsY[1][i*wx.size()+j] = wy[i];
+                abscissasX[0][i*wx.size()+j] = ax[j];
+                abscissasY[1][i*wx.size()+j] = ay[i];
             }
     }
-
-    RefinedGridX2d( unsigned multiple_x, unsigned multiple_y, 
-            double x0, double x1, double y0, double y1, 
-            double fx, double fy, 
-            unsigned n_ref, unsigned n, unsigned Nx, unsigned Ny, 
-            bc bcx = dg::PER, bc bcy = dg::PER) : 
-        dg::GridX2d( x0, x1, y0, y1, fx, fy, n_ref, multiple_x*Nx, multiple_y*Ny, bcx, bcy), 
-        wx_(size()), wy_(size()), absX_(size()), absY_(size()),
-        g_assoc_( x0, x1, y0, y1, fx, fy, n, Nx, Ny, bcx, bcy)
-    {
-        Grid1d  gx( x0, x1, n_ref, Nx, bcx);
-        Grid1d  gy( y0, y1, n_ref, Ny, bcy);
-        thrust::host_vector<double> wx, ax, wy, ay;
-        detail::linear_ref(  multiple_x, gx, wx, ax);
-        detail::linear_ref(  multiple_y, gy, wy, ay);
-        //now make product space
-        for( unsigned i=0; i<wy.size(); i++)
-            for( unsigned j=0; j<wx.size(); j++)
-            {
-                wx_[i*wx.size()+j] = wx[j];
-                wy_[i*wx.size()+j] = wy[i];
-                absX_[i*wx.size()+j] = ax[j];
-                absY_[i*wx.size()+j] = ay[i];
-            }
-    }
-
-    /**
-     * @brief Reduce from a  3d grid 
-     *
-     * This is possible because all our grids are product space grids. 
-     *
-     * @param g The 3d grid
-     */
-    RefinedGridX2d( const dg::RefinedGridX3d& g);
-
-    /**
-     * @brief The grid that this object refines
-     *
-     * This function is vitual so that derived classes can also construct the associated grid 
-     * @return  2d grid
-     */
-    virtual const dg::GridX2d& associated()const {return g_assoc_;}
-    /**
-     * @brief Return the abscissas in X-direction 
-     *
-     * @return A 2d vector
-     */
-    const thrust::host_vector<double>& abscissasX() const {return absX_;} 
-    /**
-     * @brief Return the abscissas in Y-direction 
-     *
-     * @return A 2d vector
-     */
-    const thrust::host_vector<double>& abscissasY() const {return absY_;} 
-    /**
-     * @brief Return the weights in X-direction 
-     *
-     * @return A 2d vector
+    /*! @brief the new number of cells
+     * @param N_old the old number of cells
+     * @param fx Factor to partition x-axis
      */
-    const thrust::host_vector<double>& weightsX() const {return wx_;} 
-    /**
-     * @brief Return the weights in Y-direction 
-     *
-     * @return A 2d vector
-     */
-    const thrust::host_vector<double>& weightsY() const {return wy_;} 
-    protected:
-    void init_X_boundaries( double x0, double x1)
+    unsigned nx_new( unsigned Nx_old, double fx_old) const
     {
-        double alpha = (x1-x0)/this->lx();
-        double beta = (x0*this->x1()-x1*this->x0())/this->lx();
-        //weights are invariant
-        for( unsigned i=0; i<absX_.size(); i++)
-            absX_[i]=alpha*absX_[i]+beta;
-        dg::GridX2d::init_X_boundaries( x0, x1);
+        return do_Nx_new(Nx_old, fx_old);
     }
-
-    private:
-    unsigned nx_new( unsigned Nx, unsigned add_x, double fx)
+    unsigned ny_new( unsigned Ny_old, double fy_old) const
     {
-        if( fx==0 ) return Nx + add_x ; 
-        return Nx + 2*add_x;
+        return do_Ny_new(Ny_old, fy_old);
     }
-    unsigned ny_new( unsigned Ny, unsigned add_y, double fy)
+    unsigned fx_new( unsigned Nx_old, double fx_old) const
     {
-        if( fy==0 ) return Ny * 2*add_y; 
-        return Ny + 4*add_y;
+        return do_fx_new(Nx_old, fx_old);
     }
-    double fx_new( unsigned Nx, unsigned add_x, double fx)
+    unsigned fy_new( unsigned Ny_old, double fy_old) const
     {
-        if( fx==0 ) return 0; 
-        return (fx*(double)Nx + (double)add_x)/((double)Nx+2.*add_x);
+        return do_fy_new(Ny_old, fy_old);
     }
-    double fy_new( unsigned Ny, unsigned add_y, double fy)
-    {
-        if( fy==0 ) return 0; 
-        return (fy*(double)Ny + (double)add_y)/((double)Ny+4.*add_y);
+    virtual aRefinementX2d* clone()const=0;
+    virtual ~aRefinementX2d(){}
+    protected:
+    aRefinementX2d(){}
+    aRefinementX2d(const aRefinementX2d& src){}
+    aRefinementX2d& operator=(const aRefinementX2d& src){
+        return *this;
     }
-    thrust::host_vector<double> wx_, wy_; //weights
-    thrust::host_vector<double> absX_, absY_; //abscissas 
-    dg::GridX2d g_assoc_;
+    private:
+    virtual void do_generateX( const Grid1d& gx, unsigned nodeXX, thrust::host_vector<double>& weights, thrust::host_vector<double>& abscissas) const =0;
+    virtual void do_generateY( const GridX1d& gy, thrust::host_vector<double>& weights, thrust::host_vector<double>& abscissas) const =0;
+    virtual unsigned do_Nx_new( unsigned Nx_old, double fx) const =0;
+    virtual unsigned do_Ny_new( unsigned Ny_old, double fy) const =0;
+    virtual unsigned do_fx_new( unsigned Nx_old, double fx) const =0;
+    virtual unsigned do_fy_new( unsigned Ny_old, double fy) const =0;
 };
 
 /**
- * @brief Refined grid 
- * @ingroup grid
+* @brief No refinement
+*/
+struct IdentityXRefinement : public aRefinementX2d
+{
+    IdentityXRefinement* clone()const{return new IdentityRefinementX();}
+    private:
+    virtual void do_generateX( const GridX1d& g, unsigned nodeXX, thrust::host_vector<double>& weights, thrust::host_vector<double>& abscissas) const {
+        weights=dg::create::weights(g);
+        abscissas=dg::create::abscissas(g);
+    }
+    virtual void do_generateY( const Grid1d& g, thrust::host_vector<double>& weights, thrust::host_vector<double>& abscissas) const {
+        weights=dg::create::weights(g);
+        abscissas=dg::create::abscissas(g);
+    }
+    virtual unsigned do_Nx_new( unsigned Nx_old, double fx) const { return Nx_old; }
+    virtual unsigned do_Ny_new( unsigned Ny_old, double fy) const { return Ny_old; }
+    virtual unsigned do_fx_new( unsigned Nx_old, double fx) const { return fx; }
+    virtual unsigned do_fy_new( unsigned Ny_old, double fy) const { return fy; }
+};
+/**
+ * @brief Equidistant cell refinement around the X-point
  */
-struct RefinedGridX3d : public dg::GridX3d
+struct EquidistXRefinement : public aRefinementX2d
 {
-    /**
-     * @brief Refine the X-point
-     *
-     * No refinement in the third dimension
-     * @param add_x Add number of cells to the existing one
-     * @param add_y Add number of cells to the existing one
-     * @param howmanyX howmany cells are refined in x
-     * @param howmanyY howmany cells are refined in x
-     * @param n_ref number of polynomial coefficients in the refined grid
-     * @copydetails GridX3d::GridX3d()
-     */
-    RefinedGridX3d( unsigned add_x, unsigned add_y, unsigned howmanyX,  unsigned howmanyY,
-            double x0, double x1, double y0, double y1, double z0, double z1,
-            double fx, double fy, 
-            unsigned n_ref, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz,
-            bc bcx = dg::PER, bc bcy = dg::NEU, bc bcz = dg::PER) : 
-               dg::GridX3d( x0, x1, y0, y1, z0, z1,
-                fx_new(Nx, add_x*howmanyX, fx), fy_new(Ny, add_y*howmanyY, fy), n_ref, nx_new(Nx, add_x*howmanyX, fx), ny_new(Ny, add_y*howmanyY, fy), Nz, bcx, bcy, bcz), 
-        wx_(size()), wy_(size()), absX_(size()), absY_(size()),
-        g_assoc_( x0, x1, y0, y1,z0,z1, fx, fy, n, Nx, Ny, Nz, bcx, bcy, bcz)
+    EquidistXRefinement( unsigned add_x, unsigned add_y, unsigned howmanyX=1, unsigned howmanyY=1): add_x_(add_x), howm_x_(howmanyX), add_y_(add_y), howm_y_(howmanyY){ }
+    EquidistXRefinement* clone()const{return new EquidistXRefinement(*this);}
+    private:
+    unsigned add_x_, howm_x_, addy_, howm_y_;
+    virtual void do_generateX( const Grid1d& gx, unsigned nodeXX, thrust::host_vector<double>& weights, thrust::host_vector<double>& abscissas) const 
     {
-        Grid1d  gx( x0, x1, n_ref, Nx, bcx);
-        GridX1d         gy( y0, y1, fy, n_ref, Ny, bcy);
-        thrust::host_vector<double> wx, ax, wy, ay;
-        detail::equidist_ref(  add_x, g_assoc_.inner_Nx(), gx, wx, ax, howmanyX);
-        detail::equidist_Xref( add_y, gy, wy, ay, howmanyY);
-        //detail::exponential_ref(  add_x, g_assoc_.inner_Nx(), gx, wx, ax);
-        //detail::exponential_Xref( add_y, gy, wy, ay);
-        //now make product space
-        for( unsigned s=0; s<Nz; s++)
-            for( unsigned i=0; i<wy.size(); i++)
-                for( unsigned j=0; j<wx.size(); j++)
-                {
-                    wx_[(s*wy.size()+i)*wx.size()+j] = wx[j];
-                    wy_[(s*wy.size()+i)*wx.size()+j] = wy[i];
-                    absX_[(s*wy.size()+i)*wx.size()+j] = ax[j];
-                    absY_[(s*wy.size()+i)*wx.size()+j] = ay[i];
-                }
+        EquidistRefinement equi(add_x, nodeXX, howm_x_);
+        equi(gx,weights,abscissas);
     }
-    RefinedGridX3d( unsigned multiple_x, unsigned multiple_y, 
-            double x0, double x1, double y0, double y1, double z0, double z1,
-            double fx, double fy, 
-            unsigned n_ref, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz,
-            bc bcx = dg::PER, bc bcy = dg::NEU, bc bcz = dg::PER) : 
-        dg::GridX3d( x0, x1, y0, y1, z0, z1, fx, fy, n_ref, multiple_x*Nx, multiple_y*Ny, Nz, bcx, bcy, bcz), 
-        wx_(size()), wy_(size()), absX_(size()), absY_(size()),
-        g_assoc_( x0, x1, y0, y1, z0, z1,fx, fy, n, Nx, Ny, Nz, bcx, bcy, bcz)
+    virtual void do_generateY( const GridX1d& g, thrust::host_vector<double>& weights, thrust::host_vector<double>& abscissas) const 
     {
-        Grid1d  gx( x0, x1, n_ref, Nx, bcx);
-        Grid1d  gy( y0, y1, n_ref, Ny, bcy);
-        thrust::host_vector<double> wx, ax, wy, ay;
-        detail::linear_ref(  multiple_x, gx, wx, ax);
-        detail::linear_ref(  multiple_y, gy, wy, ay);
-        //now make product space
-        for( unsigned s=0; s<Nz; s++)
-            for( unsigned i=0; i<wy.size(); i++)
-                for( unsigned j=0; j<wx.size(); j++)
-                {
-                    wx_[(s*wy.size()+i)*wx.size()+j] = wx[j];
-                    wy_[(s*wy.size()+i)*wx.size()+j] = wy[i];
-                    absX_[(s*wy.size()+i)*wx.size()+j] = ax[j];
-                    absY_[(s*wy.size()+i)*wx.size()+j] = ay[i];
-                }
+        EquidistRefinement equi0(add_y_,0,howm_y_);
+        if( add_y_ == 0 || howm_y_ == 0) { 
+            equi0(g.grid(), weights, abscissas);
+            return;
+        }
+        if( g.f() == 0) { 
+            equi0( Grid1d( g.x0(), g.x1(), g.n(), g.N(), dg::PER), weights, abscissas); 
+            return;
+        }
+        thrust::host_vector<double> w1, w2, w3;
+        thrust::host_vector<double> a1, a2, a3;
+        unsigned node1 = g.outer_N();
+        EquidistRefinement equi1(add_y_, node1, howm_y_);
+        equi1.generate(Grid1d(g.x0(),g.x1(), g.n(), g.outer_N(), dg::DIR), w1,a1); //left side
+        equi0.generate( Grid1d(g.x0(),g.x1(), g.n(), g.inner_N(), dg::PER), w2,a2);//inner side
+        equi0.generate( Grid1d(g.x0(),g.x1(), g.n(), g.outer_N(), dg::DIR), w3,a3);//right side
+        //now combine unnormalized weights
+        thrust::host_vector<double> wtot( w1.size() + w2.size() + w3.size());
+        for( unsigned i=0; i<w1.size() ; i++)
+            wtot[i] = w1[i];
+        for( unsigned i=0; i<w2.size(); i++)
+            wtot[w1.size()+i] = w2[i];
+        for( unsigned i=0; i<w3.size(); i++)
+            wtot[w1.size()+w2.size()+i] = w3[i];
+        weights = wtot;
+        abscissas = detail::normalize_weights_and_compute_abscissas( g.grid(), weights);
     }
-
-    /**
-     * @brief The grid that this object refines
-     *
-     * This function is vitual so that derived classes can also construct the associated grid 
-     * @return  2d grid
-     */
-    virtual const dg::GridX3d& associated()const {return g_assoc_;}
-    /**
-     * @brief Return the abscissas in X-direction 
-     *
-     * @return A 2d vector
-     */
-    const thrust::host_vector<double>& abscissasX() const {return absX_;} 
-    /**
-     * @brief Return the abscissas in Y-direction 
-     *
-     * @return A 2d vector
-     */
-    const thrust::host_vector<double>& abscissasY() const {return absY_;} 
-    /**
-     * @brief Return the weights in X-direction 
-     *
-     * @return A 2d vector
-     */
-    const thrust::host_vector<double>& weightsX() const {return wx_;} 
-    /**
-     * @brief Return the weights in Y-direction 
-     *
-     * @return A 2d vector
-     */
-    const thrust::host_vector<double>& weightsY() const {return wy_;} 
-    protected:
-    void init_X_boundaries( double x0, double x1)
-    {
-        double alpha = (x1-x0)/this->lx();
-        double beta = (x0*this->x1()-x1*this->x0())/this->lx();
-        //weights are invariant under linear transformation
-        for( unsigned i=0; i<absX_.size(); i++)
-            absX_[i]=alpha*absX_[i]+beta;
-        dg::GridX3d::init_X_boundaries( x0, x1);
+    virtual unsigned do_Ny_new( unsigned Ny, double fy) const {
+        if( fy==0 ) return Ny + 2*add_y_; 
+        return Ny + 4*add_y_;
+    }
+    virtual unsigned do_Nx_new( unsigned Nx, double fx) const {
+        if( fx==0 ) return Nx + add_x_; 
+        return Nx + 2*add_x_;
     }
+    virtual unsigned do_fx_new( unsigned Nx, double fx) const {
+        if( fx==0 ) return 0; 
+        return (fx*(double)Nx + (double)add_x)/(double)(Nx+2.*add_x);
+    }
+    virtual unsigned do_fy_new( unsigned Ny, double fy) const { 
+        if( fy==0 ) return 0; 
+        return (fy*(double)Ny + (double)add_y)/(double)(Ny+4.*add_y);
+    }
+};
 
+/**
+ * @brief The exponential refinement around the X-point
+ */
+struct ExponentialXRefinement : public aRefinementX2d
+{
+    ExponentialXRefinement( unsigned add_x, unsigned add_y, unsigned howmanyX=1, unsigned howmanyY=1): add_x_(add_x), howm_x_(howmanyX), add_y_(add_y), howm_y_(howmanyY){ }
+    ExponentialXRefinement* clone()const{return new ExponentialXRefinement(*this);}
     private:
-    unsigned nx_new( unsigned Nx, unsigned add_x, double fx)
+    unsigned add_x_, howm_x_, addy_, howm_y_;
+    virtual void do_generateX( const Grid1d& gx, unsigned nodeXX, thrust::host_vector<double>& weights, thrust::host_vector<double>& abscissas) const 
     {
-        if( fx==0 ) return Nx + add_x; 
-        return Nx + 2*add_x;
+        EquidistRefinement equi(add_x, nodeXX, howm_x_);
+        equi(gx,weights,abscissas);
     }
-    unsigned ny_new( unsigned Ny, unsigned add_y, double fy)
+    virtual void do_generateY( const GridX1d& g, thrust::host_vector<double>& weights, thrust::host_vector<double>& abscissas) const 
     {
-        if( fy==0 ) return Ny + 2*add_y; 
-        return Ny + 4*add_y;
+        ExponentialRefinement expo0( add_x, 0);
+        if( add_y_ == 0) { return expo0.generate( g.grid(), weights, abscissas); }
+        if( g.f()  == 0) { return expo0.generate( Grid1d( g.x0(), g.x1(), g.n(), g.N(), dg::PER), weights, abscissas); }
+        thrust::host_vector<double> w1, w2, w3;
+        thrust::host_vector<double> a1, a2, a3;
+        unsigned node1 = g.outer_N();
+        ExponentialRefinement expo1( add_y_, node1);
+        expo1.generate( Grid1d(g.x0(),g.x1(),g.n(), g.outer_N(), dg::DIR), w1,a1); //left side
+        expo0.generate( Grid1d(g.x0(),g.x1(),g.n(), g.inner_N(), dg::PER), w2,a2); //inner side
+        expo0.generate( Grid1d(g.x0(),g.x1(),g.n(), g.outer_N(), dg::DIR), w3,a3); //right side
+        //now combine unnormalized weights
+        thrust::host_vector<double> wtot( g.size() + 4*g.n()*add_x);
+        for( unsigned i=0; i<w1.size() ; i++)
+            wtot[i] = w1[i];
+        for( unsigned i=0; i<w2.size(); i++)
+            wtot[w1.size()+i] = w2[i];
+        for( unsigned i=0; i<w3.size(); i++)
+            wtot[w1.size()+w2.size()+i] = w3[i];
+        weights = wtot;
+
+        abscissas = normalize_weights_and_compute_abscissas( g.grid(), weights);
     }
-    double fx_new( unsigned Nx, unsigned add_x, double fx)
-    {
+    virtual unsigned do_Ny_new( unsigned Ny, double fy) const {
+        if( fy==0 ) return Ny + 2*add_y_; 
+        return Ny + 4*add_y_;
+    }
+    virtual unsigned do_Nx_new( unsigned Nx, double fx) const {
+        if( fx==0 ) return Nx + add_x_; 
+        return Nx + 2*add_x_;
+    }
+    virtual unsigned do_fx_new( unsigned Nx, double fx) const {
         if( fx==0 ) return 0; 
         return (fx*(double)Nx + (double)add_x)/(double)(Nx+2.*add_x);
     }
-    double fy_new( unsigned Ny, unsigned add_y, double fy)
-    {
+    virtual unsigned do_fy_new( unsigned Ny, double fy) const { 
         if( fy==0 ) return 0; 
         return (fy*(double)Ny + (double)add_y)/(double)(Ny+4.*add_y);
     }
-    thrust::host_vector<double> wx_, wy_; //weights
-    thrust::host_vector<double> absX_, absY_; //abscissas 
-    dg::GridX3d g_assoc_;
 };
 
-RefinedGridX2d::RefinedGridX2d( const dg::RefinedGridX3d& g) : 
-    dg::GridX2d( g.x0(), g.x1(), g.y0(), g.y1(), g.fx(), g.fy(), g.n(), g.Nx(), g.Ny(), g.bcx(), g.bcy()),
-    wx_( this->size()), wy_(this->size()), absX_(this->size()), absY_(this->size()),
-    g_assoc_( g.associated())
+
+/**
+ * @brief Refined X-point grid 
+ * @ingroup grid
+ */
+struct CartesianRefinedGridX2d : public dg::aGeometryX2d
 {
-    for(unsigned i=0; i<this->size(); i++)
+    CartesianRefinedGridX2d( const aRefinementX2d& ref,
+            double x0, double x1, double y0, double y1, 
+            double fx, double fy, 
+            unsigned n, unsigned Nx, unsigned Ny, 
+            bc bcx = dg::PER, bc bcy = dg::PER) : dg::aGeometryX2d( x0, x1, y0, y1, 
+                ref.fx_new(Nx, fx), ref.fy_new(Ny, fy), n, ref.nx_new(Nx, fx), ny_new(Ny, fy), bcx, bcy), w_(2), abs_(2)
     {
-        wx_[i] = g.weightsX()[i];
-        wy_[i] = g.weightsY()[i];
-        absX_[i] = g.abscissasX()[i];
-        absY_[i] = g.abscissasY()[i];
+        GridX2d g( x0,x1,y0,y1,fx,fy,n,Nx,Ny,bcx,bcy);
+        ref.generate(g,w_[0],w_[1],abs_[0],abs_[1]);
     }
-}
-
-namespace create{
 
-cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const dg::RefinedGridX2d& g_fine)
-{
-    dg::GridX2d g = g_fine.associated();
-    thrust::host_vector<double> x = g_fine.abscissasX();
-    thrust::host_vector<double> y = g_fine.abscissasY();
-    return dg::create::interpolation( x,y, g.grid());
-
-}
-cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const dg::RefinedGridX2d& g_fine)
-{
-    cusp::coo_matrix<int, double, cusp::host_memory> temp = interpolation( g_fine), A;
-    cusp::transpose( temp, A);
-    return A;
-}
-
-cusp::coo_matrix<int, double, cusp::host_memory> projection( const dg::RefinedGridX2d& g_fine)
-{
-    //form the adjoint
-    thrust::host_vector<double> w_f = dg::create::weights( g_fine);
-    thrust::host_vector<double> v_c = dg::create::inv_weights( g_fine.associated() );
-    cusp::coo_matrix<int, double, cusp::host_memory> Wf( w_f.size(), w_f.size(), w_f.size());
-    cusp::coo_matrix<int, double, cusp::host_memory> Vc( v_c.size(), v_c.size(), v_c.size());
-    for( int i =0; i<(int)w_f.size(); i++)
-    {
-        Wf.row_indices[i] = Wf.column_indices[i] = i;
-        Wf.values[i] = w_f[i]/g_fine.weightsX()[i]/g_fine.weightsY()[i];
+    CartesianRefinedGridX2d* clone()const{return new CartesianRefinedGridX2d(*this);}
+    private:
+    std::vector<thrust::host_vector<double> > w_,abs_;
+    virtual SparseTensor<thrust::host_vector<double> > do_compute_metric()const {
+        SparseTensor<thrust::host_vector<double> > t(w_);
+        dg::blas1::pointwiseDot( w_[0], w_[0], t.value(0));
+        dg::blas1::pointwiseDot( w_[1], w_[1], t.value(1));
+        t.idx(0,0)=0, t.idx(1,1)=1; 
+        return t;
     }
-    for( int i =0; i<(int)v_c.size(); i++)
-    {
-        Vc.row_indices[i] = Vc.column_indices[i] = i;
-        Vc.values[i] = v_c[i];
+    virtual SparseTensor<thrust::host_vector<double> > do_compute_jacobian()const {
+        SparseTensor<thrust::host_vector<double> > t(w_);
+        t.idx(0,0)=0, t.idx(1,1)=1; 
+        return t;
     }
-    cusp::coo_matrix<int, double, cusp::host_memory> temp = interpolation( g_fine), A;
-    cusp::transpose( temp, A);
-    cusp::multiply( A, Wf, temp);
-    cusp::multiply( Vc, temp, A);
-    A.sort_by_row_and_column();
-    return A;
-}
-
-cusp::coo_matrix<int, double, cusp::host_memory> smoothing( const dg::RefinedGridX2d& g)
-{
-    cusp::coo_matrix<int, double, cusp::host_memory> A = interpolation(g);
-    cusp::coo_matrix<int, double, cusp::host_memory> B = projection(g);
-    cusp::coo_matrix<int, double, cusp::host_memory> C;
-    cusp::multiply( A, B, C);
-    C.sort_by_row_and_column();
-    return C; 
-}
-
-cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const dg::RefinedGridX3d& g_fine)
-{
-    dg::GridX3d g = g_fine.associated();
-    thrust::host_vector<double> x = g_fine.abscissasX();
-    thrust::host_vector<double> y = g_fine.abscissasY();
-    thrust::host_vector<double> z = dg::evaluate( dg::cooZ3d, g_fine.grid());
-    return dg::create::interpolation( x,y,z, g.grid());
-}
-
-cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const dg::RefinedGridX3d& g_fine)
-{
-    cusp::coo_matrix<int, double, cusp::host_memory> temp = interpolation( g_fine), A;
-    cusp::transpose( temp, A);
-    return A;
-}
-
-cusp::coo_matrix<int, double, cusp::host_memory> projection( const dg::RefinedGridX3d& g_fine)
+    virtual std::vector<thrust::host_vector<double> > do_compute_map()const{
+        return abs_;
+    }
+};
+/**
+ * @brief Refined X-point grid 
+ * @ingroup grid
+ */
+struct CartesianRefinedGridX3d : public dg::aGeometryX3d
 {
-    //form the adjoint
-    thrust::host_vector<double> w_f = dg::create::weights( g_fine);
-    thrust::host_vector<double> v_c = dg::create::inv_weights( g_fine.associated() );
-    cusp::coo_matrix<int, double, cusp::host_memory> Wf( w_f.size(), w_f.size(), w_f.size());
-    cusp::coo_matrix<int, double, cusp::host_memory> Vc( v_c.size(), v_c.size(), v_c.size());
-    for( int i =0; i<(int)w_f.size(); i++)
+    CartesianRefinedGridX3d( const aRefinementX2d& ref,
+            double x0, double x1, double y0, double y1, double z0, double z1,
+            double fx, double fy, 
+            unsigned n, unsigned Nx, unsigned Ny, unsigned Nz,
+            bc bcx = dg::PER, bc bcy = dg::PER) : dg::aGeometryX3d( x0, x1, y0, y1,z0,z1,
+                ref.fx_new(Nx, fx), ref.fy_new(Ny, fy), n, ref.nx_new(Nx, fx), ny_new(Ny, fy),Nz, bcx, bcy,bcz), w_(2), abs_(2)
     {
-        Wf.row_indices[i] = Wf.column_indices[i] = i;
-        Wf.values[i] = w_f[i]/g_fine.weightsX()[i]/g_fine.weightsY()[i];
+        GridX2d g( x0,x1,y0,y1,fx,fy,n,Nx,Ny,bcx,bcy);
+        ref.generate(g,w_[0],w_[1],abs_[0],abs_[1]);
+        //lift to 3d
+        w_[0].resize(size()), w_[1].resize(size()), abs_[0].resize(size()), abs_[1].resize(size());
+        unsigned size2d=n()*n()*Nx()*Ny();
+        for( unsigned i=1; i<Nz; i++)
+            for(unsigned k=0; k<size2d; k++)
+            {
+                w_[0][i*size2d+k]=w_[0][k];
+                w_[1][i*size2d+k]=w_[1][k];
+                abs_[0][i*size2d+k]=abs_[0][k];
+                abs_[1][i*size2d+k]=abs_[1][k];
+            }
     }
-    for( int i =0; i<(int)v_c.size(); i++)
-    {
-        Vc.row_indices[i] = Vc.column_indices[i] = i;
-        Vc.values[i] = v_c[i];
+
+    CartesianRefinedGridX3d* clone()const{return new CartesianRefinedGridX3d(*this);}
+    private:
+    std::vector<thrust::host_vector<double> > w_,abs_;
+    virtual SparseTensor<thrust::host_vector<double> > do_compute_metric()const {
+        SparseTensor<thrust::host_vector<double> > t(w_);
+        dg::blas1::pointwiseDot( w_[0], w_[0], t.value(0));
+        dg::blas1::pointwiseDot( w_[1], w_[1], t.value(1));
+        t.idx(0,0)=0, t.idx(1,1)=1; 
+        return t;
+    }
+    virtual SparseTensor<thrust::host_vector<double> > do_compute_jacobian()const {
+        SparseTensor<thrust::host_vector<double> > t(w_);
+        t.idx(0,0)=0, t.idx(1,1)=1; 
+        return t;
     }
-    cusp::coo_matrix<int, double, cusp::host_memory> temp = interpolation( g_fine), A;
-    cusp::transpose( temp, A);
-    cusp::multiply( A, Wf, temp);
-    cusp::multiply( Vc, temp, A);
-    A.sort_by_row_and_column();
-    return A;
-}
+    virtual std::vector<thrust::host_vector<double> > do_compute_map()const{
+        return abs_;
+    }
+};
 
-cusp::coo_matrix<int, double, cusp::host_memory> smoothing( const dg::RefinedGridX3d& g)
-{
-    cusp::coo_matrix<int, double, cusp::host_memory> A = interpolation(g);
-    cusp::coo_matrix<int, double, cusp::host_memory> B = projection(g);
-    cusp::coo_matrix<int, double, cusp::host_memory> C;
-    cusp::multiply( A, B, C);
-    C.sort_by_row_and_column();
-    return C; 
-}
-}//namespace create
 
 }//namespace dg
-- 
GitLab


From 8d6ef09dd33bfa4e6f990978f7509102764e85b3 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 14 Aug 2017 07:04:31 -0700
Subject: [PATCH 154/453] update docu a bit and work on refinement

---
 inc/dg/dg_doc.h                        |  2 +-
 inc/dg/geometry/curvilinear.h          |  2 +-
 inc/dg/geometry/curvilinearX.h         |  2 +-
 inc/dg/geometry/mpi_curvilinear.h      |  2 +-
 inc/dg/geometry/refined_curvilinearX.h | 32 +++-----------------------
 inc/dg/geometry/refined_grid.h         |  4 ++--
 inc/dg/geometry/refined_gridX.h        | 10 +++++---
 7 files changed, 16 insertions(+), 38 deletions(-)

diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index 43dba4025..62e31e9ed 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -72,7 +72,7 @@
  *        @defgroup basicgeometry Geometry base classes
  *        @defgroup pullback pullback and pushforward
  *        @defgroup metric create volume
- *        @defgroup utilities Fieldalignment and Averaging
+ *        @defgroup utilities Averaging
  *        @defgroup generators Grid Generator classes
  *            The classes to perform field line integration for DS and averaging classes
  *    @}
diff --git a/inc/dg/geometry/curvilinear.h b/inc/dg/geometry/curvilinear.h
index 0bb5746c8..530ca501e 100644
--- a/inc/dg/geometry/curvilinear.h
+++ b/inc/dg/geometry/curvilinear.h
@@ -7,7 +7,7 @@
 
 namespace dg
 {
-///@addtogroup basicgrids
+///@addtogroup geometry
 ///@{
 
 ///@cond
diff --git a/inc/dg/geometry/curvilinearX.h b/inc/dg/geometry/curvilinearX.h
index 63583fe2b..d0be2fceb 100644
--- a/inc/dg/geometry/curvilinearX.h
+++ b/inc/dg/geometry/curvilinearX.h
@@ -8,7 +8,7 @@
 
 namespace dg
 {
-///@addtogroup basicgrids
+///@addtogroup geometry
 ///@{
 
 ///@cond
diff --git a/inc/dg/geometry/mpi_curvilinear.h b/inc/dg/geometry/mpi_curvilinear.h
index 50fdbacbf..71ba098df 100644
--- a/inc/dg/geometry/mpi_curvilinear.h
+++ b/inc/dg/geometry/mpi_curvilinear.h
@@ -17,7 +17,7 @@ namespace dg
 struct CurvilinearMPIGrid2d; 
 ///@endcond
 //
-///@addtogroup grids
+///@addtogroup geometry
 ///@{
 
 /**
diff --git a/inc/dg/geometry/refined_curvilinearX.h b/inc/dg/geometry/refined_curvilinearX.h
index 1209a3a51..2dbd12fdb 100644
--- a/inc/dg/geometry/refined_curvilinearX.h
+++ b/inc/dg/geometry/refined_curvilinearX.h
@@ -1,29 +1,23 @@
 #pragma once
 
-#include "dg/geometry/refined_gridX.h"
-#include "orthogonalX.h"
+#include "refined_gridX.h"
 
 namespace dg
 {
 
 ///@cond
-template< class container>
 struct CurvilinearRefinedGridX2d;
 ///@endcond
 
-///@addtogroup grids
+///@addtogroup geometry
 ///@{
 
 /**
  * @brief A three-dimensional refined X-Grid
  */
-template< class container>
-struct CurvilinearRefinedGridX3d : public dg::RefinedGridX3d
+struct CurvilinearRefinedGridX3d : public dg::aGeometryX3d
 {
-    typedef dg::CurvilinearCylindricalTag metric_category;
-    typedef dg::CurvilinearRefinedGridX2d<container> perpendicular_grid;
 
-    template <class Generator>
     CurvilinearRefinedGridX3d( unsigned add_x, unsigned add_y, unsigned howmanyX, unsigned howmanyY, const Generator& generator, double psi_0, double fx, double fy, unsigned n, unsigned n_old, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx, dg::bc bcy): 
         dg::RefinedGridX3d( add_x, add_y, howmanyX, howmanyY, 0,1, -2.*M_PI*fy/(1.-2.*fy), 2.*M_PI*(1.+fy/(1.-2.*fy)), 0., 2*M_PI, fx, fy, n, n_old, Nx, Ny, Nz, bcx, bcy, dg::PER),
         r_(this->size()), z_(r_), xr_(r_), xz_(r_), yr_(r_), yz_(r_),
@@ -33,22 +27,7 @@ struct CurvilinearRefinedGridX3d : public dg::RefinedGridX3d
         construct( generator, psi_0, fx);
     }
 
-    perpendicular_grid perp_grid() const { return perpendicular_grid(*this);}
-    const dg::CurvilinearGridX3d<container>& associated() const{ return g_assoc_;}
 
-    const thrust::host_vector<double>& r()const{return r_;}
-    const thrust::host_vector<double>& z()const{return z_;}
-    const thrust::host_vector<double>& xr()const{return xr_;}
-    const thrust::host_vector<double>& yr()const{return yr_;}
-    const thrust::host_vector<double>& xz()const{return xz_;}
-    const thrust::host_vector<double>& yz()const{return yz_;}
-
-    const container& g_xx()const{return g_xx_;}
-    const container& g_yy()const{return g_yy_;}
-    const container& g_xy()const{return g_xy_;}
-    const container& g_pp()const{return g_pp_;}
-    const container& vol()const{return vol_;}
-    const container& perpVol()const{return vol2d_;}
     private:
     template<class Generator>
     void construct( Generator generator, double psi_0, double fx)
@@ -112,18 +91,13 @@ struct CurvilinearRefinedGridX3d : public dg::RefinedGridX3d
         dg::blas1::pointwiseDivide( tempxx, r_, tempxx); //1/R^2
         g_pp_=tempxx;
     }
-    thrust::host_vector<double> r_, z_, xr_, xz_, yr_, yz_; //3d vector
-    container g_xx_, g_xy_, g_yy_, g_pp_, vol_, vol2d_;
-    dg::CurvilinearGridX3d<container> g_assoc_;
 };
 
 /**
  * @brief A two-dimensional refined X-Grid
  */
-template< class container>
 struct CurvilinearRefinedGridX2d : public dg::RefinedGridX2d
 {
-    typedef dg::CurvilinearCylindricalTag metric_category;
 
     template<class Generator>
     CurvilinearRefinedGridX2d( unsigned add_x, unsigned add_y, unsigned howmanyX, unsigned howmanyY, const Generator& generator, double psi_0, double fx, double fy, unsigned n, unsigned n_old, unsigned Nx, unsigned Ny, dg::bc bcx, dg::bc bcy): 
diff --git a/inc/dg/geometry/refined_grid.h b/inc/dg/geometry/refined_grid.h
index d2d3da28d..1def6a112 100644
--- a/inc/dg/geometry/refined_grid.h
+++ b/inc/dg/geometry/refined_grid.h
@@ -290,7 +290,7 @@ struct ExponentialRefinement : public aRefinement1d
 
 /**
  * @brief Refined Cartesian grid 
- * @ingroup grid
+ * @ingroup geometry
  */
 struct CartesianRefinedGrid2d : public dg::aGeometry2d
 {
@@ -346,7 +346,7 @@ struct CartesianRefinedGrid2d : public dg::aGeometry2d
 
 /**
  * @brief Refined Cartesian grid 
- * @ingroup grid
+ * @ingroup geometry
  */
 struct CartesianRefinedGrid3d : public dg::aGeometry3d
 {
diff --git a/inc/dg/geometry/refined_gridX.h b/inc/dg/geometry/refined_gridX.h
index c0843008a..156a26f6c 100644
--- a/inc/dg/geometry/refined_gridX.h
+++ b/inc/dg/geometry/refined_gridX.h
@@ -9,8 +9,10 @@
 
 namespace dg
 {
+///@addtogroup generators
+///@{
 /**
-* @brief Abstract base class for 1d grid refinement that increases the number of grid cells of a fixed basis grid
+* @brief Abstract base class for 2d grid refinement that increases the number of grid cells of a fixed basis grid
 */
 struct aRefinementX2d
 {
@@ -212,11 +214,12 @@ struct ExponentialXRefinement : public aRefinementX2d
         return (fy*(double)Ny + (double)add_y)/(double)(Ny+4.*add_y);
     }
 };
+///@}
 
 
 /**
  * @brief Refined X-point grid 
- * @ingroup grid
+ * @ingroup geometry
  */
 struct CartesianRefinedGridX2d : public dg::aGeometryX2d
 {
@@ -250,9 +253,10 @@ struct CartesianRefinedGridX2d : public dg::aGeometryX2d
         return abs_;
     }
 };
+
 /**
  * @brief Refined X-point grid 
- * @ingroup grid
+ * @ingroup geometry
  */
 struct CartesianRefinedGridX3d : public dg::aGeometryX3d
 {
-- 
GitLab


From ba586193f691bae7bd2ce361a721a265d94eea4f Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 14 Aug 2017 07:07:11 -0700
Subject: [PATCH 155/453] go back one commit

---
 inc/dg/geometry/refined_grid.h | 221 ++++++++++++++++++---------------
 1 file changed, 122 insertions(+), 99 deletions(-)

diff --git a/inc/dg/geometry/refined_grid.h b/inc/dg/geometry/refined_grid.h
index 3b5da7a93..baa913f18 100644
--- a/inc/dg/geometry/refined_grid.h
+++ b/inc/dg/geometry/refined_grid.h
@@ -6,7 +6,8 @@
 #include "dg/backend/interpolation.cuh"
 #include "dg/blas.h"
 
-#include "base_geometry.h"
+#include "cartesian.h"
+#include "cylindrical.h"
 
 
 namespace dg
@@ -251,57 +252,15 @@ int linear_ref( unsigned multiple_x, const Grid1d& g, thrust::host_vector<double
 ///@endcond
 }//namespace detail
 
+struct RefinedGrid3d;
 ///@endcond
 /**
- * @brief aRefined grid 
+ * @brief Refined grid 
  * @deprecated
  * @ingroup grid
  */
-struct aRefinedGrid2d : public dg::aGeometry2d
+struct RefinedGrid2d : public dg::aTopology2d
 {
-
-    /**
-     * @brief The grid that this object refines
-     *
-     * This function is vitual so that derived classes can also construct the associated grid 
-     * @return  2d grid
-     */
-    virtual const dg::Grid2d& associated()const {return g_assoc_;}
-    /**
-     * @brief Return the abscissas in X-direction 
-     *
-     * @return A 2d vector
-     */
-    const thrust::host_vector<double>& abscissasX() const {return absX_;} 
-    /**
-     * @brief Return the abscissas in Y-direction 
-     *
-     * @return A 2d vector
-     */
-    const thrust::host_vector<double>& abscissasY() const {return absY_;} 
-    /**
-     * @brief Return the weights in X-direction 
-     *
-     * @return A 2d vector
-     */
-    const thrust::host_vector<double>& weightsX() const {return wx_;} 
-    /**
-     * @brief Return the weights in Y-direction 
-     *
-     * @return A 2d vector
-     */
-    const thrust::host_vector<double>& weightsY() const {return wy_;} 
-
-    protected:
-    void init_X_boundaries( double x0, double x1)
-    {
-        double alpha = (x1-x0)/this->lx();
-        double beta = (x0*this->x1()-x1*this->x0())/this->lx();
-        //weights are invariant
-        for( unsigned i=0; i<absX_.size(); i++)
-            absX_[i]=alpha*absX_[i]+beta;
-        dg::Grid2d::init_X_boundaries( x0, x1);
-    }
     /**
      * @brief Refine a corner of a grid
      *
@@ -313,10 +272,10 @@ struct aRefinedGrid2d : public dg::aGeometry2d
      * @param howmanyY Add number of cells to the existing one
      * @copydetails Grid2d::Grid2d()
      */
-    aRefinedGrid2d( unsigned node_x, unsigned node_y, unsigned add_x, unsigned add_y, 
+    RefinedGrid2d( unsigned node_x, unsigned node_y, unsigned add_x, unsigned add_y, 
             unsigned howmanyX, unsigned howmanyY,
             double x0, double x1, double y0, double y1, 
-            unsigned n, unsigned Nx, unsigned Ny, bc bcx = dg::PER, bc bcy = dg::PER) : dg::aGeometry2d( x0, x1, y0, y1, n, n_new(Nx, add_x*howmanyX, bcx), n_new(Ny, add_y*howmanyY, bcy), bcx, bcy), 
+            unsigned n, unsigned Nx, unsigned Ny, bc bcx = dg::PER, bc bcy = dg::PER) : dg::aTopology2d( x0, x1, y0, y1, n, n_new(Nx, add_x*howmanyX, bcx), n_new(Ny, add_y*howmanyY, bcy), bcx, bcy), 
         wx_(size()), wy_(size()), absX_(size()), absY_(size()),
         g_assoc_( x0, x1, y0, y1, n, Nx, Ny, bcx, bcy)
     {
@@ -346,9 +305,9 @@ struct aRefinedGrid2d : public dg::aGeometry2d
      * @param n_old the polynomials in the old grid
      * @copydetails Grid2d::Grid2d()
      */
-    aRefinedGrid2d( unsigned multiple_x, unsigned multiple_y,
+    RefinedGrid2d( unsigned multiple_x, unsigned multiple_y,
             double x0, double x1, double y0, double y1, unsigned n,
-            unsigned n_old, unsigned Nx, unsigned Ny, bc bcx = dg::PER, bc bcy = dg::PER) : dg::aGeometry2d( x0, x1, y0, y1, n, multiple_x*Nx, multiple_y*Ny, bcx, bcy), 
+            unsigned n_old, unsigned Nx, unsigned Ny, bc bcx = dg::PER, bc bcy = dg::PER) : dg::aTopology2d( x0, x1, y0, y1, n, multiple_x*Nx, multiple_y*Ny, bcx, bcy), 
         wx_(size()), wy_(size()), absX_(size()), absY_(size()),
         g_assoc_( x0, x1, y0, y1, n_old, Nx, Ny, bcx, bcy)
     {
@@ -368,31 +327,21 @@ struct aRefinedGrid2d : public dg::aGeometry2d
             }
     }
 
-    private:
-    unsigned n_new( unsigned N, unsigned factor, dg::bc bc)
-    {
-        if( bc == dg::PER) return N + 2*factor; 
-        return N + factor;
-    }
-    thrust::host_vector<double> wx_, wy_; //weights
-    thrust::host_vector<double> absX_, absY_; //abscissas 
-    dg::Grid2d g_assoc_;
-};
-
-/**
- * @brief aRefined grid 
- * @deprecated
- * @ingroup grid
- */
-struct aRefinedGrid3d : public dg::aGeometry3d
-{
+    /**
+     * @brief Reduce from a 3d grid 
+     *
+     * This is possible because all our grids are product space grids. 
+     *
+     * @param g The 3d grid
+     */
+    RefinedGrid2d( const dg::RefinedGrid3d& g);
     /**
      * @brief The grid that this object refines
      *
      * This function is vitual so that derived classes can also construct the associated grid 
      * @return  2d grid
      */
-    virtual const dg::Grid3d& associated()const {return g_assoc_;}
+    virtual const dg::Grid2d& associated()const {return g_assoc_;}
     /**
      * @brief Return the abscissas in X-direction 
      *
@@ -417,16 +366,36 @@ struct aRefinedGrid3d : public dg::aGeometry3d
      * @return A 2d vector
      */
     const thrust::host_vector<double>& weightsY() const {return wy_;} 
+
     protected:
     void init_X_boundaries( double x0, double x1)
     {
         double alpha = (x1-x0)/this->lx();
         double beta = (x0*this->x1()-x1*this->x0())/this->lx();
-        //weights are invariant under linear transformation
+        //weights are invariant
         for( unsigned i=0; i<absX_.size(); i++)
             absX_[i]=alpha*absX_[i]+beta;
-        dg::Grid3d::init_X_boundaries( x0, x1);
+        dg::Grid2d::init_X_boundaries( x0, x1);
     }
+
+    private:
+    unsigned n_new( unsigned N, unsigned factor, dg::bc bc)
+    {
+        if( bc == dg::PER) return N + 2*factor; 
+        return N + factor;
+    }
+    thrust::host_vector<double> wx_, wy_; //weights
+    thrust::host_vector<double> absX_, absY_; //abscissas 
+    dg::Grid2d g_assoc_;
+};
+
+/**
+ * @brief Refined grid 
+ * @deprecated
+ * @ingroup grid
+ */
+struct RefinedGrid3d : public dg::aTopology3d
+{
     /**
      * @brief Refine a corner of a grid
      *
@@ -438,10 +407,10 @@ struct aRefinedGrid3d : public dg::aGeometry3d
      * @param howmanyY howmany cells should be refined in y
      * @copydetails Grid3d::Grid3d()
      */
-    aRefinedGrid3d( unsigned node_x, unsigned node_y, unsigned add_x, unsigned add_y, 
+    RefinedGrid3d( unsigned node_x, unsigned node_y, unsigned add_x, unsigned add_y, 
             unsigned howmanyX, unsigned howmanyY,
             double x0, double x1, double y0, double y1, double z0, double z1, 
-            unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = dg::PER, bc bcy = dg::PER, bc bcz = dg::PER) : dg::aGeometry3d( x0, x1, y0, y1, z0, z1, n, n_new(Nx, add_x*howmanyX, bcx), n_new(Ny, add_y*howmanyY, bcy), Nz, bcx, bcy, bcz), 
+            unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = dg::PER, bc bcy = dg::PER, bc bcz = dg::PER) : dg::aTopology3d( x0, x1, y0, y1, z0, z1, n, n_new(Nx, add_x*howmanyX, bcx), n_new(Ny, add_y*howmanyY, bcy), Nz, bcx, bcy, bcz), 
         wx_(size()), wy_(size()), absX_(size()), absY_(size()),
         g_assoc_( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, bcx, bcy, bcz)
     {
@@ -471,12 +440,12 @@ struct aRefinedGrid3d : public dg::aGeometry3d
      * @param multiple_y Multiply all cells in y - direction
      * @copydetails Grid3d::Grid3d()
      */
-    aRefinedGrid3d( unsigned multiple_x, unsigned multiple_y,
+    RefinedGrid3d( unsigned multiple_x, unsigned multiple_y,
             double x0, double x1, double y0, double y1, double z0, double z1, 
             unsigned n,
             unsigned n_old, unsigned Nx, unsigned Ny, unsigned Nz, 
             bc bcx = dg::PER, bc bcy = dg::PER, bc bcz = dg::PER) : 
-        dg::aGeometry3d( x0, x1, y0, y1, z0, z1, n, multiple_x*Nx, multiple_y*Ny, Nz, bcx, bcy, bcz), 
+        dg::aTopology3d( x0, x1, y0, y1, z0, z1, n, multiple_x*Nx, multiple_y*Ny, Nz, bcx, bcy, bcz), 
         wx_(size()), wy_(size()), absX_(size()), absY_(size()),
         g_assoc_( x0, x1, y0, y1, z0, z1, n_old, Nx, Ny, Nz, bcx, bcy, bcz)
     {
@@ -496,6 +465,47 @@ struct aRefinedGrid3d : public dg::aGeometry3d
                     absY_[(s*wy.size()+i)*wx.size()+j] = ay[i];
                 }
     }
+    /**
+     * @brief The grid that this object refines
+     *
+     * This function is vitual so that derived classes can also construct the associated grid 
+     * @return  2d grid
+     */
+    virtual const dg::Grid3d& associated()const {return g_assoc_;}
+    /**
+     * @brief Return the abscissas in X-direction 
+     *
+     * @return A 2d vector
+     */
+    const thrust::host_vector<double>& abscissasX() const {return absX_;} 
+    /**
+     * @brief Return the abscissas in Y-direction 
+     *
+     * @return A 2d vector
+     */
+    const thrust::host_vector<double>& abscissasY() const {return absY_;} 
+    /**
+     * @brief Return the weights in X-direction 
+     *
+     * @return A 2d vector
+     */
+    const thrust::host_vector<double>& weightsX() const {return wx_;} 
+    /**
+     * @brief Return the weights in Y-direction 
+     *
+     * @return A 2d vector
+     */
+    const thrust::host_vector<double>& weightsY() const {return wy_;} 
+    protected:
+    void init_X_boundaries( double x0, double x1)
+    {
+        double alpha = (x1-x0)/this->lx();
+        double beta = (x0*this->x1()-x1*this->x0())/this->lx();
+        //weights are invariant under linear transformation
+        for( unsigned i=0; i<absX_.size(); i++)
+            absX_[i]=alpha*absX_[i]+beta;
+        dg::Grid3d::init_X_boundaries( x0, x1);
+    }
 
     private:
     unsigned n_new( unsigned N, unsigned factor, dg::bc bc)
@@ -506,32 +516,38 @@ struct aRefinedGrid3d : public dg::aGeometry3d
     thrust::host_vector<double> wx_, wy_; //weights
     thrust::host_vector<double> absX_, absY_; //abscissas 
     dg::Grid3d g_assoc_;
-    virtual SparseTensor<thrust::host_vector<double> > do_compute_metric()const {
 
-        return SparseTensor<thrust::host_vector<double> >();
-    }
-    virtual SparseTensor<thrust::host_vector<double> > do_compute_jacobian()const {
-        return SparseTensor<thrust::host_vector<double> >();
-    }
-    virtual std::vector<thrust::host_vector<double> > do_compute_map()const{
-        std::vector<thrust::host_vector<double> > map(2);
-        map[0] = dg::evaluate(dg::cooX2d, *this);
-        map[1] = dg::evaluate(dg::cooY2d, *this);
-        return map;
+};
+
+RefinedGrid2d::RefinedGrid2d( const dg::RefinedGrid3d& g) : 
+    dg::Grid2d( g.x0(), g.x1(), g.y0(), g.y1(), g.n(), g.Nx(), g.Ny(), g.bcx(), g.bcy()),
+    wx_( this->size()), wy_(this->size()), absX_(this->size()), absY_(this->size()),
+    g_assoc_( g.associated())
+{
+    for(unsigned i=0; i<this->size(); i++)
+    {
+        wx_[i] = g.weightsX()[i];
+        wy_[i] = g.weightsY()[i];
+        absX_[i] = g.abscissasX()[i];
+        absY_[i] = g.abscissasY()[i];
     }
+}
+
+
 
-};
 
 /**
  * @brief A refined cartesian grid
  *
  * @ingroup basicgrids
  * @deprecated
+ * @tparam container
  */
-struct CartesianRefinedGrid2d : public dg::aRefinedGrid2d
+template<class container>
+struct CartesianRefinedGrid2d : public dg::RefinedGrid2d
 {
     typedef CurvilinearPerpTag metric_category; 
-    CartesianaRefinedGrid2d( unsigned multiple_x, unsigned multiple_y, double x0, double x1, double y0, double y1, unsigned n, unsigned n_old, unsigned Nx, unsigned Ny, bc bcx = PER, bc bcy = PER): dg::aRefinedGrid2d(multiple_x, multiple_y,x0,x1,y0,y1,n,n_old,Nx,Ny,bcx,bcy), g_assoc_(x0,x1,y0,y1,n_old,Nx,Ny,bcx,bcy){ 
+    CartesianRefinedGrid2d( unsigned multiple_x, unsigned multiple_y, double x0, double x1, double y0, double y1, unsigned n, unsigned n_old, unsigned Nx, unsigned Ny, bc bcx = PER, bc bcy = PER): dg::RefinedGrid2d(multiple_x, multiple_y,x0,x1,y0,y1,n,n_old,Nx,Ny,bcx,bcy), g_assoc_(x0,x1,y0,y1,n_old,Nx,Ny,bcx,bcy){ 
         dg::blas1::transfer( weightsX(), g_xx_);
         dg::blas1::transfer( weightsY(), g_yy_);
         dg::blas1::transfer( weightsX(), vol2d_);
@@ -541,17 +557,24 @@ struct CartesianRefinedGrid2d : public dg::aRefinedGrid2d
         dg::blas1::pointwiseDivide( vol2d_, g_xx_, vol2d_);
         dg::blas1::pointwiseDivide( vol2d_, g_yy_, vol2d_);
     }
-
+    const thrust::host_vector<double>& r()const{return this->abscissasX();}
+    const thrust::host_vector<double>& z()const{return this->abscissasY();}
     const dg::CartesianGrid2d& associated() const {return g_assoc_;}
-    virtual CartesianRefinedGrid2d* clone()const{return new CartesianRefinedGrid2d(*this);}
+    const container& g_xx()const{return g_xx_;}
+    const container& g_yy()const{return g_yy_;}
+    const container& vol()const{return vol2d_;}
+    const container& perpVol()const{return vol2d_;}
+    bool isOrthogonal()const{return true;}
+    bool isConformal()const{return false;}
     private:
+    container g_xx_, g_yy_, vol2d_;
     dg::CartesianGrid2d g_assoc_;
 };
 
 namespace create{
 
 ///@deprecated
-cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const dg::aRefinedGrid2d& g_fine)
+cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const dg::RefinedGrid2d& g_fine)
 {
     dg::Grid2d g = g_fine.associated();
     thrust::host_vector<double> x = g_fine.abscissasX();
@@ -560,7 +583,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const dg::aRefin
 
 }
 ///@deprecated
-cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const dg::aRefinedGrid2d& g_fine)
+cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const dg::RefinedGrid2d& g_fine)
 {
     cusp::coo_matrix<int, double, cusp::host_memory> temp = interpolation( g_fine), A;
     cusp::transpose( temp, A);
@@ -568,7 +591,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const dg::aRefi
 }
 
 ///@deprecated
-cusp::coo_matrix<int, double, cusp::host_memory> projection( const dg::aRefinedGrid2d& g_fine)
+cusp::coo_matrix<int, double, cusp::host_memory> projection( const dg::RefinedGrid2d& g_fine)
 {
     //form the adjoint
     thrust::host_vector<double> w_f = dg::create::weights( g_fine);
@@ -594,7 +617,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> projection( const dg::aRefinedG
 }
 
 ///@deprecated
-cusp::coo_matrix<int, double, cusp::host_memory> smoothing( const dg::aRefinedGrid2d& g)
+cusp::coo_matrix<int, double, cusp::host_memory> smoothing( const dg::RefinedGrid2d& g)
 {
     cusp::coo_matrix<int, double, cusp::host_memory> A = interpolation(g);
     cusp::coo_matrix<int, double, cusp::host_memory> B = projection(g);
@@ -605,7 +628,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> smoothing( const dg::aRefinedGr
 }
 
 ///@deprecated
-cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const dg::aRefinedGrid3d& g_fine)
+cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const dg::RefinedGrid3d& g_fine)
 {
     dg::Grid3d g = g_fine.associated();
     thrust::host_vector<double> x = g_fine.abscissasX();
@@ -615,7 +638,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const dg::aRefin
 }
 
 ///@deprecated
-cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const dg::aRefinedGrid3d& g_fine)
+cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const dg::RefinedGrid3d& g_fine)
 {
     cusp::coo_matrix<int, double, cusp::host_memory> temp = interpolation( g_fine), A;
     cusp::transpose( temp, A);
@@ -623,7 +646,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const dg::aRefi
 }
 
 ///@deprecated
-cusp::coo_matrix<int, double, cusp::host_memory> projection( const dg::aRefinedGrid3d& g_fine)
+cusp::coo_matrix<int, double, cusp::host_memory> projection( const dg::RefinedGrid3d& g_fine)
 {
     //form the adjoint
     thrust::host_vector<double> w_f = dg::create::weights( g_fine);
@@ -649,7 +672,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> projection( const dg::aRefinedG
 }
 
 ///@deprecated
-cusp::coo_matrix<int, double, cusp::host_memory> smoothing( const dg::aRefinedGrid3d& g)
+cusp::coo_matrix<int, double, cusp::host_memory> smoothing( const dg::RefinedGrid3d& g)
 {
     cusp::coo_matrix<int, double, cusp::host_memory> A = interpolation(g);
     cusp::coo_matrix<int, double, cusp::host_memory> B = projection(g);
-- 
GitLab


From 087994e121161ac785c772da8bc13a9289c7faf3 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Mon, 14 Aug 2017 21:00:46 +0200
Subject: [PATCH 156/453] finalize refined X Grid

---
 inc/dg/geometry/curvilinearX.h         |   4 -
 inc/dg/geometry/refined_curvilinearX.h | 248 +++++++++++++------------
 inc/dg/geometry/refined_gridX.h        |   8 +-
 3 files changed, 137 insertions(+), 123 deletions(-)

diff --git a/inc/dg/geometry/curvilinearX.h b/inc/dg/geometry/curvilinearX.h
index d0be2fceb..9e2a9138b 100644
--- a/inc/dg/geometry/curvilinearX.h
+++ b/inc/dg/geometry/curvilinearX.h
@@ -11,10 +11,6 @@ namespace dg
 ///@addtogroup geometry
 ///@{
 
-///@cond
-struct CurvilinearGridX2d; 
-///@endcond
-
 /**
  * @brief A three-dimensional grid based on curvilinear coordinates
  * 
diff --git a/inc/dg/geometry/refined_curvilinearX.h b/inc/dg/geometry/refined_curvilinearX.h
index 2dbd12fdb..35127e268 100644
--- a/inc/dg/geometry/refined_curvilinearX.h
+++ b/inc/dg/geometry/refined_curvilinearX.h
@@ -1,149 +1,167 @@
 #pragma once
 
+#include "generatorX.h"
 #include "refined_gridX.h"
 
 namespace dg
 {
-
-///@cond
-struct CurvilinearRefinedGridX2d;
-///@endcond
-
 ///@addtogroup geometry
 ///@{
 
 /**
- * @brief A three-dimensional refined X-Grid
+ * @brief A three-dimensional grid based on curvilinear coordinates
+ * 
+ * The base coordinate system is the cylindrical coordinate system R,Z,phi
  */
-struct CurvilinearRefinedGridX3d : public dg::aGeometryX3d
+struct CurvilinearProductRefinedGridX3d : public dg::aGeometryX3d
 {
-
-    CurvilinearRefinedGridX3d( unsigned add_x, unsigned add_y, unsigned howmanyX, unsigned howmanyY, const Generator& generator, double psi_0, double fx, double fy, unsigned n, unsigned n_old, unsigned Nx, unsigned Ny, unsigned Nz, dg::bc bcx, dg::bc bcy): 
-        dg::RefinedGridX3d( add_x, add_y, howmanyX, howmanyY, 0,1, -2.*M_PI*fy/(1.-2.*fy), 2.*M_PI*(1.+fy/(1.-2.*fy)), 0., 2*M_PI, fx, fy, n, n_old, Nx, Ny, Nz, bcx, bcy, dg::PER),
-        r_(this->size()), z_(r_), xr_(r_), xz_(r_), yr_(r_), yz_(r_),
-        g_assoc_( generator, psi_0, fx, fy, n_old, Nx, Ny, Nz, bcx, bcy)
-    {
-        assert( generator.isCurvilinear());
-        construct( generator, psi_0, fx);
+    /*!@brief Constructor
+    
+     * the coordinates of the computational space are called x,y,z
+     * @param generator must generate a grid
+     * @param n number of %Gaussian nodes in x and y
+     * @param fx
+     * @param fy
+     * @param Nx number of cells in x
+     * @param Ny number of cells in y 
+     * @param Nz  number of cells z
+     * @param bcx boundary condition in x
+     * @param bcy boundary condition in y
+     * @param bcz boundary condition in z
+     */
+    CurvilinearProductRefinedGridX3d( const aRefinementX2d& ref, const aGeneratorX2d& generator, 
+        double fx, double fy, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, double fx, double fy, bc bcx=dg::DIR, bc bcy=dg::PER, bc bcz=dg::PER):
+        dg::aGeometryX3d( generator.zeta0(), generator.zeta1(), generator.eta0(), generator.eta1(), 0., 2.*M_PI, ref.fx_new(Nx,fx),ref.fy_new(Ny,fy),n, ref.nx_new(Nx,fx), ref.ny_new(Ny,fy), Nz, bcx, bcy, bcz), map_(3)
+    { 
+        handle_ = generator;
+        ref_=ref;
+        constructPerp( fx,fy,n, Nx, Ny);
+        constructParallel(Nz);
     }
 
-
+    const aGeneratorX2d & generator() const{return handle_.get();}
+    virtual CurvilinearProductRefinedGridX3d* clone()const{return new CurvilinearProductRefinedGridX3d(*this);}
     private:
-    template<class Generator>
-    void construct( Generator generator, double psi_0, double fx)
-    { 
-        std::cout << "FIND X FOR PSI_0\n";
-        const double x_0 = generator.f0()*psi_0;
-        const double x_1 = -fx/(1.-fx)*x_0;
-        std::cout << "X0 is "<<x_0<<" and X1 is "<<x_1<<"\n";
-        init_X_boundaries( x_0, x_1);
-        ////////////compute psi(x) for a grid on x 
-        thrust::host_vector<double> x_vec(this->n()*this->Nx()); 
-        for(unsigned i=0; i<x_vec.size(); i++) {
-            x_vec[i] = this->abscissasX()[i];
-        }
-        thrust::host_vector<double> rvec, zvec, xrvec, xzvec, yrvec, yzvec;
-        thrust::host_vector<double> y_vec(this->n()*this->Ny());
-        for(unsigned i=0; i<y_vec.size(); i++) y_vec[i] = this->abscissasY()[i*x_vec.size()];
-        std::cout << "In construct function \n";
-        generator( x_vec, y_vec, 
-                this->n()*this->outer_Ny(), 
-                this->n()*(this->inner_Ny()+this->outer_Ny()), 
-                rvec, zvec, xrvec, xzvec, yrvec, yzvec);
-        std::cout << "In construct function \n";
-        unsigned Mx = this->n()*this->Nx(), My = this->n()*this->Ny();
-        //now lift to 3D grid and multiply with refined weights
-        thrust::host_vector<double> wx = this->weightsX();
-        thrust::host_vector<double> wy = this->weightsY();
-        for( unsigned k=0; k<this->Nz(); k++)
-            for( unsigned i=0; i<Mx*My; i++)
+    //construct phi and lift rest to 3d
+    void constructParallel(unsigned Nz)
+    {
+        map_[2]=dg::evaluate(dg::cooZ3d, *this);
+        unsigned size = this->size();
+        unsigned size2d = this->n()*this->n()*this->Nx()*this->Ny();
+        //resize for 3d values
+        for( unsigned r=0; r<4;r++)
+            jac_.value(r).resize(size);
+        map_[0].resize(size);
+        map_[1].resize(size);
+        //lift to 3D grid
+        for( unsigned k=1; k<Nz; k++)
+            for( unsigned i=0; i<size2d; i++)
             {
-                r_[k*Mx*My+i] = rvec[i];
-                z_[k*Mx*My+i] = zvec[i];
-                yr_[k*Mx*My+i] = yrvec[i]*wy[k*Mx*My+i];
-                yz_[k*Mx*My+i] = yzvec[i]*wy[k*Mx*My+i];
-                xr_[k*Mx*My+i] = xrvec[i]*wx[k*Mx*My+i];
-                xz_[k*Mx*My+i] = xzvec[i]*wx[k*Mx*My+i];
+                for(unsigned r=0; r<4; r++)
+                    jac_.value(r)[k*size2d+i] = jac_.value(r)[(k-1)*size2d+i];
+                map_[0][k*size2d+i] = map_[0][(k-1)*size2d+i];
+                map_[1][k*size2d+i] = map_[1][(k-1)*size2d+i];
             }
-        construct_metric();
     }
-    //compute metric elements from xr, xz, yr, yz, r and z
-    void construct_metric()
+    //construct 2d plane
+    void constructPerp( double fx, double fy, unsigned n, unsigned Nx, unsigned Ny)
+    {
+        std::vector<thrust::host_vector<double> > w(2),abs(2);
+        GridX2d g( x0(),x1(),y0(),y1(),fx,fy,n,Nx,Ny,bcx,bcy);
+        ref.generate(g,w[0],w[1],abs[0],abs[1]);
+        thrust::host_vector<double> x_vec(n()*Nx()), y_vec(n()*Ny());
+        for( unsigned i=0; i<x_vec.size(); i++)
+            x_vec[i] = abs[0][i];
+        for( unsigned i=0; i<y_vec.size(); i++)
+            x_vec[i] = abs[1][i*x_vec.size()];
+        handle_.get().generate( x_vec, y_vec, n()*outer_Ny(), n()*(inner_Ny()+outer_Ny()), map_[0], map_[1], jac_.value(0), jac_.value(1), jac_.value(2), jac_.value(3));
+        //multiply by weights
+        dg::blas1::pointwiseDot( jac.value(0), w[0], jac.value(0));
+        dg::blas1::pointwiseDot( jac.value(1), w[0], jac.value(1));
+        dg::blas1::pointwiseDot( jac.value(2), w[1], jac.value(2));
+        dg::blas1::pointwiseDot( jac.value(3), w[1], jac.value(3));
+        jac_.idx(0,0) = 0, jac_.idx(0,1) = 1, jac_.idx(1,0)=2, jac_.idx(1,1) = 3;
+    }
+    virtual SparseTensor<thrust::host_vector<double> > do_compute_jacobian( ) const {
+        return jac_;
+    }
+    virtual SparseTensor<thrust::host_vector<double> > do_compute_metric( ) const
     {
-        std::cout << "CONSTRUCTING METRIC\n";
-        thrust::host_vector<double> tempxx( r_), tempxy(r_), tempyy(r_), tempvol(r_);
-        unsigned Nx = this->n()*this->Nx(), Ny = this->n()*this->Ny();
-        for( unsigned k=0; k<this->Nz(); k++)
-            for( unsigned i=0; i<Ny; i++)
-                for( unsigned j=0; j<Nx; j++)
-                {
-                    unsigned idx = k*Ny*Nx+i*Nx+j;
-                    tempxx[idx] = (xr_[idx]*xr_[idx]+xz_[idx]*xz_[idx]);
-                    tempxy[idx] = (yr_[idx]*xr_[idx]+yz_[idx]*xz_[idx]);
-                    tempyy[idx] = (yr_[idx]*yr_[idx]+yz_[idx]*yz_[idx]);
-                    tempvol[idx] = r_[idx]/sqrt(tempxx[idx]*tempyy[idx]-tempxy[idx]*tempxy[idx]);
-                }
-        g_xx_=tempxx, g_xy_=tempxy, g_yy_=tempyy, vol_=tempvol;
-        dg::blas1::pointwiseDivide( tempvol, r_, tempvol);
-        vol2d_ = tempvol;
-        thrust::host_vector<double> ones = dg::evaluate( dg::one, *this);
-        dg::blas1::pointwiseDivide( ones, r_, tempxx);
-        dg::blas1::pointwiseDivide( tempxx, r_, tempxx); //1/R^2
-        g_pp_=tempxx;
+        thrust::host_vector<double> tempxx( size()), tempxy(size()), tempyy(size()), temppp(size());
+        for( unsigned i=0; i<size(); i++)
+        {
+            tempxx[i] = (jac_.value(0,0)[i]*jac_.value(0,0)[i]+jac_.value(0,1)[i]*jac_.value(0,1)[i]);
+            tempxy[i] = (jac_.value(0,0)[i]*jac_.value(1,0)[i]+jac_.value(0,1)[i]*jac_.value(1,1)[i]);
+            tempyy[i] = (jac_.value(1,0)[i]*jac_.value(1,0)[i]+jac_.value(1,1)[i]*jac_.value(1,1)[i]);
+            temppp[i] = 1./map_[2][i]/map_[2][i]; //1/R^2
+        }
+        SparseTensor<thrust::host_vector<double> > metric;
+        metric.idx(0,0) = 0; metric.value(0) = tempxx;
+        metric.idx(1,1) = 1; metric.value(1) = tempyy;
+        metric.idx(2,2) = 2; metric.value(2) = temppp;
+        if( !handle_.get().isOrthogonal())
+        {
+            metric.idx(0,1) = metric.idx(1,0) = 3; 
+            metric.value(3) = tempxy;
+        }
+        return metric;
     }
+    virtual std::vector<thrust::host_vector<double> > do_compute_map()const{return map_;}
+    std::vector<thrust::host_vector<double> > map_;
+    SparseTensor<thrust::host_vector<double> > jac_;
+    dg::Handle<aGeneratorX2d> handle_;
+    dg::Handle<aRefinementX2d> ref_;
 };
 
 /**
- * @brief A two-dimensional refined X-Grid
+ * @brief A two-dimensional grid based on curvilinear coordinates
  */
-struct CurvilinearRefinedGridX2d : public dg::RefinedGridX2d
+struct CurvilinearRefinedGridX2d : public dg::aGeometryX2d
 {
-
-    template<class Generator>
-    CurvilinearRefinedGridX2d( unsigned add_x, unsigned add_y, unsigned howmanyX, unsigned howmanyY, const Generator& generator, double psi_0, double fx, double fy, unsigned n, unsigned n_old, unsigned Nx, unsigned Ny, dg::bc bcx, dg::bc bcy): 
-        dg::RefinedGridX2d( add_x, add_y, howmanyX, howmanyY, 0, 1,-fy*2.*M_PI/(1.-2.*fy), 2*M_PI+fy*2.*M_PI/(1.-2.*fy), fx, fy, n, n_old, Nx, Ny, bcx, bcy),
-        g_assoc_( generator, psi_0, fx, fy, n_old, Nx, Ny, bcx, bcy) 
+    /*!@brief Constructor
+    
+     * @param generator must generate an orthogonal grid (class takes ownership of the pointer)
+     * @param fx
+     * @param fy
+     * @param n number of polynomial coefficients
+     * @param Nx number of cells in first coordinate
+     * @param Ny number of cells in second coordinate
+     * @param bcx boundary condition in first coordinate
+     * @param bcy boundary condition in second coordinate
+     */
+    CurvilinearRefinedGridX2d( const aRefinementX2d& ref, const aGeneratorX2d& generator, double fx, double fy, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx=dg::DIR, bc bcy=dg::PER):
+        dg::aGeometryX2d( generator.zeta0(), generator.zeta1(), generator.eta0(), generator.eta1(),ref.fx_new(Nx,fx),ref.fy_new(Ny,fy),n, ref.nx_new(Nx,fx), ref.ny_new(Ny,fy), bcx, bcy)
     {
-        dg::CurvilinearRefinedGridX3d<container> g(add_x, add_y, howmanyX, howmanyY, generator, psi_0, fx,fy, n,n_old,Nx,Ny,1,bcx,bcy);
-        init_X_boundaries( g.x0(), g.x1());
-        r_=g.r(), z_=g.z(), xr_=g.xr(), xz_=g.xz(), yr_=g.yr(), yz_=g.yz();
-        g_xx_=g.g_xx(), g_xy_=g.g_xy(), g_yy_=g.g_yy();
-        vol2d_=g.perpVol();
+        handle_ = generator;
+        ref_=ref;
+        construct( double fx,double fy,n,Nx,Ny);
     }
-    CurvilinearRefinedGridX2d( const CurvilinearRefinedGridX3d<container>& g):
-        dg::RefinedGridX2d(g), g_assoc_(g.associated())
+
+    const aGeneratorX2d& generator() const{return handle_.get();}
+    virtual CurvilinearGridX2d* clone()const{return new CurvilinearGridX2d(*this);}
+    private:
+    void construct(double fx, double fy, unsigned n, unsigned Nx, unsigned Ny)
     {
-        unsigned s = this->size();
-        r_.resize( s), z_.resize(s), xr_.resize(s), xz_.resize(s), yr_.resize(s), yz_.resize(s);
-        g_xx_.resize( s), g_xy_.resize(s), g_yy_.resize(s), vol2d_.resize(s);
-        for( unsigned i=0; i<s; i++)
-        { r_[i]=g.r()[i], z_[i]=g.z()[i], xr_[i]=g.xr()[i], xz_[i]=g.xz()[i], yr_[i]=g.yr()[i], yz_[i]=g.yz()[i]; }
-        thrust::copy( g.g_xx().begin(), g.g_xx().begin()+s, g_xx_.begin());
-        thrust::copy( g.g_xy().begin(), g.g_xy().begin()+s, g_xy_.begin());
-        thrust::copy( g.g_yy().begin(), g.g_yy().begin()+s, g_yy_.begin());
-        thrust::copy( g.perpVol().begin(), g.perpVol().begin()+s, vol2d_.begin());
+        CurvilinearProductRefinedGridX3d g( ref_.get(), handle_.get(),fx,fy,n,Nx,Ny,1,bcx());
+        jac_=g.jacobian();
+        map_=g.map();
+        metric_=g.metric();
     }
-    const dg::CurvilinearGridX2d<container>& associated()const{return g_assoc_;}
-    const thrust::host_vector<double>& r()const{return r_;}
-    const thrust::host_vector<double>& z()const{return z_;}
-    const thrust::host_vector<double>& xr()const{return xr_;}
-    const thrust::host_vector<double>& yr()const{return yr_;}
-    const thrust::host_vector<double>& xz()const{return xz_;}
-    const thrust::host_vector<double>& yz()const{return yz_;}
-    thrust::host_vector<double> x()const{
-        dg::Grid1d gx( x0(), x1(), n(), Nx());
-        return dg::create::abscissas(gx);}
-    const container& g_xx()const{return g_xx_;}
-    const container& g_yy()const{return g_yy_;}
-    const container& g_xy()const{return g_xy_;}
-    const container& vol()const{return vol2d_;}
-    const container& perpVol()const{return vol2d_;}
-    private:
-    thrust::host_vector<double> r_, z_, xr_, xz_, yr_, yz_; //2d vector
-    container g_xx_, g_xy_, g_yy_, vol2d_;
-    dg::CurvilinearGridX2d<container> g_assoc_;
+    virtual SparseTensor<thrust::host_vector<double> > do_compute_jacobian( ) const {
+        return jac_;
+    }
+    virtual SparseTensor<thrust::host_vector<double> > do_compute_metric( ) const {
+        return metric_;
+    }
+    virtual std::vector<thrust::host_vector<double> > do_compute_map()const{return map_;}
+    dg::SparseTensor<thrust::host_vector<double> > jac_, metric_;
+    std::vector<thrust::host_vector<double> > map_;
+    dg::Handle<aGeneratorX2d> handle_;
+    dg::Handle<aRefinementX2d> ref_;
 };
+
 ///@}
 
+
 } //namespace dg
 
diff --git a/inc/dg/geometry/refined_gridX.h b/inc/dg/geometry/refined_gridX.h
index 156a26f6c..d58cb8bc5 100644
--- a/inc/dg/geometry/refined_gridX.h
+++ b/inc/dg/geometry/refined_gridX.h
@@ -35,10 +35,10 @@ struct aRefinementX2d
         for( unsigned i=0; i<wy.size(); i++)
             for( unsigned j=0; j<wx.size(); j++)
             {
-                weightsX[0][i*wx.size()+j] = wx[j];
-                weightsY[1][i*wx.size()+j] = wy[i];
-                abscissasX[0][i*wx.size()+j] = ax[j];
-                abscissasY[1][i*wx.size()+j] = ay[i];
+                weightsX[i*wx.size()+j] = wx[j];
+                weightsY[i*wx.size()+j] = wy[i];
+                abscissasX[i*wx.size()+j] = ax[j];
+                abscissasY[i*wx.size()+j] = ay[i];
             }
     }
     /*! @brief the new number of cells
-- 
GitLab


From 893901131c394a9c37e87ed66e31972949ac361e Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Mon, 14 Aug 2017 21:44:22 +0200
Subject: [PATCH 157/453] arakawa_mpib compiles again

---
 inc/dg/backend/gridX.h                 |  16 +--
 inc/dg/backend/mpi_evaluation.h        |   6 +-
 inc/dg/geometry/mpi_base.h             |  27 ++--
 inc/dg/geometry/mpi_curvilinear.h      | 171 +++++++++++++------------
 inc/dg/geometry/refined_curvilinearX.h |   2 +
 inc/dg/geometry/transform.h            |   8 +-
 6 files changed, 119 insertions(+), 111 deletions(-)

diff --git a/inc/dg/backend/gridX.h b/inc/dg/backend/gridX.h
index da927d121..0b8e5e9d4 100644
--- a/inc/dg/backend/gridX.h
+++ b/inc/dg/backend/gridX.h
@@ -265,13 +265,13 @@ struct aTopologyX2d
      */
     double hy() const {return ly()/(double)Ny_;}
     /**
-     * @brief Factor
+     * @brief partition factor in x
      *
      * @return 
      */
     double fx() const {return fx_;}
     /**
-     * @brief Factor
+     * @brief partition factor in y
      *
      * @return 
      */
@@ -442,8 +442,8 @@ struct aTopologyX2d
      * @param x1 right boundary in x 
      * @param y0 lower boundary in y
      * @param y1 upper boundary in y 
-     * @param fx factor for x-direction (fx*Nx must be a natural number)
-     * @param fy factor for y-direction (fy*Ny must be a natural number)
+     * @param fx factor for the partition in x-direction (fx*Nx will be rounded)
+     * @param fy factor for the partition in y-direction (fy*Ny will be rounded)
      * @param n  # of polynomial coefficients per dimension
      * @param Nx # of points in x 
      * @param Ny # of points in y
@@ -595,13 +595,13 @@ struct aTopologyX3d
      */
     double hz() const {return lz()/(double)Nz_;}
     /**
-     * @brief Factor
+     * @brief partition factor in x
      *
      * @return 
      */
     double fx() const {return fx_;}
     /**
-     * @brief Factor
+     * @brief partition factor in y
      *
      * @return 
      */
@@ -753,8 +753,8 @@ struct aTopologyX3d
      * @param y1 upper boundary in y 
      * @param z0 lower boundary in z
      * @param z1 upper boundary in z 
-     * @param fx factor for x-direction
-     * @param fy factor for y-direction
+     * @param fx factor for the partition in x-direction
+     * @param fy factor for the partition in y-direction
      * @param n  # of polynomial coefficients per (x-,y-) dimension
      * @param Nx # of points in x 
      * @param Ny # of points in y
diff --git a/inc/dg/backend/mpi_evaluation.h b/inc/dg/backend/mpi_evaluation.h
index c10060a62..49e35beb9 100644
--- a/inc/dg/backend/mpi_evaluation.h
+++ b/inc/dg/backend/mpi_evaluation.h
@@ -81,7 +81,7 @@ MPI_Vector<thrust::host_vector<double> > evaluate( double(f)(double, double, dou
  */
 MPI_Vector<thrust::host_vector<double> > global2local( const thrust::host_vector<double>& global, const aMPITopology3d& g)
 {
-    assert( in.size() == g.global().size());
+    assert( global.size() == g.global().size());
     thrust::host_vector<double> temp(g.size());
     int dims[3], periods[3], coords[3];
     MPI_Cart_get( g.communicator(), 3, dims, periods, coords);
@@ -107,7 +107,7 @@ MPI_Vector<thrust::host_vector<double> > global2local( const thrust::host_vector
  */
 MPI_Vector<thrust::host_vector<double> > global2local( const thrust::host_vector<double>& global, const aMPITopology2d& g)
 {
-    assert( in.size() == g.global().size());
+    assert( global.size() == g.global().size());
     thrust::host_vector<double> temp(g.size());
     int dims[2], periods[2], coords[2];
     MPI_Cart_get( g.communicator(), 2, dims, periods, coords);
@@ -116,7 +116,7 @@ MPI_Vector<thrust::host_vector<double> > global2local( const thrust::host_vector
             //for( unsigned px=0; px<dims[0]; px++)
                 for( unsigned j=0; j<g.n()*g.Nx(); j++)
                 {
-                    unsigned idx1 = (i*g.n()*g.Nx() + j;
+                    unsigned idx1 = i*g.n()*g.Nx() + j;
                     unsigned idx2 = ((coords[1]*g.n()*g.Ny()+i)*dims[0] + coords[0])*g.n()*g.Nx() + j;
                     temp[idx1] = global[idx2];
                 }
diff --git a/inc/dg/geometry/mpi_base.h b/inc/dg/geometry/mpi_base.h
index 3d1716997..2d0269ea9 100644
--- a/inc/dg/geometry/mpi_base.h
+++ b/inc/dg/geometry/mpi_base.h
@@ -21,7 +21,7 @@ struct aMPIGeometry2d : public aMPITopology2d
     SparseTensor<host_vector > metric()const {
         return do_compute_metric();
     }
-    std::vector<host_vec > map()const{
+    std::vector<host_vector > map()const{
         return do_compute_map();
     }
     ///Geometries are cloneable
@@ -29,6 +29,9 @@ struct aMPIGeometry2d : public aMPITopology2d
     ///allow deletion through base class pointer
     virtual ~aMPIGeometry2d(){}
     protected:
+    aMPIGeometry2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx, bc bcy, MPI_Comm comm):
+        aMPITopology2d( x0, x1, y0, y1, n, Nx, Ny, bcx, bcy, comm)
+    { }
     aMPIGeometry2d( const aMPIGeometry2d& src):aMPITopology2d(src){}
     aMPIGeometry2d& operator=( const aMPIGeometry2d& src){
         aMPITopology2d::operator=(src);
@@ -36,10 +39,10 @@ struct aMPIGeometry2d : public aMPITopology2d
     }
     private:
     virtual SparseTensor<host_vector > do_compute_metric()const {
-        return SharedContainer<host_vector >();
+        return SparseTensor<host_vector >();
     }
     virtual SparseTensor<host_vector > do_compute_jacobian()const {
-        return SharedContainer<host_vector >();
+        return SparseTensor<host_vector >();
     }
     virtual std::vector<host_vector > do_compute_map()const{
         std::vector<host_vector> map(2);
@@ -69,6 +72,8 @@ struct aMPIGeometry3d : public aMPITopology3d
     ///allow deletion through base class pointer
     virtual ~aMPIGeometry3d(){}
     protected:
+    aMPIGeometry3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz, MPI_Comm comm):
+        aMPITopology3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, bcx, bcy, bcz, comm){}
     aMPIGeometry3d( const aMPIGeometry3d& src):aMPITopology3d(src){}
     aMPIGeometry3d& operator=( const aMPIGeometry3d& src){
         aMPITopology3d::operator=(src);
@@ -76,10 +81,10 @@ struct aMPIGeometry3d : public aMPITopology3d
     }
     private:
     virtual SparseTensor<host_vector > do_compute_metric()const {
-        return SharedContainer<host_vector >();
+        return SparseTensor<host_vector >();
     }
     virtual SparseTensor<host_vector > do_compute_jacobian()const {
-        return SharedContainer<host_vector >();
+        return SparseTensor<host_vector >();
     }
     virtual std::vector<host_vector > do_compute_map()const{
         std::vector<host_vector> map(3);
@@ -113,7 +118,7 @@ struct CartesianMPIGrid2d : public aMPIGeometry2d
      * @note the paramateres given in the constructor are global parameters 
      */
     CartesianMPIGrid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx, bc bcy, MPI_Comm comm):dg::aMPIGeometry2d( x0, x1, y0, y1, n, Nx, Ny,bcx, bcy, comm){}
-    CartesianMPIGrid2d( const dg::MPIGrid2d& g): aMPIGeometry2d( g.x0(),g.x1(),g.y0(),g.y1(),g.n(),g.Nx(),g.Ny(),g.bcx(),g.bcy(),g.comm()){}
+    CartesianMPIGrid2d( const dg::MPIGrid2d& g): aMPIGeometry2d( g.x0(),g.x1(),g.y0(),g.y1(),g.n(),g.Nx(),g.Ny(),g.bcx(),g.bcy(),g.communicator()){}
     virtual CartesianMPIGrid2d* clone()const{return new CartesianMPIGrid2d(*this);}
     private:
     virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny){
@@ -132,7 +137,7 @@ struct CartesianMPIGrid3d : public aMPIGeometry3d
      * @param comm a three-dimensional Cartesian communicator
      * @note the paramateres given in the constructor are global parameters 
      */
-    CartesianMPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, MPI_Comm comm): aMPIGeometry3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, dg::PER,dg::PER,dg::PER comm){}
+    CartesianMPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, MPI_Comm comm): aMPIGeometry3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, dg::PER,dg::PER,dg::PER, comm){}
 
     /**
      * @copydoc MPIGrid3d::MPIGrid3d()
@@ -141,7 +146,7 @@ struct CartesianMPIGrid3d : public aMPIGeometry3d
      */
     CartesianMPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz, MPI_Comm comm):aMPIGeometry3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, bcx, bcy, bcz, comm){}
 
-    CartesianMPIGrid3d( const dg::MPIGrid3d& g): aMPIGeometry3d( g.x0(),g.x1(),g.y0(),g.y1(),g.z0(),g.z1(),g.n(),g.Nx(),g.Ny(),g.Nz(),g.bcx(),g.bcy(),g.bcz(),g.comm()){}
+    CartesianMPIGrid3d( const dg::MPIGrid3d& g): aMPIGeometry3d( g.x0(),g.x1(),g.y0(),g.y1(),g.z0(),g.z1(),g.n(),g.Nx(),g.Ny(),g.Nz(),g.bcx(),g.bcy(),g.bcz(),g.communicator()){}
     virtual CartesianMPIGrid3d* clone()const{return new CartesianMPIGrid3d(*this);}
     private:
     virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz){
@@ -154,15 +159,15 @@ struct CartesianMPIGrid3d : public aMPIGeometry3d
  */
 struct CylindricalMPIGrid3d: public aMPIGeometry3d
 {
-    CylindricalMPIGrid3d( double R0, double R1, double Z0, double Z1, double phi0, double phi1, unsigned n, unsigned NR, unsigned NZ, unsigned Nphi, bc bcR, bc bcZ, bc bcphi, MPI_Comm comm): dg::aGeometry3d(R0,R1,Z0,Z1,phi0,phi1,n,NR,NZ,Nphi,bcR,bcZ,bcphi,comm){}
+    CylindricalMPIGrid3d( double R0, double R1, double Z0, double Z1, double phi0, double phi1, unsigned n, unsigned NR, unsigned NZ, unsigned Nphi, bc bcR, bc bcZ, bc bcphi, MPI_Comm comm): dg::aMPIGeometry3d(R0,R1,Z0,Z1,phi0,phi1,n,NR,NZ,Nphi,bcR,bcZ,bcphi,comm){}
     ///take PER for bcphi
-    CylindricalMPIGrid3d( double R0, double R1, double Z0, double Z1, double phi0, double phi1, unsigned n, unsigned NR, unsigned NZ, unsigned Nphi, bc bcR, bc bcZ, MPI_Comm comm): dg::aGeometry3d(R0,R1,Z0,Z1,phi0,phi1,n,NR,NZ,Nphi,bcR,bcZ,dg::PER,comm){}
+    CylindricalMPIGrid3d( double R0, double R1, double Z0, double Z1, double phi0, double phi1, unsigned n, unsigned NR, unsigned NZ, unsigned Nphi, bc bcR, bc bcZ, MPI_Comm comm): dg::aMPIGeometry3d(R0,R1,Z0,Z1,phi0,phi1,n,NR,NZ,Nphi,bcR,bcZ,dg::PER,comm){}
 
     virtual CylindricalMPIGrid3d* clone()const{return new CylindricalMPIGrid3d(*this);}
     private:
     virtual SparseTensor<host_vector > do_compute_metric()const{
         SparseTensor<host_vector> metric(1);
-        host_vector R = dg::evaluate(dg::coo1, *this);
+        host_vector R = dg::evaluate(dg::cooX3d, *this);
         for( unsigned i = 0; i<size(); i++)
             R.data()[i] = 1./R.data()[i]/R.data()[i];
         metric.idx(2,2)=0;
diff --git a/inc/dg/geometry/mpi_curvilinear.h b/inc/dg/geometry/mpi_curvilinear.h
index 71ba098df..f00469c01 100644
--- a/inc/dg/geometry/mpi_curvilinear.h
+++ b/inc/dg/geometry/mpi_curvilinear.h
@@ -14,11 +14,71 @@ namespace dg
 {
 
 ///@cond
-struct CurvilinearMPIGrid2d; 
+struct CurvilinearProductMPIGrid3d; 
 ///@endcond
 //
 ///@addtogroup geometry
 ///@{
+/**
+ * @brief A two-dimensional MPI grid based on curvilinear coordinates
+ */
+struct CurvilinearMPIGrid2d : public dg::aMPIGeometry2d
+{
+    CurvilinearMPIGrid2d( const aGenerator2d& generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx, dg::bc bcy, MPI_Comm comm2d): 
+        dg::aMPIGeometry2d( 0, generator.width(), 0., generator.height(), n, Nx, Ny, bcx, bcy, comm2d), handle_(generator)
+    {
+        //generate global 2d grid and then reduce to local 
+        dg::CurvilinearGrid2d g(generator, n, Nx, Ny);
+        divide_and_conquer(g);
+    }
+    explicit CurvilinearMPIGrid2d( const CurvilinearProductMPIGrid3d& g);
+
+    const aGenerator2d& generator() const{return handle_.get();}
+    virtual CurvilinearMPIGrid2d* clone()const{return new CurvilinearMPIGrid2d(*this);}
+    private:
+    virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny)
+    {
+        dg::aMPITopology2d::do_set(new_n, new_Nx, new_Ny);
+        dg::CurvilinearGrid2d g( handle_.get(), new_n, new_Nx, new_Ny);
+        divide_and_conquer(g);//distribute to processes
+    }
+    MPI_Comm get_reduced_comm( MPI_Comm src)
+    {
+        MPI_Comm planeComm;
+        int remain_dims[] = {true,true,false}; //true true false
+        MPI_Cart_sub( src, remain_dims, &planeComm);
+        return planeComm;
+    }
+    void divide_and_conquer(const dg::CurvilinearGrid2d& g_)
+    {
+        dg::SparseTensor<thrust::host_vector<double> > jacobian=g_.jacobian(); 
+        dg::SparseTensor<thrust::host_vector<double> > metric=g_.metric(); 
+        std::vector<thrust::host_vector<double> > map = g_.map();
+        for( unsigned i=0; i<3; i++)
+            for( unsigned j=0; j<3; j++)
+            {
+                metric_.idx(i,j) = metric.idx(i,j);
+                jac_.idx(i,j) = jacobian.idx(i,j);
+            }
+        for( unsigned i=0; i<jacobian.values().size(); i++)
+            jac_.value(i) = global2local( jacobian.value(i), *this);
+        for( unsigned i=0; i<metric.values().size(); i++)
+            metric_.value(i) = global2local( metric.value(i), *this);
+        for( unsigned i=0; i<map.size(); i++)
+            map_[i] = global2local( map[i], *this);
+    }
+
+    virtual SparseTensor<host_vector> do_compute_jacobian( ) const {
+        return jac_;
+    }
+    virtual SparseTensor<host_vector> do_compute_metric( ) const {
+        return metric_;
+    }
+    virtual std::vector<host_vector > do_compute_map()const{return map_;}
+    dg::SparseTensor<host_vector > jac_, metric_;
+    std::vector<host_vector > map_;
+    dg::Handle<aGenerator2d> handle_;
+};
 
 /**
  * This is s 2x1 product space MPI grid
@@ -26,9 +86,8 @@ struct CurvilinearMPIGrid2d;
 struct CurvilinearProductMPIGrid3d : public dg::aMPIGeometry3d
 {
     typedef dg::CurvilinearMPIGrid2d perpendicular_grid; //!< the two-dimensional grid
-    typedef typename MPIContainer::container_type LocalContainer; //!< the local container type
-    CurvilinearMPIGrid3d( const geo::aGenerator& generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz, MPI_Comm comm): 
-        dg::aMPITopology3d( 0, generator.width(), 0., generator.height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, bcy, bcz, comm),
+    CurvilinearProductMPIGrid3d( const aGenerator2d& generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz, MPI_Comm comm): 
+        dg::aMPIGeometry3d( 0, generator.width(), 0., generator.height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, bcy, bcz, comm),
         handle_( generator)
     {
         map_.resize(3);
@@ -53,15 +112,15 @@ struct CurvilinearProductMPIGrid3d : public dg::aMPIGeometry3d
         dg::aMPITopology3d::do_set(new_n, new_Nx, new_Ny, new_Nz);
         if( !( new_n == n() && new_Nx == Nx() && new_Ny == Ny() ) )
         {
-            CurvilinearMPIGrid2d g(handle.get(),new_n,new_Nx,new_Ny, this->bcx(), this->bcy(), get_reduced_comm(communicator()));
+            CurvilinearMPIGrid2d g(handle_.get(),new_n,new_Nx,new_Ny, this->bcx(), this->bcy(), get_reduced_comm(communicator()));
             constructPerp( g);
         }
         constructParallel(this->Nz());
     }
     void constructPerp( CurvilinearMPIGrid2d& g2d)
     {
-        jac_=g.jacobian();
-        map_=g.map();
+        jac_=g2d.jacobian();
+        map_=g2d.map();
     }
     void constructParallel( unsigned localNz )
     {
@@ -72,7 +131,7 @@ struct CurvilinearProductMPIGrid3d : public dg::aMPIGeometry3d
         for( unsigned r=0; r<4;r++)
         {
             jac_.value(r).data().resize(size);
-            jac_.communicator() = communicator();
+            jac_.value(r).communicator() = communicator();
         }
         map_[0].data().resize(size); 
         map_[0].communicator() = communicator();
@@ -100,7 +159,7 @@ struct CurvilinearProductMPIGrid3d : public dg::aMPIGeometry3d
             tempyy[i] = (jac_.value(1,0).data()[i]*jac_.value(1,0).data()[i]+jac_.value(1,1).data()[i]*jac_.value(1,1).data()[i]);
             temppp[i] = 1./map_[2][i]/map_[2][i]; //1/R^2
         }
-        SparseTensor<thrust::host_vector<double> > metric;
+        SparseTensor<host_vector > metric;
         metric.idx(0,0) = 0; metric.value(0) = host_vector(tempxx, communicator());
         metric.idx(1,1) = 1; metric.value(1) = host_vector(tempyy, communicator());
         metric.idx(2,2) = 2; metric.value(2) = host_vector(temppp, communicator());
@@ -114,85 +173,27 @@ struct CurvilinearProductMPIGrid3d : public dg::aMPIGeometry3d
     virtual std::vector<host_vector > do_compute_map()const{return map_;}
     dg::SparseTensor<host_vector > jac_;
     std::vector<host_vector > map_;
-    Handle<dg::geo::aGenerator> handle_;
+    Handle<dg::aGenerator2d> handle_;
 };
-
-/**
- * @brief A two-dimensional MPI grid based on curvilinear coordinates
- */
-struct CurvilinearMPIGrid2d : public dg::aMPIGeometry2d
+///@cond
+CurvilinearMPIGrid2d::CurvilinearMPIGrid2d( const CurvilinearProductMPIGrid3d& g):
+    dg::aMPIGeometry2d( g.global().x0(), g.global().x1(), g.global().y0(), g.global().y1(), g.global().n(), g.global().Nx(), g.global().Ny(), g.global().bcx(), g.global().bcy(), get_reduced_comm( g.communicator() )),
+    handle_(g.generator())
 {
-    typedef typename MPIContainer::container_type LocalContainer; //!< the local container type
-    CurvilinearMPIGrid2d( const aGenerator2d& generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx, dg::bc bcy, MPI_Comm comm2d): 
-        dg::aMPIGeometry2d( 0, generator.width(), 0., generator.height(), n, Nx, Ny, bcx, bcy, comm2d), handle_(generator)
-    {
-        //generate global 2d grid and then reduce to local 
-        dg::CurvilinearGrid2d<thrust::host_vector<double> > g(generator, n, Nx, Ny);
-        divide_and_conquer(g);
-    }
-    explicit CurvilinearMPIGrid2d( const CurvilinearMPIGrid3d<LocalContainer>& g):
-        dg::aMPIGeometry2d( g.global().x0(), g.global().x1(), g.global().y0(), g.global().y1(), g.global().n(), g.global().Nx(), g.global().Ny(), g.global().bcx(), g.global().bcy(), get_reduced_comm( g.communicator() )),
-        handle_(g.generator())
-    {
-        map_=g.map();
-        jac_=g.jacobian();
-        metric_=g.metric();
-        //now resize to 2d
-        unsigned s = this->size();
-        for( unsigned i=0; i<jac_.values().size(); i++)
-            jac_.value(i).resize(s);
-        for( unsigned i=0; i<metric_.values().size(); i++)
-            metric_.value(i).resize(s);
-        for( unsigned i=0; i<map_.size(); i++)
-            map_[i].resize(s);
-    }
-
-    const aGenerator2d& generator() const{return g.generator();}
-    virtual CurvilinearMPIGrid2d* clone()const{return new CurvilinearMPIGrid2d(*this);}
-    private:
-    virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny)
-    {
-        dg::aMPITopology2d::do_set(new_n, new_Nx, new_Ny);
-        dg::CurvilinearGrid2d<thrust::host_vector<double> > g( handle_.get(), new_n, new_Nx, new_Ny);
-        divide_and_conquer(g);//distribute to processes
-    }
-    MPI_Comm get_reduced_comm( MPI_Comm src)
-    {
-        MPI_Comm planeComm;
-        int remain_dims[] = {true,true,false}; //true true false
-        MPI_Cart_sub( src, remain_dims, &planeComm);
-        return planeComm;
-    }
-    void divide_and_conquer(const dg::CurvilinearGrid2d<thrust::host_vector<double> >& g_)
-    {
-        dg::SparseTensor<host_vector > jacobian=g_.jacobian(); 
-        dg::SparseTensor<host_vector > metric=g_.metric(); 
-        std::vector<host_vector > map = g_.map();
-        for( unsigned i=0; i<3; i++)
-            for( unsigned j=0; j<3; j++)
-            {
-                metric_(i,j) = metric(i,j)
-                jac_(i,j) = jacobian(i,j)
-            }
-        for( unsigned i=0; i<jacobian.values().size(); i++)
-            jac_.value(i) = global2local( jacobian.value(i), *this);
-        for( unsigned i=0; i<metric.values().size(); i++)
-            metric_.value(i) = global2local( metric.value(i), *this);
-        for( unsigned i=0; i<map.size(); i++)
-            map_[i] = global2local( map[i]);
-    }
+    map_=g.map();
+    jac_=g.jacobian();
+    metric_=g.metric();
+    //now resize to 2d
+    unsigned s = this->size();
+    for( unsigned i=0; i<jac_.values().size(); i++)
+        jac_.value(i).data().resize(s);
+    for( unsigned i=0; i<metric_.values().size(); i++)
+        metric_.value(i).data().resize(s);
+    for( unsigned i=0; i<map_.size(); i++)
+        map_[i].data().resize(s);
+}
+///@endcond
 
-    virtual SparseTensor<host_vector> do_compute_jacobian( ) const {
-        return jac_;
-    }
-    virtual SparseTensor<host_vector> do_compute_metric( ) const {
-        return metric_;
-    }
-    virtual std::vector<host_vector > do_compute_map()const{return map_;}
-    dg::SparseTensor<host_vector > jac_, metric_;
-    std::vector<host_vector > map_;
-    dg::Handle<dg::geo::aGenerator2d> handle_;
-};
 ///@}
 }//namespace dg
 
diff --git a/inc/dg/geometry/refined_curvilinearX.h b/inc/dg/geometry/refined_curvilinearX.h
index 35127e268..6eef96e36 100644
--- a/inc/dg/geometry/refined_curvilinearX.h
+++ b/inc/dg/geometry/refined_curvilinearX.h
@@ -18,6 +18,7 @@ struct CurvilinearProductRefinedGridX3d : public dg::aGeometryX3d
     /*!@brief Constructor
     
      * the coordinates of the computational space are called x,y,z
+     * @param ref a X-point refinement
      * @param generator must generate a grid
      * @param n number of %Gaussian nodes in x and y
      * @param fx
@@ -121,6 +122,7 @@ struct CurvilinearRefinedGridX2d : public dg::aGeometryX2d
     /*!@brief Constructor
     
      * @param generator must generate an orthogonal grid (class takes ownership of the pointer)
+     * @param ref a X-point refinement
      * @param fx
      * @param fy
      * @param n number of polynomial coefficients
diff --git a/inc/dg/geometry/transform.h b/inc/dg/geometry/transform.h
index e59f9239a..014fc7ef3 100644
--- a/inc/dg/geometry/transform.h
+++ b/inc/dg/geometry/transform.h
@@ -75,11 +75,11 @@ struct MemoryTraits< MPITag>
 template< class Functor>
 MPI_Vector<thrust::host_vector<double> > pullback( Functor f, const aMPIGeometry2d& g)
 {
-    std::vector<MPI_Vector<thrust::host_vector<double> > map = g.map();
+    std::vector<MPI_Vector<thrust::host_vector<double> > > map = g.map();
     thrust::host_vector<double> vec( g.size());
     for( unsigned i=0; i<g.size(); i++)
         vec[i] = f( map[0].data()[i], map[1].data()[i]);
-    return vec;
+    return MPI_Vector<thrust::host_vector<double> >( vec, g.communicator());
 }
 
 ///@copydoc pullback(Functor,const aGeometry2d&)
@@ -87,11 +87,11 @@ MPI_Vector<thrust::host_vector<double> > pullback( Functor f, const aMPIGeometry
 template< class Functor>
 MPI_Vector<thrust::host_vector<double> > pullback( Functor f, const aMPIGeometry3d& g)
 {
-    std::vector<MPI_Vector<thrust::host_vector<double> > map = g.map();
+    std::vector<MPI_Vector<thrust::host_vector<double> > > map = g.map();
     thrust::host_vector<double> vec( g.size());
     for( unsigned i=0; i<g.size(); i++)
         vec[i] = f( map[0].data()[i], map[1].data()[i], map[2].data()[i]);
-    return vec;
+    return MPI_Vector<thrust::host_vector<double> >( vec, g.communicator());
 }
 
 #endif //MPI_VERSION
-- 
GitLab


From 6946382964520d0b7fd7226a0476c0dbcb892d06 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Mon, 14 Aug 2017 22:24:13 +0200
Subject: [PATCH 158/453] elliptic_b compiles but something is wrong with
 arakawa_mpib

---
 inc/dg/elliptic.h    | 216 ++++++++++++++++++++++---------------------
 inc/dg/elliptic_b.cu |   8 +-
 2 files changed, 117 insertions(+), 107 deletions(-)

diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index a63c271c5..17958edb4 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -1,7 +1,7 @@
 #pragma once
 
 #include "blas.h"
-#include "geometry.h"
+#include "geometry/geometry.h"
 #include "enums.h"
 #include "backend/evaluation.cuh"
 #include "backend/derivatives.h"
@@ -39,9 +39,7 @@ namespace dg
  Also note that a forward discretization has more diffusion than a centered discretization.
 
  * @tparam Geometry The geometry sets the metric of the grid
- * @tparam Matrix The Matrix class to use
- * @tparam Vector The Vector class to use
- * @tparam Vector The Vector class to use
+ * @copydoc hide_matrix_container
  * This class has the SelfMadeMatrixTag so it can be used in blas2::symv functions 
  * and thus in a conjugate gradient solver. 
  * @note The constructors initialize \f$ \chi=1\f$ so that a negative laplacian operator
@@ -51,16 +49,16 @@ namespace dg
  * @note the jump term \f$ \alpha J\f$  adds artificial numerical diffusion
  * @attention Pay attention to the negative sign 
  */
-template <class Geometry, class Matrix, class Vector>
+template <class Geometry, class Matrix, class container>
 class Elliptic
 {
     public:
     /**
      * @brief Construct from Grid
      *
-     * @tparam Geometry The Grid class. A call to dg::evaluate( one, g) must return an instance of the Vector class, 
+     * @tparam Geometry The Grid class. A call to dg::evaluate( one, g) must return an instance of the container class, 
      * a call to dg::create::weights(g) and dg::create::inv_weights(g)
-     * must return instances of the Vector class and 
+     * must return instances of the container class and 
      * calls to dg::create::dx( g, no, backward) and jump2d( g, bcx, bcy, no) are made.
      * @param g The Grid, boundary conditions are taken from here
      * @param no Not normed for elliptic equations, normed else
@@ -70,7 +68,7 @@ class Elliptic
      * @note chi is assumed 1 per default
      */
     Elliptic( Geometry g, norm no = not_normed, direction dir = forward, double jfactor=1.): 
-        no_(no), g_(g), jfactor_(jfactor)
+        no_(no), jfactor_(jfactor)
     { 
         construct( g, g.bcx(), g.bcy(), dir);
     }
@@ -78,9 +76,9 @@ class Elliptic
     /**
      * @brief Construct from grid and boundary conditions
      *
-     * @tparam Geometry The Grid class. A call to dg::evaluate( one, g) must return an instance of the Vector class, 
+     * @tparam Geometry The Grid class. A call to dg::evaluate( one, g) must return an instance of the container class, 
      * a call to dg::create::weights(g) and dg::create::inv_weights(g)
-     * must return instances of the Vector class and 
+     * must return instances of the container class and 
      * calls to dg::create::dx( g, no, backward) and jump2d( g, bcx, bcy, no) are made.
      * @param g The Grid
      * @param bcx boundary condition in x
@@ -91,7 +89,7 @@ class Elliptic
      * @param jfactor scale jump terms (1 is a good value but in some cases 0.1 or 0.01 might be better)
      */
     Elliptic( Geometry g, bc bcx, bc bcy, norm no = not_normed, direction dir = forward, double jfactor=1.): 
-        no_(no), g_(g), jfactor_(jfactor)
+        no_(no), jfactor_(jfactor)
     { 
         construct( g, bcx, bcy, dir);
     }
@@ -99,14 +97,20 @@ class Elliptic
     /**
      * @brief Change Chi 
      *
-     * @param chi The new chi
+     * @param chi The new chi (all elements must be >0)
      * @note There is no get_chi because chi is multiplied with volume elements
      */
-    void set_chi( const Vector& chi)
+    void set_chi( const container& chi)
     {
-        xchi = chi;
-        dg::geo::multiplyVolume( xchi, g_); 
-        dg::geo::dividePerpVolume( xchi, g_); 
+        if( !chi_old_.isSet()) 
+        {
+            dg::tensor::scal( chi_, chi);
+            chi_old_.value() = chi;
+            return;
+        }
+        dg::blas1::pointwiseDivide(chi, chi_old_.value(), tempx);
+        dg::tensor::scal( chi_, tempx);
+        chi_old_.value()=chi;
     }
 
     /**
@@ -115,7 +119,7 @@ class Elliptic
      * i.e. the volume form 
      * @return weights (volume form including dG weights)
      */
-    const Vector& weights()const {return weights_;}
+    const container& weights()const {return weights_;}
     /**
      * @brief Returns the default preconditioner to use in conjugate gradient
      *
@@ -124,7 +128,7 @@ class Elliptic
      * @note a better preconditioner might be the inverse of \f$\chi\f$ especially 
      * when \f$ \chi\f$ exhibits large amplitudes or variations
      */
-    const Vector& precond()const {return precond_;}
+    const container& precond()const {return precond_;}
     /**
      * @brief Set the currently used jfactor
      *
@@ -144,24 +148,21 @@ class Elliptic
      * @param x left-hand-side
      * @param y result
      */
-    void symv( const Vector& x, Vector& y) 
+    void symv( const container& x, container& y) 
     {
         //compute gradient
-        dg::blas2::gemv( rightx, x, tempx); //R_x*f 
-        dg::blas2::gemv( righty, x, tempy); //R_y*f
+        dg::blas2::gemv( rightx, x, gradx); //R_x*f 
+        dg::blas2::gemv( righty, x, y); //R_y*f
 
-        dg::geo::volRaisePerpIndex( tempx, tempy, gradx, y, g_);
-
-        //multiply with chi 
-        dg::blas1::pointwiseDot( xchi, gradx, gradx); //Chi*R_x*x 
-        dg::blas1::pointwiseDot( xchi, y, y); //Chi*R_x*x 
+        //multiply with tensor
+        dg::tensor::multiply2d_inplace(chi_, gradx, y, tempx);
 
         //now take divergence
         dg::blas2::gemv( leftx, gradx, tempx);  
         dg::blas2::gemv( lefty, y, tempy);  
         dg::blas1::axpby( -1., tempx, -1., tempy, y); //-D_xx - D_yy 
         if( no_ == normed)
-            dg::geo::divideVolume( y, g_);
+            dg::tensor::pointwiseDivide( y, vol_, y);
 
         //add jump terms
         dg::blas2::symv( jumpX, x, tempx);
@@ -173,7 +174,7 @@ class Elliptic
 
     }
     private:
-    void construct( Geometry g, bc bcx, bc bcy, direction dir)
+    void construct( const Geometry& g, bc bcx, bc bcy, direction dir)
     {
         dg::blas2::transfer( dg::create::dx( g, inverse( bcx), inverse(dir)), leftx);
         dg::blas2::transfer( dg::create::dy( g, inverse( bcy), inverse(dir)), lefty);
@@ -181,15 +182,16 @@ class Elliptic
         dg::blas2::transfer( dg::create::dy( g, bcy, dir), righty);
         dg::blas2::transfer( dg::create::jumpX( g, bcx),   jumpX);
         dg::blas2::transfer( dg::create::jumpY( g, bcy),   jumpY);
+
         dg::blas1::transfer( dg::create::volume(g),        weights_);
         dg::blas1::transfer( dg::create::inv_weights(g),   precond_); //weights are better preconditioners than volume
-        dg::blas1::transfer( dg::evaluate( dg::one, g),    xchi);
-        tempx = tempy = gradx = xchi;
-        //this has to be done due to the perp vol multiplication in volRaisePerpIndex
-        dg::geo::multiplyVolume( xchi, g_); 
-        dg::geo::dividePerpVolume( xchi, g_);
-        dg::blas1::transfer( dg::create::volume(g),        weights_wo_vol);
-        dg::geo::divideVolume( weights_wo_vol, g_);
+        tempx = tempy = gradx = weights_;
+        chi_=g.metric();
+        vol_=dg::tensor::determinant(chi_);
+        dg::tensor::invert(vol_);
+        dg::tensor::sqrt(vol_); //now we have volume element
+        dg::tensor::scal( chi_, vol_);
+        dg::blas1::transfer( dg::create::weights(g), weights_wo_vol);
     }
     bc inverse( bc bound)
     {
@@ -206,10 +208,11 @@ class Elliptic
         return centered;
     }
     Matrix leftx, lefty, rightx, righty, jumpX, jumpY;
-    Vector weights_, precond_, weights_wo_vol; 
-    Vector xchi, tempx, tempy, gradx;
+    container weights_, precond_, weights_wo_vol; 
+    container tempx, tempy, gradx;
     norm no_;
-    Geometry g_;
+    SparseTensor<container> chi_;
+    SparseElement<container> chi_old_, vol_;
     double jfactor_;
 };
 
@@ -230,23 +233,21 @@ class Elliptic
  *  \f] 
  * is discretized, with \f$ b^i\f$ being the contravariant components of \f$\mathbf b\f$ . 
  * @tparam Geometry The Geometry class to use
- * @tparam Matrix The Matrix class to use
- * @tparam Vector The Vector class to use
- * @tparam Vector The Vector class to use
+ * @copydoc hide_matrix_container
  * This class has the SelfMadeMatrixTag so it can be used in blas2::symv functions 
  * and thus in a conjugate gradient solver. 
  * @note The constructors initialize \f$ b^x = b^y = b^z=1\f$ 
  * @attention Pay attention to the negative sign 
  */
-template< class Geometry, class Matrix, class Vector> 
+template< class Geometry, class Matrix, class container> 
 struct GeneralElliptic
 {
     /**
      * @brief Construct from Grid
      *
-     * @tparam Geometry The Grid class. A call to dg::evaluate( one, g) must return an instance of the Vector class, 
+     * @tparam Geometry The Grid class. A call to dg::evaluate( one, g) must return an instance of the container class, 
      * a call to dg::create::weights(g) and dg::create::inv_weights(g)
-     * must return instances of the Vector class and 
+     * must return instances of the container class and 
      * calls to dg::create::dx( g, no, backward) and jump2d( g, bcx, bcy, no) are made.
      * @param g The Grid, boundary conditions are taken from here
      * @param no Not normed for elliptic equations, normed else
@@ -265,13 +266,17 @@ struct GeneralElliptic
         xchi( dg::evaluate( one, g) ), ychi( xchi), zchi( xchi), 
         xx(xchi), yy(xx), zz(xx), temp0( xx), temp1(temp0),
         no_(no), g_(g)
-    { }
+    { 
+        vol_=dg::tensor::determinant(g.metric());
+        dg::tensor::invert(vol_);
+        dg::tensor::sqrt(vol_); //now we have volume element
+    }
     /**
      * @brief Construct from Grid and bc 
      *
-     * @tparam Grid The Grid class. A call to dg::evaluate( one, g) must return an instance of the Vector class, 
+     * @tparam Grid The Grid class. A call to dg::evaluate( one, g) must return an instance of the container class, 
      * a call to dg::create::weights(g) and dg::create::inv_weights(g)
-     * must return instances of the Vector class and 
+     * must return instances of the container class and 
      * calls to dg::create::dx( g, no, backward) and jump2d( g, bcx, bcy, no) are made.
      * @param g The Grid
      * @param bcx boundary condition in x
@@ -293,13 +298,17 @@ struct GeneralElliptic
         xchi( dg::evaluate( one, g) ), ychi( xchi), zchi( xchi), 
         xx(xchi), yy(xx), zz(xx), temp0( xx), temp1(temp0),
         no_(no), g_(g)
-    { }
+    { 
+        vol_=dg::tensor::determinant(g.metric());
+        dg::tensor::invert(vol_);
+        dg::tensor::sqrt(vol_); //now we have volume element
+    }
     /**
      * @brief Set x-component of \f$ chi\f$
      *
      * @param chi new x-component
      */
-    void set_x( const Vector& chi)
+    void set_x( const container& chi)
     {
         xchi = chi;
     }
@@ -308,7 +317,7 @@ struct GeneralElliptic
      *
      * @param chi new y-component
      */
-    void set_y( const Vector& chi)
+    void set_y( const container& chi)
     {
         ychi = chi;
     }
@@ -317,7 +326,7 @@ struct GeneralElliptic
      *
      * @param chi new z-component
      */
-    void set_z( const Vector& chi)
+    void set_z( const container& chi)
     {
         zchi = chi;
     }
@@ -327,7 +336,7 @@ struct GeneralElliptic
      *
      * @param chi chi[0] is new x-component, chi[1] the new y-component, chi[2] z-component
      */
-    void set( const std::vector<Vector>& chi)
+    void set( const std::vector<container>& chi)
     {
         xchi = chi[0];
         ychi = chi[1];
@@ -340,14 +349,14 @@ struct GeneralElliptic
      * in this case the volume element
      * @return weights (the volume element including dG weights)
      */
-    const Vector& weights()const {return weights_;}
+    const container& weights()const {return weights_;}
     /**
      * @brief Returns the preconditioner to use in conjugate gradient
      *
      * In this case inverse weights (without volume element) are returned
      * @return inverse weights
      */
-    const Vector& precond()const {return precond_;}
+    const container& precond()const {return precond_;}
 
     /**
      * @brief Computes the polarisation term
@@ -355,7 +364,7 @@ struct GeneralElliptic
      * @param x left-hand-side
      * @param y result
      */
-    void symv( Vector& x, Vector& y) 
+    void symv( container& x, container& y) 
     {
         dg::blas2::gemv( rightx, x, temp0); //R_x*x 
         dg::blas1::pointwiseDot( xchi, temp0, xx); //Chi_x*R_x*x 
@@ -369,7 +378,7 @@ struct GeneralElliptic
         dg::blas1::axpby( 1., xx, 1., yy, temp0);
         dg::blas1::axpby( 1., zz, 1., temp0, temp0); //gradpar x 
 
-        dg::geo::multiplyVolume( temp0, g_);
+        dg::tensor::pointwiseDot( vol_, temp0, temp0);
 
         dg::blas1::pointwiseDot( xchi, temp0, temp1); 
         dg::blas2::gemv( leftx, temp1, xx); 
@@ -383,7 +392,7 @@ struct GeneralElliptic
         dg::blas1::axpby( -1., xx, -1., yy, y);
         dg::blas1::axpby( -1., zz, +1., y, y); 
         if( no_==normed) 
-            dg::geo::divideVolume( y, g_);
+            dg::tensor::pointwiseDivide( temp0, vol_, temp0);
         
         dg::blas2::symv( jumpX, x, temp0);
         dg::blas1::axpby( +1., temp0, 1., y, y); 
@@ -391,7 +400,7 @@ struct GeneralElliptic
         dg::blas1::axpby( +1., temp0, 1., y, y); 
         if( no_==not_normed)//multiply weights w/o volume
         {
-            dg::geo::divideVolume( y, g_);
+            dg::tensor::pointwiseDivide( y, vol_, y);
             dg::blas2::symv( weights_, y, y);
         }
     }
@@ -411,9 +420,10 @@ struct GeneralElliptic
         return centered;
     }
     Matrix leftx, lefty, leftz, rightx, righty, rightz, jumpX, jumpY;
-    Vector weights_, precond_; //contain coeffs for chi multiplication
-    Vector xchi, ychi, zchi, xx, yy, zz, temp0, temp1;
+    container weights_, precond_; //contain coeffs for chi multiplication
+    container xchi, ychi, zchi, xx, yy, zz, temp0, temp1;
     norm no_;
+    SparseElement<container> vol_;
     Geometry g_;
 };
 
@@ -434,23 +444,21 @@ struct GeneralElliptic
  *  \f] 
  * is discretized, with \f$ b^i\f$ being the contravariant components of \f$\mathbf b\f$ . 
  * @tparam Geometry The Geometry class to use
- * @tparam Matrix The Matrix class to use
- * @tparam Vector The Vector class to use
- * @tparam Vector The Vector class to use
+ * @copydoc hide_matrix_container
  * This class has the SelfMadeMatrixTag so it can be used in blas2::symv functions 
  * and thus in a conjugate gradient solver. 
  * @note The constructors initialize \f$ \chi_x = \chi_y = \chi_z=1\f$ 
  * @attention Pay attention to the negative sign 
  */
-template<class Geometry, class Matrix, class Vector> 
+template<class Geometry, class Matrix, class container> 
 struct GeneralEllipticSym
 {
     /**
      * @brief Construct from Grid
      *
-     * @tparam Grid The Grid class. A call to dg::evaluate( one, g) must return an instance of the Vector class, 
+     * @tparam Grid The Grid class. A call to dg::evaluate( one, g) must return an instance of the container class, 
      * a call to dg::create::weights(g) and dg::create::inv_weights(g)
-     * must return instances of the Vector class and 
+     * must return instances of the container class and 
      * calls to dg::create::dx( g, no, backward) and jump2d( g, bcx, bcy, no) are made.
      * @param g The Grid, boundary conditions are taken from here
      * @param no Not normed for elliptic equations, normed else
@@ -465,9 +473,9 @@ struct GeneralEllipticSym
         /**
      * @brief Construct from Grid and bc
      *
-     * @tparam Grid The Grid class. A call to dg::evaluate( one, g) must return an instance of the Vector class, 
+     * @tparam Grid The Grid class. A call to dg::evaluate( one, g) must return an instance of the container class, 
      * a call to dg::create::weights(g) and dg::create::inv_weights(g)
-     * must return instances of the Vector class and 
+     * must return instances of the container class and 
      * calls to dg::create::dx( g, no, backward) and jump2d( g, bcx, bcy, no) are made.
      * @param g The Grid
      * @param bcx boundary condition in x
@@ -487,7 +495,7 @@ struct GeneralEllipticSym
      *
      * @param chi new x-component
      */
-    void set_x( const Vector& chi)
+    void set_x( const container& chi)
     {
         ellipticForward_.set_x( chi);
         ellipticBackward_.set_x( chi);
@@ -497,7 +505,7 @@ struct GeneralEllipticSym
      *
      * @param chi new y-component
      */
-    void set_y( const Vector& chi)
+    void set_y( const container& chi)
     {
         ellipticForward_.set_y( chi);
         ellipticBackward_.set_y( chi);
@@ -507,7 +515,7 @@ struct GeneralEllipticSym
      *
      * @param chi new z-component
      */
-    void set_z( const Vector& chi)
+    void set_z( const container& chi)
     {
         ellipticForward_.set_z( chi);
         ellipticBackward_.set_z( chi);
@@ -518,7 +526,7 @@ struct GeneralEllipticSym
      *
      * @param chi chi[0] is new x-component, chi[1] the new y-component, chi[2] z-component
      */
-    void set( const std::vector<Vector>& chi)
+    void set( const std::vector<container>& chi)
     {
         ellipticForward_.set( chi);
         ellipticBackward_.set( chi);
@@ -529,14 +537,14 @@ struct GeneralEllipticSym
      *
      * @return weights
      */
-    const Vector& weights()const {return weights_;}
+    const container& weights()const {return weights_;}
     /**
      * @brief Returns the preconditioner to use in conjugate gradient
      *
      * In this case inverse weights are the best choice
      * @return inverse weights
      */
-    const Vector& precond()const {return precond_;}
+    const container& precond()const {return precond_;}
 
     /**
      * @brief Computes the polarisation term
@@ -544,7 +552,7 @@ struct GeneralEllipticSym
      * @param x left-hand-side
      * @param y result
      */
-    void symv( Vector& x, Vector& y) 
+    void symv( container& x, container& y) 
     {
         ellipticForward_.symv( x,y);
         ellipticBackward_.symv( x,temp_);
@@ -557,9 +565,9 @@ struct GeneralEllipticSym
         if( dir == backward) return forward;
         return centered;
     }
-    dg::GeneralElliptic<Geometry, Matrix, Vector> ellipticForward_, ellipticBackward_;
-    Vector weights_, precond_; //contain coeffs for chi multiplication
-    Vector temp_;
+    dg::GeneralElliptic<Geometry, Matrix, container> ellipticForward_, ellipticBackward_;
+    container weights_, precond_; //contain coeffs for chi multiplication
+    container temp_;
 };
 
 /**
@@ -579,23 +587,21 @@ struct GeneralEllipticSym
  *  \end{align}
  *  \f] 
  * @tparam Geometry The Geometry class to use
- * @tparam Matrix The Matrix class to use
- * @tparam Vector The Vector class to use
- * @tparam Vector The Vector class to use
+ * @copydoc hide_matrix_container
  * This class has the SelfMadeMatrixTag so it can be used in blas2::symv functions 
  * and thus in a conjugate gradient solver. 
  * @note The constructors initialize \f$ \chi = I\f$ 
  * @attention Pay attention to the negative sign 
  */
-template< class Geometry, class Matrix, class Vector> 
+template< class Geometry, class Matrix, class container> 
 struct TensorElliptic
 {
     /**
      * @brief Construct from Grid
      *
-     * @tparam Geometry The Grid class. A call to dg::evaluate( one, g) must return an instance of the Vector class, 
+     * @tparam Geometry The Grid class. A call to dg::evaluate( one, g) must return an instance of the container class, 
      * a call to dg::create::weights(g) and dg::create::inv_weights(g)
-     * must return instances of the Vector class and 
+     * must return instances of the container class and 
      * calls to dg::create::dx( g, no, backward) and jump2d( g, bcx, bcy, no) are made.
      * @param g The Grid, boundary conditions are taken from here
      * @param no Not normed for elliptic equations, normed else
@@ -609,9 +615,9 @@ struct TensorElliptic
     /**
      * @brief Construct from Grid and bc 
      *
-     * @tparam Grid The Grid class. A call to dg::evaluate( one, g) must return an instance of the Vector class, 
+     * @tparam Grid The Grid class. A call to dg::evaluate( one, g) must return an instance of the container class, 
      * a call to dg::create::weights(g) and dg::create::inv_weights(g)
-     * must return instances of the Vector class and 
+     * must return instances of the container class and 
      * calls to dg::create::dx( g, no, backward) and jump2d( g, bcx, bcy, no) are made.
      * @param g The Grid
      * @param bcx boundary condition in x
@@ -633,15 +639,15 @@ struct TensorElliptic
      * @param chiYY The new yy component
      * @note Components need to be already transformed into the current coordinate system
      */
-    template<class Vector2>
-    void set( const Vector2& chiXX, const Vector2& chiXY, const Vector2& chiYY)
+    template<class container2>
+    void set( const container2& chiXX, const container2& chiXY, const container2& chiYY)
     {
         dg::blas1::transfer( chiXX, chixx_);
         dg::blas1::transfer( chiXY, chixy_);
         dg::blas1::transfer( chiYY, chiyy_);
-        dg::geo::multiplyVolume( chixx_, g_);
-        dg::geo::multiplyVolume( chixy_, g_);
-        dg::geo::multiplyVolume( chiyy_, g_);
+        dg::tensor::pointwiseDot( vol_, chixx_,chixx_);
+        dg::tensor::pointwiseDot( vol_, chixy_,chixy_);
+        dg::tensor::pointwiseDot( vol_, chiyy_,chiyy_);
     }
 
     /**
@@ -652,7 +658,7 @@ struct TensorElliptic
     void set( ChiRR chiRR, ChiRZ chiRZ, ChiZZ chiZZ)
     {
         typename GeometryTraits<Geometry>::host_vector chiXX, chiXY, chiYY;
-        dg::geo::pushForwardPerp( chiRR, chiRZ, chiZZ, chiXX, chiXY, chiYY, g_);
+        dg::pushForwardPerp( chiRR, chiRZ, chiZZ, chiXX, chiXY, chiYY, g_);
         set( chiXX, chiXY, chiYY);
     }
 
@@ -661,14 +667,14 @@ struct TensorElliptic
      *
      * @return weights
      */
-    const Vector& weights()const {return weights_;}
+    const container& weights()const {return weights_;}
     /**
      * @brief Returns the preconditioner to use in conjugate gradient
      *
      * In this case inverse weights are the best choice
      * @return inverse weights
      */
-    const Vector& precond()const {return precond_;}
+    const container& precond()const {return precond_;}
 
     /**
      * @brief Computes the polarisation term
@@ -676,7 +682,7 @@ struct TensorElliptic
      * @param x left-hand-side
      * @param y result
      */
-    void symv( Vector& x, Vector& y) 
+    void symv( container& x, container& y) 
     {
         //compute gradient
         dg::blas2::gemv( rightx, x, tempx_); //R_x*f 
@@ -693,7 +699,7 @@ struct TensorElliptic
         dg::blas2::gemv( lefty, y, tempy_);  
         dg::blas1::axpby( -1., tempx_, -1., tempy_, y); //-D_xx - D_yy 
         if( no_ == normed)
-            dg::geo::divideVolume( y, g_);
+            dg::tensor::pointwiseDivide( y, vol_,y);
 
         //add jump terms
         dg::blas2::symv( jumpX, x, tempx_);
@@ -718,11 +724,14 @@ struct TensorElliptic
         dg::blas1::transfer( dg::evaluate( dg::zero,g),    chixy_);
         dg::blas1::transfer( dg::evaluate( dg::one, g),    chiyy_);
         tempx_ = tempy_ = gradx_ = chixx_;
-        dg::geo::multiplyVolume( chixx_, g_); 
-        dg::geo::multiplyVolume( chixy_, g_); 
-        dg::geo::multiplyVolume( chiyy_, g_); 
-        dg::blas1::transfer( dg::create::volume(g), weights_wo_vol);
-        dg::geo::divideVolume( weights_wo_vol, g_);
+        dg::blas1::transfer( dg::create::weights(g), weights_wo_vol);
+
+        vol_=dg::tensor::determinant(g.metric());
+        dg::tensor::invert(vol_);
+        dg::tensor::sqrt(vol_); //now we have volume element
+        dg::tensor::pointwiseDot( vol_, chixx_, chixx_); 
+        dg::tensor::pointwiseDot( vol_, chixy_, chixy_); 
+        dg::tensor::pointwiseDot( vol_, chiyy_, chiyy_); 
     }
     bc inverse( bc bound)
     {
@@ -739,8 +748,9 @@ struct TensorElliptic
         return centered;
     }
     Matrix leftx, lefty, rightx, righty, jumpX, jumpY;
-    Vector weights_, weights_wo_vol, precond_; //contain coeffs for chi multiplication
-    Vector chixx_, chixy_, chiyy_, tempx_, tempy_, gradx_;
+    container weights_, weights_wo_vol, precond_; //contain coeffs for chi multiplication
+    container chixx_, chixy_, chiyy_, tempx_, tempy_, gradx_;
+    SparseTensor<container> vol_;
     norm no_;
     Geometry g_;
 };
diff --git a/inc/dg/elliptic_b.cu b/inc/dg/elliptic_b.cu
index 205ae5090..46abf0b7d 100644
--- a/inc/dg/elliptic_b.cu
+++ b/inc/dg/elliptic_b.cu
@@ -37,7 +37,7 @@ int main()
     double eps;
     std::cout << "Type epsilon! \n";
     std::cin >> eps;
-    dg::CylindricalGrid3d<dg::DVec> grid( R_0, R_0+lx, 0, ly, 0,lz, n, Nx, Ny,Nz, bcx, bcy, dg::PER);
+    dg::CylindricalGrid3d grid( R_0, R_0+lx, 0, ly, 0,lz, n, Nx, Ny,Nz, bcx, bcy, dg::PER);
     dg::DVec w3d = dg::create::volume( grid);
     dg::DVec v3d = dg::create::inv_volume( grid);
     dg::DVec x = dg::evaluate( initial, grid);
@@ -45,8 +45,8 @@ int main()
     std::cout << "TEST CYLINDRICAL LAPLACIAN\n";
     std::cout << "Create Laplacian\n";
     t.tic();
-    dg::Elliptic<dg::CylindricalGrid3d<dg::DVec>, dg::DMatrix, dg::DVec> laplace(grid, dg::not_normed, dg::centered);
-    dg::Elliptic<dg::CylindricalGrid3d<dg::fDVec>, dg::fDMatrix, dg::fDVec> flaplace(grid, dg::not_normed, dg::centered);
+    dg::Elliptic<dg::CylindricalGrid3d, dg::DMatrix, dg::DVec> laplace(grid, dg::not_normed, dg::centered);
+    dg::Elliptic<dg::CylindricalGrid3d, dg::fDMatrix, dg::fDVec> flaplace(grid, dg::not_normed, dg::centered);
     dg::DMatrix DX = dg::create::dx( grid);
     t.toc();
     std::cout<< "Creation took "<<t.diff()<<"s\n";
@@ -61,7 +61,7 @@ int main()
     dg::blas2::symv( w3d, b, b);
     dg::fDVec fx;
     dg::blas1::transfer(x,fx);
-    dg::Inverse<dg::Elliptic<dg::CylindricalGrid3d<dg::fDVec>, dg::fDMatrix, dg::fDVec>, dg::fDVec> inverse( flaplace, fx, 10, 1e-15, 0);
+    dg::Inverse<dg::Elliptic<dg::CylindricalGrid3d, dg::fDMatrix, dg::fDVec>, dg::fDVec> inverse( flaplace, fx, 10, 1e-15, 0);
     
     std::cout << "For a precision of "<< eps<<" ..."<<std::endl;
     t.tic();
-- 
GitLab


From 9c4ed7bf24f557e3e0f291ac629e767a2a9ba65b Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Mon, 14 Aug 2017 22:30:44 +0200
Subject: [PATCH 159/453] helmholtz_b compiles

---
 inc/dg/helmholtz.h | 84 +++++++++++++++++++++-------------------------
 1 file changed, 38 insertions(+), 46 deletions(-)

diff --git a/inc/dg/helmholtz.h b/inc/dg/helmholtz.h
index ddf8684e7..7d9a48df9 100644
--- a/inc/dg/helmholtz.h
+++ b/inc/dg/helmholtz.h
@@ -19,12 +19,11 @@ namespace dg{
  * Unnormed discretization of \f[ (\chi+\alpha\Delta) \f]
  * where \f$ \chi\f$ is a function and \f$\alpha\f$ a scalar.
  * Can be used by the Invert class
- * @tparam Geometry The geometry you want to use
- * @tparam Matrix The cusp-matrix class you want to use
- * @tparam Vector The Vector class you want to use
+ * @tparam container The container class you want to use
+ * @copydoc hide_matrix_container
  * @attention The Laplacian in this formula is positive as opposed to the negative sign in the Elliptic operator
  */
-template< class Geometry, class Matrix, class Vector> 
+template< class Geometry, class Matrix, class container> 
 struct Helmholtz
 {
     /**
@@ -38,8 +37,8 @@ struct Helmholtz
      */
     Helmholtz( Geometry g, double alpha = 1., direction dir = dg::forward, double jfactor=1.):
         laplaceM_(g, normed, dir, jfactor), 
-        temp_(dg::evaluate(dg::one, g)), chi_(temp_),
-        alpha_(alpha), isSet(false)
+        temp_(dg::evaluate(dg::one, g)),
+        alpha_(alpha)
     { 
     }
     /**
@@ -55,8 +54,8 @@ struct Helmholtz
      */
     Helmholtz( Geometry g, bc bcx, bc bcy, double alpha = 1., direction dir = dg::forward, double jfactor=1.):
         laplaceM_(g, bcx,bcy,normed, dir, jfactor), 
-        temp_(dg::evaluate(dg::one, g)), chi_(temp_),
-        alpha_(alpha), isSet(false)
+        temp_(dg::evaluate(dg::one, g)), 
+        alpha_(alpha)
     { 
     }
     /**
@@ -68,12 +67,9 @@ struct Helmholtz
      * @param y rhs contains solution
      * @note Takes care of sign in laplaceM and thus multiplies by -alpha
      */
-    void symv( Vector& x, Vector& y) 
+    void symv( container& x, container& y) 
     {
-        if( isSet)
-            blas1::pointwiseDot( chi_, x, temp_);
-        else
-            blas1::transfer( x, temp_);
+        tensor::pointwiseDot( chi_, x, temp_);
         if( alpha_ != 0)
             blas2::symv( laplaceM_, x, y);
 
@@ -86,13 +82,13 @@ struct Helmholtz
      *
      * @return weights
      */
-    const Vector& weights()const {return laplaceM_.weights();}
+    const container& weights()const {return laplaceM_.weights();}
     /**
-     * @brief Vector to use in conjugate gradient solvers
+     * @brief container to use in conjugate gradient solvers
      *
-     * @return Vector
+     * @return container
      */
-    const Vector& precond()const {return laplaceM_.precond();}
+    const container& precond()const {return laplaceM_.precond();}
     /**
      * @brief Change alpha
      *
@@ -108,24 +104,24 @@ struct Helmholtz
     /**
      * @brief Set Chi in the above formula
      *
-     * @param chi new Vector
+     * @param chi new container
      */
-    void set_chi( const Vector& chi) {chi_=chi; isSet =true;}
+    void set_chi( const container& chi) {chi_.value()=chi;}
     /**
      * @brief Sets chi back to one
      */
-    void reset_chi(){isSet = false;}
+    void reset_chi(){chi_.clear();}
     /**
      * @brief Access chi
      *
      * @return chi
      */
-    const Vector& chi() const{return chi_;}
+    const SparseElement<container>& chi() const{return chi_;}
   private:
-    Elliptic<Geometry, Matrix, Vector> laplaceM_;
-    Vector temp_, chi_;
+    Elliptic<Geometry, Matrix, container> laplaceM_;
+    container temp_;
+    SparseElement<container> chi_;
     double alpha_;
-    bool isSet;
 };
 
 /**
@@ -138,13 +134,12 @@ struct Helmholtz
  * where \f$ \chi\f$ is a function and \f$\alpha\f$ a scalar.
  * Can be used by the Invert class
  * @tparam Geometry The geometry class you want to use
- * @tparam Matrix The cusp-matrix class you want to use
- * @tparam Vector The Vector class you want to use
+ * @copydoc hide_matrix_container
  * @attention The Laplacian in this formula is positive as opposed to the negative sign in the Elliptic operator
  * @attention It might be better to solve the normal Helmholtz operator twice
  * consecutively than solving the Helmholtz2 operator once. 
  */
-template< class Geometry, class Matrix, class Vector> 
+template< class Geometry, class Matrix, class container> 
 struct Helmholtz2
 {
     /**
@@ -159,7 +154,7 @@ struct Helmholtz2
     Helmholtz2( Geometry g, double alpha = 1., direction dir = dg::forward, double jfactor=1.):
         laplaceM_(g, normed, dir, jfactor), 
         temp1_(dg::evaluate(dg::one, g)),temp2_(temp1_), chi_(temp1_),
-        alpha_(alpha), isSet(false)
+        alpha_(alpha)
     { 
     }
     /**
@@ -176,7 +171,7 @@ struct Helmholtz2
     Helmholtz2( Geometry g, bc bcx, bc bcy, double alpha = 1., direction dir = dg::forward, double jfactor=1.):
         laplaceM_(g, bcx,bcy,normed, dir, jfactor), 
         temp1_(dg::evaluate(dg::one, g)), temp2_(temp1_),chi_(temp1_),
-        alpha_(alpha), isSet(false)
+        alpha_(alpha)
     { 
     }
     /**
@@ -188,18 +183,15 @@ struct Helmholtz2
      * @param y rhs contains solution
      * @note Takes care of sign in laplaceM and thus multiplies by -alpha
      */
-    void symv( Vector& x, Vector& y) 
+    void symv( container& x, container& y) 
     {
         if( alpha_ != 0)
         {
             blas2::symv( laplaceM_, x, temp1_); // temp1_ = -nabla_perp^2 x
-            blas1::pointwiseDivide(temp1_, chi_, y); //temp2_ = (chi^-1)*W*nabla_perp^2 x
+            tensor::pointwiseDivide(temp1_, chi_, y); //temp2_ = (chi^-1)*W*nabla_perp^2 x
             blas2::symv( laplaceM_, y, temp2_);//temp2_ = nabla_perp^2 *(chi^-1)*nabla_perp^2 x            
         }
-        if( isSet)
-            blas1::pointwiseDot( chi_, x, y); //y = chi*x
-        else 
-            blas1::transfer( x, y);
+        tensor::pointwiseDot( chi_, x, y); //y = chi*x
         blas1::axpby( 1., y, -2.*alpha_, temp1_, y); 
         blas1::axpby( alpha_*alpha_, temp2_, 1., y, y);
         blas1::pointwiseDot( laplaceM_.weights(), y, y);//Helmholtz is never normed
@@ -209,14 +201,14 @@ struct Helmholtz2
      *
      * @return weights
      */
-    const Vector& weights()const {return laplaceM_.weights();}
+    const container& weights()const {return laplaceM_.weights();}
     /**
-     * @brief Vector to use in conjugate gradient solvers
+     * @brief container to use in conjugate gradient solvers
      *
      * multiply result by these coefficients to get the normed result
-     * @return Vector
+     * @return container
      */
-    const Vector& precond()const {return laplaceM_.precond();}
+    const container& precond()const {return laplaceM_.precond();}
     /**
      * @brief Change alpha
      *
@@ -232,24 +224,24 @@ struct Helmholtz2
     /**
      * @brief Set Chi in the above formula
      *
-     * @param chi new Vector
+     * @param chi new container
      */
-    void set_chi( const Vector& chi) {chi_=chi; isSet =true;}
+    void set_chi( const container& chi) {chi_.value()=chi; }
     /**
      * @brief Sets chi back to one
      */
-    void reset_chi(){isSet = false;}
+    void reset_chi(){chi_.clear();}
     /**
      * @brief Access chi
      *
      * @return chi
      */
-    const Vector& chi()const {return chi_;}
+    const SparseElement<container>& chi()const {return chi_;}
   private:
-    Elliptic<Geometry, Matrix, Vector> laplaceM_;
-    Vector temp1_, temp2_, chi_;
+    Elliptic<Geometry, Matrix, container> laplaceM_;
+    container temp1_, temp2_;
+    SparseElement<container> chi_;
     double alpha_;
-    bool isSet;
 };
 ///@cond
 template< class G, class M, class V>
-- 
GitLab


From cb3e96aac7825d87d035a8d39be2e1739ab57883 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Mon, 14 Aug 2017 22:45:45 +0200
Subject: [PATCH 160/453] place dense in tensor namespace to make SparseTensor
 independent on blas1

---
 inc/dg/geometry/multiply.h    |  6 ++-
 inc/dg/geometry/multiply_t.cu |  3 ++
 inc/dg/geometry/tensor.h      | 94 +++++++++++++++++------------------
 inc/dg/geometry/tensor_t.cu   |  3 --
 4 files changed, 54 insertions(+), 52 deletions(-)

diff --git a/inc/dg/geometry/multiply.h b/inc/dg/geometry/multiply.h
index 818b79d88..37498f8da 100644
--- a/inc/dg/geometry/multiply.h
+++ b/inc/dg/geometry/multiply.h
@@ -315,7 +315,7 @@ template<class container>
 SparseElement<container> determinant( const SparseTensor<container>& t)
 {
     if(t.isEmpty())  return SparseElement<container>();
-    SparseTensor<container> d = t.dense();
+    SparseTensor<container> d = dense(t);
     container det = d.value(0,0);
     std::vector<container> sub_det(3,det);
     dg::blas1::transform( det, det, dg::CONSTANT(0));
@@ -337,6 +337,8 @@ SparseElement<container> determinant( const SparseTensor<container>& t)
     return SparseElement<container>(det);
 }
 
+
+
 ///@cond
 //alias always allowed
 template<class container>
@@ -357,7 +359,7 @@ void multiply3d( const CholeskyTensor<container>& ch, const container& in0, cons
 template<class container>
 SparseElement<container> determinant( const CholeskyTensor<container>& ch)
 {
-    SparseTensor<container> diag = ch.diag().dense();
+    SparseTensor<container> diag = dense(ch.diag() );
     SparseElement<container> det;
     if(diag.isEmpty()) return det;
     else det.value()=diag.value(0,0);
diff --git a/inc/dg/geometry/multiply_t.cu b/inc/dg/geometry/multiply_t.cu
index 7d137f644..d557e8205 100644
--- a/inc/dg/geometry/multiply_t.cu
+++ b/inc/dg/geometry/multiply_t.cu
@@ -48,6 +48,9 @@ int main()
     std::cout << "Scale with 1/5 \n";print(t);
     dg::tensor::scal(t,two); 
     std::cout << "Scale with container(2) \n";print(t);
+    std::cout << "explicit dense Tensor \n";
+    dg::SparseTensor<thrust::host_vector<double> > dense3d = dg::tensor::dense(t);
+    if(dense3d.isDense())print( dense3d);
 
     std::cout << "Test Element multiplies \n";
     dg::SparseElement<thrust::host_vector<double> > sqr(nine);
diff --git a/inc/dg/geometry/tensor.h b/inc/dg/geometry/tensor.h
index 046ad5435..a75e8eb7b 100644
--- a/inc/dg/geometry/tensor.h
+++ b/inc/dg/geometry/tensor.h
@@ -78,8 +78,7 @@ struct SparseElement
 if negative the value of the T is assumed to be 1, except for the off-diagonal entries
     in the matrix where it is assumed to be 0.
 * We then only need to store non-trivial and non-repetitive Ts.
-* @tparam T must be default constructible and copyable. If dense is called the dg::blas1::transform function 
-* needs to be callable.
+* @tparam T must be default constructible and copyable.
 * @ingroup misc
 */
 template<class T>
@@ -252,14 +251,6 @@ struct SparseTensor
      
      ///construct an empty Tensor
      SparseTensor empty()const{return SparseTensor();}
-     /**
-     * @brief Construct a tensor with all unset values filled with explicit 0 or 1
-     *
-     * T must support the dg::blas1::transform function 
-     * @return a dense tensor
-     * @note undefined if isEmpty() returns true
-     */
-     SparseTensor dense()const;
      ///copy and erase all values in the third dimension
      ///@note calls clear_unused_values() to get rid of the elements
      SparseTensor perp()const;
@@ -280,7 +271,48 @@ struct SparseTensor
     void unique_insert(std::vector<int>& indices, int& idx);
 };
 
-
+namespace tensor
+{
+ /**
+ * @brief Construct a tensor with all unset values filled with explicit 0 or 1
+ *
+ * @copydoc hide_container_lvl1
+ * @return a dense tensor
+ * @note undefined if t.isEmpty() returns true
+ */
+template<class container> 
+SparseTensor<container> dense(const SparseTensor<container>& tensor)
+{
+    SparseTensor<container> t(tensor);
+    if( t.isEmpty()) throw Error(Message(_ping_)<< "Can't make an empty tensor dense! ") ;
+    container tmp = t.values()[0];
+    //1. diagonal
+    size_t size= t.values().size();
+    bool diagonalIsSet=true;
+    for(unsigned i=0; i<3; i++)
+        if(!t.isSet(i,i)) diagonalIsSet = false;
+    if (!diagonalIsSet){
+        dg::blas1::transform( tmp, tmp, dg::CONSTANT(1));
+        t.value(size)=tmp;
+        for(unsigned i=0; i<3; i++)
+            if(!t.isSet(i,i)) t.idx(i,i) = size;
+    }
+    //2. off-diagonal
+    size = t.values().size();
+    bool offIsSet=true;
+    for(unsigned i=0; i<3; i++)
+        for(unsigned j=0; j<3; j++)
+            if( !t.isSet(i,j) ) offIsSet=false;
+    if (!offIsSet){
+        dg::blas1::transform( tmp, tmp, dg::CONSTANT(0));
+        t.value(size)=tmp;
+        for(unsigned i=0; i<3; i++)
+            for(unsigned j=0; j<3; j++)
+                if(!t.isSet(i,j) ) t.idx(i,j) = size;
+    }
+    return t;
+}
+}//namespace tensor
 
 /**
  * @brief data structure to hold the LDL^T decomposition of a symmetric positive definite matrix
@@ -321,7 +353,7 @@ struct CholeskyTensor
      */
     void decompose( const SparseTensor<container>& in)
     {
-        SparseTensor<container> denseIn=in.dense();
+        SparseTensor<container> denseIn=dg::tensor::dense(in);
         /*
          * One nice property of positive definite is that the diagonal elements are 
          * greater than zero.
@@ -349,7 +381,7 @@ struct CholeskyTensor
 
         if( q_.isSet(1,0) || in.isSet(1,1))
         {
-            SparseTensor<container> denseL = q_.dense();
+            SparseTensor<container> denseL=dg::tensor::dense(q_);
             container tmp=denseL.value(1,0);
             dg::blas1::pointwiseDot(tmp,tmp,tmp);
             if(diag_.isSet(0,0)) dg::blas1::pointwiseDot(tmp,diag_.value(0,0),tmp);
@@ -360,7 +392,7 @@ struct CholeskyTensor
 
         if( in.isSet(2,1) || (q_.isSet(2,0)&&q_.isSet(1,0)))
         {
-            SparseTensor<container> denseL = q_.dense();
+            SparseTensor<container> denseL=dg::tensor::dense(q_);
             container tmp=denseIn.value(2,1);
             dg::blas1::pointwiseDot(denseL.value(2,0), denseL.value(1,0), tmp);
             if(diag_.isSet(0,0))dg::blas1::pointwiseDot(tmp, diag_.value(0,0), tmp);
@@ -371,7 +403,7 @@ struct CholeskyTensor
         }
         if( in.isSet(2,2) || q_.isSet(2,0) || q_.isSet(2,1))
         {
-            SparseTensor<container> denseL = q_.dense();
+            SparseTensor<container> denseL=dg::tensor::dense(q_);
             container tmp=denseL.value(2,0), tmp1=denseL.value(2,1);
             dg::blas1::pointwiseDot(tmp,tmp,tmp);
             if(diag_.isSet(0,0))dg::blas1::pointwiseDot(diag_.value(0,0),tmp,tmp);
@@ -416,38 +448,6 @@ struct CholeskyTensor
 };
 
 ///@cond
-template<class container>
-SparseTensor<container> SparseTensor<container>::dense() const
-{
-    SparseTensor<container> t(*this);
-    if( isEmpty()) throw Error(Message(_ping_)<< "Can't make an empty tensor dense! ") ;
-    container tmp = t.values_[0];
-    //1. diagonal
-    size_t size= values_.size();
-    bool diagonalIsSet=true;
-    for(unsigned i=0; i<3; i++)
-        if(!t.isSet(i,i)) diagonalIsSet = false;
-    if (!diagonalIsSet){
-        dg::blas1::transform( tmp, tmp, dg::CONSTANT(1));
-        t.values_.push_back(tmp);
-        for(unsigned i=0; i<3; i++)
-            if(!t.isSet(i,i)) t.mat_idx_(i,i) = size;
-    }
-    //2. off-diagonal
-    size = t.values_.size();
-    bool offIsSet=true;
-    for(unsigned i=0; i<3; i++)
-        for(unsigned j=0; j<3; j++)
-            if( !t.isSet(i,j) ) offIsSet=false;
-    if (!offIsSet){
-        dg::blas1::transform( tmp, tmp, dg::CONSTANT(0));
-        t.values_.push_back(tmp);
-        for(unsigned i=0; i<3; i++)
-            for(unsigned j=0; j<3; j++)
-                if(!t.isSet(i,j) ) t.mat_idx_(i,j) = size;
-    }
-    return t;
-}
 
 template<class container>
 void SparseTensor<container>::unique_insert(std::vector<int>& indices, int& idx) 
diff --git a/inc/dg/geometry/tensor_t.cu b/inc/dg/geometry/tensor_t.cu
index 520bbfdd5..ba6f79846 100644
--- a/inc/dg/geometry/tensor_t.cu
+++ b/inc/dg/geometry/tensor_t.cu
@@ -63,9 +63,6 @@ int main()
     if(!sparse3d.isDiagonal()) print( sparse3d);
     std::cout << "empty Tensor \n";
     if(sparse3d.empty().isDiagonal()) print( sparse3d.empty());
-    std::cout << "explicit dense Tensor \n";
-    dg::SparseTensor<thrust::host_vector<double> > dense3d = sparse3d.dense();
-    if(dense3d.isDense())print( dense3d);
     sparse3d = sparse3d_D;
     std::cout << "original stored on device \n";
     if(!sparse3d.isDense())print( sparse3d);
-- 
GitLab


From 9bd0bf07291e181614b98e8b74cb703677f820a3 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Tue, 15 Aug 2017 02:26:59 -0700
Subject: [PATCH 161/453] applied Harald Servats Patch file

---
 inc/dg/backend/sparseblockmat_omp_kernels.h | 150 +++++++-------------
 1 file changed, 52 insertions(+), 98 deletions(-)

diff --git a/inc/dg/backend/sparseblockmat_omp_kernels.h b/inc/dg/backend/sparseblockmat_omp_kernels.h
index 2cb637f29..e1cdaca1f 100644
--- a/inc/dg/backend/sparseblockmat_omp_kernels.h
+++ b/inc/dg/backend/sparseblockmat_omp_kernels.h
@@ -4,12 +4,12 @@ namespace dg{
 // multiply kernel
 template<class value_type>
 void ell_multiply_kernel(
-         const value_type* data, const int* cols_idx, const int* data_idx, 
+         const value_type * restrict data, const int * restrict cols_idx, const int * restrict data_idx, 
          const int num_rows, const int num_cols, const int blocks_per_line,
          const int n, 
          const int left_size, const int right_size, 
-         const int* right_range,
-         const value_type* x, value_type *y
+         const int * restrict right_range,
+         const value_type * restrict x, value_type * restrict y
          )
 {
     //simplest implementation
@@ -35,13 +35,17 @@ void ell_multiply_kernel(
 // multiply kernel n=3, 3 blocks per line
 template<class value_type>
 void ell_multiply_kernel33(
-         const value_type* data, const int* cols_idx, const int* data_idx, 
+         const value_type * restrict data, const int * restrict cols_idx, const int * restrict data_idx, 
          const int num_rows, const int num_cols, 
          const int left_size, const int right_size, 
          const int* right_range,
-         const value_type* x, value_type *y
+         const value_type * restrict x, value_type * restrict y
          )
 {
+ // left_size = 1
+ // right_range[1]-right_range[0] = 2304
+ // num_rows = 768
+
     bool trivial = true;
     for( int i=1; i<num_rows-1; i++)
         for( int d=0; d<3; d++)
@@ -51,87 +55,40 @@ void ell_multiply_kernel33(
         }
     if( trivial)
     {
-#pragma omp parallel for 
-    for( int s=0; s<left_size; s++)
-    for( int i=0; i<1; i++)
-    for( int k=0; k<3; k++)
-    for( int j=right_range[0]; j<right_range[1]; j++)
-    {
-        value_type temp = 0;
-        int B0 = (data_idx[i*3+0]*3+k)*3;
-        int B1 = (data_idx[i*3+1]*3+k)*3;
-        int B2 = (data_idx[i*3+2]*3+k)*3;
-        int J0 = (s*num_cols+cols_idx[i*3+0])*3;
-        int J1 = (s*num_cols+cols_idx[i*3+1])*3;
-        int J2 = (s*num_cols+cols_idx[i*3+2])*3;
-        temp +=data[ B0+0]* x[(J0+0)*right_size+j];
-        temp +=data[ B0+1]* x[(J0+1)*right_size+j];
-        temp +=data[ B0+2]* x[(J0+2)*right_size+j];
-
-        temp +=data[ B1+0]* x[(J1+0)*right_size+j];
-        temp +=data[ B1+1]* x[(J1+1)*right_size+j];
-        temp +=data[ B1+2]* x[(J1+2)*right_size+j];
-
-        temp +=data[ B2+0]* x[(J2+0)*right_size+j];
-        temp +=data[ B2+1]* x[(J2+1)*right_size+j];
-        temp +=data[ B2+2]* x[(J2+2)*right_size+j];
-        int I = ((s*num_rows + i)*3+k)*right_size+j;
-        y[I]=temp;
-    }
-#pragma omp parallel for collapse(2)
-    for( int s=0; s<left_size; s++)
-    for( int i=1; i<num_rows-1; i++)
-    for( int k=0; k<3; k++)
-    for( int j=right_range[0]; j<right_range[1]; j++)
-    {
-        value_type temp = 0;
-        int B0 = (0*3+k)*3;
-        int B1 = (1*3+k)*3;
-        int B2 = (2*3+k)*3;
-        int J0 = (s*num_cols+i+0-1)*3;
-        int J1 = (s*num_cols+i+1-1)*3;
-        int J2 = (s*num_cols+i+2-1)*3;
-        temp +=data[ B0+0]* x[(J0+0)*right_size+j];
-        temp +=data[ B0+1]* x[(J0+1)*right_size+j];
-        temp +=data[ B0+2]* x[(J0+2)*right_size+j];
-
-        temp +=data[ B1+0]* x[(J1+0)*right_size+j];
-        temp +=data[ B1+1]* x[(J1+1)*right_size+j];
-        temp +=data[ B1+2]* x[(J1+2)*right_size+j];
-
-        temp +=data[ B2+0]* x[(J2+0)*right_size+j];
-        temp +=data[ B2+1]* x[(J2+1)*right_size+j];
-        temp +=data[ B2+2]* x[(J2+2)*right_size+j];
-        int I = ((s*num_rows + i)*3+k)*right_size+j;
-        y[I]=temp;
-    }
-#pragma omp parallel for 
-    for( int s=0; s<left_size; s++)
-    for( int i=num_rows-1; i<num_rows; i++)
-    for( int k=0; k<3; k++)
-    for( int j=right_range[0]; j<right_range[1]; j++)
-    {
-        value_type temp = 0;
-        int B0 = (data_idx[i*3+0]*3+k)*3;
-        int B1 = (data_idx[i*3+1]*3+k)*3;
-        int B2 = (data_idx[i*3+2]*3+k)*3;
-        int J0 = (s*num_cols+cols_idx[i*3+0])*3;
-        int J1 = (s*num_cols+cols_idx[i*3+1])*3;
-        int J2 = (s*num_cols+cols_idx[i*3+2])*3;
-        temp +=data[ B0+0]* x[(J0+0)*right_size+j];
-        temp +=data[ B0+1]* x[(J0+1)*right_size+j];
-        temp +=data[ B0+2]* x[(J0+2)*right_size+j];
-
-        temp +=data[ B1+0]* x[(J1+0)*right_size+j];
-        temp +=data[ B1+1]* x[(J1+1)*right_size+j];
-        temp +=data[ B1+2]* x[(J1+2)*right_size+j];
-
-        temp +=data[ B2+0]* x[(J2+0)*right_size+j];
-        temp +=data[ B2+1]* x[(J2+1)*right_size+j];
-        temp +=data[ B2+2]* x[(J2+2)*right_size+j];
-        int I = ((s*num_rows + i)*3+k)*right_size+j;
-        y[I]=temp;
-    }
+	#pragma omp parallel for collapse(2)
+	for( int s=0; s<left_size; s++)
+	{
+		for( int i=0; i<num_rows; i++)
+		{
+		        int J0 = (i==0 || i==num_rows-1)?(s*num_cols+cols_idx[i*3+0])*3:(s*num_cols+i+0-1)*3;
+		        int J1 = (i==0 || i==num_rows-1)?(s*num_cols+cols_idx[i*3+1])*3:(s*num_cols+i+1-1)*3;
+		        int J2 = (i==0 || i==num_rows-1)?(s*num_cols+cols_idx[i*3+2])*3:(s*num_cols+i+2-1)*3;
+			for( int k=0; k<3; k++)
+			{
+				int B0 = (i==0 || i==num_rows-1)?(data_idx[i*3+0]*3+k)*3:(0*3+k)*3;
+			        int B1 = (i==0 || i==num_rows-1)?(data_idx[i*3+1]*3+k)*3:(1*3+k)*3;
+			        int B2 = (i==0 || i==num_rows-1)?(data_idx[i*3+2]*3+k)*3:(2*3+k)*3;
+				#pragma vector nontemporal(y)
+				for( int j=right_range[0]; j<right_range[1]; j++)
+				{
+				    value_type temp = 0;
+				    temp +=data[ B0+0]* x[(J0+0)*right_size+j];
+				    temp +=data[ B0+1]* x[(J0+1)*right_size+j];
+				    temp +=data[ B0+2]* x[(J0+2)*right_size+j];
+				
+				    temp +=data[ B1+0]* x[(J1+0)*right_size+j];
+				    temp +=data[ B1+1]* x[(J1+1)*right_size+j];
+				    temp +=data[ B1+2]* x[(J1+2)*right_size+j];
+				
+				    temp +=data[ B2+0]* x[(J2+0)*right_size+j];
+				    temp +=data[ B2+1]* x[(J2+1)*right_size+j];
+				    temp +=data[ B2+2]* x[(J2+2)*right_size+j];
+				    int I = ((s*num_rows + i)*3+k)*right_size+j;
+				    y[I]=temp;
+				}
+			}
+		}
+	}
     }
     else 
         ell_multiply_kernel( data, cols_idx, data_idx, num_rows, num_cols, 3, 3, left_size, right_size, right_range,  x, y);
@@ -140,11 +97,11 @@ void ell_multiply_kernel33(
 // multiply kernel, n=3, 2 blocks per line
 template<class value_type>
 void ell_multiply_kernel32(
-         const value_type* data, const int* cols_idx, const int* data_idx, 
+         const value_type * restrict data, const int * restrict cols_idx, const int * restrict data_idx, 
          const int num_rows, const int num_cols,
          const int left_size, const int right_size, 
-         const int* right_range,
-         const value_type* x, value_type *y
+         const int * restrict right_range,
+         const value_type * restrict x, value_type * restrict y
          )
 {
     bool forward = true, backward = true;
@@ -227,10 +184,10 @@ void ell_multiply_kernel32(
 // multiply kernel, n=3, 3 blocks per line, right_size = 1
 template<class value_type>
 void ell_multiply_kernel33x(
-         const value_type* data, const int* cols_idx, const int* data_idx, 
+         const value_type * restrict data, const int * restrict cols_idx, const int * restrict data_idx, 
          const int num_rows, const int num_cols,
          const int left_size,
-         const value_type* x, value_type *y
+         const value_type * restrict x, value_type * restrict y
          )
 {
     bool trivial = true;
@@ -244,6 +201,7 @@ void ell_multiply_kernel33x(
     {
 #pragma omp parallel for
     for( int s=0; s<left_size; s++)
+    {
     for( int i=0; i<1; i++)
     for( int k=0; k<3; k++)
     {
@@ -268,9 +226,6 @@ void ell_multiply_kernel33x(
         int I = ((s*num_rows + i)*3+k);
         y[I]=temp;
     }
-
-#pragma omp parallel for// collapse(2)
-    for( int s=0; s<left_size; s++)
     for( int i=1; i<num_rows-1; i++)
     for( int k=0; k<3; k++)
     {
@@ -294,8 +249,6 @@ void ell_multiply_kernel33x(
         int I = ((s*num_rows + i)*3+k);
         y[I]=temp;
     }
-#pragma omp parallel for
-    for( int s=0; s<left_size; s++)
     for( int i=num_rows-1; i<num_rows; i++)
     for( int k=0; k<3; k++)
     {
@@ -322,6 +275,7 @@ void ell_multiply_kernel33x(
         y[I]=temp;
     }
     }
+    }
     else 
     {
         int right_range[2] = {0,1};
@@ -332,10 +286,10 @@ void ell_multiply_kernel33x(
 // multiply kernel, n=3, 2 blocks per line, right_size = 1
 template<class value_type>
 void ell_multiply_kernel32x(
-         const value_type* data, const int* cols_idx, const int* data_idx, 
+         const value_type * restrict data, const int * restrict cols_idx, const int * restrict data_idx, 
          const int num_rows, const int num_cols,
          const int left_size, 
-         const value_type* x, value_type *y
+         const value_type * restrict x, value_type * restrict y
          )
 {
     bool forward = true, backward = true;
-- 
GitLab


From 2e46ee3871855a91de41788441ecfd89d7033416 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Tue, 15 Aug 2017 04:05:42 -0700
Subject: [PATCH 162/453] initial setup of marconi and mic device backend

---
 config/devices/devices.mk | 1 +
 config/marconi.mk         | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/config/devices/devices.mk b/config/devices/devices.mk
index 0bbd99955..d6ed50bb2 100644
--- a/config/devices/devices.mk
+++ b/config/devices/devices.mk
@@ -28,4 +28,5 @@ CFLAGS+=-Wall -x c++
 CFLAGS+= -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_OMP
 CFLAGS+= $(OMPFLAG) -mmic 
 MPICFLAGS+=$(CFLAGS) #includes values in CFLAGS defined later
+OPT+=-xMIC-AVX512 -fma -finline-functions -align
 endif #device=mic
diff --git a/config/marconi.mk b/config/marconi.mk
index e00b7bd1f..bf18c9cff 100644
--- a/config/marconi.mk
+++ b/config/marconi.mk
@@ -5,7 +5,7 @@ INCLUDE += -I$(NETCDF_INC) -I$(HDF5_INC)
 GLFLAGS  = -lm
 CC=icc
 MPICC=mpiicc
-OPT=-O3 -xHost -fma -xMIC-AVX512 -finline-functions -align
+OPT=-O3 -xHost -restrict
 #MPICFLAGS+= -DMPICH_IGNORE_CXX_SEEK
 OMPFLAG=-qopenmp
 JSONLIB=-L$(HOME)/include/json/../../src/lib_json -ljsoncpp # json library for input parameters
-- 
GitLab


From 39aad4814de564b44e685f076370b1c87807bc84 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Tue, 15 Aug 2017 05:13:55 -0700
Subject: [PATCH 163/453] begin to adapt X generation to BinaryFunctors classes

---
 inc/geometries/adaption.h              |  3 +-
 inc/geometries/fluxfunctions.h         | 63 +++++++++++++++++++++--
 inc/geometries/ribeiroX.h              | 41 +++++++--------
 inc/geometries/separatrix_orthogonal.h | 59 ++++++++--------------
 inc/geometries/utilities.h             |  2 +-
 inc/geometries/utilitiesX.h            | 70 +++++++++++---------------
 6 files changed, 129 insertions(+), 109 deletions(-)

diff --git a/inc/geometries/adaption.h b/inc/geometries/adaption.h
index 9638b8d95..7d79de018 100644
--- a/inc/geometries/adaption.h
+++ b/inc/geometries/adaption.h
@@ -175,8 +175,7 @@ struct Liseikin_XY: public aCloneableBinaryFunctor<Liseikin_XY>
 
     private:
     double k_, eps_;
-    PsiX psiX_;
-    PsiY psiY_;
+    BinaryFunctorsLvl1 psi_;
 };
 
 /**
diff --git a/inc/geometries/fluxfunctions.h b/inc/geometries/fluxfunctions.h
index 9a99641f2..3951ea4c8 100644
--- a/inc/geometries/fluxfunctions.h
+++ b/inc/geometries/fluxfunctions.h
@@ -97,6 +97,10 @@ struct BinaryFunctorAdapter : public dg::geo::aCloneableBinaryFunctor<Adapter>
 temmplate<class BinaryFunctor>
 aBinaryFunctor* make_aBinaryFunctor(const BinaryFunctor& f){return new BinaryFunctorAdapter<BinaryFunctor>(f);}
 
+///@cond
+struct BinaryFunctorsLvl2;
+///@endcond
+
 /**
 * @brief This struct bundles a function and its first derivatives
 */
@@ -111,16 +115,28 @@ struct BinaryFunctorsLvl1
     * @param fx \f$ \partial f / \partial x \f$ its derivative in the first coordinate
     * @param fy \f$ \partial f / \partial y \f$ its derivative in the second coordinate
     */
-    BinaryFunctorsLvl1( const aBinaryFunctor* f, const aBinaryFunctor* fx, const aBinaryFunctor* fy)
-    {
+    BinaryFunctorsLvl1( const aBinaryFunctor* f, const aBinaryFunctor* fx, const aBinaryFunctor* fy) {
+        reset(f,fx,fy);
+    }
+    ///clone given functors
+    BinaryFunctorsLvl1( const aBinaryFunctor& f, const aBinaryFunctor& fx, const aBinaryFunctor& fy) {
         reset(f,fx,fy);
     }
+    BinaryFunctorsLvl1( const BinaryFunctorsLvl2& func);
+    ///Take ownership of given pointers
     void reset( const aBinaryFunctor* f, const aBinaryFunctor* fx, const aBinaryFunctor* fy)
     {
         p_[0].reset(f);
         p_[1].reset(fx);
         p_[2].reset(fy);
     }
+    ///clone given references
+    void reset( const aBinaryFunctor& f, const aBinaryFunctor& fx, const aBinaryFunctor& fy)
+    {
+        p_[0].reset(f);
+        p_[1].reset(fx);
+        p_[2].reset(fy);
+    }
     /// \f$ f \f$
     const aBinaryFunctor& f()const{return p_[0].get();}
     /// \f$ \partial f / \partial x \f$ 
@@ -145,9 +161,17 @@ struct BinaryFunctorsLvl2
     */
     BinaryFunctorsLvl2( const aBinaryFunctor* f, const aBinaryFunctor* fx, const aBinaryFunctor* fy, const aBinaryFunctor* fxx, const aBinaryFunctor* fxy, const aBinaryFunctor* fyy): f(f,fx,fy), f1(fxx,fxy,fyy) 
     { }
+    ///clone given Functors
+    BinaryFunctorsLvl2( const aBinaryFunctor& f, const aBinaryFunctor& fx, const aBinaryFunctor& fy, const aBinaryFunctor& fxx, const aBinaryFunctor& fxy, const aBinaryFunctor& fyy): f(f,fx,fy), f1(fxx,fxy,fyy) 
+    { }
+    ///Take ownership of given pointers
     void reset( const aBinaryFunctor* f, const aBinaryFunctor* fx, const aBinaryFunctor* fy, const aBinaryFunctor* fxx, const aBinaryFunctor* fxy, const aBinaryFunctor* fyy){ 
         f.reset(f,fx,fy), f1.reset(fxx,fxy,fyy) 
     }
+    ///clone given pointers
+    void reset( const aBinaryFunctor& f, const aBinaryFunctor& fx, const aBinaryFunctor& fy, const aBinaryFunctor& fxx, const aBinaryFunctor& fxy, const aBinaryFunctor& fyy){ 
+        f.reset(f,fx,fy), f1.reset(fxx,fxy,fyy) 
+    }
     operator BinaryFunctorsLvl1 ()const {return f;}
     /// \f$ f \f$
     const aBinaryFunctor& f()const{return f.f();}
@@ -164,6 +188,12 @@ struct BinaryFunctorsLvl2
     private:
     BinaryFunctorsLvl1 f,f1;
 };
+///@cond
+BinaryFunctorsLvl1::BinaryFunctorsLvl1( const BinaryFunctorsLvl2& func)
+{
+    reset(func.f(),func.dfx(),func.dfy());
+}
+///@endcond
 
 /// A symmetric 2d tensor field and its divergence
 struct BinarySymmTensorLvl1
@@ -183,6 +213,12 @@ struct BinarySymmTensorLvl1
     {
         reset(chi_xx,chi_xy,chi_yy,divChiX,divChiY);
     }
+    ///clone given functors
+    BinarySymmTensorLvl1( const aBinaryFunctor& chi_xx, const aBinaryFunctor& chi_xy, const aBinaryFunctor& chi_yy, const aBinaryFunctor& divChiX, const aBinaryFunctor& divChiY)
+    {
+        reset(chi_xx,chi_xy,chi_yy,divChiX,divChiY);
+    }
+    ///Take ownership of pointers and release any  currently held ones
     void reset( const aBinaryFunctor* chi_xx, const aBinaryFunctor* chi_xy, const aBinaryFunctor* chi_yy, const aBinaryFunctor* divChiX, const aBinaryFunctor* divChiY)
     {
         p_[0].reset( chi_xx);
@@ -191,6 +227,15 @@ struct BinarySymmTensorLvl1
         p_[3].reset( divChiX);
         p_[4].reset( divChiY);
     }
+    ///clone given references 
+    void reset( const aBinaryFunctor& chi_xx, const aBinaryFunctor& chi_xy, const aBinaryFunctor& chi_yy, const aBinaryFunctor& divChiX, const aBinaryFunctor& divChiY)
+    {
+        p_[0].reset( chi_xx);
+        p_[1].reset( chi_xy);
+        p_[2].reset( chi_yy);
+        p_[3].reset( divChiX);
+        p_[4].reset( divChiY);
+    }
     ///xy component \f$ \chi^{xx}\f$ 
     const aBinaryFunctor& xx()const{return p_[0].get();}
     ///xy component \f$ \chi^{xy}\f$ 
@@ -209,11 +254,19 @@ struct BinarySymmTensorLvl1
 struct BinaryVectorLvl0
 {
     BinaryVectorLvl0(){}
-    BinaryVectorLvl0( const aBinaryFunctor* v_x, const aBinaryFunctor* v_y, const aBinaryFunctor* v_z)
+    ///Take ownership of given pointers
+    BinaryVectorLvl0( const aBinaryFunctor* v_x, const aBinaryFunctor* v_y, const aBinaryFunctor* v_z) { reset(v_x,v_y,v_z); }
+    ///clone given references
+    BinaryVectorLvl0( const aBinaryFunctor& v_x, const aBinaryFunctor& v_y, const aBinaryFunctor& v_z) { reset(v_x,v_y,v_z); }
+    ///Take ownership of given pointers and release any currently held ones
+    void reset( const aBinaryFunctor* v_x, const aBinaryFunctor* v_y, const aBinaryFunctor* v_z)
     {
-        reset(v_x,v_y,v_z);
+        p_[0].reset(v_x);
+        p_[1].reset(v_y);
+        p_[2].reset(v_z);
     }
-    void reset( const aBinaryFunctor* v_x, const aBinaryFunctor* v_y, const aBinaryFunctor* v_z)
+    ///clone given references
+    void reset( const aBinaryFunctor& v_x, const aBinaryFunctor& v_y, const aBinaryFunctor& v_z)
     {
         p_[0].reset(v_x);
         p_[1].reset(v_y);
diff --git a/inc/geometries/ribeiroX.h b/inc/geometries/ribeiroX.h
index d6478fd51..aa1b413e9 100644
--- a/inc/geometries/ribeiroX.h
+++ b/inc/geometries/ribeiroX.h
@@ -22,11 +22,10 @@ namespace detail
 {
 //This leightweights struct and its methods finds the initial R and Z values and the coresponding f(\psi) as 
 //good as it can, i.e. until machine precision is reached
-template< class Psi, class PsiX, class PsiY>
 struct FpsiX
 {
-    FpsiX( Psi psi, PsiX psiX, PsiY psiY, double xX, double yX, double x0, double y0): 
-        initX_(psi, psiX, psiY, xX, yX), fieldRZYT_(psiX, psiY, x0, y0), fieldRZYZ_(psiX, psiY)
+    FpsiX( const BinaryFunctorsLvl1& psi, double xX, double yX, double x0, double y0): 
+        initX_(psi, xX, yX), fieldRZYT_(psi, x0, y0), fieldRZYZ_(psi)
     { }
     //for a given psi finds the four starting points for the integration in y direction on the perpendicular lines through the X-point
     void find_initial( double psi, double* R_0, double* Z_0) 
@@ -162,18 +161,17 @@ struct FpsiX
         return fprime_old;
     }
     private:
-    dg::geo::orthogonal::detail::InitialX<Psi, PsiX, PsiY> initX_;
-    const dg::geo::ribeiro::FieldRZYT<PsiX, PsiY> fieldRZYT_;
-    const dg::geo::ribeiro::FieldRZYZ<PsiX, PsiY> fieldRZYZ_;
+    dg::geo::orthogonal::detail::InitialX initX_;
+    const dg::geo::ribeiro::FieldRZYT fieldRZYT_;
+    const dg::geo::ribeiro::FieldRZYZ fieldRZYZ_;
 
 };
 
 //This struct computes -2pi/f with a fixed number of steps for all psi
-template<class Psi, class PsiX, class PsiY>
 struct XFieldFinv
 {
-    XFieldFinv( Psi psi, PsiX psiX, PsiY psiY, double xX, double yX, double x0, double y0, unsigned N_steps = 500): 
-        fpsi_(psi, psiX, psiY, xX, yX, x0, y0), fieldRZYT_(psiX, psiY, x0, y0), fieldRZYZ_(psiX, psiY) , N_steps(N_steps)
+    XFieldFinv( const BinaryFunctorsLvl1& psi, double xX, double yX, double x0, double y0, unsigned N_steps = 500): 
+        fpsi_(psi, xX, yX, x0, y0), fieldRZYT_(psi, x0, y0), fieldRZYZ_(psi) , N_steps(N_steps)
             { xAtOne_ = fpsi_.find_x(0.1); }
     void operator()(const thrust::host_vector<double>& psi, thrust::host_vector<double>& fpsiM) 
     { 
@@ -225,9 +223,9 @@ struct XFieldFinv
     }
 
     private:
-    FpsiX<Psi, PsiX, PsiY> fpsi_;
-    dg::geo::ribeiro::FieldRZYT<PsiX, PsiY> fieldRZYT_;
-    dg::geo::ribeiro::FieldRZYZ<PsiX, PsiY> fieldRZYZ_;
+    FpsiX fpsi_;
+    dg::geo::ribeiro::FieldRZYT fieldRZYT_;
+    dg::geo::ribeiro::FieldRZYZ fieldRZYZ_;
     thrust::host_vector<double> fpsi_neg_inv;
     unsigned N_steps;
     double xAtOne_;
@@ -241,12 +239,11 @@ struct XFieldFinv
  * @ingroup generators
  * @tparam Psi All the template parameters must model aBinaryOperator i.e. the bracket operator() must be callable with two arguments and return a double. 
  */
-template< class Psi, class PsiX, class PsiY, class PsiXX, class PsiXY, class PsiYY>
 struct RibeiroX
 {
-    RibeiroX( Psi psi, PsiX psiX, PsiY psiY, PsiXX psiXX, PsiXY psiXY, PsiYY psiYY, double psi_0, double fx, 
+    RibeiroX( const BinaryFunctorsLvl2& psi, double psi_0, double fx, 
             double xX, double yX, double x0, double y0):
-        psiX_(psiX), psiY_(psiY), psiXX_(psiXX), psiXY_(psiXY), psiYY_(psiYY), fpsi_(psi, psiX, psiY, xX, yX, x0,y0), fpsiMinv_(psi, psiX, psiY, xX, yX, x0,y0, 500)
+        psi_(psi), fpsi_(psi, xX, yX, x0,y0), fpsiMinv_(psi, xX, yX, x0,y0, 500)
     {
         assert( psi_0 < 0 );
         zeta0_ = fpsi_.find_x( psi_0);
@@ -295,7 +292,7 @@ struct RibeiroX
     @note the \f$ \zeta\f$ coordinate is contiguous in memory
      * @note All the resulting vectors are write-only and get properly resized
     */
-    void operator()( 
+    void generate( 
          const thrust::host_vector<double>& zeta1d, 
          const thrust::host_vector<double>& eta1d, 
          const unsigned nodeX0, const unsigned nodeX1, 
@@ -314,7 +311,7 @@ struct RibeiroX
         psi_1_numerical_ = dg::geo::detail::construct_psi_values( fpsiMinv_, psi0_, zeta0_, zeta1d, zeta1_, inside, psi_x);
 
         //std::cout << "In grid function:\n";
-        dg::geo::ribeiro::FieldRZYRYZY<PsiX, PsiY, PsiXX, PsiXY, PsiYY> fieldRZYRYZYribeiro(psiX_, psiY_, psiXX_, psiXY_, psiYY_);
+        dg::geo::ribeiro::FieldRZYRYZY fieldRZYRYZYribeiro(psi_);
         unsigned size = zeta1d.size()*eta1d.size();
         x.resize(size), y.resize(size);
         zetaX = zetaY = etaX = etaY =x ;
@@ -335,13 +332,9 @@ struct RibeiroX
         }
     }
     private:
-    PsiX psiX_;
-    PsiY psiY_;
-    PsiXX psiXX_;
-    PsiXY psiXY_;
-    PsiYY psiYY_;
-    dg::geo::ribeiro::detail::XFieldFinv<Psi, PsiX, PsiY> fpsiMinv_; 
-    dg::geo::ribeiro::detail::FpsiX<Psi, PsiX, PsiY> fpsi_;
+    BinaryFunctorsLvl2 psi_;
+    dg::geo::ribeiro::detail::XFieldFinv fpsiMinv_; 
+    dg::geo::ribeiro::detail::FpsiX fpsi_;
     double f0_, psi_1_numerical_;
     thrust::host_vector<double> fx_;
     double zeta0_, zeta1_;
diff --git a/inc/geometries/separatrix_orthogonal.h b/inc/geometries/separatrix_orthogonal.h
index a5faf4b72..18da2cb43 100644
--- a/inc/geometries/separatrix_orthogonal.h
+++ b/inc/geometries/separatrix_orthogonal.h
@@ -25,8 +25,7 @@ namespace detail
 
 //compute the vector of r and z - values that form one psi surface
 //assumes y_0 = 0
-template <class PsiX, class PsiY>
-void computeX_rzy( PsiX psiX, PsiY psiY, 
+void computeX_rzy( const BinaryFunctorsLvl1& psi,
         const thrust::host_vector<double>& y_vec, 
         const unsigned nodeX0, const unsigned nodeX1,
         thrust::host_vector<double>& r, //output r - values
@@ -40,8 +39,8 @@ void computeX_rzy( PsiX psiX, PsiY psiY,
     r.resize( y_vec.size()), z.resize(y_vec.size());
     thrust::host_vector<double> begin( 2, 0), end(begin), temp(begin);
     begin[0] = R_init[0], begin[1] = Z_init[0];
-    dg::geo::ribeiro::FieldRZY<PsiX, PsiY> fieldRZYconf(psiX, psiY);
-    dg::geo::equalarc::FieldRZY<PsiX, PsiY> fieldRZYequi(psiX, psiY);
+    dg::geo::ribeiro::FieldRZY fieldRZYconf(psi.dfx(), psi.dfy());
+    dg::geo::equalarc::FieldRZY fieldRZYequi(psi.dfx(), psi.dfy());
     fieldRZYconf.set_f(f_psi);
     fieldRZYequi.set_f(f_psi);
     unsigned steps = 1; double eps = 1e10, eps_old=2e10;
@@ -118,31 +117,26 @@ void computeX_rzy( PsiX psiX, PsiY psiY,
  * @brief Choose points on inside or outside line (models aGeneratorX)
  *
  * @ingroup generators
- * @tparam Psi models aBinaryOperator
- * @tparam PsiX models aBinaryOperator
- * @tparam PsiY models aBinaryOperator
- * @tparam LaplacePsi models aBinaryOperator
  */
-template< class Psi, class PsiX, class PsiY, class LaplacePsi>
 struct SimpleOrthogonalX : public aGeneratorX2d
 {
     SimpleOrthogonalX(): f0_(1), firstline_(0){}
-    SimpleOrthogonalX( Psi psi, PsiX psiX, PsiY psiY, LaplacePsi laplacePsi, double psi_0, //psi_0 must be the closed surface, 0 the separatrix
+    SimpleOrthogonalX( const BinaryFunctorsLvl2& psi, double psi_0, //psi_0 must be the closed surface, 0 the separatrix
             double xX, double yX, double x0, double y0, int firstline =0):
-        psiX_(psiX), psiY_(psiY), laplacePsi_(laplacePsi)
+        psi_(psi)
     {
         firstline_ = firstline;
-        orthogonal::detail::Fpsi<Psi, PsiX, PsiY> fpsi(psi, psiX, psiY, x0, y0, firstline);
+        orthogonal::detail::Fpsi fpsi(psi_, x0, y0, firstline);
         double R0, Z0; 
         f0_ = fpsi.construct_f( psi_0, R0, Z0);
         zeta0_=f0_*psi_0;
-        dg::geo::orthogonal::detail::InitialX<Psi, PsiX, PsiY> initX(psi, psiX, psiY, xX, yX);
+        dg::geo::orthogonal::detail::InitialX initX(psi_, xX, yX);
         initX.find_initial(psi_0, R0_, Z0_);
     }
     bool isConformal()const{return false;}
     bool isOrthogonal()const{return true;}
     double f0() const{return f0_;}
-    void operator()( //this one doesn't know if the separatrix comes to lie on a cell boundary or not
+    virtual void generate( //this one doesn't know if the separatrix comes to lie on a cell boundary or not
          const thrust::host_vector<double>& zeta1d, 
          const thrust::host_vector<double>& eta1d, 
          const unsigned nodeX0, const unsigned nodeX1,
@@ -155,8 +149,8 @@ struct SimpleOrthogonalX : public aGeneratorX2d
     {
 
         thrust::host_vector<double> r_init, z_init;
-        orthogonal::detail::computeX_rzy( psiX_, psiY_, eta1d, nodeX0, nodeX1, r_init, z_init, R0_, Z0_, f0_, firstline_);
-        dg::geo::orthogonal::detail::Nemov<PsiX, PsiY, LaplacePsi> nemov(psiX_, psiY_, laplacePsi_, f0_, firstline_);
+        orthogonal::detail::computeX_rzy( psi_, eta1d, nodeX0, nodeX1, r_init, z_init, R0_, Z0_, f0_, firstline_);
+        dg::geo::orthogonal::detail::Nemov nemov(psi_, f0_, firstline_);
         thrust::host_vector<double> h;
         orthogonal::detail::construct_rz(nemov, zeta0_, zeta1d, r_init, z_init, x, y, h);
         unsigned size = x.size();
@@ -164,8 +158,8 @@ struct SimpleOrthogonalX : public aGeneratorX2d
         etaX.resize(size), etaY.resize(size);
         for( unsigned idx=0; idx<size; idx++)
         {
-            double psipR = psiX_(x[idx], y[idx]);
-            double psipZ = psiY_(x[idx], y[idx]);
+            double psipR = psi_.dfx()(x[idx], y[idx]);
+            double psipZ = psi_.dfy()(x[idx], y[idx]);
             zetaX[idx] = f0_*psipR;
             zetaY[idx] = f0_*psipZ;
             etaX[idx] = -h[idx]*psipZ;
@@ -178,9 +172,7 @@ struct SimpleOrthogonalX : public aGeneratorX2d
         const double x_0 = generator.f0()*psi_0;
         const double x_1 = -fx/(1.-fx)*x_0;
     private:
-    PsiX psiX_;
-    PsiY psiY_;
-    LaplacePsi laplacePsi_;
+    BinaryFunctorsLvl2 psi_;
     double R0_[2], Z0_[2];
     double zeta0_, f0_;
     int firstline_;
@@ -190,9 +182,7 @@ struct SimpleOrthogonalX : public aGeneratorX2d
  * @brief Choose points on separatrix (models aGeneratorX)
  *
  * @ingroup generators
- * @tparam Psi All the template parameters must model aBinaryOperator i.e. the bracket operator() must be callable with two arguments and return a double. 
  */
-template< class Psi, class PsiX, class PsiY, class LaplacePsi>
 struct SeparatrixOrthogonal : public aGeneratorX2d
 {
     typedef dg::OrthogonalTag metric_category;
@@ -200,9 +190,6 @@ struct SeparatrixOrthogonal : public aGeneratorX2d
      * @brief Construct 
      *
      * @param psi
-     * @param psiX
-     * @param psiY
-     * @param laplacePsi
      * @param psi_0
      * @param xX the X-point
      * @param yX the X-point
@@ -210,10 +197,10 @@ struct SeparatrixOrthogonal : public aGeneratorX2d
      * @param y0
      * @param firstline =0 means conformal, =1 means equalarc discretization
      */
-    SeparatrixOrthogonal( Psi psi, PsiX psiX, PsiY psiY, LaplacePsi laplacePsi, double psi_0, //psi_0 must be the closed surface, 0 the separatrix
+    SeparatrixOrthogonal( const BinaryFunctorsLvl2& psi, double psi_0, //psi_0 must be the closed surface, 0 the separatrix
             double xX, double yX, double x0, double y0, int firstline ):
-        psiX_(psiX), psiY_(psiY), laplacePsi_(laplacePsi),
-        sep_( psi, psiX, psiY, xX, yX, x0, y0, firstline)
+        psi_(psi)
+        sep_( psi, xX, yX, x0, y0, firstline)
     {
         firstline_ = firstline;
         f0_ = sep_.get_f();
@@ -221,7 +208,7 @@ struct SeparatrixOrthogonal : public aGeneratorX2d
     bool isConformal()const{return false;}
     bool isOrthogonal()const{return true;}
     double f0() const{return sep_.get_f();}
-    void operator()(  //this one doesn't know if the separatrix comes to lie on a cell boundary or not
+    virtual void generate(  //this one doesn't know if the separatrix comes to lie on a cell boundary or not
          const thrust::host_vector<double>& zeta1d, 
          const thrust::host_vector<double>& eta1d, 
          const unsigned nodeX0, const unsigned nodeX1, 
@@ -235,7 +222,7 @@ struct SeparatrixOrthogonal : public aGeneratorX2d
 
         thrust::host_vector<double> r_init, z_init;
         sep_.compute_rzy( eta1d, nodeX0, nodeX1, r_init, z_init);
-        dg::geo::orthogonal::detail::Nemov<PsiX, PsiY, LaplacePsi> nemov(psiX_, psiY_, laplacePsi_, f0_, firstline_);
+        dg::geo::orthogonal::detail::Nemov nemov(psi_, f0_, firstline_);
 
         //separate integration of inside and outside
         unsigned inside=0;
@@ -327,8 +314,8 @@ struct SeparatrixOrthogonal : public aGeneratorX2d
         etaX.resize(size), etaY.resize(size);
         for( unsigned idx=0; idx<size; idx++)
         {
-            double psipX = psiX_(x[idx], y[idx]);
-            double psipY = psiY_(x[idx], y[idx]);
+            double psipX = psi_.dfx()(x[idx], y[idx]);
+            double psipY = psi_.dfy()(x[idx], y[idx]);
             zetaX[idx] = f0_*psipX;
             zetaY[idx] = f0_*psipY;
             etaX[idx] = -h[idx]*psipY;
@@ -340,10 +327,8 @@ struct SeparatrixOrthogonal : public aGeneratorX2d
     double R0_[2], Z0_[2];
     double f0_;
     int firstline_;
-    PsiX psiX_;
-    PsiY psiY_;
-    LaplacePsi laplacePsi_;
-    dg::geo::detail::SeparatriX<Psi, PsiX, PsiY> sep_;
+    BinaryFunctorsLvl2 psi_;
+    dg::geo::detail::SeparatriX sep_;
 };
 
 // /**
diff --git a/inc/geometries/utilities.h b/inc/geometries/utilities.h
index 9bcab190b..784bad63a 100644
--- a/inc/geometries/utilities.h
+++ b/inc/geometries/utilities.h
@@ -358,7 +358,7 @@ struct FieldRZtau
 
 struct HessianRZtau
 {
-    HessianRZtau( const BinaryFunctorsLvl2): norm_(false), quad_(1), psip_(psip){}
+    HessianRZtau( const BinaryFunctorsLvl2& psip): norm_(false), quad_(1), psip_(psip){}
     // if true goes into positive Z - direction and X else
     void set_quadrant( int quadrant) {quad_ = quadrant;}
     void set_norm( bool normed) {norm_ = normed;}
diff --git a/inc/geometries/utilitiesX.h b/inc/geometries/utilitiesX.h
index 5af4b59f1..ad2dd22be 100644
--- a/inc/geometries/utilitiesX.h
+++ b/inc/geometries/utilitiesX.h
@@ -10,11 +10,6 @@ namespace geo
  * @brief This function finds the X-point via Newton iteration applied to the gradient of psi, 
  *
  * The inverse of the Hessian matrix is computed analytically
- * @tparam PsiR models aBinaryOperator
- * @tparam PsiZ models aBinaryOperator
- * @tparam PsiRR models aBinaryOperator
- * @tparam PsiRZ models aBinaryOperator
- * @tparam PsiZZ models aBinaryOperator
     @param psiR \f$ \partial_R \psi(R,Z)\f$, where R, Z are cylindrical coordinates
     @param psiZ \f$ \partial_Z \psi(R,Z)\f$, where R, Z are cylindrical coordinates
     @param psiRR \f$ \partial_R\partial_R \psi(R,Z)\f$, where R, Z are cylindrical coordinates
@@ -24,10 +19,9 @@ namespace geo
  * @param Z_X start value on input, X-point on output
  * @ingroup misc
  */
-template<class PsiR, class PsiZ, class PsiRR, class PsiRZ, class PsiZZ>
-void findXpoint( const PsiR& psiR, const PsiZ& psiZ, const PsiRR& psiRR, const PsiRZ& psiRZ, const PsiZZ& psiZZ, double& R_X, double& Z_X)
+void findXpoint( const BinaryFunctorsLvl2& psi, double& R_X, double& Z_X)
 {
-    dg::geo::HessianRZtau<PsiR, PsiZ, PsiRR, PsiRZ, PsiZZ> hessianRZtau(  psiR, psiZ, psiRR, psiRZ, psiZZ);
+    dg::geo::HessianRZtau hessianRZtau(  psi);
     thrust::host_vector<double> X(2,0), XN(X), X_OLD(X);
     X[0] = R_X, X[1] = Z_X;
     double eps = 1e10, eps_old= 2e10;
@@ -49,11 +43,10 @@ namespace detail
 /**
  * @brief This struct finds and stores the X-point and can act in a root finding routine to find points on the perpendicular line through the X-point 
  */
-template<class Psi, class PsiR, class PsiZ>
 struct XCross
 {
     
-    XCross( Psi psi, PsiR psiR, PsiZ psiZ, double R_X, double Z_X, double distance=1): fieldRZtau_(psiR, psiZ), psip_(psi), dist_(distance)
+    XCross( const BinaryFunctorsLvl1& psi, double R_X, double Z_X, double distance=1): fieldRZtau_(psi), psip_(psi), dist_(distance)
     {
         R_X_ = R_X, Z_X_ = Z_X;
         //std::cout << "X-point set at "<<R_X_<<" "<<Z_X_<<"\n";
@@ -77,7 +70,7 @@ struct XCross
         if( quad_ == 0 || quad_ == 2) { begin[1] += x;}
         else if( quad_ == 1 || quad_ == 3) { begin[0] += x;}
 
-        double psi0 = psip_(begin[0], begin[1]);
+        double psi0 = psip_.f()(begin[0], begin[1]);
         while( (eps < eps_old || eps > 1e-4 ) && eps > 1e-10)
         {
             eps_old = eps; end_old = end;
@@ -105,8 +98,8 @@ struct XCross
 
     private:
     int quad_;
-    dg::geo::FieldRZtau<PsiR, PsiZ> fieldRZtau_;
-    Psi psip_;
+    dg::geo::FieldRZtau fieldRZtau_;
+    BinaryFunctorsLvl1 psip_;
     double R_X_, Z_X_;
     double R_i[4], Z_i[4];
     double dist_;
@@ -274,32 +267,30 @@ double construct_psi_values( XFieldFinv fpsiMinv,
 
 
 //!ATTENTION: choosing h on separatrix is a mistake if LaplacePsi does not vanish at X-point
-template< class Psi>
 struct PsipSep
 {
-    PsipSep( Psi psi): psip_(psi), Z_(0){}
+    PsipSep( const aBinaryFunctor& psi): psip_(psi), Z_(0){}
     void set_Z( double z){ Z_=z;}
-    double operator()(double R) { return psip_(R, Z_);}
+    double operator()(double R) { return psip_.get()(R, Z_);}
     private:
-    Psi psip_;
+    Handle<aBinaryFunctor> psip_;
     double Z_;
 };
 
 //!ATTENTION: choosing h on separatrix is a mistake if LaplacePsi does not vanish at X-point
 //This leightweights struct and its methods finds the initial R and Z values and the coresponding f(\psi) as 
 //good as it can, i.e. until machine precision is reached (like FpsiX just for separatrix)
-template< class Psi, class PsiX, class PsiY>
 struct SeparatriX
 {
-    SeparatriX( Psi psi, PsiX psiX, PsiY psiY, double xX, double yX, double x0, double y0, int firstline): 
+    SeparatriX( const BinaryFunctorsLvl1& psi, double xX, double yX, double x0, double y0, int firstline): 
         mode_(firstline),
-        fieldRZYequi_(psiX, psiY), fieldRZYTequi_(psiX, psiY, x0, y0), fieldRZYZequi_(psiX, psiY),
-        fieldRZYconf_(psiX, psiY), fieldRZYTconf_(psiX, psiY, x0, y0), fieldRZYZconf_(psiX, psiY)
+        fieldRZYequi_(psi), fieldRZYTequi_(psi, x0, y0), fieldRZYZequi_(psi),
+        fieldRZYconf_(psi), fieldRZYTconf_(psi, x0, y0), fieldRZYZconf_(psi)
     {
         //find four points on the separatrix and construct y coordinate at those points and at last construct f 
         //////////////////////////////////////////////
         double R_X = xX; double Z_X = yX;
-        PsipSep<Psi> psip_sep( psi);
+        PsipSep psip_sep( psi.f());
         psip_sep.set_Z( Z_X + 1.);
         double R_min = R_X, R_max = R_X + 10;
         dg::bisection1d( psip_sep, R_min, R_max, 1e-13);
@@ -467,12 +458,12 @@ struct SeparatriX
         return f_psi_;
     }
     int mode_;
-    dg::geo::equalarc::FieldRZY<PsiX, PsiY>   fieldRZYequi_;
-    dg::geo::equalarc::FieldRZYT<PsiX, PsiY> fieldRZYTequi_;
-    dg::geo::equalarc::FieldRZYZ<PsiX, PsiY> fieldRZYZequi_;
-    dg::geo::ribeiro::FieldRZY<PsiX, PsiY>    fieldRZYconf_;
-    dg::geo::ribeiro::FieldRZYT<PsiX, PsiY>  fieldRZYTconf_;
-    dg::geo::ribeiro::FieldRZYZ<PsiX, PsiY>  fieldRZYZconf_;
+    dg::geo::equalarc::FieldRZY  fieldRZYequi_;
+    dg::geo::equalarc::FieldRZYT fieldRZYTequi_;
+    dg::geo::equalarc::FieldRZYZ fieldRZYZequi_;
+    dg::geo::ribeiro::FieldRZY   fieldRZYconf_;
+    dg::geo::ribeiro::FieldRZYT  fieldRZYTconf_;
+    dg::geo::ribeiro::FieldRZYZ  fieldRZYZconf_;
     unsigned N_steps_;
     double R_i[4], Z_i[4], y_i[4];
     double f_psi_;
@@ -484,16 +475,15 @@ namespace orthogonal
 namespace detail
 {
 //find points on the perp line through the X-point
-template< class Psi, class PsiX, class PsiY>
 struct InitialX
 {
 
-    InitialX( Psi psi, PsiX psiX, PsiY psiY, double xX, double yX): 
-        psip_(psi), fieldRZtau_(psiX, psiY), 
-        xpointer_(psi, psiX, psiY, xX, yX, 1e-4)
+    InitialX( const BinaryFunctorsLvl1& psi, double xX, double yX): 
+        psip_(psi), fieldRZtau_(psi), 
+        xpointer_(psi, xX, yX, 1e-4)
     {
         //constructor finds four points around X-point and integrates them a bit away from it
-        dg::geo::FieldRZtau<PsiX, PsiY> fieldRZtau_(psiX, psiY);
+        dg::geo::FieldRZtau fieldRZtau_(psi);
         thrust::host_vector<double> begin( 2, 0), end(begin), temp(begin), end_old(end);
         double eps[] = {1e-11, 1e-12, 1e-11, 1e-12};
         for( unsigned i=0; i<4; i++)
@@ -507,7 +497,7 @@ struct InitialX
             begin[0] = R_i_[i], begin[1] = Z_i_[i];
             double eps = 1e10, eps_old = 2e10;
             unsigned N=10;
-            double psi0 = psip_(begin[0], begin[1]), psi1 = 1e3*psi0; 
+            double psi0 = psip_.f()(begin[0], begin[1]), psi1 = 1e3*psi0; 
             while( (eps < eps_old || eps > 1e-5 ) && eps > 1e-9)
             {
                 eps_old = eps; end_old = end;
@@ -520,7 +510,7 @@ struct InitialX
             R_i_[i] = end_old[0], Z_i_[i] = end_old[1];
             begin[0] = R_i_[i], begin[1] = Z_i_[i];
             eps = 1e10, eps_old = 2e10; N=10;
-            psi0 = psip_(begin[0], begin[1]), psi1 = -0.01; 
+            psi0 = psip_.f()(begin[0], begin[1]), psi1 = -0.01; 
             if( i==0||i==2)psi1*=-1.;
             while( (eps < eps_old || eps > 1e-5 ) && eps > 1e-9)
             {
@@ -532,7 +522,7 @@ struct InitialX
                 //std::cout << " for N "<< N<<" eps is "<<eps<<"\n";
             }
             R_i_[i] = end_old[0], Z_i_[i] = end_old[1];
-            std::cout << "Quadrant "<<i<<" Found initial point: "<<R_i_[i]<<" "<<Z_i_[i]<<" "<<psip_(R_i_[i], Z_i_[i])<<"\n";
+            std::cout << "Quadrant "<<i<<" Found initial point: "<<R_i_[i]<<" "<<Z_i_[i]<<" "<<psip_.f()(R_i_[i], Z_i_[i])<<"\n";
 
         }
     }
@@ -561,7 +551,7 @@ struct InitialX
             while( (eps < eps_old||eps > 1e-7) && eps > 1e-11)
             {
                 eps_old = eps; end_old = end;
-                dg::stepperRK17( fieldRZtau_, begin, end, psip_(begin[0], begin[1]), psi, steps);
+                dg::stepperRK17( fieldRZtau_, begin, end, psip_.f()(begin[0], begin[1]), psi, steps);
                 eps = sqrt( (end[0]-end_old[0])*(end[0]- end_old[0]) + (end[1]-end_old[1])*(end[1]-end_old[1]));
                 //std::cout << "rel. error is "<<eps<<" with "<<steps<<" steps\n";
                 if( std::isnan(eps)) { eps = eps_old/2.; end = end_old; }
@@ -582,9 +572,9 @@ struct InitialX
 
 
     private:
-    Psi psip_;
-    const dg::geo::FieldRZtau<PsiX, PsiY> fieldRZtau_;
-    dg::geo::detail::XCross<Psi, PsiX, PsiY> xpointer_;
+    BinaryFunctorsLvl1 psip_;
+    const dg::geo::FieldRZtau fieldRZtau_;
+    dg::geo::detail::XCross xpointer_;
     double R_i_[4], Z_i_[4];
 
 };
-- 
GitLab


From 15b25815edb3ca6c144120a5e83334b049da2a1d Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Tue, 15 Aug 2017 05:36:31 -0700
Subject: [PATCH 164/453] updated X Generators

---
 inc/dg/geometry/curvilinearX.h         |  4 +--
 inc/dg/geometry/generatorX.h           | 18 +++++------
 inc/dg/geometry/refined_curvilinearX.h |  4 +--
 inc/geometries/ribeiroX.h              | 44 ++++++--------------------
 inc/geometries/separatrix_orthogonal.h | 34 +++++++++++---------
 5 files changed, 42 insertions(+), 62 deletions(-)

diff --git a/inc/dg/geometry/curvilinearX.h b/inc/dg/geometry/curvilinearX.h
index 9e2a9138b..5d4b1bc82 100644
--- a/inc/dg/geometry/curvilinearX.h
+++ b/inc/dg/geometry/curvilinearX.h
@@ -34,7 +34,7 @@ struct CurvilinearProductGridX3d : public dg::aGeometryX3d
      */
     CurvilinearProductGridX3d( const aGeneratorX2d& generator, 
         double fx, double fy, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, double fx, double fy, bc bcx=dg::DIR, bc bcy=dg::PER, bc bcz=dg::PER):
-        dg::aGeometryX3d( generator.zeta0(), generator.zeta1(), generator.eta0(), generator.eta1(), 0., 2.*M_PI, fx,fy,n, Nx, Ny, Nz, bcx, bcy, bcz)
+        dg::aGeometryX3d( generator.zeta0(fx), generator.zeta1(fx), generator.eta0(fy), generator.eta1(fy), 0., 2.*M_PI, fx,fy,n, Nx, Ny, Nz, bcx, bcy, bcz)
     { 
         map_.resize(3);
         handle_ = generator;
@@ -123,7 +123,7 @@ struct CurvilinearGridX2d : public dg::aGeometryX2d
      * @param bcy boundary condition in second coordinate
      */
     CurvilinearGridX2d( const aGeneratorX2d& generator, double fx, double fy, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx=dg::DIR, bc bcy=dg::PER):
-        dg::aGeometryX2d( generator.zeta0(), generator.zeta1(), generator.eta0(), generator.eta1(),fx,fy, n, Nx, Ny, bcx, bcy), handle_(generator)
+        dg::aGeometryX2d( generator.zeta0(fx), generator.zeta1(fx), generator.eta0(fy), generator.eta1(fy),fx,fy, n, Nx, Ny, bcx, bcy), handle_(generator)
     {
         construct( n,Nx,Ny);
     }
diff --git a/inc/dg/geometry/generatorX.h b/inc/dg/geometry/generatorX.h
index 638c0bc78..da3037b60 100644
--- a/inc/dg/geometry/generatorX.h
+++ b/inc/dg/geometry/generatorX.h
@@ -14,10 +14,10 @@ is a product space.
 */
 struct aGeneratorX2d
 {
-    double zeta0() const{return do_zeta0();}
-    double zeta1() const{return do_zeta1();}
-    double eta0() const{return do_eta0();}
-    double eta1() const{return do_eta1();}
+    unsigned zeta0(double fx) const{return do_zeta0(fx);}
+    unsigned zeta1(double fx) const{return do_zeta1(fx);}
+    unsigned eta0(double fy) const{return do_eta0(fy);}
+    unsigned eta1(double fy) const{return do_eta1(fy);}
     ///@brief sparsity pattern for metric
     bool isOrthogonal() const { return doIsOrthogonal(); }
 
@@ -79,11 +79,11 @@ struct aGeneratorX2d
          thrust::host_vector<double>& zetaY, 
          thrust::host_vector<double>& etaX, 
          thrust::host_vector<double>& etaY) const = 0;
-    virtual bool doIsOrthogonal()const{return false;}
-    virtual double do_zeta0() const=0;
-    virtual double do_zeta1() const=0;
-    virtual double do_eta0() const=0;
-    virtual double do_eta1() const=0;
+    virtual bool do_isOrthogonal()const{return false;}
+    virtual unsigned do_zeta0(double fx) const=0;
+    virtual unsigned do_zeta1(double fx) const=0;
+    virtual unsigned do_eta0(double fy) const=0;
+    virtual unsigned do_eta1(double fy) const=0;
 
 
 };
diff --git a/inc/dg/geometry/refined_curvilinearX.h b/inc/dg/geometry/refined_curvilinearX.h
index 6eef96e36..46b117dc2 100644
--- a/inc/dg/geometry/refined_curvilinearX.h
+++ b/inc/dg/geometry/refined_curvilinearX.h
@@ -32,7 +32,7 @@ struct CurvilinearProductRefinedGridX3d : public dg::aGeometryX3d
      */
     CurvilinearProductRefinedGridX3d( const aRefinementX2d& ref, const aGeneratorX2d& generator, 
         double fx, double fy, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, double fx, double fy, bc bcx=dg::DIR, bc bcy=dg::PER, bc bcz=dg::PER):
-        dg::aGeometryX3d( generator.zeta0(), generator.zeta1(), generator.eta0(), generator.eta1(), 0., 2.*M_PI, ref.fx_new(Nx,fx),ref.fy_new(Ny,fy),n, ref.nx_new(Nx,fx), ref.ny_new(Ny,fy), Nz, bcx, bcy, bcz), map_(3)
+        dg::aGeometryX3d( generator.zeta0(fx), generator.zeta1(fx), generator.eta0(fy), generator.eta1(fy), 0., 2.*M_PI, ref.fx_new(Nx,fx),ref.fy_new(Ny,fy),n, ref.nx_new(Nx,fx), ref.ny_new(Ny,fy), Nz, bcx, bcy, bcz), map_(3)
     { 
         handle_ = generator;
         ref_=ref;
@@ -132,7 +132,7 @@ struct CurvilinearRefinedGridX2d : public dg::aGeometryX2d
      * @param bcy boundary condition in second coordinate
      */
     CurvilinearRefinedGridX2d( const aRefinementX2d& ref, const aGeneratorX2d& generator, double fx, double fy, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx=dg::DIR, bc bcy=dg::PER):
-        dg::aGeometryX2d( generator.zeta0(), generator.zeta1(), generator.eta0(), generator.eta1(),ref.fx_new(Nx,fx),ref.fy_new(Ny,fy),n, ref.nx_new(Nx,fx), ref.ny_new(Ny,fy), bcx, bcy)
+        dg::aGeometryX2d( generator.zeta0(fx), generator.zeta1(fx), generator.eta0(fy), generator.eta1(fy),ref.fx_new(Nx,fx),ref.fy_new(Ny,fy),n, ref.nx_new(Nx,fx), ref.ny_new(Ny,fy), bcx, bcy)
     {
         handle_ = generator;
         ref_=ref;
diff --git a/inc/geometries/ribeiroX.h b/inc/geometries/ribeiroX.h
index aa1b413e9..0383a6a79 100644
--- a/inc/geometries/ribeiroX.h
+++ b/inc/geometries/ribeiroX.h
@@ -5,6 +5,7 @@
 #include "dg/backend/evaluationX.cuh"
 #include "dg/backend/weightsX.cuh"
 #include "dg/runge_kutta.h"
+#include "dg/geometry/generatorX.h"
 #include "utilitiesX.h"
 #include "ribeiro.h"
 
@@ -239,7 +240,7 @@ struct XFieldFinv
  * @ingroup generators
  * @tparam Psi All the template parameters must model aBinaryOperator i.e. the bracket operator() must be callable with two arguments and return a double. 
  */
-struct RibeiroX
+struct RibeiroX : public aGeneratorX2d
 {
     RibeiroX( const BinaryFunctorsLvl2& psi, double psi_0, double fx, 
             double xX, double yX, double x0, double y0):
@@ -251,24 +252,10 @@ struct RibeiroX
         zeta1_= -fx/(1.-fx)*zeta0_;
         x0_=x0, y0_=y0, psi0_=psi_0;
     }
+    private:
     bool isConformal()const{return false;}
-    bool isOrthogonal()const{return false;}
+    bool do_isOrthogonal()const{return false;}
     double f0() const{return f0_;}
-    /**
-     * @brief The length of the zeta-domain
-     *
-     * Call before discretizing the zeta domain
-     * @return length of zeta-domain (f0*(psi_1-psi_0))
-     * @note the length is always positive
-     */
-    double width() const{return lx_;}
-    /**
-     * @brief 2pi (length of the eta domain)
-     *
-     * Always returns 2pi
-     * @return 2pi 
-     */
-    double height() const{return 2.*M_PI;}
     /**
      * @brief The vector f(x)
      *
@@ -276,23 +263,7 @@ struct RibeiroX
      */
     thrust::host_vector<double> fx() const{ return fx_;}
     double psi1() const {return psi_1_numerical_;}
-    /**
-    * @brief Generate grid points and elements of the Jacobian 
-    *
-    * @param zeta1d (input) a list of \f$ N_\zeta\f$ points \f$ f_0\psi_0<\zeta_i< -f_\zeta\zeta_0/(1-f_\zeta)\f$
-    * @param eta1d (input) a list of \f$ N_\eta\f$ points \f$ 0<\eta_j<\f$height() 
-    * @param x (output) the list of \f$ N_\eta N_\zeta\f$ coordinates \f$ x(\zeta_i, \eta_j)\f$ 
-    * @param y (output) the list of \f$ N_\eta N_\zeta\f$ coordinates \f$ y(\zeta_i, \eta_j)\f$ 
-    * @param nodeX0 is the index of the first point in eta1d  after the first jump in topology in \f$ \eta\f$
-    * @param nodeX1 is the index of the first point in eta1d  after the second jump in topology in \f$ \eta\f$
-    * @param zetaX (output) the list of \f$ N_\eta N_\zeta\f$ elements \f$ \partial\zeta/\partial x (\zeta_i, \eta_j)\f$ 
-    * @param zetaY (output) the list of \f$ N_\eta N_\zeta\f$ elements \f$ \partial\zeta/\partial y (\zeta_i, \eta_j)\f$ 
-    * @param etaX (output) the list of \f$ N_\eta N_\zeta\f$ elements \f$ \partial\eta/\partial x (\zeta_i, \eta_j)\f$ 
-    * @param etaY (output) the list of \f$ N_\eta N_\zeta\f$ elements \f$ \partial\eta/\partial y (\zeta_i, \eta_j)\f$ 
-    @note the \f$ \zeta\f$ coordinate is contiguous in memory
-     * @note All the resulting vectors are write-only and get properly resized
-    */
-    void generate( 
+    void do_generate( 
          const thrust::host_vector<double>& zeta1d, 
          const thrust::host_vector<double>& eta1d, 
          const unsigned nodeX0, const unsigned nodeX1, 
@@ -331,6 +302,11 @@ struct RibeiroX
             }
         }
     }
+
+    virtual unsigned do_zeta0(double fx) const { return zeta0_; }
+    virtual unsigned do_zeta1(double fx) const { return zeta1_;}
+    virtual unsigned do_eta0(double fy) const { return -2.*M_PI*fy/(1.-2.*fy); }
+    virtual unsigned do_eta1(double fy) const { return 2.*M_PI*(1.+fy/(1.-2.*fy));}
     private:
     BinaryFunctorsLvl2 psi_;
     dg::geo::ribeiro::detail::XFieldFinv fpsiMinv_; 
diff --git a/inc/geometries/separatrix_orthogonal.h b/inc/geometries/separatrix_orthogonal.h
index 18da2cb43..af4187353 100644
--- a/inc/geometries/separatrix_orthogonal.h
+++ b/inc/geometries/separatrix_orthogonal.h
@@ -121,9 +121,9 @@ void computeX_rzy( const BinaryFunctorsLvl1& psi,
 struct SimpleOrthogonalX : public aGeneratorX2d
 {
     SimpleOrthogonalX(): f0_(1), firstline_(0){}
-    SimpleOrthogonalX( const BinaryFunctorsLvl2& psi, double psi_0, //psi_0 must be the closed surface, 0 the separatrix
-            double xX, double yX, double x0, double y0, int firstline =0):
-        psi_(psi)
+    ///psi_0 must be the closed surface, 0 the separatrix
+    SimpleOrthogonalX( const BinaryFunctorsLvl2& psi, double psi_0, 
+            double xX, double yX, double x0, double y0, int firstline =0): psi_(psi)
     {
         firstline_ = firstline;
         orthogonal::detail::Fpsi fpsi(psi_, x0, y0, firstline);
@@ -133,10 +133,11 @@ struct SimpleOrthogonalX : public aGeneratorX2d
         dg::geo::orthogonal::detail::InitialX initX(psi_, xX, yX);
         initX.find_initial(psi_0, R0_, Z0_);
     }
+    private:
     bool isConformal()const{return false;}
-    bool isOrthogonal()const{return true;}
+    bool do_isOrthogonal()const{return true;}
     double f0() const{return f0_;}
-    virtual void generate( //this one doesn't know if the separatrix comes to lie on a cell boundary or not
+    virtual void do_generate( //this one doesn't know if the separatrix comes to lie on a cell boundary or not
          const thrust::host_vector<double>& zeta1d, 
          const thrust::host_vector<double>& eta1d, 
          const unsigned nodeX0, const unsigned nodeX1,
@@ -166,12 +167,10 @@ struct SimpleOrthogonalX : public aGeneratorX2d
             etaY[idx] = +h[idx]*psipR;
         }
     }
-    double laplace(double x, double y) {return f0_*laplacePsi_(x,y);}
-        y_0= -2.*M_PI*fy/(1.-2.*fy); 
-        y_1= 2.*M_PI*(1.+fy/(1.-2.*fy));
-        const double x_0 = generator.f0()*psi_0;
-        const double x_1 = -fx/(1.-fx)*x_0;
-    private:
+    unsigned do_zeta0(double fx) const { return zeta0_; }
+    unsigned do_zeta1(double fx) const { return -fx/(1.-fx)*zeta0_;}
+    unsigned do_eta0(double fy) const { return -2.*M_PI*fy/(1.-2.*fy); }
+    unsigned do_eta1(double fy) const { return 2.*M_PI*(1.+fy/(1.-2.*fy));}
     BinaryFunctorsLvl2 psi_;
     double R0_[2], Z0_[2];
     double zeta0_, f0_;
@@ -204,11 +203,13 @@ struct SeparatrixOrthogonal : public aGeneratorX2d
     {
         firstline_ = firstline;
         f0_ = sep_.get_f();
+        psi_0_=psi_0;
     }
+    private:
     bool isConformal()const{return false;}
-    bool isOrthogonal()const{return true;}
+    bool do_isOrthogonal()const{return true;}
     double f0() const{return sep_.get_f();}
-    virtual void generate(  //this one doesn't know if the separatrix comes to lie on a cell boundary or not
+    virtual void do_generate(  //this one doesn't know if the separatrix comes to lie on a cell boundary or not
          const thrust::host_vector<double>& zeta1d, 
          const thrust::host_vector<double>& eta1d, 
          const unsigned nodeX0, const unsigned nodeX1, 
@@ -322,10 +323,13 @@ struct SeparatrixOrthogonal : public aGeneratorX2d
             etaY[idx] = +h[idx]*psipX;
         }
     }
-    double laplace(double x, double y) {return f0_*laplacePsi_(x,y);}
+    virtual unsigned do_zeta0(double fx) const { return f0_*psi_0_; }
+    virtual unsigned do_zeta1(double fx) const { return -fx/(1.-fx)*f0_*psi_0_;}
+    virtual unsigned do_eta0(double fy) const { return -2.*M_PI*fy/(1.-2.*fy); }
+    virtual unsigned do_eta1(double fy) const { return 2.*M_PI*(1.+fy/(1.-2.*fy));}
     private:
     double R0_[2], Z0_[2];
-    double f0_;
+    double f0_, psi_0_;
     int firstline_;
     BinaryFunctorsLvl2 psi_;
     dg::geo::detail::SeparatriX sep_;
-- 
GitLab


From b824fce9fbd5453999ecace54ea800d47fdac2db Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Tue, 15 Aug 2017 06:53:50 -0700
Subject: [PATCH 165/453] made flux_t compile: corrected fluxfunctions and
 Cloneable

---
 inc/dg/backend/{tensor.cuh => dgtensor.cuh} |  0
 inc/dg/backend/interpolation.cuh            |  2 +-
 inc/dg/backend/xspacelib.cuh                |  8 +--
 inc/dg/geometry/generator.h                 |  4 +-
 inc/dg/geometry/generatorX.h                |  2 +-
 inc/geometries/flux.h                       | 55 +++++------------
 inc/geometries/flux_t.cu                    | 46 +++++++-------
 inc/geometries/fluxfunctions.h              | 67 +++++++++------------
 inc/geometries/guenther.h                   |  4 +-
 inc/geometries/hector.h                     | 39 ++++--------
 inc/geometries/magnetic_field.h             | 23 ++++---
 inc/geometries/ribeiro.h                    | 21 +++----
 inc/geometries/simple_orthogonal.h          | 29 ++-------
 inc/geometries/solovev.h                    | 16 ++---
 inc/geometries/taylor.h                     |  2 +-
 inc/geometries/toroidal.h                   | 31 +++-------
 inc/geometries/utilities.h                  |  6 +-
 17 files changed, 138 insertions(+), 217 deletions(-)
 rename inc/dg/backend/{tensor.cuh => dgtensor.cuh} (100%)

diff --git a/inc/dg/backend/tensor.cuh b/inc/dg/backend/dgtensor.cuh
similarity index 100%
rename from inc/dg/backend/tensor.cuh
rename to inc/dg/backend/dgtensor.cuh
diff --git a/inc/dg/backend/interpolation.cuh b/inc/dg/backend/interpolation.cuh
index 24f6b22fe..dcf1f4d5b 100644
--- a/inc/dg/backend/interpolation.cuh
+++ b/inc/dg/backend/interpolation.cuh
@@ -7,7 +7,7 @@
 #include "evaluation.cuh"
 #include "functions.h"
 #include "creation.cuh"
-#include "tensor.cuh"
+#include "dgtensor.cuh"
 #include "operator_tensor.cuh"
 
 /*! @file
diff --git a/inc/dg/backend/xspacelib.cuh b/inc/dg/backend/xspacelib.cuh
index e8ccb3fcf..57087b11a 100644
--- a/inc/dg/backend/xspacelib.cuh
+++ b/inc/dg/backend/xspacelib.cuh
@@ -11,7 +11,7 @@
 #include "dlt.h"
 #include "operator.h"
 #include "operator_tensor.cuh"
-#include "tensor.cuh"
+#include "dgtensor.cuh"
 #include "interpolation.cuh" //makes typedefs available
 
 
@@ -94,8 +94,8 @@ cusp::coo_matrix<int, double, cusp::host_memory> backscatter( const aTopology2d&
     dg::Operator<double> forward( g.dlt().forward());
     dg::Operator<double> backward1d = backwardeq*forward;
 
-    Matrix transformX = dg::tensor( g.Nx(), backward1d);
-    Matrix transformY = dg::tensor( g.Ny(), backward1d);
+    Matrix transformX = dg::tensorproduct( g.Nx(), backward1d);
+    Matrix transformY = dg::tensorproduct( g.Ny(), backward1d);
     Matrix backward = dg::dgtensor( g.n(), transformY, transformX);
 
     //thrust::host_vector<int> map = dg::create::gatherMap( g.n(), g.Nx(), g.Ny());
@@ -120,7 +120,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> backscatter( const aTopology3d&
 {
     Grid2d g2d( g.x0(), g.x1(), g.y0(), g.y1(), g.n(), g.Nx(), g.Ny(), g.bcx(), g.bcy());
     cusp::coo_matrix<int,double, cusp::host_memory> back2d = backscatter( g2d);
-    return dgtensor<double>( 1, tensor<double>( g.Nz(), delta(1)), back2d);
+    return dgtensor<double>( 1, tensorproduct<double>( g.Nz(), delta(1)), back2d);
 }
 
 
diff --git a/inc/dg/geometry/generator.h b/inc/dg/geometry/generator.h
index 2ca67b251..ba81a44d4 100644
--- a/inc/dg/geometry/generator.h
+++ b/inc/dg/geometry/generator.h
@@ -19,7 +19,7 @@ struct aGenerator2d
     ///@brief length in \f$ \eta\f$ of the computational space
     double height() const{return do_height();}
     ///@brief sparsity pattern for metric
-    bool isOrthogonal() const { return doIsOrthogonal(); }
+    bool isOrthogonal() const { return do_isOrthogonal(); }
 
     /**
     * @brief Generate grid points and elements of the Jacobian 
@@ -77,7 +77,7 @@ struct aGenerator2d
          thrust::host_vector<double>& etaY) const = 0;
      virtual double do_width() const =0;
      virtual double do_height() const =0;
-     virtual bool doIsOrthogonal()const{return false;}
+     virtual bool do_isOrthogonal()const{return false;}
 
 
 };
diff --git a/inc/dg/geometry/generatorX.h b/inc/dg/geometry/generatorX.h
index da3037b60..fa9aa6245 100644
--- a/inc/dg/geometry/generatorX.h
+++ b/inc/dg/geometry/generatorX.h
@@ -19,7 +19,7 @@ struct aGeneratorX2d
     unsigned eta0(double fy) const{return do_eta0(fy);}
     unsigned eta1(double fy) const{return do_eta1(fy);}
     ///@brief sparsity pattern for metric
-    bool isOrthogonal() const { return doIsOrthogonal(); }
+    bool isOrthogonal() const { return do_isOrthogonal(); }
 
     /**
     * @brief Generate grid points and elements of the Jacobian 
diff --git a/inc/geometries/flux.h b/inc/geometries/flux.h
index d221a8cc0..550cef491 100644
--- a/inc/geometries/flux.h
+++ b/inc/geometries/flux.h
@@ -5,13 +5,12 @@
 #include "dg/backend/interpolation.cuh"
 #include "dg/backend/operator.h"
 #include "dg/backend/derivatives.h"
+#include "dg/geometry/geometry.h"
 #include "dg/functors.h"
 #include "dg/runge_kutta.h"
 #include "dg/nullstelle.h"
-#include "dg/geometry.h"
 #include "fluxfunctions.h"
 #include "ribeiro.h"
-#include "generator.h"
 
 
 namespace dg
@@ -126,7 +125,7 @@ struct Fpsi
  * @brief A symmetry flux generator
  * @ingroup generators
  */
-struct FluxGenerator : public aGridGenerator
+struct FluxGenerator : public aGenerator2d
 {
     /**
      * @brief Construct a symmetry flux grid generator
@@ -161,25 +160,13 @@ struct FluxGenerator : public aGridGenerator
         //std::cout << "lx_ = "<<lx_<<"\n";
     }
 
-    /**
-     * @brief The length of the zeta-domain
-     *
-     * Call before discretizing the zeta domain
-     * @return length of zeta-domain (f0*(psi_1-psi_0))
-     * @note the length is always positive
-     */
-    virtual double width() const{return lx_;}
-    /**
-     * @brief 2pi (length of the eta domain)
-     *
-     * Always returns 2pi
-     * @return 2pi 
-     */
-    virtual double height() const{return 2.*M_PI;}
     virtual FluxGenerator* clone() const{return new FluxGenerator(*this);}
 
     private:
-    virtual void generate( 
+    // length of zeta-domain (f0*(psi_1-psi_0))
+    virtual double do_width() const{return lx_;}
+    virtual double do_height() const{return 2.*M_PI;}
+    virtual void do_generate( 
          const thrust::host_vector<double>& zeta1d, 
          const thrust::host_vector<double>& eta1d, 
          thrust::host_vector<double>& x, 
@@ -228,7 +215,7 @@ struct FluxGenerator : public aGridGenerator
  * @brief Same as the Ribeiro class just but uses psi as a flux label directly
  * @ingroup generators
  */
-struct RibeiroFluxGenerator : public aGridGenerator
+struct RibeiroFluxGenerator : public aGenerator2d
 {
     /**
      * @brief Construct a flux aligned grid generator
@@ -241,7 +228,7 @@ struct RibeiroFluxGenerator : public aGridGenerator
      * @param mode This parameter indicates the adaption type used to create the grid: 0 is no adaption, 1 is an equalarc adaption
      */
     RibeiroFluxGenerator( const BinaryFunctorsLvl2& psi, double psi_0, double psi_1, double x0, double y0, int mode=0):
-        psi_(psi), mode_(mode)
+        psip_(psi), mode_(mode)
     {
         psi0_ = psi_0, psi1_ = psi_1;
         assert( psi_1 != psi_0);
@@ -252,25 +239,13 @@ struct RibeiroFluxGenerator : public aGridGenerator
         x0_=x0, y0_=y0, psi0_=psi_0, psi1_=psi_1;
         //std::cout << "lx_ = "<<lx_<<"\n";
     }
-    /**
-     * @brief The length of the zeta-domain
-     *
-     * Call before discretizing the zeta domain
-     * @return length of zeta-domain (f0*(psi_1-psi_0))
-     * @note the length is always positive
-     */
-    virtual double width() const{return lx_;}
-    /**
-     * @brief 2pi (length of the eta domain)
-     *
-     * Always returns 2pi
-     * @return 2pi 
-     */
-    virtual double height() const{return 2.*M_PI;}
     virtual RibeiroFluxGenerator* clone() const{return new RibeiroFluxGenerator(*this);}
 
     private:
-    virtual void generate( 
+    //length of zeta-domain (f0*(psi_1-psi_0))
+    virtual double do_width() const{return lx_;}
+    virtual double do_height() const{return 2.*M_PI;}
+    virtual void do_generate( 
          const thrust::host_vector<double>& zeta1d, 
          const thrust::host_vector<double>& eta1d, 
          thrust::host_vector<double>& x, 
@@ -285,9 +260,9 @@ struct RibeiroFluxGenerator : public aGridGenerator
         for( unsigned i=0; i<psi_x.size(); i++)
             psi_x[i] = zeta1d[i]/f0_ +psi0_;
 
-        ribeiro::detail::Fpsi fpsi(psi_, x0_, y0_, mode_);
-        dg::geo::ribeiro::FieldRZYRYZY fieldRZYRYZYribeiro(psi_);
-        dg::geo::equalarc::FieldRZYRYZY fieldRZYRYZYequalarc(psi_);
+        ribeiro::detail::Fpsi fpsi(psip_, x0_, y0_, mode_);
+        dg::geo::ribeiro::FieldRZYRYZY fieldRZYRYZYribeiro(psip_);
+        dg::geo::equalarc::FieldRZYRYZY fieldRZYRYZYequalarc(psip_);
         thrust::host_vector<double> fx_;
         fx_.resize( zeta1d.size());
         thrust::host_vector<double> f_p(fx_);
diff --git a/inc/geometries/flux_t.cu b/inc/geometries/flux_t.cu
index be4234c13..32d331e4b 100644
--- a/inc/geometries/flux_t.cu
+++ b/inc/geometries/flux_t.cu
@@ -9,7 +9,7 @@
 #include "dg/functors.h"
 
 #include "dg/backend/timer.cuh"
-#include "curvilinear.h"
+#include "dg/geometry/curvilinear.h"
 //#include "guenther.h"
 #include "solovev.h"
 #include "flux.h"
@@ -72,13 +72,12 @@ int main( int argc, char* argv[])
     //solovev::detail::Fpsi fpsi( gp, -10);
     std::cout << "Constructing flux grid ... \n";
     t.tic();
-    MagneticField c( gp);
-    dg::geo::FluxGenerator<Psip, PsipR, PsipZ, PsipRR, PsipRZ, PsipZZ, Ipol, IpolR, IpolZ>
-        flux( c.psip, c.psipR, c.psipZ, c.psipRR, c.psipRZ, c.psipZZ, c.ipol, c.ipolR, c.ipolZ, psi_0, psi_1, gp.R_0, 0., 1);
-    dg::CurvilinearGrid3d<dg::HVec> g3d(&flux, n, Nx, Ny,Nz, dg::DIR);
-    dg::CurvilinearGrid2d<dg::HVec> g2d = g3d.perp_grid();
-    //OrthogonalGrid3d<dg::HVec> g3d(gp, psi_0, psi_1, n, Nx, Ny,Nz, dg::DIR);
-    //OrthogonalGrid2d<dg::HVec> g2d = g3d.perp_grid();
+    dg::geo::TokamakMagneticField c = createMagField( gp);
+    dg::geo::FluxGenerator flux( c.get_psip(), c.get_ipol(), psi_0, psi_1, gp.R_0, 0., 1);
+    dg::CurvilinearProductGrid3d g3d(flux, n, Nx, Ny,Nz, dg::DIR);
+    dg::CurvilinearGrid2d g2d = g3d.perp_grid();
+    //OrthogonalGrid3d g3d(gp, psi_0, psi_1, n, Nx, Ny,Nz, dg::DIR);
+    //OrthogonalGrid2d g2d = g3d.perp_grid();
     dg::Grid2d g2d_periodic(g2d.x0(), g2d.x1(), g2d.y0(), g2d.y1(), g2d.n(), g2d.Nx(), g2d.Ny()+1); 
     t.toc();
     std::cout << "Construction took "<<t.diff()<<"s"<<std::endl;
@@ -103,8 +102,8 @@ int main( int argc, char* argv[])
     dg::HVec X( g2d.size()), Y(X); //P = dg::pullback( dg::coo3, g);
     for( unsigned i=0; i<g2d.size(); i++)
     {
-        X[i] = g2d.r()[i];
-        Y[i] = g2d.z()[i];
+        X[i] = g2d.map()[0][i];
+        Y[i] = g2d.map()[1][i];
     }
 
     dg::HVec temp0( g2d.size()), temp1(temp0);
@@ -115,13 +114,18 @@ int main( int argc, char* argv[])
     //err = nc_put_var_double( ncid, coordsID[2], g.z().data());
 
     //compute and write deformation into netcdf
-    dg::blas1::pointwiseDivide( g2d.g_xy(), g2d.g_xx(), temp0);
+    dg::SparseTensor<dg::HVec> metric = g2d.metric();
+    dg::HVec g_xx = metric.value(0,0), g_xy = metric.value(0,1), g_yy=metric.value(1,1);
+    dg::SparseElement<dg::HVec> vol_ = dg::tensor::determinant(metric);
+    dg::tensor::invert(vol_);
+    dg::tensor::sqrt(vol_);
+    dg::blas1::pointwiseDivide( g_xy, g_xx, temp0);
     const dg::HVec ones = dg::evaluate( dg::one, g2d);
-    X=g2d.g_yy();
+    X=g_yy;
     err = nc_put_var_double( ncid, defID, periodify(X, g2d_periodic).data());
     //compute and write conformalratio into netcdf
-    dg::blas1::pointwiseDivide( g2d.g_yy(), g2d.g_xx(), temp0);
-    X=g2d.g_xx();
+    dg::blas1::pointwiseDivide( g_yy, g_xx, temp0);
+    X=g_xx;
     err = nc_put_var_double( ncid, confID, periodify(X, g2d_periodic).data());
 
     std::cout << "Construction successful!\n";
@@ -141,20 +145,20 @@ int main( int argc, char* argv[])
     //double error = sqrt( dg::blas2::dot( temp0, w3d, temp0)/dg::blas2::dot( temp1, w3d, temp1));
     //std::cout<< "Rel Error in Determinant is "<<error<<"\n";
 
-    dg::blas1::pointwiseDot( g2d.g_xx(), g2d.g_yy(), temp0);
-    dg::blas1::pointwiseDot( g2d.g_xy(), g2d.g_xy(), temp1);
+    dg::blas1::pointwiseDot( g_xx, g_yy, temp0);
+    dg::blas1::pointwiseDot( g_xy, g_xy, temp1);
     dg::blas1::axpby( 1., temp0, -1., temp1, temp0);
     dg::blas1::transform( temp0, temp0, dg::SQRT<double>()); //temp0=1/sqrt(g) = sqrt(g^xx g^yy - g^xy^2)
     dg::blas1::pointwiseDivide( ones, temp0, temp0); //temp0=sqrt(g)
     X=temp0;
     err = nc_put_var_double( ncid, volID, periodify(X, g2d_periodic).data());
-    dg::blas1::axpby( 1., temp0, -1., g2d.vol(), temp0); //temp0 = sqrt(g)-vol
-    double error = sqrt(dg::blas2::dot( temp0, w3d, temp0)/dg::blas2::dot( g2d.vol(), w3d, g2d.vol()));
+    dg::blas1::axpby( 1., temp0, -1., vol_.value(), temp0); //temp0 = sqrt(g)-vol
+    double error = sqrt(dg::blas2::dot( temp0, w3d, temp0)/dg::blas2::dot(vol_.value(), w3d, vol_.value()));
     std::cout << "Rel Consistency  of volume is "<<error<<"\n";
 
     //dg::blas1::pointwiseDivide(ones,fby,temp1); //=sqrt(g)
-    //dg::blas1::axpby( 1., temp1, -1., g2d.vol(), temp0);
-    //error=sqrt(dg::blas2::dot( temp0, w3d, temp0))/sqrt( dg::blas2::dot(g2d.vol(), w3d, g2d.vol()));
+    //dg::blas1::axpby( 1., temp1, -1., vol_, temp0);
+    //error=sqrt(dg::blas2::dot( temp0, w3d, temp0))/sqrt( dg::blas2::dot(vol_, w3d, vol_));
     //std::cout << "Rel Error of volume form is "<<error<<"\n";
 
     const dg::HVec vol = dg::create::volume( g3d);
@@ -164,7 +168,7 @@ int main( int argc, char* argv[])
     std::cout << "TEST VOLUME IS:\n";
     if( psi_0 < psi_1) gp.psipmax = psi_1, gp.psipmin = psi_0;
     else               gp.psipmax = psi_0, gp.psipmin = psi_1;
-    dg::geo::Iris<Psip> iris( c.psip, gp.psipmin, gp.psipmax);
+    dg::geo::Iris<const dg::geo::aBinaryFunctor&> iris( c.psip(), gp.psipmin, gp.psipmax);
     //dg::CylindricalGrid3d<dg::HVec> g3d( gp.R_0 -2.*gp.a, gp.R_0 + 2*gp.a, -2*gp.a, 2*gp.a, 0, 2*M_PI, 3, 2200, 2200, 1, dg::PER, dg::PER, dg::PER);
     dg::CartesianGrid2d g2dC( gp.R_0 -2.0*gp.a, gp.R_0 + 2.0*gp.a, -2.0*gp.a,2.0*gp.a,1, 2e3, 2e3, dg::PER, dg::PER);
     dg::HVec vec  = dg::evaluate( iris, g2dC);
diff --git a/inc/geometries/fluxfunctions.h b/inc/geometries/fluxfunctions.h
index 3951ea4c8..b7fcf54ab 100644
--- a/inc/geometries/fluxfunctions.h
+++ b/inc/geometries/fluxfunctions.h
@@ -43,6 +43,7 @@ struct aBinaryFunctor
     virtual aBinaryFunctor* clone()const=0;
     virtual ~aBinaryFunctor(){}
     protected:
+    aBinaryFunctor(){}
     /**
     * @brief We do not allow object slicing so the copy is protected
     */
@@ -50,7 +51,7 @@ struct aBinaryFunctor
     /**
     * @brief We do not allow object slicing so the assignment is protected
     */
-    aBinaryFunctor& operator=(const aBinaryFunctor&){return *this}
+    aBinaryFunctor& operator=(const aBinaryFunctor&){return *this;}
 };
 
 /**
@@ -59,14 +60,14 @@ struct aBinaryFunctor
     https://katyscode.wordpress.com/2013/08/22/c-polymorphic-cloning-and-the-crtp-curiously-recurring-template-pattern/
 */
 template<class Derived>
-struct dg::geo::aCloneableBinaryFunctor : public dg::geo::aBinaryFunctor
+struct aCloneableBinaryFunctor : public aBinaryFunctor
 {
     /**
     * @brief Returns a copy of the functor dynamically allocated on the heap
     *
     * @return new copy of the functor
     */
-    virtual Derived* clone() const
+    virtual aBinaryFunctor* clone() const
     {
         return new Derived(static_cast<Derived const &>(*this));
     }
@@ -78,7 +79,7 @@ struct dg::geo::aCloneableBinaryFunctor : public dg::geo::aBinaryFunctor
  * double operator()(double,double)const;
  */
 template<class BinaryFunctor>
-struct BinaryFunctorAdapter : public dg::geo::aCloneableBinaryFunctor<Adapter>
+struct BinaryFunctorAdapter : public aCloneableBinaryFunctor<BinaryFunctorAdapter<BinaryFunctor> >
 {
     BinaryFunctorAdapter( const BinaryFunctor& f):f_(f){}
     double operator()(double x, double y)const{return f_(x,y);}
@@ -94,13 +95,9 @@ struct BinaryFunctorAdapter : public dg::geo::aCloneableBinaryFunctor<Adapter>
  * @return a newly allocated instance of aBinaryFunctor on the heap
  * @note the preferred way is to derive your Functor from aCloneableBinaryFunctor but if you can't or don't want to for whatever reason then use this to make one
  */
-temmplate<class BinaryFunctor>
+template<class BinaryFunctor>
 aBinaryFunctor* make_aBinaryFunctor(const BinaryFunctor& f){return new BinaryFunctorAdapter<BinaryFunctor>(f);}
 
-///@cond
-struct BinaryFunctorsLvl2;
-///@endcond
-
 /**
 * @brief This struct bundles a function and its first derivatives
 */
@@ -115,16 +112,15 @@ struct BinaryFunctorsLvl1
     * @param fx \f$ \partial f / \partial x \f$ its derivative in the first coordinate
     * @param fy \f$ \partial f / \partial y \f$ its derivative in the second coordinate
     */
-    BinaryFunctorsLvl1( const aBinaryFunctor* f, const aBinaryFunctor* fx, const aBinaryFunctor* fy) {
+    BinaryFunctorsLvl1(  aBinaryFunctor* f,  aBinaryFunctor* fx,  aBinaryFunctor* fy) {
         reset(f,fx,fy);
     }
     ///clone given functors
     BinaryFunctorsLvl1( const aBinaryFunctor& f, const aBinaryFunctor& fx, const aBinaryFunctor& fy) {
         reset(f,fx,fy);
     }
-    BinaryFunctorsLvl1( const BinaryFunctorsLvl2& func);
     ///Take ownership of given pointers
-    void reset( const aBinaryFunctor* f, const aBinaryFunctor* fx, const aBinaryFunctor* fy)
+    void reset(  aBinaryFunctor* f,  aBinaryFunctor* fx,  aBinaryFunctor* fy)
     {
         p_[0].reset(f);
         p_[1].reset(fx);
@@ -159,41 +155,36 @@ struct BinaryFunctorsLvl2
     * @param fxy \f$ \partial^2 f / \partial x \partial y\f$ second mixed derivative 
     * @param fyy \f$ \partial^2 f / \partial y^2\f$ second derivative in second coordinate
     */
-    BinaryFunctorsLvl2( const aBinaryFunctor* f, const aBinaryFunctor* fx, const aBinaryFunctor* fy, const aBinaryFunctor* fxx, const aBinaryFunctor* fxy, const aBinaryFunctor* fyy): f(f,fx,fy), f1(fxx,fxy,fyy) 
+    BinaryFunctorsLvl2(  aBinaryFunctor* f,  aBinaryFunctor* fx,  aBinaryFunctor* fy,  aBinaryFunctor* fxx,  aBinaryFunctor* fxy,  aBinaryFunctor* fyy): f0(f,fx,fy), f1(fxx,fxy,fyy) 
     { }
     ///clone given Functors
-    BinaryFunctorsLvl2( const aBinaryFunctor& f, const aBinaryFunctor& fx, const aBinaryFunctor& fy, const aBinaryFunctor& fxx, const aBinaryFunctor& fxy, const aBinaryFunctor& fyy): f(f,fx,fy), f1(fxx,fxy,fyy) 
+    BinaryFunctorsLvl2( const aBinaryFunctor& f, const aBinaryFunctor& fx, const aBinaryFunctor& fy, const aBinaryFunctor& fxx, const aBinaryFunctor& fxy, const aBinaryFunctor& fyy): f0(f,fx,fy), f1(fxx,fxy,fyy) 
     { }
     ///Take ownership of given pointers
-    void reset( const aBinaryFunctor* f, const aBinaryFunctor* fx, const aBinaryFunctor* fy, const aBinaryFunctor* fxx, const aBinaryFunctor* fxy, const aBinaryFunctor* fyy){ 
-        f.reset(f,fx,fy), f1.reset(fxx,fxy,fyy) 
+    void reset(  aBinaryFunctor* f,  aBinaryFunctor* fx,  aBinaryFunctor* fy,  aBinaryFunctor* fxx,  aBinaryFunctor* fxy,  aBinaryFunctor* fyy){ 
+        f0.reset(f,fx,fy), f1.reset(fxx,fxy,fyy);
     }
     ///clone given pointers
     void reset( const aBinaryFunctor& f, const aBinaryFunctor& fx, const aBinaryFunctor& fy, const aBinaryFunctor& fxx, const aBinaryFunctor& fxy, const aBinaryFunctor& fyy){ 
-        f.reset(f,fx,fy), f1.reset(fxx,fxy,fyy) 
+        f0.reset(f,fx,fy), f1.reset(fxx,fxy,fyy);
     }
-    operator BinaryFunctorsLvl1 ()const {return f;}
+    ///type conversion: Lvl2 can also be used as Lvl1
+    operator BinaryFunctorsLvl1 ()const {return f0;}
     /// \f$ f \f$
-    const aBinaryFunctor& f()const{return f.f();}
+    const aBinaryFunctor& f()const{return f0.f();}
     /// \f$ \partial f / \partial x \f$ 
-    const aBinaryFunctor& dfx()const{return f.dfx();}
+    const aBinaryFunctor& dfx()const{return f0.dfx();}
     /// \f$ \partial f / \partial y\f$
-    const aBinaryFunctor& dfy()const{return f.dfy();}
+    const aBinaryFunctor& dfy()const{return f0.dfy();}
     /// \f$ \partial^2f/\partial x^2\f$
     const aBinaryFunctor& dfxx()const{return f1.f();}
     /// \f$ \partial^2 f / \partial x \partial y\f$
-    const aBinaryFunctor& dfxy()const{return f1.fx();}
+    const aBinaryFunctor& dfxy()const{return f1.dfx();}
     /// \f$ \partial^2f/\partial y^2\f$
-    const aBinaryFunctor& dfyy()const{return f1.fy();}
+    const aBinaryFunctor& dfyy()const{return f1.dfy();}
     private:
-    BinaryFunctorsLvl1 f,f1;
+    BinaryFunctorsLvl1 f0,f1;
 };
-///@cond
-BinaryFunctorsLvl1::BinaryFunctorsLvl1( const BinaryFunctorsLvl2& func)
-{
-    reset(func.f(),func.dfx(),func.dfy());
-}
-///@endcond
 
 /// A symmetric 2d tensor field and its divergence
 struct BinarySymmTensorLvl1
@@ -209,7 +200,7 @@ struct BinarySymmTensorLvl1
      * @param divChiX \f$ \partial_x \chi^{xx} + \partial_y\chi^{yx}\f$ is the x-component of the divergence of the tensor \f$ \chi\f$
      * @param divChiY \f$ \partial_x \chi^{xy} + \partial_y\chi^{yy}\f$ is the y-component of the divergence of the tensor \f$ \chi \f$
     */
-    BinarySymmTensorLvl1( const aBinaryFunctor* chi_xx, const aBinaryFunctor* chi_xy, const aBinaryFunctor* chi_yy, const aBinaryFunctor* divChiX, const aBinaryFunctor* divChiY)
+    BinarySymmTensorLvl1(  aBinaryFunctor* chi_xx,  aBinaryFunctor* chi_xy,  aBinaryFunctor* chi_yy,  aBinaryFunctor* divChiX,  aBinaryFunctor* divChiY)
     {
         reset(chi_xx,chi_xy,chi_yy,divChiX,divChiY);
     }
@@ -219,7 +210,7 @@ struct BinarySymmTensorLvl1
         reset(chi_xx,chi_xy,chi_yy,divChiX,divChiY);
     }
     ///Take ownership of pointers and release any  currently held ones
-    void reset( const aBinaryFunctor* chi_xx, const aBinaryFunctor* chi_xy, const aBinaryFunctor* chi_yy, const aBinaryFunctor* divChiX, const aBinaryFunctor* divChiY)
+    void reset(  aBinaryFunctor* chi_xx,  aBinaryFunctor* chi_xy,  aBinaryFunctor* chi_yy,  aBinaryFunctor* divChiX,  aBinaryFunctor* divChiY)
     {
         p_[0].reset( chi_xx);
         p_[1].reset( chi_xy);
@@ -255,11 +246,11 @@ struct BinaryVectorLvl0
 {
     BinaryVectorLvl0(){}
     ///Take ownership of given pointers
-    BinaryVectorLvl0( const aBinaryFunctor* v_x, const aBinaryFunctor* v_y, const aBinaryFunctor* v_z) { reset(v_x,v_y,v_z); }
+    BinaryVectorLvl0(  aBinaryFunctor* v_x,  aBinaryFunctor* v_y,  aBinaryFunctor* v_z) { reset(v_x,v_y,v_z); }
     ///clone given references
     BinaryVectorLvl0( const aBinaryFunctor& v_x, const aBinaryFunctor& v_y, const aBinaryFunctor& v_z) { reset(v_x,v_y,v_z); }
     ///Take ownership of given pointers and release any currently held ones
-    void reset( const aBinaryFunctor* v_x, const aBinaryFunctor* v_y, const aBinaryFunctor* v_z)
+    void reset(  aBinaryFunctor* v_x,  aBinaryFunctor* v_y,  aBinaryFunctor* v_z)
     {
         p_[0].reset(v_x);
         p_[1].reset(v_y);
@@ -273,16 +264,16 @@ struct BinaryVectorLvl0
         p_[2].reset(v_z);
     }
     /// x-component of the vector
-    const aBinaryFunctor& x()const{return p_[0];}
+    const aBinaryFunctor& x()const{return p_[0].get();}
     /// y-component of the vector
-    const aBinaryFunctor& y()const{return p_[1];}
+    const aBinaryFunctor& y()const{return p_[1].get();}
     /// z-component of the vector
-    const aBinaryFunctor& z()const{return p_[2];}
+    const aBinaryFunctor& z()const{return p_[2].get();}
     private:
     Handle<aBinaryFunctor> p_[3];
 };
 
-struct Constant:public aCloneableBinaryOperator<Constant> 
+struct Constant: public aCloneableBinaryFunctor<Constant> 
 { 
     Constant(double c):c_(c){}
     double operator()(double R,double Z)const{return c_;}
diff --git a/inc/geometries/guenther.h b/inc/geometries/guenther.h
index 1395f754f..b1e2ac1ab 100644
--- a/inc/geometries/guenther.h
+++ b/inc/geometries/guenther.h
@@ -145,9 +145,9 @@ BinaryFunctorsLvl1 createIpol( GeomParameters gp)
     BinaryFunctorsLvl1 ipol( new Ipol(gp), new IpolR(gp), new IpolZ(gp))
     return ipol;
 }
-MagneticField createMagField( GeomParameters gp)
+TokamakMagneticField createMagField( GeomParameters gp)
 {
-    return MagneticField( gp.R_0, createPsip(gp), createIpol(gp));
+    return TokamakMagneticField( gp.R_0, createPsip(gp), createIpol(gp));
 }
 ///@}
 
diff --git a/inc/geometries/hector.h b/inc/geometries/hector.h
index f24a183a7..272a553b9 100644
--- a/inc/geometries/hector.h
+++ b/inc/geometries/hector.h
@@ -304,33 +304,6 @@ struct Hector : public aGenerator
         //dg::blas1::transfer( u, u_);
     }
 
-    /**
-     * @brief The length of the u-domain
-     *
-     * Call before discretizing the u domain
-     * @return length of u-domain
-     * @note the length is always positive
-     */
-    virtual double width() const {return lu_;}
-    /**
-     * @brief 2pi
-     *
-     * Always returns 2pi
-     * @return 2pi 
-     */
-    virtual double height() const {return 2.*M_PI;}
-    /**
-     * @brief True if conformal constructor was used
-     *
-     * @return true if conformal constructor was used
-     */
-    virtual bool isConformal() const {return conformal_;}
-    /**
-     * @brief True if orthogonal constructor was used
-     *
-     * @return true if orthogonal constructor was used
-     */
-    virtual bool isOrthogonal() const {return orthogonal_;}
 
     /**
      * @brief Return the internally used orthogonal grid
@@ -339,8 +312,17 @@ struct Hector : public aGenerator
      */
     const dg::CurvilinearGrid2d<container>& internal_grid() const {return g2d_;}
     virtual Hector* clone() const{return new Hector(*this);}
+    bool isConformal() const {return conformal_;}
     private:
-    virtual void generate( const thrust::host_vector<double>& u1d, 
+    virtual double do_width() const {return lu_;}
+    virtual double do_height() const {return 2.*M_PI;}
+    /**
+     * @brief True if orthogonal constructor was used
+     *
+     * @return true if orthogonal constructor was used
+     */
+    virtual bool do_isOrthogonal() const {return orthogonal_;}
+    virtual void do_generate( const thrust::host_vector<double>& u1d, 
                      const thrust::host_vector<double>& v1d, 
                      thrust::host_vector<double>& x, 
                      thrust::host_vector<double>& y, 
@@ -519,6 +501,7 @@ struct Hector : public aGenerator
         dg::blas1::transfer( etaU, etaU_);
         dg::blas1::transfer( zetaU, zetaU_);
     }
+    private:
     bool conformal_, orthogonal_;
     double c0_, lu_;
     thrust::host_vector<double> u_, ux_, uy_, vx_, vy_;
diff --git a/inc/geometries/magnetic_field.h b/inc/geometries/magnetic_field.h
index 221d137eb..589d4ceee 100644
--- a/inc/geometries/magnetic_field.h
+++ b/inc/geometries/magnetic_field.h
@@ -44,21 +44,21 @@ struct TokamakMagneticField
     /// \f$ \psi_p(R,Z)\f$, where R, Z are cylindrical coordinates
     const aBinaryFunctor& psip()const{return psip_.f();}
     /// \f$ \partial_R \psi_p(R,Z)\f$, where R, Z are cylindrical coordinates
-    const aBinaryFunctor& psipR()const{return psip_.fx();}
+    const aBinaryFunctor& psipR()const{return psip_.dfx();}
     /// \f$ \partial_Z \psi_p(R,Z)\f$, where R, Z are cylindrical coordinates
-    const aBinaryFunctor& psipZ()const{return psip_.fy();}
+    const aBinaryFunctor& psipZ()const{return psip_.dfy();}
     /// \f$ \partial_R\partial_R \psi_p(R,Z)\f$, where R, Z are cylindrical coordinates
-    const aBinaryFunctor& psipRR()const{return psip_.fxx();}
+    const aBinaryFunctor& psipRR()const{return psip_.dfxx();}
     /// \f$ \partial_R\partial_Z \psi_p(R,Z)\f$, where R, Z are cylindrical coordinates
-    const aBinaryFunctor& psipRZ()const{return psip_.fxy();}
+    const aBinaryFunctor& psipRZ()const{return psip_.dfxy();}
     /// \f$ \partial_Z\partial_Z \psi_p(R,Z)\f$, where R, Z are cylindrical coordinates
-    const aBinaryFunctor& psipZZ()const{return psip_.fyy();}
+    const aBinaryFunctor& psipZZ()const{return psip_.dfyy();}
     /// \f$ I(\psi_p) \f$ the current
     const aBinaryFunctor& ipol()const{return ipol_.f();}
     /// \f$ \partial_R I(\psi_p) \f$ 
-    const aBinaryFunctor& ipolR()const{return ipol_.fx();}
+    const aBinaryFunctor& ipolR()const{return ipol_.dfx();}
     /// \f$ \partial_Z I(\psi_p) \f$ 
-    const aBinaryFunctor& ipolZ()const{return ipol_.fy();}
+    const aBinaryFunctor& ipolZ()const{return ipol_.dfy();}
 
     const BinaryFunctorsLvl2& get_psip() const{return psip_;}
     const BinaryFunctorsLvl1& get_ipol() const{return ipol_;}
@@ -132,7 +132,6 @@ struct LnB : public aCloneableBinaryFunctor<LnB>
  * @brief \f[  \frac{\partial |\hat{B}| }{ \partial \hat{R}}  \f]
  */ 
 struct BR: public aCloneableBinaryFunctor<BR>
-
 {
     BR(const TokamakMagneticField& mag): invB_(mag), mag_(mag) { }
 /**
@@ -146,7 +145,7 @@ struct BR: public aCloneableBinaryFunctor<BR>
     double operator()(double R, double Z) const
     { 
         double Rn;
-        Rn = R/mag.R0();
+        Rn = R/mag_.R0();
         //sign before A changed to +
         //return -( Rn*Rn/invB_(R,Z)/invB_(R,Z)+ qampl_*qampl_*Rn *A_*psipR_(R,Z) - R  *(psipZ_(R,Z)*psipRZ_(R,Z)+psipR_(R,Z)*psipRR_(R,Z)))/(R*Rn*Rn/invB_(R,Z));
         return -1./R/invB_(R,Z) + invB_(R,Z)/Rn/Rn*(mag_.ipol()(R,Z)*mag_.ipolR()(R,Z) + mag_.psipR()(R,Z)*mag_.psipRR()(R,Z) + mag_.psipZ()(R,Z)*mag_.psipRZ()(R,Z));
@@ -174,7 +173,7 @@ struct BZ: public aCloneableBinaryFunctor<BZ>
         Rn = R/mag_.R0();
         //sign before A changed to -
         //return (-qampl_*qampl_*A_/R_0_*psipZ_(R,Z) + psipR_(R,Z)*psipRZ_(R,Z)+psipZ_(R,Z)*psipZZ_(R,Z))/(Rn*Rn/invB_(R,Z));
-        return (invB_(R,Z)/Rn/Rn)*(mag_.ipol()(R,Z)*mag_.ipolZ(R,Z) + mag_.psipR()(R,Z)*mag_.psipRZ()(R,Z) + mag_.psipZ()(R,Z)*mag_.psipZZ()(R,Z));
+        return (invB_(R,Z)/Rn/Rn)*(mag_.ipol()(R,Z)*mag_.ipolZ()(R,Z) + mag_.psipR()(R,Z)*mag_.psipRZ()(R,Z) + mag_.psipZ()(R,Z)*mag_.psipZZ()(R,Z));
     }
   private:
     TokamakMagneticField mag_;
@@ -297,7 +296,7 @@ struct FieldP: public aCloneableBinaryFunctor<LnB>
     FieldP( const TokamakMagneticField& mag): mag_(mag){}
     double operator()( double R, double Z, double phi) const
     {
-        return mag.R0()*mag_.ipol()(R,Z)/R/R;
+        return mag_.R0()*mag_.ipol()(R,Z)/R/R;
     }
     
     private:
@@ -312,7 +311,7 @@ struct FieldR: public aCloneableBinaryFunctor<FieldR>
     FieldR( const TokamakMagneticField& mag): mag_(mag){}
     double operator()( double R, double Z) const
     {
-        return  mag.R0()/R*mag_.psipZ()(R,Z);
+        return  mag_.R0()/R*mag_.psipZ()(R,Z);
     }
     private:
     TokamakMagneticField mag_;
diff --git a/inc/geometries/ribeiro.h b/inc/geometries/ribeiro.h
index 91ef81b07..c81622e40 100644
--- a/inc/geometries/ribeiro.h
+++ b/inc/geometries/ribeiro.h
@@ -5,12 +5,11 @@
 #include "dg/backend/interpolation.cuh"
 #include "dg/backend/operator.h"
 #include "dg/backend/derivatives.h"
+#include "dg/geometry/geometry.h"
 #include "dg/functors.h"
 #include "dg/runge_kutta.h"
 #include "dg/nullstelle.h"
-#include "dg/geometry.h"
 #include "utilities.h"
-#include "generator.h"
 
 
 
@@ -28,7 +27,7 @@ namespace detail
 //good as it can, i.e. until machine precision is reached
 struct Fpsi
 {
-    Fpsi( const BinaryFuncotsLvl1& psi_, double x0, double y0, int mode): 
+    Fpsi( const BinaryFunctorsLvl1& psi, double x0, double y0, int mode): 
         psip_(psi), fieldRZYTribeiro_(psi,x0, y0),fieldRZYTequalarc_(psi, x0, y0), fieldRZtau_(psi), mode_(mode)
     {
         R_init = x0; Z_init = y0;
@@ -185,7 +184,7 @@ struct FieldFinv
  * @brief A two-dimensional grid based on "almost-conformal" coordinates by %Ribeiro and Scott 2010 (models aGenerator)
  * @ingroup generators
  */
-struct Ribeiro : public aGenerator
+struct Ribeiro : public aGenerator2d
 {
     /**
      * @brief Construct a near-conformal grid generator
@@ -207,6 +206,9 @@ struct Ribeiro : public aGenerator
         x0_=x0, y0_=y0, psi0_=psi_0, psi1_=psi_1;
         //std::cout << "lx_ = "<<lx_<<"\n";
     }
+    virtual Ribeiro* clone() const{return new Ribeiro(*this);}
+
+    private:
     /**
      * @brief The length of the zeta-domain
      *
@@ -214,18 +216,15 @@ struct Ribeiro : public aGenerator
      * @return length of zeta-domain (f0*(psi_1-psi_0))
      * @note the length is always positive
      */
-    virtual double width() const{return lx_;}
+    virtual double do_width() const{return lx_;}
     /**
      * @brief 2pi (length of the eta domain)
      *
      * Always returns 2pi
      * @return 2pi 
      */
-    virtual double height() const{return 2.*M_PI;}
-    virtual Ribeiro* clone() const{return new Ribeiro(*this);}
-
-    private:
-    virtual void generate( 
+    virtual double do_height() const{return 2.*M_PI;}
+    virtual void do_generate( 
          const thrust::host_vector<double>& zeta1d, 
          const thrust::host_vector<double>& eta1d, 
          thrust::host_vector<double>& x, 
@@ -236,7 +235,7 @@ struct Ribeiro : public aGenerator
          thrust::host_vector<double>& etaY) const
     {
         //compute psi(x) for a grid on x and call construct_rzy for all psi
-        ribeiro::detail::FieldFinv fpsiMinv_(psi_, psiX_, psiY_, x0_,y0_, 500, mode_);
+        ribeiro::detail::FieldFinv fpsiMinv_(psi_, x0_,y0_, 500, mode_);
         thrust::host_vector<double> psi_x, fx_;
         dg::geo::detail::construct_psi_values( fpsiMinv_, psi0_, psi1_, 0., zeta1d, lx_, psi_x, fx_);
 
diff --git a/inc/geometries/simple_orthogonal.h b/inc/geometries/simple_orthogonal.h
index 651e64937..d9be0f82e 100644
--- a/inc/geometries/simple_orthogonal.h
+++ b/inc/geometries/simple_orthogonal.h
@@ -276,7 +276,7 @@ void construct_rz( Nemov nemov,
  * Psi is the radial coordinate and you can choose various discretizations of the first line
  * @ingroup generators
  */
-struct SimpleOrthogonal : public aGridGenerator
+struct SimpleOrthogonal : public aGenerator2d
 {
     /**
      * @brief Construct a simple orthogonal grid 
@@ -305,31 +305,14 @@ struct SimpleOrthogonal : public aGridGenerator
      * @return f_0 is the grid constant  s.a. width() )
      */
     double f0() const{return f0_;}
-    /**
-     * @brief The length of the zeta-domain
-     *
-     * Call before discretizing the zeta domain
-     * @return length of zeta-domain (f0*(psi_1-psi_0))
-     * @note the length is always positive
-     */
-    virtual double width() const{return lz_;}
-    /**
-     * @brief 2pi (length of the eta domain)
-     *
-     * Always returns 2pi
-     * @return 2pi 
-     */
-    virtual double height() const{return 2.*M_PI;}
-    /**
-     * @brief Indicate orthogonality
-     *
-     * @return true
-     */
-    virtual bool isOrthogonal() const{return true;}
     virtual SimpleOrthogonal* clone() const{return new SimpleOrthogonal(*this);}
 
     private:
-    virtual void generate( 
+     // length of zeta-domain (f0*(psi_1-psi_0))
+    virtual double do_width() const{return lz_;}
+    virtual double do_height() const{return 2.*M_PI;}
+    virtual bool do_isOrthogonal() const{return true;}
+    virtual void do_generate( 
          const thrust::host_vector<double>& zeta1d, 
          const thrust::host_vector<double>& eta1d, 
          thrust::host_vector<double>& x, 
diff --git a/inc/geometries/solovev.h b/inc/geometries/solovev.h
index baf2b2d69..84615f60f 100644
--- a/inc/geometries/solovev.h
+++ b/inc/geometries/solovev.h
@@ -447,12 +447,12 @@ BinaryFunctorsLvl2 createPsip( GeomParameters gp)
 }
 BinaryFunctorsLvl1 createIpol( GeomParameters gp)
 {
-    BinaryFunctorsLvl1 ipol( new Ipol(gp), new IpolR(gp), new IpolZ(gp))
+    BinaryFunctorsLvl1 ipol( new Ipol(gp), new IpolR(gp), new IpolZ(gp));
     return ipol;
 }
-MagneticField createMagField( GeomParameters gp)
+TokamakMagneticField createMagField( GeomParameters gp)
 {
-    return MagneticField( gp.R_0, createPsip(gp), createIpol(gp));
+    return TokamakMagneticField( gp.R_0, createPsip(gp), createIpol(gp));
 }
 ///@}
 
@@ -460,7 +460,7 @@ MagneticField createMagField( GeomParameters gp)
 namespace mod
 {
 
-struct Psip: public aCloneableBinaryFunctor
+struct Psip: public aCloneableBinaryFunctor<Psip>
 {
     Psip( GeomParameters gp): R_X( gp.R_0-1.1*gp.triangularity*gp.a), Z_X(-1.1*gp.elongation*gp.a),
         psip_(gp), psipRR_(gp), psipRZ_(gp), psipZZ_(gp), cauchy_( R_X, Z_X, 50, 50,1.)
@@ -486,7 +486,7 @@ struct Psip: public aCloneableBinaryFunctor
     solovev::PsipZZ psipZZ_;
     dg::Cauchy cauchy_;
 };
-struct PsipR: public aCloneableBinaryFunctor
+struct PsipR: public aCloneableBinaryFunctor<PsipR>
 {
     PsipR( GeomParameters gp): R_X( gp.R_0-1.1*gp.triangularity*gp.a), Z_X(-1.1*gp.elongation*gp.a),
         psip_(gp), psipR_(gp), psipRR_(gp), psipRZ_(gp), psipZZ_(gp), cauchy_( R_X, Z_X, 50, 50,1.)
@@ -514,7 +514,7 @@ struct PsipR: public aCloneableBinaryFunctor
     solovev::PsipZZ psipZZ_;
     dg::Cauchy cauchy_;
 };
-struct PsipZ: public aCloneableBinaryFunctor
+struct PsipZ: public aCloneableBinaryFunctor<PsipZ>
 {
     PsipZ( GeomParameters gp): R_X( gp.R_0-1.1*gp.triangularity*gp.a), Z_X(-1.1*gp.elongation*gp.a),
         psip_(gp), psipZ_(gp), psipRR_(gp), psipRZ_(gp), psipZZ_(gp), cauchy_( R_X, Z_X, 50, 50, 1)
@@ -543,7 +543,7 @@ struct PsipZ: public aCloneableBinaryFunctor
     dg::Cauchy cauchy_;
 };
 
-struct PsipZZ: public aCloneableBinaryFunctor
+struct PsipZZ: public aCloneableBinaryFunctor<PsipZZ>
 {
     PsipZZ( GeomParameters gp): R_X( gp.R_0-1.1*gp.triangularity*gp.a), Z_X(-1.1*gp.elongation*gp.a),
         psip_(gp), psipZ_(gp), psipRR_(gp), psipRZ_(gp), psipZZ_(gp), cauchy_( R_X, Z_X, 50, 50, 1)
@@ -572,7 +572,7 @@ struct PsipZZ: public aCloneableBinaryFunctor
     solovev::PsipZZ psipZZ_;
     dg::Cauchy cauchy_;
 };
-struct PsipRR: public aCloneableBinaryFunctor
+struct PsipRR: public aCloneableBinaryFunctor<PsipRR>
 {
     PsipRR( GeomParameters gp): R_X( gp.R_0-1.1*gp.triangularity*gp.a), Z_X(-1.1*gp.elongation*gp.a),
         psip_(gp), psipR_(gp), psipRR_(gp), psipRZ_(gp), psipZZ_(gp), cauchy_( R_X, Z_X, 50, 50, 1)
diff --git a/inc/geometries/taylor.h b/inc/geometries/taylor.h
index 867897dbb..19c58d38a 100644
--- a/inc/geometries/taylor.h
+++ b/inc/geometries/taylor.h
@@ -320,7 +320,7 @@ BinaryFunctorsLvl1 createIpol( solovev::GeomParameters gp)
     BinaryFunctorsLvl1 ipol( new Ipol(gp), new IpolR(gp), new IpolZ(gp))
     return ipol;
 }
-MagneticField createMagField( solovev::GeomParameters gp)
+TokamakMagneticField createMagField( solovev::GeomParameters gp)
 {
     return MagneticField( gp.R_0, createPsip(gp), createIpol(gp));
 }
diff --git a/inc/geometries/toroidal.h b/inc/geometries/toroidal.h
index 3cf21f951..2b31f5213 100644
--- a/inc/geometries/toroidal.h
+++ b/inc/geometries/toroidal.h
@@ -6,26 +6,6 @@ namespace dg{
 namespace geo{
 namespace toroidal{
 
-/**
- * @brief Models a slab toroidal field 
- *
- * \f$ B=\frac{R_0}{R}\f$, \f$ \psi_p = 1\f$ and \f$ I = 1\f$.
- @note The solovev field can also be made to model a todoidal slab field
- */
-struct MagneticField : public dg::geo::aTokamakMagneticField
-{
-    MagneticField( double R0): aTokamakMagneticField(R0, 
-        new Constant(1), 
-        new Constant(0), 
-        new Constant(0), 
-        new Constant(0), 
-        new Constant(0), 
-        new Constant(0), 
-        new Constant(0), 
-        new Constant(1), 
-        new Constant(0), 
-        new Constant(0)){}
-};
 BinaryFunctorsLvl2 createPsip( )
 {
     BinaryFunctorsLvl2 psip( new Constant(1), new Constant(0), new Constant(0),new Constant(0), new Constant(0), new Constant(0));
@@ -36,9 +16,16 @@ BinaryFunctorsLvl1 createIpol( )
     BinaryFunctorsLvl1 ipol( new Constant(1), new Constant(0), new Constant(0))
     return ipol;
 }
-MagneticField createMagField( double R0)
+
+/**
+ * @brief Models a slab toroidal field 
+ *
+ * \f$ B=\frac{R_0}{R}\f$, \f$ \psi_p = 1\f$ and \f$ I = 1\f$.
+ @note The solovev field can also be made to model a todoidal slab field
+ */
+TokamakMagneticField createMagField( double R0)
 {
-    return MagneticField( R0, createPsip(), createIpol());
+    return TokamakMagneticField( R0, createPsip(), createIpol());
 }
 
 }//namespace toroidal
diff --git a/inc/geometries/utilities.h b/inc/geometries/utilities.h
index 784bad63a..d51271bed 100644
--- a/inc/geometries/utilities.h
+++ b/inc/geometries/utilities.h
@@ -133,7 +133,7 @@ namespace ribeiro{
 
 struct FieldRZYT
 {
-    FieldRZYT( const BinaryFunctorsLvl1& psip, double R0, double Z0): R_0_(R0), Z_0_(Z0), psip_(psip)
+    FieldRZYT( const BinaryFunctorsLvl1& psip, double R0, double Z0): R_0_(R0), Z_0_(Z0), psip_(psip){}
     void operator()( const dg::HVec& y, dg::HVec& yp) const
     {
         double psipR = psip_.dfx()(y[0], y[1]), psipZ = psip_.dfy()(y[0],y[1]);
@@ -364,7 +364,7 @@ struct HessianRZtau
     void set_norm( bool normed) {norm_ = normed;}
     void operator()( const dg::HVec& y, dg::HVec& yp) const
     {
-        double psipRZ = psip_.dfxy(y[0], y[1]);
+        double psipRZ = psip_.dfxy()(y[0], y[1]);
         if( psipRZ == 0)
         {
             if(      quad_ == 0) { yp[0] = 1; yp[1] = 0; }
@@ -374,7 +374,7 @@ struct HessianRZtau
         }
         else
         {
-            double psipRR = psip_.dfxx(y[0], y[1]), psipZZ = psip_.dfyy(y[0],y[1]);
+            double psipRR = psip_.dfxx()(y[0], y[1]), psipZZ = psip_.dfyy()(y[0],y[1]);
             double T = psipRR + psipZZ; 
             double D = psipZZ*psipRR - psipRZ*psipRZ;
             double L1 = 0.5*T+sqrt( 0.25*T*T-D); // > 0
-- 
GitLab


From 44ff8c36b11a6ba3f75d1a1f49b0d345486f82a5 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Tue, 15 Aug 2017 06:54:24 -0700
Subject: [PATCH 166/453] remove pc40-c722.mk because this machine is no longer
 in use

---
 config/pc40-c722.mk | 14 --------------
 1 file changed, 14 deletions(-)
 delete mode 100644 config/pc40-c722.mk

diff --git a/config/pc40-c722.mk b/config/pc40-c722.mk
deleted file mode 100644
index 884b62323..000000000
--- a/config/pc40-c722.mk
+++ /dev/null
@@ -1,14 +0,0 @@
-
-
-
-ifeq ($(strip $(shell hostname)),pc40-c722)#    # uniquely identify system
-INCLUDE = -I$(HOME)/include#                    # cusp and thrust libraries
-GLFLAGS =$$(pkg-config --static --libs glfw3)#  # flags for glfw3
-CC=g++#                                         # the host c++ compiler
-MPICC=mpic++#                                   # the mpi compiler
-OPT=-O3#                                        # the optimization flag for the host
-NVCCARCH=-arch sm_20#                           # nvcc gpu compute capability
-OMPFLAG=-fopenmp#                               # openmp flag for CC and MPICC
-LIBS= -lnetcdf -lhdf5 -lhdf5_hl#                 # netcdf library for file output
-JSONLIB=-L$(HOME)/include/json/../../src/lib_json -ljsoncpp # json library for input parameters
-endif
-- 
GitLab


From fbbda23d6188a0732409642806d0c50e96a24bcc Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Wed, 16 Aug 2017 06:24:59 -0700
Subject: [PATCH 167/453] debugging

---
 inc/dg/geometry/curvilinear.h |  4 ++--
 inc/geometries/adaption.h     | 40 +++++++++++++++++------------------
 inc/geometries/hector.h       | 33 ++++++++++++++---------------
 inc/geometries/hector_t.cu    |  2 +-
 4 files changed, 39 insertions(+), 40 deletions(-)

diff --git a/inc/dg/geometry/curvilinear.h b/inc/dg/geometry/curvilinear.h
index 530ca501e..892cfa7e5 100644
--- a/inc/dg/geometry/curvilinear.h
+++ b/inc/dg/geometry/curvilinear.h
@@ -83,8 +83,8 @@ struct CurvilinearProductGrid3d : public dg::aGeometry3d
     //construct 2d plane
     void constructPerp( unsigned n, unsigned Nx, unsigned Ny)
     {
-        dg::Grid1d gX1d( 0., x0(), n, Nx);
-        dg::Grid1d gY1d( 0., y0(), n, Ny);
+        dg::Grid1d gX1d( x0(), x1(), n, Nx);
+        dg::Grid1d gY1d( y0(), y1(), n, Ny);
         thrust::host_vector<double> x_vec = dg::evaluate( dg::cooX1d, gX1d);
         thrust::host_vector<double> y_vec = dg::evaluate( dg::cooX1d, gY1d);
         handle_.get().generate( x_vec, y_vec, map_[0], map_[1], jac_.value(0), jac_.value(1), jac_.value(2), jac_.value(3));
diff --git a/inc/geometries/adaption.h b/inc/geometries/adaption.h
index 7d79de018..4ac8f4fe8 100644
--- a/inc/geometries/adaption.h
+++ b/inc/geometries/adaption.h
@@ -18,7 +18,7 @@ namespace detail{
 struct LaplaceAdaptPsi: public aCloneableBinaryFunctor<LaplaceAdaptPsi>
 {
     LaplaceAdaptPsi( const BinaryFunctorsLvl2& psi, const BinaryFunctorsLvl1& chi) : psi_(psi), chi_(chi){}
-    double operator()(double x, double y)
+    double operator()(double x, double y)const
     {
         return  psi_.dfx()(x,y)*chi_.dfx()(x,y) +
                 psi_.dfy()(x,y)*chi_.dfy()(x,y) +
@@ -31,9 +31,9 @@ struct LaplaceAdaptPsi: public aCloneableBinaryFunctor<LaplaceAdaptPsi>
 
 struct LaplaceChiPsi: public aCloneableBinaryFunctor<LaplaceChiPsi>
 {
-    LaplaceChiPsi( const BinaryFunctorsLvl2& psi, const BinarySymmTensorLvl1& chi)
+    LaplaceChiPsi( const BinaryFunctorsLvl2& psi, const BinarySymmTensorLvl1& chi):
         psi_(psi), chi_(chi){}
-    double operator()(double x, double y)
+    double operator()(double x, double y)const
     {
         return psi_.dfxx()(x,y)*chi_.xx()(x,y)+2.*psi_.dfxy()(x,y)*chi_.xy()(x,y)+psi_.dfyy()(x,y)*chi_.yy()(x,y)
             + chi_.divX()(x,y)*psi_.dfx()(x,y) + chi_.divY()(x,y)*psi_.dfy()(x,y);
@@ -47,7 +47,7 @@ struct LaplaceChiPsi: public aCloneableBinaryFunctor<LaplaceChiPsi>
 struct LaplacePsi: public aCloneableBinaryFunctor<LaplacePsi>
 {
     LaplacePsi( const BinaryFunctorsLvl2& psi): psi_(psi){}
-    double operator()(double x, double y){return psi_.dfxx()(x,y)+psi_.dfyy()(x,y);}
+    double operator()(double x, double y)const{return psi_.dfxx()(x,y)+psi_.dfyy()(x,y);}
     private:
     BinaryFunctorsLvl2 psi_;
 };
@@ -74,7 +74,7 @@ struct NablaPsiInv: public aCloneableBinaryFunctor<NablaPsiInv>
      * @brief  A weight function for the Hector algorithm
      * \f[ |\nabla\psi|^{-1} = (\psi_x^2 + \psi_y^2)^{-1/2} \f]
      */
-    double operator()(double x, double y)
+    virtual double operator()(double x, double y)const
     {
         double psiX = psi_.dfx()(x,y), psiY = psi_.dfy()(x,y);
         return 1./sqrt(psiX*psiX+psiY*psiY);
@@ -90,7 +90,7 @@ struct NablaPsiInv: public aCloneableBinaryFunctor<NablaPsiInv>
 struct NablaPsiInvX: public aCloneableBinaryFunctor<NablaPsiInvX>
 {
     NablaPsiInvX( const BinaryFunctorsLvl2& psi):psi_(psi) {}
-    double operator()(double x, double y)
+    virtual double operator()(double x, double y)const
     {
         double psiX = psi_.dfx()(x,y), psiY = psi_.dfy()(x,y);
         double psiXX = psi_.dfxx()(x,y), psiXY = psi_.dfxy()(x,y);
@@ -109,7 +109,7 @@ struct NablaPsiInvX: public aCloneableBinaryFunctor<NablaPsiInvX>
 struct NablaPsiInvY: public aCloneableBinaryFunctor<NablaPsiInvY>
 {
     NablaPsiInvY( const BinaryFunctorsLvl2& psi):psi_(psi) {}
-    double operator()(double x, double y)
+    double operator()(double x, double y)const
     {
         double psiX = psi_.dfx()(x,y), psiY = psi_.dfy()(x,y);
         double psiYY = psi_.dfyy()(x,y), psiXY = psi_.dfxy()(x,y);
@@ -141,8 +141,8 @@ BinaryFunctorsLvl1 make_NablaPsiInvCollective( const BinaryFunctorsLvl2& psi)
  */
 struct Liseikin_XX: public aCloneableBinaryFunctor<Liseikin_XX>
 {
-    Liseikin_XX(BinaryFunctorsLvl1& psi, double k, double eps):k_(k), eps_(eps), psi_(psi){}
-    double operator()(double x, double y)
+    Liseikin_XX(const BinaryFunctorsLvl1& psi, double k, double eps):k_(k), eps_(eps), psi_(psi){}
+    double operator()(double x, double y)const
     {
         double psiX = psi_.dfx()(x,y), psiY = psi_.dfy()(x,y), k2 = k_*k_;
         double psip2 = psiX*psiX+psiY*psiY;
@@ -164,8 +164,8 @@ struct Liseikin_XX: public aCloneableBinaryFunctor<Liseikin_XX>
  */
 struct Liseikin_XY: public aCloneableBinaryFunctor<Liseikin_XY>
 {
-    Liseikin_XY(BinaryFunctorsLvl1& psi, double k, double eps):k_(k), eps_(eps), psi_(psi){}
-    double operator()(double x, double y)
+    Liseikin_XY(const BinaryFunctorsLvl1& psi, double k, double eps):k_(k), eps_(eps), psi_(psi){}
+    double operator()(double x, double y)const
     {
         double psiX = psi_.dfx()(x,y), psiY = psi_.dfy()(x,y), k2 = k_*k_;
         double psip2 = psiX*psiX+psiY*psiY;
@@ -187,8 +187,8 @@ struct Liseikin_XY: public aCloneableBinaryFunctor<Liseikin_XY>
  */
 struct Liseikin_YY: public aCloneableBinaryFunctor<Liseikin_YY>
 {
-    Liseikin_YY(BinaryFunctorsLvl1& psi, double k, double eps):k_(k), eps_(eps), psi_(psi){}
-    double operator()(double x, double y)
+    Liseikin_YY(const BinaryFunctorsLvl1& psi, double k, double eps):k_(k), eps_(eps), psi_(psi){}
+    double operator()(double x, double y)const
     {
         double psiX = psi_.dfx()(x,y), psiY = psi_.dfy()(x,y), k2 = k_*k_;
         double psip2 = psiX*psiX+psiY*psiY;
@@ -207,8 +207,8 @@ struct Liseikin_YY: public aCloneableBinaryFunctor<Liseikin_YY>
  */
 struct DivLiseikinX: public aCloneableBinaryFunctor<DivLiseikinX>
 {
-    DivLiseikinX(BinaryFunctorsLvl2& psi, double k, double eps):k_(k), eps_(eps), psi_(psi){}
-    double operator()(double x, double y)
+    DivLiseikinX(const BinaryFunctorsLvl2& psi, double k, double eps): k_(k), eps_(eps), psi_(psi){}
+    double operator()(double x, double y)const
     {
         double psiX = psi_.dfx()(x,y), psiY = psi_.dfy()(x,y), k2 = k_*k_;
         double psiXX = psi_.dfxx()(x,y), psiXY = psi_.dfxy()(x,y), psiYY=psi_.dfyy()(x,y);
@@ -234,8 +234,8 @@ struct DivLiseikinX: public aCloneableBinaryFunctor<DivLiseikinX>
  */
 struct DivLiseikinY : public aCloneableBinaryFunctor<DivLiseikinY>
 {
-    DivLiseikinY(BinaryFunctorsLvl2& psi, double k, double eps):k_(k), eps_(eps), psi_(psi){}
-    double operator()(double x, double y)
+    DivLiseikinY(const BinaryFunctorsLvl2& psi, double k, double eps):k_(k), eps_(eps), psi_(psi){}
+    double operator()(double x, double y)const
     {
         double psiX = psi_.dfx()(x,y), psiY = psi_.dfy()(x,y), k2 = k_*k_;
         double psiXX = psi_.dfxx()(x,y), psiXY = psi_.dfxy()(x,y), psiYY=psi_.dfyy()(x,y);
@@ -247,7 +247,7 @@ struct DivLiseikinY : public aCloneableBinaryFunctor<DivLiseikinY>
                 k2*psiX4*psiY*(2.*psiYY-psiXX)+psiY*(eps_+k2*psiY2)
                 *(eps_*psiYY+(eps_+psiY2)*psiXX)+k2*psiX5*psiXY+
                 psiX3*(-(eps_+2.*k2*psiY2)*psiXY+eps_*(1.+k2)*psiXY) +
-                psiX*(-(eps_+2.*psiY2)*(eps_+k2*psiY2)*psiXY + (eps_*eps_-k2*psiY4)*psiXY)/sqrtG/sqrtG/sqrtG;
+                psiX*(-(eps_+2.*psiY2)*(eps_+k2*psiY2)*psiXY + (eps_*eps_-k2*psiY4)*psiXY))/sqrtG/sqrtG/sqrtG;
     }
 
     private:
@@ -255,9 +255,9 @@ struct DivLiseikinY : public aCloneableBinaryFunctor<DivLiseikinY>
     BinaryFunctorsLvl2 psi_;
 };
 
-BinarySymmTensorLvl1 make_LiseikinCollective( const BinaryFunctorsLvl2& psi)
+BinarySymmTensorLvl1 make_LiseikinCollective( const BinaryFunctorsLvl2& psi, double k, double eps)
 {
-    BinarySymmTensorLvl1 temp( new Liseikin_XX(psi), new Liseikin_XY(psi), new Liseikin_YY(psi), new DivLiseikinX(psi), new DivLiseikinY(psi));
+    BinarySymmTensorLvl1 temp( new Liseikin_XX(psi,k,eps), new Liseikin_XY(psi,k,eps), new Liseikin_YY(psi,k,eps), new DivLiseikinX(psi,k,eps), new DivLiseikinY(psi,k,eps));
     return temp;
 }
 ///@}
diff --git a/inc/geometries/hector.h b/inc/geometries/hector.h
index 272a553b9..7a07684a9 100644
--- a/inc/geometries/hector.h
+++ b/inc/geometries/hector.h
@@ -3,13 +3,12 @@
 #include <vector>
 #include "dg/backend/grid.h"
 #include "dg/backend/interpolation.cuh"
-#include "dg/geometry/curvilinear.h"
+#include "dg/geometry/geometry.h"
 #include "dg/elliptic.h"
 #include "fluxfunctions.h"
 #include "dg/cg.h"
 #include "flux.h"
 #include "adaption.h"
-#include "generator.h"
 
 
 
@@ -27,7 +26,7 @@ struct Interpolate
 {
     Interpolate( const thrust::host_vector<double>& fZeta, 
                  const thrust::host_vector<double>& fEta, 
-                 const dg::Grid2d& g2d ): 
+                 const dg::aTopology2d& g2d ): 
         iter0_( dg::create::forward_transform( fZeta, g2d) ), 
         iter1_( dg::create::forward_transform(  fEta, g2d) ), 
         g_(g2d), zeta1_(g2d.x1()), eta1_(g2d.y1()){}
@@ -54,7 +53,7 @@ struct Interpolate
 };
 
 //compute c_0 
-double construct_c0( const thrust::host_vector<double>& etaVinv, const dg::Grid2d& g2d) 
+double construct_c0( const thrust::host_vector<double>& etaVinv, const dg::aTopology2d& g2d) 
 {
     //this is a normal integration:
     thrust::host_vector<double> etaVinvL( dg::create::forward_transform(  etaVinv, g2d) );
@@ -94,7 +93,7 @@ void compute_zev(
         const thrust::host_vector<double>& etaV,
         const thrust::host_vector<double>& v_vec,
         thrust::host_vector<double>& eta, 
-        const dg::Grid2d& g2d
+        const dg::aTopology2d& g2d
         ) 
 {
     Interpolate iter( thrust::host_vector<double>( etaV.size(), 0), etaV, g2d);
@@ -134,7 +133,7 @@ void construct_grid(
         const thrust::host_vector<double>& eta_init, //1d intial values
         thrust::host_vector<double>& zeta, 
         thrust::host_vector<double>& eta, 
-        const dg::Grid2d& g2d
+        const dg::aTopology2d& g2d
     )
 {
     Interpolate inter( zetaU, etaU, g2d);
@@ -206,7 +205,7 @@ void transform(
  * @tparam container models aContainer 
  */
 template <class IMatrix = dg::IHMatrix, class Matrix = dg::HMatrix, class container = dg::HVec>
-struct Hector : public aGenerator
+struct Hector : public aGenerator2d
 {
     /**
      * @brief Construct a conformal grid 
@@ -310,7 +309,7 @@ struct Hector : public aGenerator
      *
      * @return  orthogonal zeta, eta grid
      */
-    const dg::CurvilinearGrid2d<container>& internal_grid() const {return g2d_;}
+    const dg::CurvilinearGrid2d& internal_grid() const {return g2d_;}
     virtual Hector* clone() const{return new Hector(*this);}
     bool isConformal() const {return conformal_;}
     private:
@@ -340,8 +339,8 @@ struct Hector : public aGenerator
             eta[i] = fmod(eta[i]+2.*M_PI, 2.*M_PI); 
         dg::IHMatrix Q = dg::create::interpolation( zeta, eta, g2d_);
 
-        dg::blas2::symv( Q, g2d_.r(), x);
-        dg::blas2::symv( Q, g2d_.z(), y);
+        dg::blas2::symv( Q, g2d_.map()[0], x);
+        dg::blas2::symv( Q, g2d_.map()[1], y);
         dg::blas2::symv( Q, ux_, ux);
         dg::blas2::symv( Q, uy_, uy);
         dg::blas2::symv( Q, vx_, vx);
@@ -362,9 +361,9 @@ struct Hector : public aGenerator
     {
         //first find u( \zeta, \eta)
         double eps = 1e10, eps_old = 2e10;
-        dg::CurvilinearGrid2d<container> g2d_old = g2d_;
+        dg::CurvilinearGrid2d g2d_old = g2d_;
         container adapt = dg::pullback(chi, g2d_old);
-        dg::Elliptic<dg::CurvilinearGrid2d<container>, Matrix, container> ellipticD_old( g2d_old, dg::DIR, dg::PER, dg::not_normed, dg::centered);
+        dg::Elliptic<dg::CurvilinearGrid2d, Matrix, container> ellipticD_old( g2d_old, dg::DIR, dg::PER, dg::not_normed, dg::centered);
         ellipticD_old.set_chi( adapt);
 
         container u_old = dg::evaluate( dg::zero, g2d_old), u(u_old);
@@ -376,7 +375,7 @@ struct Hector : public aGenerator
             eps = eps_old;
             g2d_.multiplyCellNumber(2,2);
             if(verbose) std::cout << "Nx "<<Nx<<" Ny "<<Ny<<std::flush;
-            dg::Elliptic<dg::CurvilinearGrid2d<container>, Matrix, container> ellipticD( g2d_, dg::DIR, dg::PER, dg::not_normed, dg::centered);
+            dg::Elliptic<dg::CurvilinearGrid2d, Matrix, container> ellipticD( g2d_, dg::DIR, dg::PER, dg::not_normed, dg::centered);
             adapt = dg::pullback(chi, g2d_);
             ellipticD.set_chi( adapt);
             lapu = dg::pullback( lapChiPsi, g2d_);
@@ -404,8 +403,8 @@ struct Hector : public aGenerator
         dg::geo::detail::LaplaceChiPsi lapChiPsi( psi, chi);
         //first find u( \zeta, \eta)
         double eps = 1e10, eps_old = 2e10;
-        dg::CurvilinearGrid2d<container> g2d_old = g2d_;
-        dg::TensorElliptic<dg::CurvilinearGrid2d<container>, Matrix, container> ellipticD_old( g2d_old, dg::DIR, dg::PER, dg::not_normed, dg::centered);
+        dg::CurvilinearGrid2d g2d_old = g2d_;
+        dg::TensorElliptic<dg::CurvilinearGrid2d, Matrix, container> ellipticD_old( g2d_old, dg::DIR, dg::PER, dg::not_normed, dg::centered);
         ellipticD_old.set( chi.xx(), chi.xy(), chi.yy());
 
         container u_old = dg::evaluate( dg::zero, g2d_old), u(u_old);
@@ -417,7 +416,7 @@ struct Hector : public aGenerator
             eps = eps_old;
             g2d_.multiplyCellNumber(2,2);
             if(verbose)std::cout << "Nx "<<Nx<<" Ny "<<Ny<<std::flush;
-            dg::TensorElliptic<dg::CurvilinearGrid2d<container>, Matrix, container> ellipticD( g2d_, dg::DIR, dg::PER, dg::not_normed, dg::centered);
+            dg::TensorElliptic<dg::CurvilinearGrid2d, Matrix, container> ellipticD( g2d_, dg::DIR, dg::PER, dg::not_normed, dg::centered);
             ellipticD.set( chi.xx(), chi.xy(), chi.yy() );
             lapu = dg::pullback( lapChiPsi, g2d_);
             const container vol2d = dg::create::weights( g2d_);
@@ -506,7 +505,7 @@ struct Hector : public aGenerator
     double c0_, lu_;
     thrust::host_vector<double> u_, ux_, uy_, vx_, vy_;
     thrust::host_vector<double> etaV_, zetaU_, etaU_;
-    dg::CurvilinearGrid2d<container> g2d_;
+    dg::CurvilinearGrid2d g2d_;
 
 };
 
diff --git a/inc/geometries/hector_t.cu b/inc/geometries/hector_t.cu
index e32b97b53..834c08dd2 100644
--- a/inc/geometries/hector_t.cu
+++ b/inc/geometries/hector_t.cu
@@ -7,11 +7,11 @@
 
 #include "dg/backend/xspacelib.cuh"
 #include "dg/functors.h"
+#include "dg/geometry/curvilinear.h"
 
 #include "dg/backend/timer.cuh"
 //#include "guenther.h"
 #include "solovev.h"
-#include "curvilinear.h"
 #include "hector.h"
 //#include "refined_conformal.h"
 #include "init.h"
-- 
GitLab


From 2e35b8f2e41417dd249cc17eb0cdab4f2b08b013 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Wed, 16 Aug 2017 07:17:45 -0700
Subject: [PATCH 168/453] replaced all Functor arguments by const Functor& in
 evaluate, pullback and push-Forward functions

---
 inc/dg/backend/evaluation.cuh   | 12 ++++++------
 inc/dg/backend/evaluationX.cuh  | 12 ++++++------
 inc/dg/backend/mpi_evaluation.h |  8 ++++----
 inc/dg/elliptic.h               |  2 +-
 inc/dg/geometry/transform.h     | 14 +++++++-------
 5 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/inc/dg/backend/evaluation.cuh b/inc/dg/backend/evaluation.cuh
index f78f72d2a..e7eda1acf 100644
--- a/inc/dg/backend/evaluation.cuh
+++ b/inc/dg/backend/evaluation.cuh
@@ -29,7 +29,7 @@ namespace dg
  * @return  A DG Host Vector with values
  */
 template< class UnaryOp>
-thrust::host_vector<double> evaluate( UnaryOp f, const Grid1d& g)
+thrust::host_vector<double> evaluate( const UnaryOp& f, const Grid1d& g)
 {
     thrust::host_vector<double> abs = create::abscissas( g);
     for( unsigned i=0; i<g.size(); i++)
@@ -39,7 +39,7 @@ thrust::host_vector<double> evaluate( UnaryOp f, const Grid1d& g)
 ///@cond
 thrust::host_vector<double> evaluate( double (f)(double), const Grid1d& g)
 {
-    thrust::host_vector<double> v = evaluate<double (double)>( f, g);
+    thrust::host_vector<double> v = evaluate<double (double)>( *f, g);
     return v;
 };
 ///@endcond
@@ -57,7 +57,7 @@ thrust::host_vector<double> evaluate( double (f)(double), const Grid1d& g)
  * @note if you don't like to copy f then just pass a (const) reference then the type should adapt
  */
 template< class BinaryOp>
-thrust::host_vector<double> evaluate( BinaryOp f, const aTopology2d& g)
+thrust::host_vector<double> evaluate( const BinaryOp& f, const aTopology2d& g)
 {
     unsigned n= g.n();
     Grid1d gx(g.x0(), g.x1(), g.n(), g.Nx());
@@ -79,7 +79,7 @@ thrust::host_vector<double> evaluate( BinaryOp f, const aTopology2d& g)
 thrust::host_vector<double> evaluate( double(f)(double, double), const aTopology2d& g)
 {
     //return evaluate<double(&)(double, double), n>( f, g );
-    return evaluate<double(double, double)>( f, g);
+    return evaluate<double(double, double)>( *f, g);
 };
 ///@endcond
 
@@ -95,7 +95,7 @@ thrust::host_vector<double> evaluate( double(f)(double, double), const aTopology
  * @note if you don't like to copy f then just pass a (const) reference then the type should adapt
  */
 template< class TernaryOp>
-thrust::host_vector<double> evaluate( TernaryOp f, const aTopology3d& g)
+thrust::host_vector<double> evaluate( const TernaryOp& f, const aTopology3d& g)
 {
     unsigned n= g.n();
     Grid1d gx(g.x0(), g.x1(), g.n(), g.Nx());
@@ -119,7 +119,7 @@ thrust::host_vector<double> evaluate( TernaryOp f, const aTopology3d& g)
 thrust::host_vector<double> evaluate( double(f)(double, double, double), const aTopology3d& g)
 {
     //return evaluate<double(&)(double, double), n>( f, g );
-    return evaluate<double(double, double, double)>( f, g);
+    return evaluate<double(double, double, double)>( *f, g);
 };
 ///@endcond
 
diff --git a/inc/dg/backend/evaluationX.cuh b/inc/dg/backend/evaluationX.cuh
index fbded8a26..67af562b3 100644
--- a/inc/dg/backend/evaluationX.cuh
+++ b/inc/dg/backend/evaluationX.cuh
@@ -26,14 +26,14 @@ namespace dg
  * @return  A DG Host Vector with values
  */
 template< class UnaryOp>
-thrust::host_vector<double> evaluate( UnaryOp f, const GridX1d& g)
+thrust::host_vector<double> evaluate( const UnaryOp& f, const GridX1d& g)
 {
     return evaluate( f, g.grid());
 };
 ///@cond
 thrust::host_vector<double> evaluate( double (f)(double), const GridX1d& g)
 {
-    return evaluate( f, g.grid());
+    return evaluate( *f, g.grid());
 };
 ///@endcond
 
@@ -50,14 +50,14 @@ thrust::host_vector<double> evaluate( double (f)(double), const GridX1d& g)
             may be constructed during function call.
  */
 template< class BinaryOp>
-thrust::host_vector<double> evaluate( BinaryOp f, const aTopologyX2d& g)
+thrust::host_vector<double> evaluate( const BinaryOp& f, const aTopologyX2d& g)
 {
     return evaluate( f, g.grid());
 };
 ///@cond
 thrust::host_vector<double> evaluate( double(f)(double, double), const aTopologyX2d& g)
 {
-    return evaluate( f, g.grid());
+    return evaluate( *f, g.grid());
 };
 ///@endcond
 
@@ -74,14 +74,14 @@ thrust::host_vector<double> evaluate( double(f)(double, double), const aTopology
             may be constructed during function call.
  */
 template< class TernaryOp>
-thrust::host_vector<double> evaluate( TernaryOp f, const aTopologyX3d& g)
+thrust::host_vector<double> evaluate( const TernaryOp& f, const aTopologyX3d& g)
 {
     return evaluate( f, g.grid());
 };
 ///@cond
 thrust::host_vector<double> evaluate( double(f)(double, double, double), const aTopologyX3d& g)
 {
-    return evaluate( f, g.grid());
+    return evaluate( *f, g.grid());
 };
 ///@endcond
 
diff --git a/inc/dg/backend/mpi_evaluation.h b/inc/dg/backend/mpi_evaluation.h
index 49e35beb9..b97987fa2 100644
--- a/inc/dg/backend/mpi_evaluation.h
+++ b/inc/dg/backend/mpi_evaluation.h
@@ -28,7 +28,7 @@ namespace dg
             may be constructed during function call.
  */
 template< class BinaryOp>
-MPI_Vector<thrust::host_vector<double> > evaluate( BinaryOp f, const aMPITopology2d& g)
+MPI_Vector<thrust::host_vector<double> > evaluate( const BinaryOp& f, const aMPITopology2d& g)
 {
     thrust::host_vector<double> w = evaluate( f, g.local());
     MPI_Vector<thrust::host_vector<double> > v( w, g.communicator());
@@ -38,7 +38,7 @@ MPI_Vector<thrust::host_vector<double> > evaluate( BinaryOp f, const aMPITopolog
 ///@cond
 MPI_Vector<thrust::host_vector<double> > evaluate( double(f)(double, double), const aMPITopology2d& g)
 {
-    return evaluate<double(double, double)>( f, g);
+    return evaluate<double(double, double)>( *f, g);
 };
 ///@endcond
 
@@ -55,7 +55,7 @@ MPI_Vector<thrust::host_vector<double> > evaluate( double(f)(double, double), co
             may be constructed during function call.
  */
 template< class TernaryOp>
-MPI_Vector<thrust::host_vector<double> > evaluate( TernaryOp f, const aMPITopology3d& g)
+MPI_Vector<thrust::host_vector<double> > evaluate( const TernaryOp& f, const aMPITopology3d& g)
 {
     thrust::host_vector<double> w = evaluate( f, g.local());
     MPI_Vector<thrust::host_vector<double> > v( w, g.communicator());
@@ -65,7 +65,7 @@ MPI_Vector<thrust::host_vector<double> > evaluate( TernaryOp f, const aMPITopolo
 ///@cond
 MPI_Vector<thrust::host_vector<double> > evaluate( double(f)(double, double, double), const aMPITopology3d& g)
 {
-    return evaluate<double(double, double, double)>( f, g);
+    return evaluate<double(double, double, double)>( *f, g);
 };
 ///@endcond
 //
diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index 17958edb4..face1b4a3 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -655,7 +655,7 @@ struct TensorElliptic
      *
      */
     template<class ChiRR, class ChiRZ, class ChiZZ>
-    void set( ChiRR chiRR, ChiRZ chiRZ, ChiZZ chiZZ)
+    void set( const ChiRR& chiRR, const ChiRZ& chiRZ, const ChiZZ& chiZZ)
     {
         typename GeometryTraits<Geometry>::host_vector chiXX, chiXY, chiYY;
         dg::pushForwardPerp( chiRR, chiRZ, chiZZ, chiXX, chiXY, chiYY, g_);
diff --git a/inc/dg/geometry/transform.h b/inc/dg/geometry/transform.h
index 014fc7ef3..7d7657490 100644
--- a/inc/dg/geometry/transform.h
+++ b/inc/dg/geometry/transform.h
@@ -39,7 +39,7 @@ struct GeometryTraits
  * @ingroup pullback
  */
 template< class Functor>
-thrust::host_vector<double> pullback( Functor f, const aGeometry2d& g)
+thrust::host_vector<double> pullback( const Functor& f, const aGeometry2d& g)
 {
     std::vector<thrust::host_vector<double> > map = g.map();
     thrust::host_vector<double> vec( g.size());
@@ -51,7 +51,7 @@ thrust::host_vector<double> pullback( Functor f, const aGeometry2d& g)
 ///@copydoc pullback(Functor,const aGeometry2d&)
 ///@ingroup pullback
 template< class Functor>
-thrust::host_vector<double> pullback( Functor f, const aGeometry3d& g)
+thrust::host_vector<double> pullback( const Functor& f, const aGeometry3d& g)
 {
     std::vector<thrust::host_vector<double> > map = g.map();
     thrust::host_vector<double> vec( g.size());
@@ -73,7 +73,7 @@ struct MemoryTraits< MPITag>
 ///@copydoc pullback(Functor,const aGeometry2d&)
 ///@ingroup pullback
 template< class Functor>
-MPI_Vector<thrust::host_vector<double> > pullback( Functor f, const aMPIGeometry2d& g)
+MPI_Vector<thrust::host_vector<double> > pullback( const Functor& f, const aMPIGeometry2d& g)
 {
     std::vector<MPI_Vector<thrust::host_vector<double> > > map = g.map();
     thrust::host_vector<double> vec( g.size());
@@ -85,7 +85,7 @@ MPI_Vector<thrust::host_vector<double> > pullback( Functor f, const aMPIGeometry
 ///@copydoc pullback(Functor,const aGeometry2d&)
 ///@ingroup pullback
 template< class Functor>
-MPI_Vector<thrust::host_vector<double> > pullback( Functor f, const aMPIGeometry3d& g)
+MPI_Vector<thrust::host_vector<double> > pullback( const Functor& f, const aMPIGeometry3d& g)
 {
     std::vector<MPI_Vector<thrust::host_vector<double> > > map = g.map();
     thrust::host_vector<double> vec( g.size());
@@ -112,7 +112,7 @@ MPI_Vector<thrust::host_vector<double> > pullback( Functor f, const aMPIGeometry
  * @ingroup pullback
  */
 template<class Functor1, class Functor2, class container, class Geometry> 
-void pushForwardPerp( Functor1 vR, Functor2 vZ, 
+void pushForwardPerp( const Functor1& vR, const Functor2& vZ, 
         container& vx, container& vy,
         const Geometry& g)
 {
@@ -142,7 +142,7 @@ void pushForwardPerp( Functor1 vR, Functor2 vZ,
  * @ingroup pullback
  */
 template<class Functor1, class Functor2, class Functor3, class container, class Geometry> 
-void pushForward( Functor1 vR, Functor2 vZ, Functor3 vPhi,
+void pushForward( const Functor1& vR, const Functor2& vZ, const Functor3& vPhi,
         container& vx, container& vy, container& vz,
         const Geometry& g)
 {
@@ -176,7 +176,7 @@ void pushForward( Functor1 vR, Functor2 vZ, Functor3 vPhi,
  * @ingroup pullback
  */
 template<class FunctorRR, class FunctorRZ, class FunctorZZ, class container, class Geometry> 
-void pushForwardPerp( FunctorRR chiRR, FunctorRZ chiRZ, FunctorZZ chiZZ,
+void pushForwardPerp( const FunctorRR& chiRR, const FunctorRZ& chiRZ, const FunctorZZ& chiZZ,
         container& chixx, container& chixy, container& chiyy,
         const Geometry& g)
 {
-- 
GitLab


From 4d0a3f0e265cd118bfac9abeccdff7d0d7f20f41 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Wed, 16 Aug 2017 07:24:05 -0700
Subject: [PATCH 169/453] replaced all Functor arguments by const Functor& in
 evaluate, pullback and push-Forward functions

---
 inc/dg/geometry/multiply.h | 13 +++++++++++++
 inc/geometries/hector.h    | 13 ++++++++-----
 2 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/inc/dg/geometry/multiply.h b/inc/dg/geometry/multiply.h
index 37498f8da..e052bdf03 100644
--- a/inc/dg/geometry/multiply.h
+++ b/inc/dg/geometry/multiply.h
@@ -88,6 +88,19 @@ void pointwiseDot( const SparseElement<container>& mu, const container& in, cont
     else
         out=in;
 }
+/**
+ * @brief Multiply container with form
+ *
+ * @copydoc hide_container_lvl1
+ * @param in input vector
+ * @param mu if mu.isEmpty() then out=in, else the input is pointwise multiplied with the value in mu
+ * @param out output vector (may alias in)
+ */
+template<class container>
+void pointwiseDot( const container& in, const SparseElement<container>& mu, container& out)
+{
+    pointwiseDot( mu, in, out);
+}
 
 /**
  * @brief Divide container with form
diff --git a/inc/geometries/hector.h b/inc/geometries/hector.h
index 7a07684a9..09863c3f9 100644
--- a/inc/geometries/hector.h
+++ b/inc/geometries/hector.h
@@ -390,7 +390,7 @@ struct Hector : public aGenerator2d
             dg::blas1::axpby( 1. ,u, -1., u_diff);
             eps = sqrt( dg::blas2::dot( u_diff, vol2d, u_diff) / dg::blas2::dot( u, vol2d, u) );
             if(verbose) std::cout <<" iter "<<number<<" error "<<eps<<"\n";
-            g2d_old = g2d;
+            g2d_old = g2d_;
             u_old = u;
             number++;//get rid of warning
         }
@@ -449,7 +449,7 @@ struct Hector : public aGenerator2d
 
 
         thrust::host_vector<double> chi_ZZ, chi_ZE, chi_EE;
-        dg::geo::pushForwardPerp( chi_XX, chi_XY, chi_YY, chi_ZZ, chi_ZE, chi_EE, g2d_);
+        dg::pushForwardPerp( chi_XX, chi_XY, chi_YY, chi_ZZ, chi_ZE, chi_EE, g2d_);
         container chiZZ, chiZE, chiEE;
         dg::blas1::transfer( chi_ZZ, chiZZ);
         dg::blas1::transfer( chi_ZE, chiZE);
@@ -470,7 +470,10 @@ struct Hector : public aGenerator2d
         //now compute etaV and its inverse
         container etaVinv(u_zeta), etaV(etaVinv);
         dg::blas1::pointwiseDot( etaVinv, chiZZ, etaVinv);
-        dg::geo::multiplyPerpVolume( etaVinv, g2d_);
+        dg::SparseElement<container> perp_vol = dg::tensor::determinant(g2d_.metric());
+        dg::tensor::sqrt(perp_vol);
+        dg::tensor::invert(perp_vol);
+        dg::tensor::pointwiseDot( etaVinv, perp_vol, etaVinv);
         dg::blas1::transform( etaVinv, etaV, dg::INVERT<double>());
         thrust::host_vector<double> etaVinv_h;
         dg::blas1::transfer( etaVinv, etaVinv_h);
@@ -478,8 +481,8 @@ struct Hector : public aGenerator2d
         container v_zeta(u), v_eta(u);
         dg::blas1::axpby( -1., temp_eta, 0.,v_zeta);
         dg::blas1::axpby( +1., temp_zeta, 0.,v_eta);
-        dg::geo::multiplyPerpVolume( v_zeta, g2d_);
-        dg::geo::multiplyPerpVolume(  v_eta, g2d_);
+        dg::tensor::pointwiseDot( v_zeta, perp_vol, v_zeta);
+        dg::tensor::pointwiseDot( v_eta, perp_vol, v_eta);
 
         //construct c0 and scale all vector components with it
         c0_ = fabs( detail::construct_c0( etaVinv_h, g2d_));
-- 
GitLab


From 9a885eeefaf20c841ac3c62b1fa667aeb1bb04d9 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Thu, 17 Aug 2017 01:33:40 -0700
Subject: [PATCH 170/453] added volume function to tensor namespace

---
 inc/dg/elliptic.h          |  8 ++------
 inc/dg/geometry/multiply.h | 22 ++++++++++++++++++++
 inc/geometries/flux_t.cu   |  4 +---
 inc/geometries/hector_t.cu | 33 ++++++++++++++++--------------
 inc/geometries/init.h      | 42 +++++++++++++++++++-------------------
 5 files changed, 64 insertions(+), 45 deletions(-)

diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index face1b4a3..3105910d6 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -187,9 +187,7 @@ class Elliptic
         dg::blas1::transfer( dg::create::inv_weights(g),   precond_); //weights are better preconditioners than volume
         tempx = tempy = gradx = weights_;
         chi_=g.metric();
-        vol_=dg::tensor::determinant(chi_);
-        dg::tensor::invert(vol_);
-        dg::tensor::sqrt(vol_); //now we have volume element
+        vol_=dg::tensor::volume(chi_);
         dg::tensor::scal( chi_, vol_);
         dg::blas1::transfer( dg::create::weights(g), weights_wo_vol);
     }
@@ -726,9 +724,7 @@ struct TensorElliptic
         tempx_ = tempy_ = gradx_ = chixx_;
         dg::blas1::transfer( dg::create::weights(g), weights_wo_vol);
 
-        vol_=dg::tensor::determinant(g.metric());
-        dg::tensor::invert(vol_);
-        dg::tensor::sqrt(vol_); //now we have volume element
+        vol_=dg::tensor::volume(g.metric());
         dg::tensor::pointwiseDot( vol_, chixx_, chixx_); 
         dg::tensor::pointwiseDot( vol_, chixy_, chixy_); 
         dg::tensor::pointwiseDot( vol_, chiyy_, chiyy_); 
diff --git a/inc/dg/geometry/multiply.h b/inc/dg/geometry/multiply.h
index e052bdf03..2ddfb18b9 100644
--- a/inc/dg/geometry/multiply.h
+++ b/inc/dg/geometry/multiply.h
@@ -350,6 +350,28 @@ SparseElement<container> determinant( const SparseTensor<container>& t)
     return SparseElement<container>(det);
 }
 
+/**
+ * @brief Compute the sqrt of the inverse determinant of a tensor
+ *
+ * This is a convenience function that is the same as
+ * @code
+    SparseElement<container> volume=determinant(g);
+    invert(volume);
+    sqrt(volume);
+    @endcode
+ * @copydoc hide_container_lvl1
+ * @param t the input tensor 
+ * @return the inverse square root of the determinant of t as a SparseElement (unset if t is empty)
+ */
+template<class container>
+SparseElement<container> volume( const SparseTensor<container>& t)
+{
+    SparseElement<container> vol=determinant(t);
+    invert(vol);
+    sqrt(vol);
+    return vol;
+
+}
 
 
 ///@cond
diff --git a/inc/geometries/flux_t.cu b/inc/geometries/flux_t.cu
index 32d331e4b..f60084edf 100644
--- a/inc/geometries/flux_t.cu
+++ b/inc/geometries/flux_t.cu
@@ -116,9 +116,7 @@ int main( int argc, char* argv[])
     //compute and write deformation into netcdf
     dg::SparseTensor<dg::HVec> metric = g2d.metric();
     dg::HVec g_xx = metric.value(0,0), g_xy = metric.value(0,1), g_yy=metric.value(1,1);
-    dg::SparseElement<dg::HVec> vol_ = dg::tensor::determinant(metric);
-    dg::tensor::invert(vol_);
-    dg::tensor::sqrt(vol_);
+    dg::SparseElement<dg::HVec> vol_ = dg::tensor::volume(metric);
     dg::blas1::pointwiseDivide( g_xy, g_xx, temp0);
     const dg::HVec ones = dg::evaluate( dg::one, g2d);
     X=g_yy;
diff --git a/inc/geometries/hector_t.cu b/inc/geometries/hector_t.cu
index 834c08dd2..e4ff75d09 100644
--- a/inc/geometries/hector_t.cu
+++ b/inc/geometries/hector_t.cu
@@ -73,27 +73,27 @@ int main( int argc, char* argv[])
     //solovev::detail::Fpsi fpsi( gp, -10);
     std::cout << "Constructing conformal grid ... \n";
     t.tic();
-    dg::geo::solovev::MagneticField c( gp); 
+    dg::geo::BinaryFunctorsLvl2 psip = dg::geo::solovev::createPsip( gp); 
     Hector<dg::IDMatrix, dg::DMatrix, dg::DVec>* hector;
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     int construction = 0;
     if( construction == 0)
     {
-        hector = new Hector<dg::IDMatrix, dg::DMatrix, dg::DVec>( c.psip, c.psipR, c.psipZ, c.psipRR, c.psipRZ, c.psipZZ, psi_0, psi_1, gp.R_0, 0., nGrid, NxGrid, NyGrid, epsHector, true);
+        hector = new Hector<dg::IDMatrix, dg::DMatrix, dg::DVec>( psip, psi_0, psi_1, gp.R_0, 0., nGrid, NxGrid, NyGrid, epsHector, true);
     }
     else if( construction == 1)
     {
-        dg::geo::NablaPsiInvCollective<solovev::PsipR, solovev::PsipZ, solovev::PsipRR, solovev::PsipRZ, solovev::PsipZZ> nc( c.psipR, c.psipZ, c.psipRR, c.psipRZ, c.psipZZ);
-        hector = new Hector<dg::IDMatrix, dg::DMatrix, dg::DVec>( c.psip, c.psipR, c.psipZ, c.psipRR, c.psipRZ, c.psipZZ, nc.nablaPsiInv, nc.nablaPsiInvX, nc.nablaPsiInvY, psi_0, psi_1, gp.R_0, 0., nGrid, NxGrid, NyGrid, epsHector, true);
+        dg::geo::BinaryFunctorsLvl1 nc = dg::geo::make_NablaPsiInvCollective nc( psip);
+        hector = new Hector<dg::IDMatrix, dg::DMatrix, dg::DVec>( psip, nc, psi_0, psi_1, gp.R_0, 0., nGrid, NxGrid, NyGrid, epsHector, true);
     }
     else
     {
-        dg::geo::LiseikinCollective<solovev::PsipR, solovev::PsipZ, solovev::PsipRR, solovev::PsipRZ, solovev::PsipZZ> lc( c.psipR, c.psipZ, c.psipRR, c.psipRZ, c.psipZZ, 0.1, 0.001);
-        hector = new Hector<dg::IDMatrix, dg::DMatrix, dg::DVec>( c.psip, c.psipR, c.psipZ, c.psipRR, c.psipRZ, c.psipZZ, lc.chi_XX, lc.chi_XY, lc.chi_YY, lc.divChiX, lc.divChiY, psi_0, psi_1, gp.R_0, 0., nGrid, NxGrid, NyGrid, epsHector, true);
+        dg::geo::BinarySymmTensor lc = dg::geo::make_LiseikinCollective lc( psip, 0.1, 0.001);
+        hector = new Hector<dg::IDMatrix, dg::DMatrix, dg::DVec>( psip,lc, psi_0, psi_1, gp.R_0, 0., nGrid, NxGrid, NyGrid, epsHector, true);
     }
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-    dg::CurvilinearGrid3d<dg::HVec> g3d(hector, n, Nx, Ny,Nz, dg::DIR);
-    dg::CurvilinearGrid2d<dg::HVec> g2d = g3d.perp_grid();
+    dg::CurvilinearGrid3d g3d(hector, n, Nx, Ny,Nz, dg::DIR);
+    dg::CurvilinearGrid2d g2d = g3d.perp_grid();
 
     dg::Grid2d g2d_periodic(g2d.x0(), g2d.x1(), g2d.y0(), g2d.y1(), g2d.n(), g2d.Nx(), g2d.Ny()+1); 
     t.toc();
@@ -120,8 +120,8 @@ int main( int argc, char* argv[])
     dg::HVec X( g2d.size()), Y(X); //P = dg::pullback( dg::coo3, g);
     for( unsigned i=0; i<g2d.size(); i++)
     {
-        X[i] = g2d.r()[i];
-        Y[i] = g2d.z()[i];
+        X[i] = g2d.map()[0][i];
+        Y[i] = g2d.map()[1][i];
     }
 
     dg::HVec temp0( g2d.size()), temp1(temp0);
@@ -132,26 +132,29 @@ int main( int argc, char* argv[])
     //err = nc_put_var_double( ncid, coordsID[2], g.z().data());
 
     //compute and write deformation into netcdf
-    dg::blas1::pointwiseDivide( g2d.g_yy(), g2d.g_xx(), temp0);
+    dg::SparseTensor<dg::HVec> metric = g2d.metric();
+    dg::HVec g_xx = metric.value(0,0), g_yy=metric.value(1,1);
+    dg::blas1::pointwiseDivide( g_yy, g_xx, temp0);
     const dg::HVec ones = dg::evaluate( dg::one, g2d);
     X=temp0;
     err = nc_put_var_double( ncid, defID, periodify(X, g2d_periodic).data());
     //compute and write conformalratio into netcdf
-    dg::blas1::pointwiseDivide( g2d.g_yy(), g2d.g_xx(), temp0);
+    dg::blas1::pointwiseDivide( g_yy, g_xx, temp0);
     X=temp0;
     err = nc_put_var_double( ncid, confID, periodify(X, g2d_periodic).data());
 
     std::cout << "Construction successful!\n";
 
     //compare determinant vs volume form
-    dg::blas1::pointwiseDot( g2d.g_xx(), g2d.g_yy(), temp0);
+    dg::blas1::pointwiseDot( g_xx, g_yy, temp0);
     dg::blas1::axpby( 1., temp0, -1., temp1, temp0);
     dg::blas1::transform( temp0, temp0, dg::SQRT<double>());
     dg::blas1::pointwiseDivide( ones, temp0, temp0);
     dg::blas1::transfer( temp0, X);
     err = nc_put_var_double( ncid, volID, periodify(X, g2d_periodic).data());
-    dg::blas1::axpby( 1., temp0, -1., g2d.vol(), temp0);
-    double error = sqrt(dg::blas2::dot( temp0, w2d, temp0)/dg::blas2::dot( g2d.vol(), w2d, g2d.vol()));
+    dg::SparseElement<dg::HVec> vol = dg::tensor::volume(metric);
+    dg::blas1::axpby( 1., temp0, -1., vol, temp0);
+    double error = sqrt(dg::blas2::dot( temp0, w2d, temp0)/dg::blas2::dot( vol, w2d, vol));
     std::cout << "Rel Consistency  of volume is "<<error<<"\n";
 
     std::cout << "TEST VOLUME IS:\n";
diff --git a/inc/geometries/init.h b/inc/geometries/init.h
index f95d3e7f4..202305748 100644
--- a/inc/geometries/init.h
+++ b/inc/geometries/init.h
@@ -33,7 +33,7 @@ struct Iris
         0  \text{ else}
      \end{cases}\f]
      */
-    double operator( )(double R, double Z)
+    double operator( )(double R, double Z)const
     {
         if( psip_(R,Z) > psipmax_) return 0.;
         if( psip_(R,Z) < psipmin_) return 0.;
@@ -42,7 +42,7 @@ struct Iris
     /**
     * @brief == operator()(R,Z)
     */
-    double operator( )(double R, double Z, double phi)
+    double operator( )(double R, double Z, double phi)const
     {
         return (*this)(R,Z);
     }
@@ -69,7 +69,7 @@ struct Pupil
         1  \text{ else}
      \end{cases}\f]
      */
-    double operator( )(double R, double Z)
+    double operator( )(double R, double Z)const
     {
         if( psip_(R,Z) > psipmaxcut_) return 0.;
         return 1.;
@@ -77,7 +77,7 @@ struct Pupil
     /**
     * @brief == operator()(R,Z)
     */
-    double operator( )(double R, double Z, double phi)
+    double operator( )(double R, double Z, double phi)const
     {
         return (*this)(R,Z);
     }
@@ -104,7 +104,7 @@ struct PsiPupil
         \psi_p(R,Z) \text{ else}
      \end{cases}\f]
      */
-    double operator( )(double R, double Z)
+    double operator( )(double R, double Z)const
     {
         if( psip_(R,Z) > psipmax_) return psipmax_;
         return  psip_(R,Z);
@@ -112,7 +112,7 @@ struct PsiPupil
     /**
     * @brief == operator()(R,Z)
     */
-    double operator( )(double R, double Z, double phi)
+    double operator( )(double R, double Z, double phi)const
     {
         return (*this)(R,Z);
     }
@@ -141,7 +141,7 @@ struct PsiLimiter
         0  \text{ else}
      \end{cases}\f]
      */
-    double operator( )(double R, double Z)
+    double operator( )(double R, double Z)const
     {
         if( psip_(R,Z) > psipmaxlim_) return 1.;
         return 0.;
@@ -149,7 +149,7 @@ struct PsiLimiter
     /**
     * @brief == operator()(R,Z)
     */
-    double operator( )(double R, double Z, double phi)
+    double operator( )(double R, double Z, double phi)const
     {
         return (*this)(R,Z);
     }
@@ -186,7 +186,7 @@ struct GaussianDamping
  \end{cases}
    \f]
      */
-    double operator( )(double R, double Z)
+    double operator( )(double R, double Z)const
     {
         if( psip_(R,Z) > psipmaxcut_ + 4.*alpha_) return 0.;
         if( psip_(R,Z) < psipmaxcut_) return 1.;
@@ -195,7 +195,7 @@ struct GaussianDamping
     /**
     * @brief == operator()(R,Z)
     */
-    double operator( )(double R, double Z, double phi)
+    double operator( )(double R, double Z, double phi)const
     {
         return (*this)(R,Z);
     }
@@ -229,7 +229,7 @@ struct GaussianProfDamping
  \end{cases}
    \f]
      */
-    double operator( )(double R, double Z)
+    double operator( )(double R, double Z)const
     {
         if( psip_(R,Z) > psipmax_ ) return 0.;
         if( psip_(R,Z) < (psipmax_-4.*alpha_)) return 1.;
@@ -238,7 +238,7 @@ struct GaussianProfDamping
     /**
     * @brief == operator()(R,Z)
     */
-    double operator( )(double R, double Z, double phi)
+    double operator( )(double R, double Z, double phi)const
     {
         return (*this)(R,Z);
     }
@@ -275,7 +275,7 @@ struct GaussianProfXDamping
  \end{cases}
    \f]
      */
-    double operator( )(double R, double Z)
+    double operator( )(double R, double Z)const
     {
         if( psip_(R,Z) > gp_.psipmax || Z<-1.1*gp_.elongation*gp_.a) return 0.;
         if( psip_(R,Z) < (gp_.psipmax-4.*gp_.alpha)) return 1.;
@@ -284,7 +284,7 @@ struct GaussianProfXDamping
     /**
     * @brief == operator()(R,Z)
     */
-    double operator( )(double R, double Z, double phi)
+    double operator( )(double R, double Z, double phi)const
     {
         return (*this)(R,Z);
     }
@@ -307,14 +307,14 @@ struct TanhSource
      * @brief \f[ 0.5\left( 1 + \tanh\left( -\frac{\psi_p(R,Z) - \psi_{p,min} + 3\alpha}{\alpha}\right)\right)
    \f]
      */
-    double operator( )(double R, double Z)
+    double operator( )(double R, double Z)const
     {
         return 0.5*(1.+tanh(-(psip_(R,Z)-psipmin_ + 3.*alpha_)/alpha_) );
     }
     /**
     * @brief == operator()(R,Z)
     */
-    double operator( )(double R, double Z, double phi)
+    double operator( )(double R, double Z, double phi)const
     {
         return 0.5*(1.+tanh(-(psip_(R,Z,phi)-psipmin_ + 3.*alpha_)/alpha_) );
     }
@@ -336,7 +336,7 @@ struct TanhSource
 //         if( psip_(R,Z) < 0.) return p_.nprofileamp+p_.bgprofamp-(gp_.psipmin-psip_(R,Z))*(p_.nprofileamp/gp_.psipmin);
 //         return p_.bgprofamp;
 //     }
-//     double operator( )(double R, double Z, double phi)
+//     double operator( )(double R, double Z, double phi)const
 //     {
 //         return (*this)(R,Z);
 // 
@@ -370,7 +370,7 @@ struct Nprofile
  \end{cases}
    \f]
      */
-   double operator( )(double R, double Z)
+   double operator( )(double R, double Z)const
     {
         if (psip_(R,Z)<gp_.psipmax) return bgamp +(psip_(R,Z)/psip_(gp_.R_0,0.0)*namp);
 	if( psip_(R,Z) > gp_.psipmax || Z<-1.1*gp_.elongation*gp_.a) return bgamp;
@@ -379,7 +379,7 @@ struct Nprofile
     /**
     * @brief == operator()(R,Z)
     */
-    double operator( )(double R, double Z, double phi)
+    double operator( )(double R, double Z, double phi)const
     {
         return (*this)(R,Z);
     }
@@ -411,7 +411,7 @@ struct ZonalFlow
  \end{cases}
    \f]
      */
-    double operator() (double R, double Z)
+    double operator() (double R, double Z)const
     {
       if (psip_(R,Z)<gp_.psipmax) 
           return (amp_*fabs(cos(2.*M_PI*psip_(R,Z)*k_)));
@@ -421,7 +421,7 @@ struct ZonalFlow
     /**
     * @brief == operator()(R,Z)
     */
-    double operator() (double R, double Z,double phi)
+    double operator() (double R, double Z,double phi)const
     {
         return (*this)(R,Z);
     }
-- 
GitLab


From 516f8cbc00e153b1106b05d9d9a3bd4db3dbfe8c Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Thu, 17 Aug 2017 02:07:16 -0700
Subject: [PATCH 171/453] made hector_t compile

---
 inc/dg/elliptic.h              |  19 ++--
 inc/dg/geometry/transform.h    |   7 +-
 inc/geometries/flux_t.cu       |   2 +-
 inc/geometries/fluxfunctions.h |   1 +
 inc/geometries/hector.h        |  17 +--
 inc/geometries/hector_t.cu     |  25 +++--
 inc/geometries/init.h          | 191 ++++++++++-----------------------
 7 files changed, 92 insertions(+), 170 deletions(-)

diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index 3105910d6..5c6e66de8 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -637,15 +637,11 @@ struct TensorElliptic
      * @param chiYY The new yy component
      * @note Components need to be already transformed into the current coordinate system
      */
-    template<class container2>
-    void set( const container2& chiXX, const container2& chiXY, const container2& chiYY)
+    void set( const container& chiXX, const container& chiXY, const container& chiYY)
     {
-        dg::blas1::transfer( chiXX, chixx_);
-        dg::blas1::transfer( chiXY, chixy_);
-        dg::blas1::transfer( chiYY, chiyy_);
-        dg::tensor::pointwiseDot( vol_, chixx_,chixx_);
-        dg::tensor::pointwiseDot( vol_, chixy_,chixy_);
-        dg::tensor::pointwiseDot( vol_, chiyy_,chiyy_);
+        dg::tensor::pointwiseDot( vol_, chiXX, chixx_);
+        dg::tensor::pointwiseDot( vol_, chiXY, chixy_);
+        dg::tensor::pointwiseDot( vol_, chiYY, chiyy_);
     }
 
     /**
@@ -657,7 +653,10 @@ struct TensorElliptic
     {
         typename GeometryTraits<Geometry>::host_vector chiXX, chiXY, chiYY;
         dg::pushForwardPerp( chiRR, chiRZ, chiZZ, chiXX, chiXY, chiYY, g_);
-        set( chiXX, chiXY, chiYY);
+        dg::blas1::transfer( chiXX, chixx_);
+        dg::blas1::transfer( chiXY, chixy_);
+        dg::blas1::transfer( chiYY, chiyy_);
+        set( chixx_, chixy_, chiyy_);
     }
 
     /**
@@ -746,7 +745,7 @@ struct TensorElliptic
     Matrix leftx, lefty, rightx, righty, jumpX, jumpY;
     container weights_, weights_wo_vol, precond_; //contain coeffs for chi multiplication
     container chixx_, chixy_, chiyy_, tempx_, tempy_, gradx_;
-    SparseTensor<container> vol_;
+    SparseElement<container> vol_;
     norm no_;
     Geometry g_;
 };
diff --git a/inc/dg/geometry/transform.h b/inc/dg/geometry/transform.h
index 7d7657490..b6f563a0c 100644
--- a/inc/dg/geometry/transform.h
+++ b/inc/dg/geometry/transform.h
@@ -103,6 +103,7 @@ MPI_Vector<thrust::host_vector<double> > pullback( const Functor& f, const aMPIG
                 v^y(x,y) = y_R (x,y) v^R(R(x,y), Z(x,y)) + y_Z v^Z(R(x,y), Z(x,y)) \f]
    where \f$ x_R = \frac{\partial x}{\partial R}\f$, ... 
  * @tparam Functor1 Binary or Ternary functor
+ * @copydoc hide_container_lvl1
  * @tparam Geometry The Geometry class
  * @param vR input R-component in cylindrical coordinates
  * @param vZ input Z-component in cylindrical coordinates
@@ -131,6 +132,7 @@ void pushForwardPerp( const Functor1& vR, const Functor2& vZ,
                 v^y(x,y) = y_R (x,y) v^R(R(x,y), Z(x,y)) + y_Z v^Z(R(x,y), Z(x,y)) \f]
    where \f$ x_R = \frac{\partial x}{\partial R}\f$, ... 
  * @tparam Functor1 Binary or Ternary functor
+ * @copydoc hide_container_lvl1
  * @tparam Geometry The Geometry class
  * @param vR input R-component in cartesian or cylindrical coordinates
  * @param vZ input Z-component in cartesian or cylindrical coordinates
@@ -165,6 +167,7 @@ void pushForward( const Functor1& vR, const Functor2& vZ, const Functor3& vPhi,
  \chi^{yy}(x,y) = y_R y_R \chi^{RR} + 2y_Ry_Z \chi^{RZ} + y_Zy_Z\chi^{ZZ} \\
                \f]
    where \f$ x_R = \frac{\partial x}{\partial R}\f$, ... 
+ * @copydoc hide_container_lvl1
  * @tparam Geometry The Geometry class
  * @param chiRR input RR-component in cylindrical coordinates
  * @param chiRZ input RZ-component in cylindrical coordinates
@@ -196,9 +199,9 @@ void pushForwardPerp( const FunctorRR& chiRR, const FunctorRZ& chiRZ, const Func
     std::vector<container> values( 3); 
     values[0] = chiRR_, values[1] = chiRZ_, values[2] = chiZZ_;
     SparseTensor<container> chi(values);
-    chi(0,0)=0, chi(0,1)=chi(1,0)=1, chi(1,1)=2;
+    chi.idx(0,0)=0, chi.idx(0,1)=chi.idx(1,0)=1, chi.idx(1,1)=2;
 
-    SparseTensor<container> d = jac.dense(); //now we have a dense tensor
+    SparseTensor<container> d = dg::tensor::dense(jac); //now we have a dense tensor
     container tmp00(d.value(0,0)), tmp01(tmp00), tmp10(tmp00), tmp11(tmp00);
     // multiply Chi*t -> tmp
     dg::tensor::multiply2d( chi, d.value(0,0), d.value(1,0), tmp00, tmp10);
diff --git a/inc/geometries/flux_t.cu b/inc/geometries/flux_t.cu
index f60084edf..668ca6058 100644
--- a/inc/geometries/flux_t.cu
+++ b/inc/geometries/flux_t.cu
@@ -166,7 +166,7 @@ int main( int argc, char* argv[])
     std::cout << "TEST VOLUME IS:\n";
     if( psi_0 < psi_1) gp.psipmax = psi_1, gp.psipmin = psi_0;
     else               gp.psipmax = psi_0, gp.psipmin = psi_1;
-    dg::geo::Iris<const dg::geo::aBinaryFunctor&> iris( c.psip(), gp.psipmin, gp.psipmax);
+    dg::geo::Iris iris( c.psip(), gp.psipmin, gp.psipmax);
     //dg::CylindricalGrid3d<dg::HVec> g3d( gp.R_0 -2.*gp.a, gp.R_0 + 2*gp.a, -2*gp.a, 2*gp.a, 0, 2*M_PI, 3, 2200, 2200, 1, dg::PER, dg::PER, dg::PER);
     dg::CartesianGrid2d g2dC( gp.R_0 -2.0*gp.a, gp.R_0 + 2.0*gp.a, -2.0*gp.a,2.0*gp.a,1, 2e3, 2e3, dg::PER, dg::PER);
     dg::HVec vec  = dg::evaluate( iris, g2dC);
diff --git a/inc/geometries/fluxfunctions.h b/inc/geometries/fluxfunctions.h
index b7fcf54ab..604863668 100644
--- a/inc/geometries/fluxfunctions.h
+++ b/inc/geometries/fluxfunctions.h
@@ -1,4 +1,5 @@
 #pragma once
+#include "dg/backend/manage.h"
 
 namespace dg
 {
diff --git a/inc/geometries/hector.h b/inc/geometries/hector.h
index 09863c3f9..83d6548b7 100644
--- a/inc/geometries/hector.h
+++ b/inc/geometries/hector.h
@@ -133,7 +133,7 @@ void construct_grid(
         const thrust::host_vector<double>& eta_init, //1d intial values
         thrust::host_vector<double>& zeta, 
         thrust::host_vector<double>& eta, 
-        const dg::aTopology2d& g2d
+        const dg::aGeometry2d& g2d
     )
 {
     Interpolate inter( zetaU, etaU, g2d);
@@ -187,10 +187,11 @@ void transform(
     thrust::host_vector<double> uh_zeta, uh_eta;
     dg::blas1::transfer( u_zeta, uh_zeta);
     dg::blas1::transfer( u_eta, uh_eta);
-    dg::blas1::pointwiseDot( uh_zeta, g2d.xr(), u_x);
-    dg::blas1::pointwiseDot( 1., uh_eta, g2d.yr(), 1., u_x);
-    dg::blas1::pointwiseDot( uh_zeta, g2d.xz(), u_y);
-    dg::blas1::pointwiseDot( 1., uh_eta, g2d.yz(), 1., u_y);
+    dg::SparseTensor<thrust::host_vector<double> > jac = g2d.jacobian();
+    dg::blas1::pointwiseDot( uh_zeta, jac.value(0,0), u_x);
+    dg::blas1::pointwiseDot( 1., uh_eta, jac.value(1,0) , 1., u_x);
+    dg::blas1::pointwiseDot( uh_zeta, jac.value(0,1), u_y);
+    dg::blas1::pointwiseDot( 1., uh_eta, jac.value(1,1), 1., u_y);
 }
 
 }//namespace detail
@@ -222,7 +223,7 @@ struct Hector : public aGenerator2d
      * @param verbose If true convergence details are printed to std::cout
      */
     Hector( const BinaryFunctorsLvl2& psi, double psi0, double psi1, double X0, double Y0, unsigned n = 13, unsigned Nx = 2, unsigned Ny = 10, double eps_u = 1e-10, bool verbose=false) : 
-        g2d_(new dg::geo::RibeiroFluxGenerator(psi, psi0, psi1, X0, Y0,1), n, Nx, Ny, dg::DIR)
+        g2d_(dg::geo::RibeiroFluxGenerator(psi, psi0, psi1, X0, Y0,1), n, Nx, Ny, dg::DIR)
     {
         //first construct u_
         container u = construct_grid_and_u( dg::geo::Constant(1), dg::geo::detail::LaplacePsi(psi), psi0, psi1, X0, Y0, n, Nx, Ny, eps_u , verbose);
@@ -253,7 +254,7 @@ struct Hector : public aGenerator2d
      * @param verbose If true convergence details are printed to std::cout
      */
     Hector( const BinaryFunctorsLvl2& psi, const BinaryFunctorsLvl1& chi, double psi0, double psi1, double X0, double Y0, unsigned n = 13, unsigned Nx = 2, unsigned Ny = 10, double eps_u = 1e-10, bool verbose=false) : 
-        g2d_(new dg::geo::RibeiroFluxGenerator(psi, psi0, psi1, X0, Y0,1), n, Nx, Ny, dg::DIR)
+        g2d_(dg::geo::RibeiroFluxGenerator(psi, psi0, psi1, X0, Y0,1), n, Nx, Ny, dg::DIR)
     {
         dg::geo::detail::LaplaceAdaptPsi lapAdaPsi( psi, chi);
         //first construct u_
@@ -287,7 +288,7 @@ struct Hector : public aGenerator2d
      */
     Hector( const BinaryFunctorsLvl2& psi,const BinarySymmTensorLvl1& chi,
             double psi0, double psi1, double X0, double Y0, unsigned n = 13, unsigned Nx = 2, unsigned Ny = 10, double eps_u = 1e-10, bool verbose=false) : 
-        g2d_(new dg::geo::RibeiroFluxGenerator(psi, psi0, psi1, X0, Y0,1), n, Nx, Ny, dg::DIR)
+        g2d_(dg::geo::RibeiroFluxGenerator(psi, psi0, psi1, X0, Y0,1), n, Nx, Ny, dg::DIR)
     {
         //first construct u_
         container u = construct_grid_and_u( psi, chi, 
diff --git a/inc/geometries/hector_t.cu b/inc/geometries/hector_t.cu
index e4ff75d09..33ccb6542 100644
--- a/inc/geometries/hector_t.cu
+++ b/inc/geometries/hector_t.cu
@@ -63,8 +63,8 @@ int main( int argc, char* argv[])
     }
     //write parameters from file into variables
     dg::geo::solovev::GeomParameters gp(js);
-    dg::geo::solovev::Psip psip( gp); 
-    std::cout << "Psi min "<<psip(gp.R_0, 0)<<"\n";
+    dg::geo::BinaryFunctorsLvl2 psip = dg::geo::solovev::createPsip( gp); 
+    std::cout << "Psi min "<<psip.f()(gp.R_0, 0)<<"\n";
     std::cout << "Type psi_0 and psi_1\n";
     double psi_0, psi_1;
     std::cin >> psi_0>> psi_1;
@@ -73,7 +73,6 @@ int main( int argc, char* argv[])
     //solovev::detail::Fpsi fpsi( gp, -10);
     std::cout << "Constructing conformal grid ... \n";
     t.tic();
-    dg::geo::BinaryFunctorsLvl2 psip = dg::geo::solovev::createPsip( gp); 
     Hector<dg::IDMatrix, dg::DMatrix, dg::DVec>* hector;
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     int construction = 0;
@@ -83,16 +82,16 @@ int main( int argc, char* argv[])
     }
     else if( construction == 1)
     {
-        dg::geo::BinaryFunctorsLvl1 nc = dg::geo::make_NablaPsiInvCollective nc( psip);
+        dg::geo::BinaryFunctorsLvl1 nc = dg::geo::make_NablaPsiInvCollective( psip);
         hector = new Hector<dg::IDMatrix, dg::DMatrix, dg::DVec>( psip, nc, psi_0, psi_1, gp.R_0, 0., nGrid, NxGrid, NyGrid, epsHector, true);
     }
     else
     {
-        dg::geo::BinarySymmTensor lc = dg::geo::make_LiseikinCollective lc( psip, 0.1, 0.001);
+        dg::geo::BinarySymmTensorLvl1 lc = dg::geo::make_LiseikinCollective( psip, 0.1, 0.001);
         hector = new Hector<dg::IDMatrix, dg::DMatrix, dg::DVec>( psip,lc, psi_0, psi_1, gp.R_0, 0., nGrid, NxGrid, NyGrid, epsHector, true);
     }
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-    dg::CurvilinearGrid3d g3d(hector, n, Nx, Ny,Nz, dg::DIR);
+    dg::CurvilinearProductGrid3d g3d(*hector, n, Nx, Ny,Nz, dg::DIR);
     dg::CurvilinearGrid2d g2d = g3d.perp_grid();
 
     dg::Grid2d g2d_periodic(g2d.x0(), g2d.x1(), g2d.y0(), g2d.y1(), g2d.n(), g2d.Nx(), g2d.Ny()+1); 
@@ -114,7 +113,7 @@ int main( int argc, char* argv[])
     err = nc_def_var( ncid, "volume", NC_DOUBLE, 2, dim3d, &volID);
     err = nc_def_var( ncid, "divB", NC_DOUBLE, 2, dim3d, &divBID);
 
-    thrust::host_vector<double> psi_p = dg::pullback( psip, g2d);
+    thrust::host_vector<double> psi_p = dg::pullback( psip.f(), g2d);
     //g.display();
     err = nc_put_var_double( ncid, onesID, periodify(psi_p, g2d_periodic).data());
     dg::HVec X( g2d.size()), Y(X); //P = dg::pullback( dg::coo3, g);
@@ -153,18 +152,18 @@ int main( int argc, char* argv[])
     dg::blas1::transfer( temp0, X);
     err = nc_put_var_double( ncid, volID, periodify(X, g2d_periodic).data());
     dg::SparseElement<dg::HVec> vol = dg::tensor::volume(metric);
-    dg::blas1::axpby( 1., temp0, -1., vol, temp0);
-    double error = sqrt(dg::blas2::dot( temp0, w2d, temp0)/dg::blas2::dot( vol, w2d, vol));
+    dg::blas1::axpby( 1., temp0, -1., vol.value(), temp0);
+    double error = sqrt(dg::blas2::dot( temp0, w2d, temp0)/dg::blas2::dot( vol.value(), w2d, vol.value()));
     std::cout << "Rel Consistency  of volume is "<<error<<"\n";
 
     std::cout << "TEST VOLUME IS:\n";
-    dg::HVec vol = dg::create::volume( g2d);
+    dg::HVec volume = dg::create::volume( g2d);
     dg::HVec ones2d = dg::evaluate( dg::one, g2d);
-    double volumeUV = dg::blas1::dot( vol, ones2d);
+    double volumeUV = dg::blas1::dot( vol.value(), ones2d);
 
-    vol = dg::create::volume( hector->internal_grid());
+    volume = dg::create::volume( hector->internal_grid());
     ones2d = dg::evaluate( dg::one, hector->internal_grid());
-    double volumeZE = dg::blas1::dot( vol, ones2d);
+    double volumeZE = dg::blas1::dot( vol.value(), ones2d);
     std::cout << "volumeUV is "<< volumeUV<<std::endl;
     std::cout << "volumeZE is "<< volumeZE<<std::endl;
     std::cout << "relative difference in volume is "<<fabs(volumeUV - volumeZE)/volumeZE<<std::endl;
diff --git a/inc/geometries/init.h b/inc/geometries/init.h
index 202305748..fb3b714c7 100644
--- a/inc/geometries/init.h
+++ b/inc/geometries/init.h
@@ -1,5 +1,6 @@
 #pragma once
 //#include "solovev.h"
+#include "fluxfunctions.h"
 #include "solovev_parameters.h"
 
 /*!@file
@@ -18,12 +19,10 @@ namespace geo
         1  \text{ if } \psi_{p,min} < \psi_p(R,Z) < \psi_{p,max}\\
         0  \text{ else}
      \end{cases}\f]
-     @tparam Psi models aBinaryOperator
  */ 
-template<class Psi>
-struct Iris
+struct Iris : public aCloneableBinaryFunctor<Iris>
 {
-    Iris( Psi psi, double psi_min, double psi_max ): 
+    Iris( const aBinaryFunctor& psi, double psi_min, double psi_max ): 
         psip_(psi), psipmin_(psi_min), psipmax_(psi_max) { }
 
     /**
@@ -35,19 +34,12 @@ struct Iris
      */
     double operator( )(double R, double Z)const
     {
-        if( psip_(R,Z) > psipmax_) return 0.;
-        if( psip_(R,Z) < psipmin_) return 0.;
+        if( psip_.get()(R,Z) > psipmax_) return 0.;
+        if( psip_.get()(R,Z) < psipmin_) return 0.;
         return 1.;
     }
-    /**
-    * @brief == operator()(R,Z)
-    */
-    double operator( )(double R, double Z, double phi)const
-    {
-        return (*this)(R,Z);
-    }
     private:
-    Psi psip_;
+    Handle<aBinaryFunctor> psip_;
     double psipmin_, psipmax_;
 };
 /**
@@ -57,10 +49,9 @@ struct Iris
         1  \text{ else}
      \end{cases}\f]
  */ 
-template<class Psi>
-struct Pupil
+struct Pupil : public aCloneableBinaryFunctor<Pupil>
 {
-    Pupil( Psi psi, double psipmaxcut): 
+    Pupil( const aBinaryFunctor& psi, double psipmaxcut): 
         psip_(psi), psipmaxcut_(psipmaxcut) { }
     /**
      * @brief 
@@ -71,18 +62,11 @@ struct Pupil
      */
     double operator( )(double R, double Z)const
     {
-        if( psip_(R,Z) > psipmaxcut_) return 0.;
+        if( psip_.get()(R,Z) > psipmaxcut_) return 0.;
         return 1.;
     }
-    /**
-    * @brief == operator()(R,Z)
-    */
-    double operator( )(double R, double Z, double phi)const
-    {
-        return (*this)(R,Z);
-    }
     private:
-    Psi psip_;
+    Handle<aBinaryFunctor> psip_;
     double psipmaxcut_;
 };
 /**
@@ -92,10 +76,9 @@ struct Pupil
         \psi_p(R,Z) \text{ else}
      \end{cases}\f]
  */ 
-template<class Psi>
-struct PsiPupil
+struct PsiPupil : public aCloneableBinaryFunctor<PsiPupil>
 {
-    PsiPupil( Psi psi, double psipmax): 
+    PsiPupil(const aBinaryFunctor& psi, double psipmax): 
         psipmax_(psipmax), psip_(psi) { } 
     /**
      * @brief 
@@ -106,19 +89,12 @@ struct PsiPupil
      */
     double operator( )(double R, double Z)const
     {
-        if( psip_(R,Z) > psipmax_) return psipmax_;
-        return  psip_(R,Z);
-    }
-    /**
-    * @brief == operator()(R,Z)
-    */
-    double operator( )(double R, double Z, double phi)const
-    {
-        return (*this)(R,Z);
+        if( psip_.get()(R,Z) > psipmax_) return psipmax_;
+        return  psip_.get()(R,Z);
     }
     private:
     double psipmax_;
-    Psi psip_;
+    Handle<aBinaryFunctor> psip_;
 };
 /**
  * @brief Sets values to one outside psipmaxcut, zero else
@@ -128,10 +104,9 @@ struct PsiPupil
      \end{cases}\f]
  *
  */ 
-template<class Psi>
-struct PsiLimiter
+struct PsiLimiter : public aCloneableBinaryFunctor<PsiLimiter>
 {
-    PsiLimiter( Psi psi, double psipmaxlim): 
+    PsiLimiter( const aBinaryFunctor& psi, double psipmaxlim): 
         psipmaxlim_(psipmaxlim), psip_(psi) { }
 
     /**
@@ -143,19 +118,12 @@ struct PsiLimiter
      */
     double operator( )(double R, double Z)const
     {
-        if( psip_(R,Z) > psipmaxlim_) return 1.;
+        if( psip_.get()(R,Z) > psipmaxlim_) return 1.;
         return 0.;
     }
-    /**
-    * @brief == operator()(R,Z)
-    */
-    double operator( )(double R, double Z, double phi)const
-    {
-        return (*this)(R,Z);
-    }
     private:
     double psipmaxlim_;
-    Psi psip_;
+    Handle<aBinaryFunctor> psip_;
 };
 
 
@@ -172,10 +140,9 @@ struct PsiLimiter
    \f]
  *
  */ 
-template< class Psi>
-struct GaussianDamping
+struct GaussianDamping : public aCloneableBinaryFunctor<GaussianDamping>
 {
-    GaussianDamping( Psi psi, double psipmaxcut, double alpha):
+    GaussianDamping( const aBinaryFunctor& psi, double psipmaxcut, double alpha):
         psip_(psi), psipmaxcut_(psipmaxcut), alpha_(alpha) { }
     /**
      * @brief 
@@ -188,19 +155,12 @@ struct GaussianDamping
      */
     double operator( )(double R, double Z)const
     {
-        if( psip_(R,Z) > psipmaxcut_ + 4.*alpha_) return 0.;
-        if( psip_(R,Z) < psipmaxcut_) return 1.;
-        return exp( -( psip_(R,Z)-psipmaxcut_)*( psip_(R,Z)-psipmaxcut_)/2./alpha_/alpha_);
-    }
-    /**
-    * @brief == operator()(R,Z)
-    */
-    double operator( )(double R, double Z, double phi)const
-    {
-        return (*this)(R,Z);
+        if( psip_.get()(R,Z) > psipmaxcut_ + 4.*alpha_) return 0.;
+        if( psip_.get()(R,Z) < psipmaxcut_) return 1.;
+        return exp( -( psip_.get()(R,Z)-psipmaxcut_)*( psip_.get()(R,Z)-psipmaxcut_)/2./alpha_/alpha_);
     }
     private:
-    Psi psip_;
+    Handle<aBinaryFunctor> psip_;
     double psipmaxcut_, alpha_;
 };
 /**
@@ -215,10 +175,9 @@ struct GaussianDamping
    \f]
  *
  */ 
-template< class Psi>
-struct GaussianProfDamping
+struct GaussianProfDamping : public aCloneableBinaryFunctor<GaussianProfDamping>
 {
-    GaussianProfDamping( Psi psi, double psipmax, double alpha):
+    GaussianProfDamping( const aBinaryFunctor& psi, double psipmax, double alpha):
         psip_(psi), psipmax_(psipmax), alpha_(alpha) { }
     /**
      * @brief 
@@ -231,19 +190,12 @@ struct GaussianProfDamping
      */
     double operator( )(double R, double Z)const
     {
-        if( psip_(R,Z) > psipmax_ ) return 0.;
-        if( psip_(R,Z) < (psipmax_-4.*alpha_)) return 1.;
-        return exp( -( psip_(R,Z)-(psipmax_-4.*alpha_))*( psip_(R,Z)-(psipmax_-4.*alpha_))/2./alpha_/alpha_);
-    }
-    /**
-    * @brief == operator()(R,Z)
-    */
-    double operator( )(double R, double Z, double phi)const
-    {
-        return (*this)(R,Z);
+        if( psip_.get()(R,Z) > psipmax_ ) return 0.;
+        if( psip_.get()(R,Z) < (psipmax_-4.*alpha_)) return 1.;
+        return exp( -( psip_.get()(R,Z)-(psipmax_-4.*alpha_))*( psip_.get()(R,Z)-(psipmax_-4.*alpha_))/2./alpha_/alpha_);
     }
     private:
-    Psi psip_;
+    Handle<aBinaryFunctor> psip_;
     double psipmax_, alpha_;
 };
 /**
@@ -259,10 +211,9 @@ struct GaussianProfDamping
    \f]
  *
  */ 
-template <class Psi>
-struct GaussianProfXDamping
+struct GaussianProfXDamping : public aCloneableBinaryFunctor<GaussianProfXDamping>
 {
-    GaussianProfXDamping( Psi psi, dg::geo::solovev::GeomParameters gp):
+    GaussianProfXDamping( const aBinaryFunctor& psi, dg::geo::solovev::GeomParameters gp):
         gp_(gp),
         psip_(psi) {
         }
@@ -277,20 +228,13 @@ struct GaussianProfXDamping
      */
     double operator( )(double R, double Z)const
     {
-        if( psip_(R,Z) > gp_.psipmax || Z<-1.1*gp_.elongation*gp_.a) return 0.;
-        if( psip_(R,Z) < (gp_.psipmax-4.*gp_.alpha)) return 1.;
-        return exp( -( psip_(R,Z)-(gp_.psipmax-4.*gp_.alpha))*( psip_(R,Z)-(gp_.psipmax-4.*gp_.alpha))/2./gp_.alpha/gp_.alpha);
-    }
-    /**
-    * @brief == operator()(R,Z)
-    */
-    double operator( )(double R, double Z, double phi)const
-    {
-        return (*this)(R,Z);
+        if( psip_.get()(R,Z) > gp_.psipmax || Z<-1.1*gp_.elongation*gp_.a) return 0.;
+        if( psip_.get()(R,Z) < (gp_.psipmax-4.*gp_.alpha)) return 1.;
+        return exp( -( psip_.get()(R,Z)-(gp_.psipmax-4.*gp_.alpha))*( psip_.get()(R,Z)-(gp_.psipmax-4.*gp_.alpha))/2./gp_.alpha/gp_.alpha);
     }
     private:
     dg::geo::solovev::GeomParameters gp_;
-    Psi psip_;
+    Handle<aBinaryFunctor> psip_;
 };
 
 /**
@@ -298,10 +242,9 @@ struct GaussianProfXDamping
  * Returns a tanh profile shifted to \f$ \psi_{p,min}-3\alpha\f$
  \f[ 0.5\left( 1 + \tanh\left( -\frac{\psi_p(R,Z) - \psi_{p,min} + 3\alpha}{\alpha}\right)\right) \f]
  */
-template<class Psi>
-struct TanhSource
+struct TanhSource : public aCloneableBinaryFunctor<TanhSource>
 {
-        TanhSource(Psi psi, double psipmin, double alpha):
+    TanhSource(const aBinaryFunctor& psi, double psipmin, double alpha):
             psipmin_(psipmin), alpha_(alpha), psip_(psi) { }
     /**
      * @brief \f[ 0.5\left( 1 + \tanh\left( -\frac{\psi_p(R,Z) - \psi_{p,min} + 3\alpha}{\alpha}\right)\right)
@@ -309,21 +252,14 @@ struct TanhSource
      */
     double operator( )(double R, double Z)const
     {
-        return 0.5*(1.+tanh(-(psip_(R,Z)-psipmin_ + 3.*alpha_)/alpha_) );
-    }
-    /**
-    * @brief == operator()(R,Z)
-    */
-    double operator( )(double R, double Z, double phi)const
-    {
-        return 0.5*(1.+tanh(-(psip_(R,Z,phi)-psipmin_ + 3.*alpha_)/alpha_) );
+        return 0.5*(1.+tanh(-(psip_.get()(R,Z)-psipmin_ + 3.*alpha_)/alpha_) );
     }
     private:
     double psipmin_, alpha_; 
-    Psi psip_;
+    Handle<aBinaryFunctor> psip_;
 };
 
-// struct Gradient
+// struct Gradient : public aCloneableBinaryFunctor<Gradient>
 // {
 //     Gradient(  eule::Parameters p, GeomParameters gp):
 //         p_(p),
@@ -332,8 +268,8 @@ struct TanhSource
 //     }
 //     double operator( )(double R, double Z)
 //     {
-//         if( psip_(R,Z) < (gp_.psipmin)) return p_.nprofileamp+p_.bgprofamp;
-//         if( psip_(R,Z) < 0.) return p_.nprofileamp+p_.bgprofamp-(gp_.psipmin-psip_(R,Z))*(p_.nprofileamp/gp_.psipmin);
+//         if( psip_.get()(R,Z) < (gp_.psipmin)) return p_.nprofileamp+p_.bgprofamp;
+//         if( psip_.get()(R,Z) < 0.) return p_.nprofileamp+p_.bgprofamp-(gp_.psipmin-psip_.get()(R,Z))*(p_.nprofileamp/gp_.psipmin);
 //         return p_.bgprofamp;
 //     }
 //     double operator( )(double R, double Z, double phi)const
@@ -344,7 +280,7 @@ struct TanhSource
 //     private:
 //     eule::Parameters p_;
 //     GeomParameters gp_;
-//     Psi psip_;
+//     Handle<aBinaryFunctor> psip_;
 // };
 
 /**
@@ -354,12 +290,10 @@ struct TanhSource
  A_{bg} \text{ else } 
  \end{cases}
    \f]
-   @tparam Psi models aBinaryOperator
  */ 
-template<class Psi>
-struct Nprofile
+struct Nprofile : public aCloneableBinaryFunctor<Nprofile>
 {
-     Nprofile( double bgprofamp, double peakamp, dg::geo::solovev::GeomParameters gp, Psi psi):
+     Nprofile( double bgprofamp, double peakamp, dg::geo::solovev::GeomParameters gp, const aBinaryFunctor& psi):
          bgamp(bgprofamp), namp( peakamp),
          gp_(gp),
          psip_(psi) { }
@@ -372,21 +306,14 @@ struct Nprofile
      */
    double operator( )(double R, double Z)const
     {
-        if (psip_(R,Z)<gp_.psipmax) return bgamp +(psip_(R,Z)/psip_(gp_.R_0,0.0)*namp);
-	if( psip_(R,Z) > gp_.psipmax || Z<-1.1*gp_.elongation*gp_.a) return bgamp;
+        if (psip_.get()(R,Z)<gp_.psipmax) return bgamp +(psip_.get()(R,Z)/psip_.get()(gp_.R_0,0.0)*namp);
+	if( psip_.get()(R,Z) > gp_.psipmax || Z<-1.1*gp_.elongation*gp_.a) return bgamp;
         return bgamp;
     }
-    /**
-    * @brief == operator()(R,Z)
-    */
-    double operator( )(double R, double Z, double phi)const
-    {
-        return (*this)(R,Z);
-    }
     private:
     double bgamp, namp;
     dg::geo::solovev::GeomParameters gp_;
-    Psi psip_;
+    Handle<aBinaryFunctor> psip_;
 };
 
 /**
@@ -397,10 +324,9 @@ struct Nprofile
  \end{cases}
    \f]
  */ 
-template<class Psi>
-struct ZonalFlow
+struct ZonalFlow : public aCloneableBinaryFunctor<ZonalFlow>
 {
-    ZonalFlow(  double amplitude, double k_psi, dg::geo::solovev::GeomParameters gp, Psi psi):
+    ZonalFlow(  double amplitude, double k_psi, dg::geo::solovev::GeomParameters gp, const aBinaryFunctor& psi):
         amp_(amplitude), k_(k_psi),
         gp_(gp),
         psip_(psi) { }
@@ -413,22 +339,15 @@ struct ZonalFlow
      */
     double operator() (double R, double Z)const
     {
-      if (psip_(R,Z)<gp_.psipmax) 
-          return (amp_*fabs(cos(2.*M_PI*psip_(R,Z)*k_)));
+      if (psip_.get()(R,Z)<gp_.psipmax) 
+          return (amp_*fabs(cos(2.*M_PI*psip_.get()(R,Z)*k_)));
       return 0.;
 
     }
-    /**
-    * @brief == operator()(R,Z)
-    */
-    double operator() (double R, double Z,double phi)const
-    {
-        return (*this)(R,Z);
-    }
     private:
     double amp_, k_;
     dg::geo::solovev::GeomParameters gp_;
-    Psi psip_;
+    Handle<aBinaryFunctor> psip_;
 };
 
 
-- 
GitLab


From 703c50bae30bd16f070193a7d1b1354da84ad856 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Thu, 17 Aug 2017 02:15:07 -0700
Subject: [PATCH 172/453] make ribeiro_t compile again

---
 inc/geometries/ribeiro_t.cu | 61 ++++++++++++++++++-------------------
 1 file changed, 29 insertions(+), 32 deletions(-)

diff --git a/inc/geometries/ribeiro_t.cu b/inc/geometries/ribeiro_t.cu
index 29f601833..6931bcb57 100644
--- a/inc/geometries/ribeiro_t.cu
+++ b/inc/geometries/ribeiro_t.cu
@@ -9,11 +9,10 @@
 #include "dg/functors.h"
 
 #include "dg/backend/timer.cuh"
+#include "dg/geometry/curvilinear.h"
 //#include "guenther.h"
 #include "solovev.h"
 #include "ribeiro.h"
-#include "curvilinear.h"
-#include "refined_curvilinear.h"
 //#include "ds.h"
 #include "init.h"
 
@@ -62,26 +61,19 @@ int main( int argc, char* argv[])
     }
     //write parameters from file into variables
     dg::geo::solovev::GeomParameters gp(js);
-    dg::geo::solovev::Psip psip( gp); 
-    std::cout << "Psi min "<<psip(gp.R_0, 0)<<"\n";
+    dg::geo::BinaryFunctorsLvl2 psip = dg::geo::solovev::createPsip( gp);
+    std::cout << "Psi min "<<psip.f()(gp.R_0, 0)<<"\n";
     std::cout << "Type psi_0 and psi_1\n";
     double psi_0, psi_1;
     std::cin >> psi_0>> psi_1;
-    std::cout << "Type new_n, multiple_x and multiple_y \n";
-    double n_ref, multiple_x, multiple_y;
-    std::cin >> n_ref>>multiple_x >> multiple_y;
     gp.display( std::cout);
     dg::Timer t;
     //solovev::detail::Fpsi fpsi( gp, -10);
     std::cout << "Constructing ribeiro grid ... \n";
     t.tic();
-    dg::geo::solovev::MagneticField c( gp);
-    dg::geo::Ribeiro<Psip, PsipR, PsipZ, PsipRR, PsipRZ, PsipZZ>
-        ribeiro( c.psip, c.psipR, c.psipZ, c.psipRR, c.psipRZ, c.psipZZ, psi_0, psi_1, gp.R_0, 0., 1);
-    //dg::CurvilinearGrid3d<dg::HVec> g3d(ribeiro, n, Nx, Ny,Nz, dg::DIR);
-    //dg::CurvilinearGrid2d<dg::HVec> g2d = g3d.perp_grid();
-    dg::CurvilinearRefinedGrid3d<dg::HVec> g3d(multiple_x, multiple_y, &ribeiro, n_ref, n, Nx, Ny,Nz, dg::DIR);
-    dg::CurvilinearRefinedGrid2d<dg::HVec> g2d = g3d.perp_grid();
+    dg::geo::Ribeiro ribeiro( psip, psi_0, psi_1, gp.R_0, 0., 1);
+    dg::CurvilinearProductGrid3d g3d(ribeiro, n, Nx, Ny,Nz, dg::DIR);
+    dg::CurvilinearGrid2d g2d = g3d.perp_grid();
     dg::Grid2d g2d_periodic(g2d.x0(), g2d.x1(), g2d.y0(), g2d.y1(), g2d.n(), g2d.Nx(), g2d.Ny()+1); 
     t.toc();
     std::cout << "Construction took "<<t.diff()<<"s"<<std::endl;
@@ -100,14 +92,14 @@ int main( int argc, char* argv[])
     err = nc_def_var( ncid, "volume", NC_DOUBLE, 2, dim3d, &volID);
     err = nc_def_var( ncid, "divB", NC_DOUBLE, 2, dim3d, &divBID);
 
-    thrust::host_vector<double> psi_p = dg::pullback( psip, g2d);
+    thrust::host_vector<double> psi_p = dg::pullback( psip.f(), g2d);
     //g.display();
     err = nc_put_var_double( ncid, onesID, periodify(psi_p, g2d_periodic).data());
     dg::HVec X( g2d.size()), Y(X); //P = dg::pullback( dg::coo3, g);
     for( unsigned i=0; i<g2d.size(); i++)
     {
-        X[i] = g2d.r()[i];
-        Y[i] = g2d.z()[i];
+        X[i] = g2d.map()[0][i];
+        Y[i] = g2d.map()[1][i];
     }
 
     dg::HVec temp0( g2d.size()), temp1(temp0);
@@ -115,56 +107,61 @@ int main( int argc, char* argv[])
 
     err = nc_put_var_double( ncid, coordsID[0], periodify(X, g2d_periodic).data());
     err = nc_put_var_double( ncid, coordsID[1], periodify(Y, g2d_periodic).data());
+
+    dg::SparseTensor<dg::HVec> metric = g2d.metric();
+    dg::HVec g_xx = metric.value(0,0), g_xy = metric.value(0,1), g_yy=metric.value(1,1);
+    dg::SparseElement<dg::HVec> vol_ = dg::tensor::volume(metric);
+    dg::HVec vol = vol_.value();
     //err = nc_put_var_double( ncid, coordsID[2], g.z().data());
     //compute and write deformation into netcdf
-    dg::blas1::pointwiseDivide( g2d.g_xy(), g2d.g_xx(), temp0);
+    dg::blas1::pointwiseDivide( g_xy, g_xx, temp0);
     const dg::HVec ones = dg::evaluate( dg::one, g2d);
-    X=g2d.g_yy();
+    X=g_yy;
     err = nc_put_var_double( ncid, defID, periodify(X, g2d_periodic).data());
     //compute and write ribeiroratio into netcdf
-    dg::blas1::pointwiseDivide( g2d.g_yy(), g2d.g_xx(), temp0);
+    dg::blas1::pointwiseDivide( g_yy, g_xx, temp0);
     X=temp0;
 
     err = nc_put_var_double( ncid, confID, periodify(X, g2d_periodic).data());
     std::cout << "Construction successful!\n";
 
     //compute error in volume element (in ribeiro grid g^xx is the volume element)
-    dg::blas1::pointwiseDot( g2d.g_xx(), g2d.g_yy(), temp0);
-    dg::blas1::pointwiseDot( g2d.g_xy(), g2d.g_xy(), temp1);
+    dg::blas1::pointwiseDot( g_xx, g_yy, temp0);
+    dg::blas1::pointwiseDot( g_xy, g_xy, temp1);
     dg::blas1::axpby( 1., temp0, -1., temp1, temp0);
-    dg::blas1::transfer( g2d.g_xx(),  temp1);
+    dg::blas1::transfer( g_xx,  temp1);
     dg::blas1::pointwiseDot( temp1, temp1, temp1);
     dg::blas1::axpby( 1., temp1, -1., temp0, temp0);
     double error = sqrt( dg::blas2::dot( temp0, w2d, temp0)/dg::blas2::dot( temp1, w2d, temp1));
     std::cout<< "Rel Error in Determinant is "<<error<<"\n";
 
     //compute error in determinant vs volume form
-    dg::blas1::pointwiseDot( g2d.g_xx(), g2d.g_yy(), temp0);
-    dg::blas1::pointwiseDot( g2d.g_xy(), g2d.g_xy(), temp1);
+    dg::blas1::pointwiseDot( g_xx, g_yy, temp0);
+    dg::blas1::pointwiseDot( g_xy, g_xy, temp1);
     dg::blas1::axpby( 1., temp0, -1., temp1, temp0);
     dg::blas1::transform( temp0, temp0, dg::SQRT<double>());
     dg::blas1::pointwiseDivide( ones, temp0, temp0);
     dg::blas1::transfer( temp0, X);
     err = nc_put_var_double( ncid, volID, periodify(X, g2d_periodic).data());
-    dg::blas1::axpby( 1., temp0, -1., g2d.vol(), temp0);
-    error = sqrt(dg::blas2::dot( temp0, w2d, temp0)/dg::blas2::dot( g2d.vol(), w2d, g2d.vol()));
+    dg::blas1::axpby( 1., temp0, -1., vol, temp0);
+    error = sqrt(dg::blas2::dot( temp0, w2d, temp0)/dg::blas2::dot( vol, w2d, vol));
     std::cout << "Rel Consistency  of volume is "<<error<<"\n";
 
     //compare g^xx to volume form
-    dg::blas1::transfer( g2d.g_xx(), temp0);
+    dg::blas1::transfer( g_xx, temp0);
     dg::blas1::pointwiseDivide( ones, temp0, temp0);
-    dg::blas1::axpby( 1., temp0, -1., g2d.vol(), temp0);
-    error=sqrt(dg::blas2::dot( temp0, w2d, temp0))/sqrt( dg::blas2::dot(g2d.vol(), w2d, g2d.vol()));
+    dg::blas1::axpby( 1., temp0, -1., vol, temp0);
+    error=sqrt(dg::blas2::dot( temp0, w2d, temp0))/sqrt( dg::blas2::dot(vol, w2d, vol));
     std::cout << "Rel Error of volume form is "<<error<<"\n";
 
-    const dg::HVec vol = dg::create::volume( g3d);
+    vol = dg::create::volume( g3d);
     dg::HVec ones3d = dg::evaluate( dg::one, g3d);
     double volume = dg::blas1::dot( vol, ones3d);
 
     std::cout << "TEST VOLUME IS:\n";
     if( psi_0 < psi_1) gp.psipmax = psi_1, gp.psipmin = psi_0;
     else               gp.psipmax = psi_0, gp.psipmin = psi_1;
-    dg::geo::Iris<Psip> iris(c.psip, gp.psipmin, gp.psipmax);
+    dg::geo::Iris iris(psip.f(), gp.psipmin, gp.psipmax);
     //dg::CylindricalGrid3d<dg::HVec> g3d( gp.R_0 -2.*gp.a, gp.R_0 + 2*gp.a, -2*gp.a, 2*gp.a, 0, 2*M_PI, 3, 2200, 2200, 1, dg::PER, dg::PER, dg::PER);
 //     dg::CartesianGrid2d g2dC( gp.R_0 -1.2*gp.a, gp.R_0 + 1.2*gp.a, -1.2*gp.a, 1.2*gp.a, 1, 1e3, 1e3, dg::PER, dg::PER);
     dg::CartesianGrid2d g2dC( gp.R_0 -2.0*gp.a, gp.R_0 + 2.0*gp.a, -2.0*gp.a, 2.0*gp.a, 1, 2e3, 2e3, dg::PER, dg::PER);
-- 
GitLab


From 8f6e788c47f2fa5dc19a45d665e1b2513179cef6 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Thu, 17 Aug 2017 02:51:14 -0700
Subject: [PATCH 173/453] great mystery why Ternary Op is not inherited??

---
 inc/geometries/ds.h                   |  2 +-
 inc/geometries/fluxfunctions.h        |  4 +-
 inc/geometries/ribeiro_mpit.cu        | 58 +++++++++++-------------
 inc/geometries/simple_orthogonal.h    |  8 ++--
 inc/geometries/simple_orthogonal_t.cu | 63 ++++++++++++++-------------
 inc/geometries/testfunctors.h         | 58 ++++++++++++------------
 6 files changed, 93 insertions(+), 100 deletions(-)

diff --git a/inc/geometries/ds.h b/inc/geometries/ds.h
index 7e353c700..7687d99e1 100644
--- a/inc/geometries/ds.h
+++ b/inc/geometries/ds.h
@@ -1,7 +1,7 @@
 #pragma once
 
 #include "dg/blas.h"
-#include "dg/geometry.h"
+#include "dg/geometry/geometry.h"
 #include "dg/backend/derivatives.h"
 #include "fieldaligned.h"
 #ifdef MPI_VERSION
diff --git a/inc/geometries/fluxfunctions.h b/inc/geometries/fluxfunctions.h
index 604863668..ef85cab99 100644
--- a/inc/geometries/fluxfunctions.h
+++ b/inc/geometries/fluxfunctions.h
@@ -32,9 +32,9 @@ struct aBinaryFunctor
     *
     * @return f(R,Z)
     */
-    double operator()(double R, double Z, double phi)
+    double operator()(double R, double Z, double phi)const
     {
-        return this->operator()(R,Z);
+        return operator()(R,Z);
     }
     /**
     * @brief abstract copy of a binary functor
diff --git a/inc/geometries/ribeiro_mpit.cu b/inc/geometries/ribeiro_mpit.cu
index 80ca7dc9b..c36e5005e 100644
--- a/inc/geometries/ribeiro_mpit.cu
+++ b/inc/geometries/ribeiro_mpit.cu
@@ -12,9 +12,9 @@
 
 #include "dg/backend/timer.cuh"
 #include "dg/backend/mpi_init.h"
+#include "dg/geometry/mpi_curvilinear.h"
 //#include "guenther.h"
 #include "solovev.h"
-#include "mpi_curvilinear.h"
 #include "ribeiro.h"
 #include "simple_orthogonal.h"
 //#include "ds.h"
@@ -28,8 +28,6 @@ double sineX( double x, double y) {return sin(x)*sin(y);}
 double cosineX( double x, double y) {return cos(x)*sin(y);}
 double sineY( double x, double y) {return sin(x)*sin(y);}
 double cosineY( double x, double y) {return sin(x)*cos(y);}
-//typedef dg::MPI_FieldAligned< dg::CurvilinearMPIGrid3d<dg::HVec> , dg::IHMatrix, dg::BijectiveComm<dg::iHVec, dg::HVec>, dg::HVec> DFA;
-typedef dg::MPI_FieldAligned< dg::OrthogonalMPIGrid3d<dg::HVec> , dg::IHMatrix, dg::BijectiveComm<dg::iHVec, dg::HVec>, dg::HVec> DFA;
 
 //should be the same as conformal_t.cu, except for the periodify
 int main( int argc, char* argv[])
@@ -53,8 +51,8 @@ int main( int argc, char* argv[])
         reader.parse(is,js,false);
     }
     GeomParameters gp(js);
-    Psip psip( gp); 
-    if(rank==0)std::cout << "Psi min "<<psip(gp.R_0, 0)<<"\n";
+    dg::geo::BinaryFunctorsLvl2 psip = createPsip( gp);
+    if(rank==0)std::cout << "Psi min "<<psip.f()(gp.R_0, 0)<<"\n";
     if(rank==0)std::cout << "Type psi_0 and psi_1\n";
     double psi_0, psi_1;
     if(rank==0)std::cin >> psi_0>> psi_1;
@@ -65,16 +63,9 @@ int main( int argc, char* argv[])
     //solovev::detail::Fpsi fpsi( gp, -10);
     if(rank==0)std::cout << "Constructing grid ... \n";
     t.tic();
-    MagneticField c( gp);
-    dg::geo::Ribeiro<Psip, PsipR, PsipZ, PsipRR, PsipRZ, PsipZZ>
-        ribeiro( c.psip, c.psipR, c.psipZ, c.psipRR, c.psipRZ, c.psipZZ, psi_0, psi_1, gp.R_0, 0., 1);
-    dg::CurvilinearMPIGrid3d<dg::HVec> g3d(&ribeiro, n, Nx, Ny,Nz, dg::DIR,comm);
-    dg::CurvilinearMPIGrid2d<dg::HVec> g2d = g3d.perp_grid();
-    //dg::SimpleOrthogonal<Psip, PsipR, PsipZ, LaplacePsip> 
-    //    orthogonal( c.psip, c.psipR, c.psipZ, c.laplacePsip, psi_0, psi_1, gp.R_0, 0., 1);
-    //dg::OrthogonalMPIGrid3d<dg::HVec> g3d(orthogonal, n, Nx, Ny,Nz, dg::DIR, comm);
-    //dg::OrthogonalMPIGrid2d<dg::HVec> g2d = g3d.perp_grid();
-    //
+    dg::geo::Ribeiro ribeiro( psip, psi_0, psi_1, gp.R_0, 0., 1);
+    dg::CurvilinearMPIGrid3d g3d(&ribeiro, n, Nx, Ny,Nz, dg::DIR,dg::PER, dg::PER,comm);
+    dg::CurvilinearMPIGrid2d g2d = g3d.perp_grid();
     t.toc();
     if(rank==0)std::cout << "Construction took "<<t.diff()<<"s"<<std::endl;
     int ncid;
@@ -109,8 +100,8 @@ int main( int argc, char* argv[])
     dg::HVec X( g2d.size()), Y(X); //P = dg::pullback( dg::coo3, g);
     for( unsigned i=0; i<g2d.size(); i++)
     {
-        X[i] = g2d.r()[i];
-        Y[i] = g2d.z()[i];
+        X[i] = g2d.map()[0].data()[i];
+        Y[i] = g2d.map()[0].data()[i];
     }
 
     dg::MHVec temp0( dg::evaluate(dg::zero, g2d)), temp1(temp0);
@@ -118,10 +109,14 @@ int main( int argc, char* argv[])
 
     err = nc_put_vara_double( ncid, coordsID[0], start,count, X.data());
     err = nc_put_vara_double( ncid, coordsID[1], start,count, Y.data());
-    //err = nc_put_vara_double( ncid, coordsID[2], g.z().data());
 
+    dg::SparseTensor<dg::HVec> metric = g2d.metric();
+    dg::MHVec g_xx = metric.value(0,0), g_xy = metric.value(0,1), g_yy=metric.value(1,1);
+    dg::SparseElement<dg::MHVec> vol_ = dg::tensor::volume(metric);
+    dg::MHVec vol = vol_.value();
+    //err = nc_put_vara_double( ncid, coordsID[2], g.z().data());
     //dg::blas1::pointwiseDivide( g2d.g_xy(), g2d.g_xx(), temp0);
-    dg::blas1::pointwiseDivide( g2d.g_yy(), g2d.g_xx(), temp0);
+    dg::blas1::pointwiseDivide( g_yy, g_xx, temp0);
     const dg::MHVec ones = dg::evaluate( dg::one, g2d);
     dg::blas1::axpby( 1., ones, -1., temp0, temp0);
     dg::blas1::transfer( temp0.data(), X);
@@ -130,35 +125,35 @@ int main( int argc, char* argv[])
     if(rank==0)std::cout << "Construction successful!\n";
 
     //compute error in volume element
-    dg::blas1::pointwiseDot( g2d.g_xx(), g2d.g_yy(), temp0);
-    dg::blas1::pointwiseDot( g2d.g_xy(), g2d.g_xy(), temp1);
+    dg::blas1::pointwiseDot( g_xx, g_yy, temp0);
+    dg::blas1::pointwiseDot( g_xy, g_xy, temp1);
     dg::blas1::axpby( 1., temp0, -1., temp1, temp0);
-    dg::blas1::transfer( g2d.g_xx(),  temp1);
+    dg::blas1::transfer( g_xx,  temp1);
     dg::blas1::pointwiseDot( temp1, temp1, temp1);
     dg::blas1::axpby( 1., temp1, -1., temp0, temp0);
     double error = sqrt( dg::blas2::dot( temp0, w2d, temp0)/dg::blas2::dot( temp1, w2d, temp1));
     if(rank==0)std::cout<< "Rel Error in Determinant is "<<error<<"\n";
 
     //compute error in determinant vs volume form
-    dg::blas1::pointwiseDot( g2d.g_xx(), g2d.g_yy(), temp0);
-    dg::blas1::pointwiseDot( g2d.g_xy(), g2d.g_xy(), temp1);
+    dg::blas1::pointwiseDot( g_xx, g_yy, temp0);
+    dg::blas1::pointwiseDot( g_xy, g_xy, temp1);
     dg::blas1::axpby( 1., temp0, -1., temp1, temp0);
     dg::blas1::transform( temp0, temp0, dg::SQRT<double>());
     dg::blas1::pointwiseDivide( ones, temp0, temp0);
     dg::blas1::transfer( temp0.data(), X);
     err = nc_put_var_double( ncid, volID, X.data());
-    dg::blas1::axpby( 1., temp0, -1., g2d.vol(), temp0);
-    error = sqrt(dg::blas2::dot( temp0, w2d, temp0)/dg::blas2::dot( g2d.vol(), w2d, g2d.vol()));
+    dg::blas1::axpby( 1., temp0, -1., vol, temp0);
+    error = sqrt(dg::blas2::dot( temp0, w2d, temp0)/dg::blas2::dot( vol, w2d, vol));
     if(rank==0)std::cout << "Rel Consistency  of volume is "<<error<<"\n";
 
     //compare g^xx to volume form
-    dg::blas1::transfer( g2d.g_xx(), temp0);
+    dg::blas1::transfer( g_xx, temp0);
     dg::blas1::pointwiseDivide( ones, temp0, temp0);
-    dg::blas1::axpby( 1., temp0, -1., g2d.vol(), temp0);
-    error=sqrt(dg::blas2::dot( temp0, w2d, temp0))/sqrt( dg::blas2::dot(g2d.vol(), w2d, g2d.vol()));
+    dg::blas1::axpby( 1., temp0, -1., vol, temp0);
+    error=sqrt(dg::blas2::dot( temp0, w2d, temp0))/sqrt( dg::blas2::dot(vol, w2d, vol));
     if(rank==0)std::cout << "Rel Error of volume form is "<<error<<"\n";
 
-    const dg::MHVec vol = dg::create::volume( g3d);
+    vol = dg::create::volume( g3d);
     dg::MHVec ones3d = dg::evaluate( dg::one, g3d);
     double volume = dg::blas1::dot( vol, ones3d);
 
@@ -189,8 +184,7 @@ int main( int argc, char* argv[])
     //t.toc();
     //if(rank==0)std::cout << "Construction took "<<t.diff()<<"s\n";
     //dg::MHVec B = dg::pullback( dg::geo::InvB(gp), g3d), divB(B);
-    //dg::MHVec lnB = dg::pullback( dg::geo::LnB(gp), g3d), gradB(B);
-    //dg::MHVec gradLnB = dg::pullback( dg::geo::GradLnB(gp), g3d);
+    //dg::MHVec lnB = dg::pullback( dg::geo::LnB(gp), g3d), gradB(B); //dg::MHVec gradLnB = dg::pullback( dg::geo::GradLnB(gp), g3d);
     //dg::blas1::pointwiseDivide( ones3d, B, B);
     //dg::MHVec function = dg::pullback( dg::geo::FuncNeu(gp), g3d), derivative(function);
     //ds( function, derivative);
diff --git a/inc/geometries/simple_orthogonal.h b/inc/geometries/simple_orthogonal.h
index d9be0f82e..c1a5cc72e 100644
--- a/inc/geometries/simple_orthogonal.h
+++ b/inc/geometries/simple_orthogonal.h
@@ -8,9 +8,8 @@
 #include "dg/functors.h"
 #include "dg/runge_kutta.h"
 #include "dg/nullstelle.h"
-#include "dg/geometry.h"
+#include "dg/geometry/geometry.h"
 #include "utilities.h"
-#include "generator.h"
 
 
 namespace dg
@@ -153,8 +152,7 @@ struct Nemov
 {
     Nemov( const BinaryFunctorsLvl2 psi, double f0, int mode):
         f0_(f0), mode_(mode),
-        psip_(psi), 
-            { }
+        psip_(psi) { }
     void initialize( 
         const thrust::host_vector<double>& r_init, //1d intial values
         const thrust::host_vector<double>& z_init, //1d intial values
@@ -323,7 +321,7 @@ struct SimpleOrthogonal : public aGenerator2d
          thrust::host_vector<double>& etaY) const
     {
         thrust::host_vector<double> r_init, z_init;
-        orthogonal::detail::compute_rzy( psiX_, psiY_, eta1d, r_init, z_init, R0_, Z0_, f0_, firstline_);
+        orthogonal::detail::compute_rzy( psi_, eta1d, r_init, z_init, R0_, Z0_, f0_, firstline_);
         orthogonal::detail::Nemov nemov(psi_, f0_, firstline_);
         thrust::host_vector<double> h;
         orthogonal::detail::construct_rz(nemov, 0., zeta1d, r_init, z_init, x, y, h);
diff --git a/inc/geometries/simple_orthogonal_t.cu b/inc/geometries/simple_orthogonal_t.cu
index 2bb3b05ae..819b0e623 100644
--- a/inc/geometries/simple_orthogonal_t.cu
+++ b/inc/geometries/simple_orthogonal_t.cu
@@ -11,11 +11,9 @@
 #include "dg/backend/timer.cuh"
 //#include "guenther.h"
 #include "solovev.h"
-#include "curvilinear.h"
-#include "refined_curvilinear.h"
 
 #include "simple_orthogonal.h"
-#include "ds.h"
+//#include "ds.h"
 #include "init.h"
 #include "testfunctors.h"
 
@@ -44,7 +42,7 @@ double sineX( double x, double y) {return sin(x)*sin(y);}
 double cosineX( double x, double y) {return cos(x)*sin(y);}
 double sineY( double x, double y) {return sin(x)*sin(y);}
 double cosineY( double x, double y) {return sin(x)*cos(y);}
-typedef dg::FieldAligned< dg::OrthogonalGrid3d<dg::HVec> , dg::IHMatrix, dg::HVec> DFA;
+//typedef dg::FieldAligned< dg::OrthogonalGrid3d<dg::HVec> , dg::IHMatrix, dg::HVec> DFA;
 
 int main( int argc, char* argv[])
 {
@@ -64,8 +62,8 @@ int main( int argc, char* argv[])
         reader.parse(is,js,false);
     }
     GeomParameters gp(js);
-    MagneticField c( gp); 
-    std::cout << "Psi min "<<c.psip(gp.R_0, 0)<<"\n";
+    dg::geo::BinaryFunctorsLvl2 psip=createPsip(gp);
+    std::cout << "Psi min "<<psip.f()(gp.R_0, 0)<<"\n";
     std::cout << "Type psi_0 and psi_1\n";
     double psi_0, psi_1;
     std::cin >> psi_0>> psi_1;
@@ -78,11 +76,9 @@ int main( int argc, char* argv[])
     std::cout << "Constructing orthogonal grid ... \n";
     t.tic();
 
-    dg::geo::SimpleOrthogonal<Psip, PsipR, PsipZ, LaplacePsip> generator( Psip(gp), PsipR(gp), PsipZ(gp), LaplacePsip(gp), psi_0, psi_1, gp.R_0, 0., 1);
-    //dg::OrthogonalGrid3d<dg::HVec> g3d(generator, n, Nx, Ny,Nz, dg::DIR);
-    //dg::OrthogonalGrid2d<dg::HVec> g2d = g3d.perp_grid();
-    dg::OrthogonalRefinedGrid3d<dg::HVec> g3d(multiple_x, multiple_y, generator, n_ref, n, Nx, Ny,Nz, dg::DIR);
-    dg::OrthogonalRefinedGrid2d<dg::HVec> g2d = g3d.perp_grid();
+    dg::geo::SimpleOrthogonal generator( psip, psi_0, psi_1, gp.R_0, 0., 1);
+    dg::CurvilinearProductGrid3d g3d(generator, n, Nx, Ny,Nz, dg::DIR);
+    dg::CurvilinearGrid2d g2d = g3d.perp_grid();
     dg::Grid2d g2d_periodic(g2d.x0(), g2d.x1(), g2d.y0(), g2d.y1(), g2d.n(), g2d.Nx(), g2d.Ny()+1); 
     t.toc();
     std::cout << "Construction took "<<t.diff()<<"s"<<std::endl;
@@ -101,14 +97,14 @@ int main( int argc, char* argv[])
     err = nc_def_var( ncid, "volume", NC_DOUBLE, 2, dim3d, &volID);
     err = nc_def_var( ncid, "divB", NC_DOUBLE, 2, dim3d, &divBID);
 
-    thrust::host_vector<double> psi_p = dg::pullback( c.psip, g2d);
+    thrust::host_vector<double> psi_p = dg::pullback( psip.f(), g2d);
     //g.display();
     err = nc_put_var_double( ncid, onesID, periodify(psi_p, g2d_periodic).data());
     dg::HVec X( g2d.size()), Y(X); //P = dg::pullback( dg::coo3, g);
     for( unsigned i=0; i<g2d.size(); i++)
     {
-        X[i] = g2d.r()[i];
-        Y[i] = g2d.z()[i];
+        X[i] = g2d.map()[0][i];
+        Y[i] = g2d.map()[1][i];
     }
 
     dg::HVec temp0( g2d.size()), temp1(temp0);
@@ -119,37 +115,41 @@ int main( int argc, char* argv[])
     //err = nc_put_var_double( ncid, coordsID[2], g.z().data());
 
     //compute and write deformation into netcdf
-    dg::blas1::pointwiseDivide( g2d.g_yy(), g2d.g_xx(), temp0);
+    dg::SparseTensor<dg::HVec> metric = g2d.metric();
+    dg::HVec g_xx = metric.value(0,0), g_xy = metric.value(0,1), g_yy=metric.value(1,1);
+    dg::SparseElement<dg::HVec> vol_ = dg::tensor::volume(metric);
+    dg::HVec vol = vol_.value();
+    dg::blas1::pointwiseDivide( g_yy, g_xx, temp0);
     const dg::HVec ones = dg::evaluate( dg::one, g2d);
     X=temp0;
     err = nc_put_var_double( ncid, defID, periodify(X, g2d_periodic).data());
     //compute and write conformalratio into netcdf
-    dg::blas1::pointwiseDivide( g2d.g_yy(), g2d.g_xx(), temp0);
+    dg::blas1::pointwiseDivide( g_yy, g_xx, temp0);
     X=temp0;
     err = nc_put_var_double( ncid, confID, periodify(X, g2d_periodic).data());
 
     std::cout << "Construction successful!\n";
 
     //compute error in volume element
-    dg::blas1::pointwiseDot( g2d.g_xy(), g2d.g_xy(), temp1); double error = sqrt( dg::blas2::dot( temp1, w2d, temp1));
+    dg::blas1::pointwiseDot( g_xy, g_xy, temp1); double error = sqrt( dg::blas2::dot( temp1, w2d, temp1));
     std::cout<< "    Error in Off-diagonal is "<<error<<"\n";
 
     //compare determinant vs volume form
-    dg::blas1::pointwiseDot( g2d.g_xx(), g2d.g_yy(), temp0);
-    dg::blas1::pointwiseDot( g2d.g_xy(), g2d.g_xy(), temp1);
+    dg::blas1::pointwiseDot( g_xx, g_yy, temp0);
+    dg::blas1::pointwiseDot( g_xy, g_xy, temp1);
     dg::blas1::axpby( 1., temp0, -1., temp1, temp0);
     dg::blas1::transform( temp0, temp0, dg::SQRT<double>());
     dg::blas1::pointwiseDivide( ones, temp0, temp0);
     dg::blas1::transfer( temp0, X);
     err = nc_put_var_double( ncid, volID, periodify(X, g2d_periodic).data());
-    dg::blas1::axpby( 1., temp0, -1., g2d.vol(), temp0);
-    error = sqrt(dg::blas2::dot( temp0, w2d, temp0)/dg::blas2::dot( g2d.vol(), w2d, g2d.vol()));
+    dg::blas1::axpby( 1., temp0, -1., vol, temp0);
+    error = sqrt(dg::blas2::dot( temp0, w2d, temp0)/dg::blas2::dot( vol, w2d, vol));
     std::cout << "Rel Consistency  of volume is "<<error<<"\n";
 
     /*
     //alternative method to compute volume
-    dg::HVec psipR_ = dg::pullback(c.psipR, g2d);
-    dg::HVec psipZ_ = dg::pullback(c.psipZ, g2d);
+    dg::HVec psipR_ = dg::pullback(psip.dfx(), g2d);
+    dg::HVec psipZ_ = dg::pullback(psip.dfy(), g2d);
     dg::HVec psip2_(psipR_);
     dg::blas1::pointwiseDot( psipR_, psipR_, psipR_);
     dg::blas1::pointwiseDot( psipZ_, psipZ_, psipZ_);
@@ -158,15 +158,15 @@ int main( int argc, char* argv[])
     const dg::HVec g_ = g2d.f2();
     dg::blas1::pointwiseDot( f_, psip2_, temp1);
     dg::blas1::pointwiseDot( f_, temp1, temp1);
-    dg::blas1::axpby( 1., g2d.g_xx(), -1., temp1, temp1);
-    error= dg::blas2::dot( temp1, w2d, temp1)/dg::blas2::dot(g2d.g_xx(),w2d,g2d.g_xx());
+    dg::blas1::axpby( 1., g_xx, -1., temp1, temp1);
+    error= dg::blas2::dot( temp1, w2d, temp1)/dg::blas2::dot(g_xx,w2d,g_xx);
     std::cout << "Rel Error of g_xx is "<<sqrt(error)<<"\n";
     dg::blas1::pointwiseDot( g_, psip2_, temp1);
     dg::blas1::pointwiseDot( g_,  temp1, temp1);
-    dg::blas1::axpby( 1., g2d.g_yy(), -1., temp1, temp1);
-    error= dg::blas2::dot( temp1, w2d, temp1)/dg::blas2::dot(g2d.g_yy(),w2d,g2d.g_yy());
+    dg::blas1::axpby( 1., g_yy, -1., temp1, temp1);
+    error= dg::blas2::dot( temp1, w2d, temp1)/dg::blas2::dot(g_yy,w2d,g_yy);
     std::cout << "Rel Error of g_yy is "<<sqrt(error)<<"\n";
-    dg::blas1::pointwiseDivide( ones, g2d.vol(), temp0);
+    dg::blas1::pointwiseDivide( ones, vol, temp0);
     dg::blas1::pointwiseDot( f_, psip2_, temp1);
     dg::blas1::pointwiseDot( g_, temp1 , temp1);
     dg::blas1::axpby( 1., temp0, -1., temp1, temp1);
@@ -175,12 +175,12 @@ int main( int argc, char* argv[])
     */
 
     std::cout << "TEST VOLUME IS:\n";
-    const dg::HVec vol = dg::create::volume( g3d);
+    vol = dg::create::volume( g3d);
     dg::HVec ones3d = dg::evaluate( dg::one, g3d);
     double volume = dg::blas1::dot( vol, ones3d);
     if( psi_0 < psi_1) gp.psipmax = psi_1, gp.psipmin = psi_0;
     else               gp.psipmax = psi_0, gp.psipmin = psi_1;
-    dg::geo::Iris<Psip> iris( c.psip, gp.psipmin, gp.psipmax);
+    dg::geo::Iris iris( psip.f(), gp.psipmin, gp.psipmax);
     dg::CartesianGrid2d g2dC( gp.R_0 -2.0*gp.a, gp.R_0 + 2.0*gp.a, -2.0*gp.a, 2.0*gp.a, 1, 2e3, 2e3, dg::PER, dg::PER);
 
     dg::HVec vec  = dg::evaluate( iris, g2dC);
@@ -218,7 +218,8 @@ int main( int argc, char* argv[])
 //     std::cout << "ana. norm of gradLnB is "<<norm<<"\n";
 //     dg::blas1::axpby( 1., gradB, -1., gradLnB, gradLnB);
      //X = g2d.lapx();
-     X = dg::pullback(dg::geo::FuncDirNeu<MagneticField>(c, psi_0, psi_1, 550, -150, 30., 1), g2d);
+    dg::geo::TokamakMagneticField c=createMagField(gp);
+     X = dg::pullback(dg::geo::FuncDirNeu(c, psi_0, psi_1, 550, -150, 30., 1), g2d);
      err = nc_put_var_double( ncid, divBID, periodify(X, g2d_periodic).data());
 //     double norm2 = sqrt(dg::blas2::dot(gradLnB, vol3d,gradLnB));
 //     std::cout << "rel. error of lnB is    "<<norm2/norm<<"\n";
diff --git a/inc/geometries/testfunctors.h b/inc/geometries/testfunctors.h
index 16acca24e..96372eb6d 100644
--- a/inc/geometries/testfunctors.h
+++ b/inc/geometries/testfunctors.h
@@ -5,7 +5,7 @@
 
 /*!@file
  *
- * aTokamakMagneticField objects 
+ * TokamakMagneticField objects 
  */
 namespace dg
 {
@@ -19,27 +19,27 @@ namespace geo
 
 struct FuncNeu
 {
-    FuncNeu( const aTokamakMagneticField& c):c_(c){}
+    FuncNeu( const TokamakMagneticField& c):c_(c){}
     double operator()(double R, double Z, double phi) const {return -c_.psip()(R,Z)*cos(phi);
     }
     private:
-    aTokamakMagneticField c_;
+    TokamakMagneticField c_;
 };
 
 struct DeriNeu
 {
-    DeriNeu( const aTokamakMagneticField& c, double R0):c_(c), bhat_(c){}
-    double operator()(double R, double Z, double phi) const {return c_.psip()(R,Z)*bhat_(R,Z,phi)*sin(phi);
+    DeriNeu( const TokamakMagneticField& c, double R0):c_(c), bhat_(c){}
+    double operator()(double R, double Z, double phi) const {return c_.psip()(R,Z,phi)*bhat_(R,Z,phi)*sin(phi);
     }
     private:
-    aTokamakMagneticField c_;
-    dg::geo::BHatP<aTokamakMagneticField> bhat_;
+    TokamakMagneticField c_;
+    dg::geo::BHatP bhat_;
 };
 
 //psi * cos(theta)
 struct FuncDirPer
 {
-    FuncDirPer( const aTokamakMagneticField& c, double psi_0, double psi_1, double k):
+    FuncDirPer( const TokamakMagneticField& c, double psi_0, double psi_1, double k):
         R_0_(c.R0()), psi0_(psi_0), psi1_(psi_1), k_(k), c_(c) {}
     double operator()(double R, double Z) const {
         double psip = c_.psip()(R,Z);
@@ -47,7 +47,7 @@ struct FuncDirPer
         return 0.1*result;
     }
     double operator()(double R, double Z, double phi) const {
-        return this->operator()(R,Z);
+        return operator()(R,Z);
     }
     double dR( double R, double Z)const
     {
@@ -105,15 +105,15 @@ struct FuncDirPer
     double thetaZZ( double R, double Z) const { return -thetaRR(R,Z);}
     double R_0_;
     double psi0_, psi1_, k_;
-    const aTokamakMagneticField c_;
+    const TokamakMagneticField c_;
 };
 
 //takes the magnetic field as chi
 struct EllipticDirPerM
 {
-    EllipticDirPerM( const aTokamakMagneticField& c, double psi_0, double psi_1, double k): func_(c, psi_0, psi_1, k), bmod_(c), br_(c), bz_(c) {}
+    EllipticDirPerM( const TokamakMagneticField& c, double psi_0, double psi_1, double k): func_(c, psi_0, psi_1, k), bmod_(c), br_(c), bz_(c) {}
     double operator()(double R, double Z, double phi) const {
-        return this->operator()(R,Z);}
+        return operator()(R,Z);}
     double operator()(double R, double Z) const {
         double bmod = bmod_(R,Z), br = br_(R,Z), bz = bz_(R,Z);
         return -(br*func_.dR(R,Z) + bz*func_.dZ(R,Z) + bmod*(func_.dRR(R,Z) + func_.dZZ(R,Z) ));
@@ -129,12 +129,12 @@ struct EllipticDirPerM
 //Blob function
 struct FuncDirNeu
 {
-    FuncDirNeu( const aTokamakMagneticField& c, double psi_0, double psi_1, double R_blob, double Z_blob, double sigma_blob, double amp_blob):
+    FuncDirNeu( const TokamakMagneticField& c, double psi_0, double psi_1, double R_blob, double Z_blob, double sigma_blob, double amp_blob):
         psi0_(psi_0), psi1_(psi_1), 
         cauchy_(R_blob, Z_blob, sigma_blob, sigma_blob, amp_blob){}
 
     double operator()(double R, double Z, double phi) const {
-        return this->operator()(R,Z);}
+        return operator()(R,Z);}
     double operator()(double R, double Z) const {
         return cauchy_(R,Z);
         //double psip = psip_(R,Z);
@@ -181,9 +181,9 @@ struct FuncDirNeu
 //takes the magnetic field multiplied by (1+0.5sin(theta)) as chi
 struct BmodTheta
 {
-    BmodTheta( const aTokamakMagneticField& c): R_0_(c.R0()), bmod_(c){}
+    BmodTheta( const TokamakMagneticField& c): R_0_(c.R0()), bmod_(c){}
     double operator()(double R,double Z, double phi) const{
-        return this->operator()(R,Z);}
+        return operator()(R,Z);}
     double operator()(double R,double Z) const{
         return bmod_(R,Z)*(1.+0.5*sin(theta(R,Z)));
     }
@@ -203,8 +203,8 @@ struct BmodTheta
 //take BmodTheta as chi
 struct EllipticDirNeuM
 {
-    EllipticDirNeuM( const aTokamakMagneticField& c, double psi_0, double psi_1, double R_blob, double Z_blob, double sigma_blob, double amp_blob): R_0_(c.R0()), 
-    func_(c, psi_0, psi_1, R_blob, Z_blob, sigma_blob,amp_blob), bmod_(c, R0), br_(c, R0), bz_(c, R0) {}
+    EllipticDirNeuM( const TokamakMagneticField& c, double psi_0, double psi_1, double R_blob, double Z_blob, double sigma_blob, double amp_blob): R_0_(c.R0()), 
+    func_(c, psi_0, psi_1, R_blob, Z_blob, sigma_blob,amp_blob), bmod_(c), br_(c), bz_(c) {}
     double operator()(double R, double Z) const {
         double bmod = bmod_(R,Z), br = br_(R,Z), bz = bz_(R,Z), theta_ = theta(R,Z);
         double chi = bmod*(1.+0.5*sin(theta_));
@@ -214,7 +214,7 @@ struct EllipticDirNeuM
 
     }
     double operator()(double R, double Z, double phi) const {
-        return this->operator()(R,Z);
+        return operator()(R,Z);
     }
     private:
     double theta( double R, double Z) const {
@@ -242,11 +242,11 @@ struct EllipticDirNeuM
 //the psi surfaces
 struct FuncXDirNeu
 {
-    FuncXDirNeu( const aTokamakMagneticField& c, double psi_0, double psi_1):
+    FuncXDirNeu( const TokamakMagneticField& c, double psi_0, double psi_1):
         c_(c), psi0_(psi_0), psi1_(psi_1){}
 
     double operator()(double R, double Z, double phi) const {
-        return this->operator()(R,Z);}
+        return operator()(R,Z);}
     double operator()(double R, double Z) const {
         double psip = c_.psip()(R,Z);
         return (psip-psi0_)*(psip-psi1_);
@@ -275,14 +275,14 @@ struct FuncXDirNeu
         return (2.*(psipZ*psipZ + psip*psipZZ) - (psi0_+psi1_)*psipZZ);
     }
     private:
-    aTokamakMagneticField c_;
+    TokamakMagneticField c_;
     double psi0_, psi1_;
 };
 
 //take Bmod as chi
 struct EllipticXDirNeuM
 {
-    EllipticXDirNeuM( const aTokamakMagneticField& c, double psi_0, double psi_1): R_0_(c.R0()), 
+    EllipticXDirNeuM( const TokamakMagneticField& c, double psi_0, double psi_1): R_0_(c.R0()), 
     func_(c, psi_0, psi_1), bmod_(c), br_(c), bz_(c) {}
     double operator()(double R, double Z) const {
         double bmod = bmod_(R,Z), br = br_(R,Z), bz = bz_(R,Z);
@@ -295,7 +295,7 @@ struct EllipticXDirNeuM
 
     }
     double operator()(double R, double Z, double phi) const {
-        return this->operator()(R,Z);
+        return operator()(R,Z);
     }
     private:
     double R_0_;
@@ -308,13 +308,13 @@ struct EllipticXDirNeuM
 //take Blob and chi=1
 struct EllipticBlobDirNeuM
 {
-    EllipticBlobDirNeuM( const aTokamakMagneticField& c, double psi_0, double psi_1, double R_blob, double Z_blob, double sigma_blob, double amp_blob): 
+    EllipticBlobDirNeuM( const TokamakMagneticField& c, double psi_0, double psi_1, double R_blob, double Z_blob, double sigma_blob, double amp_blob): 
     func_(c, psi_0, psi_1, R_blob, Z_blob, sigma_blob, amp_blob){}
     double operator()(double R, double Z) const {
         return -( func_.dRR(R,Z) + func_.dZZ(R,Z) );
     }
     double operator()(double R, double Z, double phi) const {
-        return this->operator()(R,Z);
+        return operator()(R,Z);
     }
     private:
     double R_0_;
@@ -323,7 +323,7 @@ struct EllipticBlobDirNeuM
 
 struct EllipticDirSimpleM
 {
-    EllipticDirSimpleM( const aTokamakMagneticField& c, double psi_0, double psi_1, double R_blob, double Z_blob, double sigma_blob, double amp_blob): func_(c, psi_0, psi_1, R_blob, Z_blob, sigma_blob, amp_blob) {}
+    EllipticDirSimpleM( const TokamakMagneticField& c, double psi_0, double psi_1, double R_blob, double Z_blob, double sigma_blob, double amp_blob): func_(c, psi_0, psi_1, R_blob, Z_blob, sigma_blob, amp_blob) {}
     double operator()(double R, double Z, double phi) const {
         return -(( 1./R*func_.dR(R,Z) + func_.dRR(R,Z) + func_.dZZ(R,Z) ));
 
@@ -338,7 +338,7 @@ struct EllipticDirSimpleM
  */ 
 struct TestFunction
 {
-    TestFunction( const aTokamakMagneticField& c) :  
+    TestFunction( const TokamakMagneticField& c) :  
         bhatR_(c),
         bhatZ_(c),
         bhatP_(c) {}
@@ -374,7 +374,7 @@ struct TestFunction
  */ 
 struct DeriTestFunction
 {
-    DeriTestFunction( const aTokamakMagneticField& c) :
+    DeriTestFunction( const TokamakMagneticField& c) :
         bhatR_(c),
         bhatZ_(c),
         bhatP_(c) {}
-- 
GitLab


From 22c631a56c554a2bdb382822460b9370ff3766db Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Thu, 17 Aug 2017 07:00:16 -0700
Subject: [PATCH 174/453] begin to introduce do_compute function in
 aBinaryFunctor

---
 inc/dg/backend/evaluation.cuh   |   6 +-
 inc/dg/backend/mpi_evaluation.h |   4 +-
 inc/geometries/average.h        |  12 +-
 inc/geometries/fluxfunctions.h  |  12 +-
 inc/geometries/geometry_diag.cu |  76 ++++++------
 inc/geometries/magnetic_field.h | 147 +++++++++++------------
 inc/geometries/solovev.h        | 202 +++++++++-----------------------
 inc/geometries/taylor.h         |   6 +-
 8 files changed, 183 insertions(+), 282 deletions(-)

diff --git a/inc/dg/backend/evaluation.cuh b/inc/dg/backend/evaluation.cuh
index e7eda1acf..cad9ab4a3 100644
--- a/inc/dg/backend/evaluation.cuh
+++ b/inc/dg/backend/evaluation.cuh
@@ -29,7 +29,7 @@ namespace dg
  * @return  A DG Host Vector with values
  */
 template< class UnaryOp>
-thrust::host_vector<double> evaluate( const UnaryOp& f, const Grid1d& g)
+thrust::host_vector<double> evaluate( UnaryOp& f, const Grid1d& g)
 {
     thrust::host_vector<double> abs = create::abscissas( g);
     for( unsigned i=0; i<g.size(); i++)
@@ -57,7 +57,7 @@ thrust::host_vector<double> evaluate( double (f)(double), const Grid1d& g)
  * @note if you don't like to copy f then just pass a (const) reference then the type should adapt
  */
 template< class BinaryOp>
-thrust::host_vector<double> evaluate( const BinaryOp& f, const aTopology2d& g)
+thrust::host_vector<double> evaluate( BinaryOp& f, const aTopology2d& g)
 {
     unsigned n= g.n();
     Grid1d gx(g.x0(), g.x1(), g.n(), g.Nx());
@@ -95,7 +95,7 @@ thrust::host_vector<double> evaluate( double(f)(double, double), const aTopology
  * @note if you don't like to copy f then just pass a (const) reference then the type should adapt
  */
 template< class TernaryOp>
-thrust::host_vector<double> evaluate( const TernaryOp& f, const aTopology3d& g)
+thrust::host_vector<double> evaluate( TernaryOp& f, const aTopology3d& g)
 {
     unsigned n= g.n();
     Grid1d gx(g.x0(), g.x1(), g.n(), g.Nx());
diff --git a/inc/dg/backend/mpi_evaluation.h b/inc/dg/backend/mpi_evaluation.h
index b97987fa2..a89b09800 100644
--- a/inc/dg/backend/mpi_evaluation.h
+++ b/inc/dg/backend/mpi_evaluation.h
@@ -28,7 +28,7 @@ namespace dg
             may be constructed during function call.
  */
 template< class BinaryOp>
-MPI_Vector<thrust::host_vector<double> > evaluate( const BinaryOp& f, const aMPITopology2d& g)
+MPI_Vector<thrust::host_vector<double> > evaluate( BinaryOp& f, const aMPITopology2d& g)
 {
     thrust::host_vector<double> w = evaluate( f, g.local());
     MPI_Vector<thrust::host_vector<double> > v( w, g.communicator());
@@ -55,7 +55,7 @@ MPI_Vector<thrust::host_vector<double> > evaluate( double(f)(double, double), co
             may be constructed during function call.
  */
 template< class TernaryOp>
-MPI_Vector<thrust::host_vector<double> > evaluate( const TernaryOp& f, const aMPITopology3d& g)
+MPI_Vector<thrust::host_vector<double> > evaluate( TernaryOp& f, const aMPITopology3d& g)
 {
     thrust::host_vector<double> w = evaluate( f, g.local());
     MPI_Vector<thrust::host_vector<double> > v( w, g.communicator());
diff --git a/inc/geometries/average.h b/inc/geometries/average.h
index 794a238b6..47d2ddc14 100644
--- a/inc/geometries/average.h
+++ b/inc/geometries/average.h
@@ -18,7 +18,7 @@ namespace geo
  */
 struct DeltaFunction
 {
-    DeltaFunction(const aTokamakMagneticField& c, double epsilon,double psivalue) :
+    DeltaFunction(const TokamakMagneticField& c, double epsilon,double psivalue) :
         c_(c),
         epsilon_(epsilon),
         psivalue_(psivalue){
@@ -53,7 +53,7 @@ struct DeltaFunction
         return (*this)(R,Z);
     }
     private:
-    aTokamakMagneticField c_;
+    TokamakMagneticField c_;
     double epsilon_;
     double psivalue_;
 };
@@ -65,7 +65,7 @@ struct DeltaFunction
  */
 struct Alpha
 {
-    Alpha( const aTokamakMagneticField& c):c_(c){}
+    Alpha( const TokamakMagneticField& c):c_(c){}
 
     /**
     * @brief \f[ \frac{ I_{pol}(R,Z)}{R \sqrt{\nabla\psi_p}} \f]
@@ -83,7 +83,7 @@ struct Alpha
         return operator()(R,Z);
     }
     private:
-    aTokamakMagneticField c_;
+    TokamakMagneticField c_;
 };
 
 /**
@@ -103,7 +103,7 @@ struct FluxSurfaceAverage
      * @param c contains psip, psipR and psipZ
      * @param f container for global safety factor
      */
-    FluxSurfaceAverage(const dg::Grid2d& g2d, const aTokamakMagneticField& c, const container& f) :
+    FluxSurfaceAverage(const dg::Grid2d& g2d, const TokamakMagneticField& c, const container& f) :
     g2d_(g2d),
     f_(f),
     deltaf_(geo::DeltaFunction(c,0.0,0.0)),
@@ -159,7 +159,7 @@ struct SafetyFactor
      * @param c contains psip, psipR and psipZ
      * @param f container for global safety factor
      */
-    SafetyFactor(const dg::Grid2d& g2d, const aTokamakMagneticField& c, const container& f) :
+    SafetyFactor(const dg::Grid2d& g2d, const TokamakMagneticField& c, const container& f) :
     g2d_(g2d),
     f_(f), //why not directly use Alpha??
     deltaf_(geo::DeltaFunction(c,0.0,0.0)),
diff --git a/inc/geometries/fluxfunctions.h b/inc/geometries/fluxfunctions.h
index ef85cab99..3ab6f5787 100644
--- a/inc/geometries/fluxfunctions.h
+++ b/inc/geometries/fluxfunctions.h
@@ -22,7 +22,11 @@ struct aBinaryFunctor
     *
     * @return f(R,Z)
     */
-    virtual double operator()(double R, double Z) const=0;
+    double operator()(double R, double Z) const
+    {
+        return do_compute(R,Z); //if we didn't make the virtual function have another way
+        //the operator() would hide the 3d version
+    }
     /**
     * @brief Redirects to the 2D version
     *
@@ -53,6 +57,8 @@ struct aBinaryFunctor
     * @brief We do not allow object slicing so the assignment is protected
     */
     aBinaryFunctor& operator=(const aBinaryFunctor&){return *this;}
+    private:
+    double do_compute(double R, double Z) const=0;
 };
 
 /**
@@ -83,8 +89,8 @@ template<class BinaryFunctor>
 struct BinaryFunctorAdapter : public aCloneableBinaryFunctor<BinaryFunctorAdapter<BinaryFunctor> >
 {
     BinaryFunctorAdapter( const BinaryFunctor& f):f_(f){}
-    double operator()(double x, double y)const{return f_(x,y);}
     private:
+    double do_compute(double x, double y)const{return f_(x,y);}
     BinaryFunctor f_;
 };
 /**
@@ -277,8 +283,8 @@ struct BinaryVectorLvl0
 struct Constant: public aCloneableBinaryFunctor<Constant> 
 { 
     Constant(double c):c_(c){}
-    double operator()(double R,double Z)const{return c_;}
     private:
+    double do_compute(double R,double Z)const{return c_;}
     double c_;
 };
 
diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index dcc1051dd..9486b0371 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -17,10 +17,6 @@
 #include "magnetic_field.h"
 #include "average.h"
 
-//using namespace dg::geo::solovev;
-using namespace dg::geo::taylor;
-
-
 struct Parameters
 {
     unsigned n, Nx, Ny, Nz;
@@ -112,7 +108,7 @@ int main( int argc, char* argv[])
         reader.parse( geom, geom_js, false);
     }
     const Parameters p(input_js);
-    const GeomParameters gp(geom_js);
+    const dg::geo::solovev::GeomParameters gp(geom_js);
     p.display( std::cout);
     gp.display( std::cout);
     std::string input = input_js.toStyledString();
@@ -125,7 +121,7 @@ int main( int argc, char* argv[])
     double Zmax=p.boxscaleZp*gp.a*gp.elongation;
  
     //construct all geometry quantities
-    MagneticField c(gp);
+    dg::geo::TokamakMagneticField c = dg::geo::taylor::createMagField(gp);
     const double R_X = gp.R_0-1.1*gp.triangularity*gp.a;
     const double Z_X = -1.1*gp.elongation*gp.a;
     const double R_H = gp.R_0-gp.triangularity*gp.a;
@@ -135,37 +131,37 @@ int main( int argc, char* argv[])
     const double N2 =  (1.-alpha_)/(gp.a*gp.elongation*gp.elongation)*(1.-alpha_);
     const double N3 = -gp.elongation/(gp.a*cos(alpha_)*cos(alpha_));
     std::cout << "TEST ACCURACY OF PSI\n";
-    std::cout << "psip( 1+e,0)           "<<c.psip(gp.R_0 + gp.a, 0.)<<"\n";
-    std::cout << "psip( 1-e,0)           "<<c.psip(gp.R_0 - gp.a, 0.)<<"\n";
-    std::cout << "psip( 1-de,ke)         "<<c.psip(R_H, Z_H)<<"\n";
-    std::cout << "psip( 1-1.1de,-1.1ke)  "<<c.psip(R_X, Z_X)<<"\n";
-    std::cout << "psipZ( 1+e,0)          "<<c.psipZ(gp.R_0 + gp.a, 0.)<<"\n";
-    std::cout << "psipZ( 1-e,0)          "<<c.psipZ(gp.R_0 - gp.a, 0.)<<"\n";
-    std::cout << "psipR( 1-de,ke)        "<<c.psipR(R_H,Z_H)<<"\n";
-    std::cout << "psipR( 1-1.1de,-1.1ke) "<<c.psipR(R_X,Z_X)<<"\n";
-    std::cout << "psipZ( 1-1.1de,-1.1ke) "<<c.psipZ(R_X,Z_X)<<"\n";
-    std::cout << "psipZZ( 1+e,0)         "<<c.psipZZ(gp.R_0+gp.a,0.)+N1*c.psipR(gp.R_0+gp.a,0)<<"\n";
-    std::cout << "psipZZ( 1-e,0)         "<<c.psipZZ(gp.R_0-gp.a,0.)+N2*c.psipR(gp.R_0-gp.a,0)<<"\n";
-    std::cout << "psipRR( 1-de,ke)       "<<c.psipRR(R_H,Z_H)+N3*c.psipZ(R_H,Z_H)<<"\n";
+    std::cout << "psip( 1+e,0)           "<<c.psip()(gp.R_0 + gp.a, 0.)<<"\n";
+    std::cout << "psip( 1-e,0)           "<<c.psip()(gp.R_0 - gp.a, 0.)<<"\n";
+    std::cout << "psip( 1-de,ke)         "<<c.psip()(R_H, Z_H)<<"\n";
+    std::cout << "psip( 1-1.1de,-1.1ke)  "<<c.psip()(R_X, Z_X)<<"\n";
+    std::cout << "psipZ( 1+e,0)          "<<c.psipZ()(gp.R_0 + gp.a, 0.)<<"\n";
+    std::cout << "psipZ( 1-e,0)          "<<c.psipZ()(gp.R_0 - gp.a, 0.)<<"\n";
+    std::cout << "psipR( 1-de,ke)        "<<c.psipR()(R_H,Z_H)<<"\n";
+    std::cout << "psipR( 1-1.1de,-1.1ke) "<<c.psipR()(R_X,Z_X)<<"\n";
+    std::cout << "psipZ( 1-1.1de,-1.1ke) "<<c.psipZ()(R_X,Z_X)<<"\n";
+    std::cout << "psipZZ( 1+e,0)         "<<c.psipZZ()(gp.R_0+gp.a,0.)+N1*c.psipR()(gp.R_0+gp.a,0)<<"\n";
+    std::cout << "psipZZ( 1-e,0)         "<<c.psipZZ()(gp.R_0-gp.a,0.)+N2*c.psipR()(gp.R_0-gp.a,0)<<"\n";
+    std::cout << "psipRR( 1-de,ke)       "<<c.psipRR()(R_H,Z_H)+N3*c.psipZ()(R_H,Z_H)<<"\n";
 
     //Feltor quantities
-    dg::geo::InvB<MagneticField> invB(c, gp.R_0);
-    dg::geo::BR<MagneticField> bR(c, gp.R_0);
-    dg::geo::BZ<MagneticField> bZ(c, gp.R_0);
-    dg::geo::CurvatureNablaBR<MagneticField> curvatureR(c, gp.R_0);
-    dg::geo::CurvatureNablaBZ<MagneticField> curvatureZ(c, gp.R_0);
-    dg::geo::GradLnB<MagneticField> gradLnB(c, gp.R_0);
-    dg::geo::Field<MagneticField>   field(c, gp.R_0);
-    dg::geo::FieldR<MagneticField> fieldR(c, gp.R_0);
-    dg::geo::FieldZ<MagneticField> fieldZ(c, gp.R_0);
-    dg::geo::FieldP<MagneticField> fieldP(c, gp.R_0);
-    dg::geo::Iris<Psip> iris( c.psip, gp.psipmin, gp.psipmax );
-    dg::geo::Pupil<Psip> pupil(c.psip, gp.psipmaxcut);
-    dg::geo::GaussianDamping<Psip> dampgauss(c.psip, gp.psipmaxcut, gp.alpha);
-    dg::geo::GaussianProfDamping<Psip> dampprof(c.psip,gp.psipmax, gp.alpha);
-    dg::geo::ZonalFlow<Psip> zonalflow(p.amp, p.k_psi, gp, c.psip);
-    dg::geo::PsiLimiter<Psip> psilimiter(c.psip, gp.psipmaxlim);
-    dg::geo::Nprofile<Psip> prof(p.bgprofamp, p.nprofileamp, gp, c.psip);
+    dg::geo::InvB invB(c);
+    dg::geo::BR bR(c);
+    dg::geo::BZ bZ(c);
+    dg::geo::CurvatureNablaBR curvatureR(c);
+    dg::geo::CurvatureNablaBZ curvatureZ(c);
+    dg::geo::GradLnB gradLnB(c);
+    dg::geo::FieldR  field(c);
+    dg::geo::FieldR fieldR(c);
+    dg::geo::FieldZ fieldZ(c);
+    dg::geo::FieldP fieldP(c);
+    dg::geo::Iris iris( c.psip(), gp.psipmin, gp.psipmax );
+    dg::geo::Pupil pupil(c.psip(), gp.psipmaxcut);
+    dg::geo::GaussianDamping dampgauss(c.psip(), gp.psipmaxcut, gp.alpha);
+    dg::geo::GaussianProfDamping dampprof(c.psip(),gp.psipmax, gp.alpha);
+    dg::geo::ZonalFlow zonalflow(p.amp, p.k_psi, gp, c.psip());
+    dg::geo::PsiLimiter psilimiter(c.psip(), gp.psipmaxlim);
+    dg::geo::Nprofile prof(p.bgprofamp, p.nprofileamp, gp, c.psip());
 
     dg::BathRZ bath(16,16,p.Nz,Rmin,Zmin, 30.,5.,p.amp);
 //     dg::Gaussian3d bath(gp.R_0+p.posX*gp.a, p.posY*gp.a, M_PI, p.sigma, p.sigma, p.sigma, p.amp);
@@ -177,8 +173,8 @@ int main( int argc, char* argv[])
     std::vector<dg::HVec> visual(21);
 
     //B field functions
-    hvisual[1] = dg::evaluate( c.psip, grid2d);
-    hvisual[2] = dg::evaluate( c.ipol, grid2d);
+    hvisual[1] = dg::evaluate( c.psip(), grid2d);
+    hvisual[2] = dg::evaluate( c.ipol(), grid2d);
     hvisual[3] = dg::evaluate( invB, grid2d);
     hvisual[4] = dg::evaluate( field, grid2d);
     hvisual[5] = dg::evaluate( curvatureR, grid2d);
@@ -211,14 +207,14 @@ int main( int argc, char* argv[])
     dg::blas1::pointwiseDot(hvisual[10], hvisual[19], hvisual[19]); //damped 
 
     //Compute flux average
-    dg::geo::Alpha<MagneticField> alpha(c); // = B^phi / |nabla psip |
-    dg::DVec psipog2d   = dg::evaluate( c.psip, grid2d);
+    dg::geo::Alpha alpha(c); // = B^phi / |nabla psip |
+    dg::DVec psipog2d   = dg::evaluate( c.psip(), grid2d);
     dg::DVec alphaog2d  = dg::evaluate( alpha, grid2d); 
     double psipmin = (float)thrust::reduce( psipog2d .begin(), psipog2d .end(), 0.0,thrust::minimum<double>()  );
     unsigned npsi = 3, Npsi = 150;//set number of psivalues
     psipmin += (gp.psipmax - psipmin)/(double)Npsi; //the inner value is not good
     dg::Grid1d grid1d(psipmin , gp.psipmax, npsi ,Npsi,dg::DIR);
-    dg::geo::SafetyFactor<MagneticField, dg::DVec>     qprof(grid2d, c, alphaog2d );
+    dg::geo::SafetyFactor< dg::DVec>     qprof(grid2d, c, alphaog2d );
     dg::HVec sf         = dg::evaluate( qprof,    grid1d);
     dg::HVec abs        = dg::evaluate( dg::cooX1d, grid1d);
 
diff --git a/inc/geometries/magnetic_field.h b/inc/geometries/magnetic_field.h
index 589d4ceee..ac494a999 100644
--- a/inc/geometries/magnetic_field.h
+++ b/inc/geometries/magnetic_field.h
@@ -76,73 +76,68 @@ struct TokamakMagneticField
 struct Bmodule : public aCloneableBinaryFunctor<Bmodule>
 {
     Bmodule( const TokamakMagneticField& mag): mag_(mag)  { }
-    /**
-    * @brief \f[   \hat{B} \f]
-    */ 
-    double operator()(double R, double Z) const
+  private:
+    double do_compute(double R, double Z) const
     {    
         double psipR = mag_.psipR()(R,Z), psipZ = mag_.psipZ()(R,Z), ipol = mag_.ipol()(R,Z);
         return mag_.R0()/R*sqrt(ipol*ipol+psipR*psipR +psipZ*psipZ);
     }
-  private:
     TokamakMagneticField mag_;
 };
 
 /**
  * @brief \f[  |B|^{-1} = R/R_0\sqrt{I^2+(\nabla\psi)^2}    \f]
+
+    \f[   \frac{1}{\hat{B}} = 
+        \frac{\hat{R}}{\hat{R}_0}\frac{1}{ \sqrt{ \hat{I}^2  + \left(\frac{\partial \hat{\psi}_p }{ \partial \hat{R}}\right)^2
+        + \left(\frac{\partial \hat{\psi}_p }{ \partial \hat{Z}}\right)^2}}  \f]
  */ 
 struct InvB : public aCloneableBinaryFunctor<InvB>
 {
     InvB(  const TokamakMagneticField& mag): mag_(mag){ }
-    /**
-    * @brief \f[   \frac{1}{\hat{B}} = 
-        \frac{\hat{R}}{\hat{R}_0}\frac{1}{ \sqrt{ \hat{I}^2  + \left(\frac{\partial \hat{\psi}_p }{ \partial \hat{R}}\right)^2
-        + \left(\frac{\partial \hat{\psi}_p }{ \partial \hat{Z}}\right)^2}}  \f]
-    */ 
-    double operator()(double R, double Z) const
+  private:
+    double do_compute(double R, double Z) const
     {    
         double psipR = mag_.psipR()(R,Z), psipZ = mag_.psipZ()(R,Z), ipol = mag_.ipol()(R,Z);
         return R/(mag_.R0()*sqrt(ipol*ipol + psipR*psipR +psipZ*psipZ)) ;
     }
-  private:
     TokamakMagneticField mag_;
 };
 
 /**
  * @brief \f[   \ln{|B|}  \f]
+ *
+   \f[   \ln{(   \hat{B})} = \ln{\left[
+          \frac{\hat{R}_0}{\hat{R}} \sqrt{ \hat{I}^2  + \left(\frac{\partial \hat{\psi}_p }{ \partial \hat{R}}\right)^2
+          + \left(\frac{\partial \hat{\psi}_p }{ \partial \hat{Z}}\right)^2} \right] } \f]
  */ 
 struct LnB : public aCloneableBinaryFunctor<LnB>
 {
     LnB(const TokamakMagneticField& mag): mag_(mag) { }
-    /**
-     * @brief \f[   \ln{(   \hat{B})} = \ln{\left[
-          \frac{\hat{R}_0}{\hat{R}} \sqrt{ \hat{I}^2  + \left(\frac{\partial \hat{\psi}_p }{ \partial \hat{R}}\right)^2
-          + \left(\frac{\partial \hat{\psi}_p }{ \partial \hat{Z}}\right)^2} \right] } \f]
-     */ 
-    double operator()(double R, double Z) const
+  private:
+    double do_compute(double R, double Z) const
     {    
         double psipR = mag_.psipR()(R,Z), psipZ = mag_.psipZ()(R,Z), ipol = mag_.ipol()(R,Z);
         return log(mag_.R0()/R*sqrt(ipol*ipol + psipR*psipR +psipZ*psipZ)) ;
     }
-  private:
     TokamakMagneticField mag_;
 };
 
 /**
  * @brief \f[  \frac{\partial |\hat{B}| }{ \partial \hat{R}}  \f]
- */ 
-struct BR: public aCloneableBinaryFunctor<BR>
-{
-    BR(const TokamakMagneticField& mag): invB_(mag), mag_(mag) { }
-/**
- * @brief \f[  \frac{\partial \hat{B} }{ \partial \hat{R}} = 
+ * 
+ \f[  \frac{\partial \hat{B} }{ \partial \hat{R}} = 
       -\frac{1}{\hat B \hat R}   
       +  \frac{\hat I \left(\frac{\partial\hat I}{\partial\hat R} \right) 
       + \left(\frac{\partial \hat{\psi}_p }{ \partial \hat{Z}} \right)\left(\frac{\partial^2  \hat{\psi}_p }{ \partial \hat{R} \partial\hat{Z}}\right)
       + \left( \frac{\partial \hat{\psi}_p }{ \partial \hat{R}}\right)\left( \frac{\partial^2  \hat{\psi}_p }{ \partial \hat{R}^2}\right)}
       {\hat{R}^2 \hat{R}_0^{-2}\hat{B}} \f]
  */ 
-    double operator()(double R, double Z) const
+struct BR: public aCloneableBinaryFunctor<BR>
+{
+    BR(const TokamakMagneticField& mag): invB_(mag), mag_(mag) { }
+  private:
+    double do_compute(double R, double Z) const
     { 
         double Rn;
         Rn = R/mag_.R0();
@@ -150,24 +145,23 @@ struct BR: public aCloneableBinaryFunctor<BR>
         //return -( Rn*Rn/invB_(R,Z)/invB_(R,Z)+ qampl_*qampl_*Rn *A_*psipR_(R,Z) - R  *(psipZ_(R,Z)*psipRZ_(R,Z)+psipR_(R,Z)*psipRR_(R,Z)))/(R*Rn*Rn/invB_(R,Z));
         return -1./R/invB_(R,Z) + invB_(R,Z)/Rn/Rn*(mag_.ipol()(R,Z)*mag_.ipolR()(R,Z) + mag_.psipR()(R,Z)*mag_.psipRR()(R,Z) + mag_.psipZ()(R,Z)*mag_.psipRZ()(R,Z));
     }
-  private:
     InvB invB_;
     TokamakMagneticField mag_;
 };
 
 /**
  * @brief \f[  \frac{\partial \hat{B} }{ \partial \hat{Z}}  \f]
+ *
+  \f[  \frac{\partial \hat{B} }{ \partial \hat{Z}} = 
+     \frac{ \hat I \left(\frac{\partial \hat I}{\partial\hat Z}    \right)+
+     \left(\frac{\partial \hat{\psi}_p }{ \partial \hat{R}} \right)\left(\frac{\partial^2  \hat{\psi}_p }{ \partial \hat{R} \partial\hat{Z}}\right)
+          + \left( \frac{\partial \hat{\psi}_p }{ \partial \hat{Z}} \right)\left(\frac{\partial^2  \hat{\psi}_p }{ \partial \hat{Z}^2} \right)}{\hat{R}^2 \hat{R}_0^{-2}\hat{B}} \f]
  */ 
 struct BZ: public aCloneableBinaryFunctor<BZ>
 {
     BZ(const TokamakMagneticField& mag ): mag_(mag), invB_(mag) { }
-    /**
-     * @brief \f[  \frac{\partial \hat{B} }{ \partial \hat{Z}} = 
-     \frac{ \hat I \left(\frac{\partial \hat I}{\partial\hat Z}    \right)+
-     \left(\frac{\partial \hat{\psi}_p }{ \partial \hat{R}} \right)\left(\frac{\partial^2  \hat{\psi}_p }{ \partial \hat{R} \partial\hat{Z}}\right)
-          + \left( \frac{\partial \hat{\psi}_p }{ \partial \hat{Z}} \right)\left(\frac{\partial^2  \hat{\psi}_p }{ \partial \hat{Z}^2} \right)}{\hat{R}^2 \hat{R}_0^{-2}\hat{B}} \f]
-     */ 
-    double operator()(double R, double Z) const
+  private:
+    double do_compute(double R, double Z) const
     { 
         double Rn;
         Rn = R/mag_.R0();
@@ -175,58 +169,55 @@ struct BZ: public aCloneableBinaryFunctor<BZ>
         //return (-qampl_*qampl_*A_/R_0_*psipZ_(R,Z) + psipR_(R,Z)*psipRZ_(R,Z)+psipZ_(R,Z)*psipZZ_(R,Z))/(Rn*Rn/invB_(R,Z));
         return (invB_(R,Z)/Rn/Rn)*(mag_.ipol()(R,Z)*mag_.ipolZ()(R,Z) + mag_.psipR()(R,Z)*mag_.psipRZ()(R,Z) + mag_.psipZ()(R,Z)*mag_.psipZZ()(R,Z));
     }
-  private:
     TokamakMagneticField mag_;
     InvB invB_; 
 };
 
 /**
  * @brief \f[ \mathcal{\hat{K}}^{\hat{R}}_{\nabla B} \f]
+ *
+    \f[ \mathcal{\hat{K}}^{\hat{R}}_{\nabla B} =-\frac{1}{ \hat{B}^2}  \frac{\partial \hat{B}}{\partial \hat{Z}}  \f]
  */ 
 struct CurvatureNablaBR: public aCloneableBinaryFunctor<CurvatureNablaBR>
 {
     CurvatureNablaBR(const TokamakMagneticField& mag): invB_(mag), bZ_(mag) { }
-    /**
-     * @brief \f[ \mathcal{\hat{K}}^{\hat{R}}_{\nabla B} =-\frac{1}{ \hat{B}^2}  \frac{\partial \hat{B}}{\partial \hat{Z}}  \f]
-     */ 
-    double operator()( double R, double Z) const
+    private:    
+    double do_compute( double R, double Z) const
     {
         return -invB_(R,Z)*invB_(R,Z)*bZ_(R,Z); 
     }
-    private:    
     InvB invB_;
     BZ bZ_;    
 };
 
 /**
  * @brief \f[  \mathcal{\hat{K}}^{\hat{Z}}_{\nabla B}  \f]
+ *
+   \f[  \mathcal{\hat{K}}^{\hat{Z}}_{\nabla B} =\frac{1}{ \hat{B}^2}   \frac{\partial \hat{B}}{\partial \hat{R}} \f]
  */ 
 struct CurvatureNablaBZ: public aCloneableBinaryFunctor<CurvatureNablaBZ>
 {
     CurvatureNablaBZ( const TokamakMagneticField& mag): invB_(mag), bR_(mag) { }
-    /**
-     * @brief \f[  \mathcal{\hat{K}}^{\hat{Z}}_{\nabla B} =\frac{1}{ \hat{B}^2}   \frac{\partial \hat{B}}{\partial \hat{R}} \f]
-     */    
-    double operator()( double R, double Z) const
+    private:    
+    double do_compute( double R, double Z) const
     {
         return invB_(R,Z)*invB_(R,Z)*bR_(R,Z);
     }
-    private:    
     InvB invB_;
     BR bR_;   
 };
 
 /**
  * @brief \f[ \mathcal{\hat{K}}^{\hat{R}}_{\vec{\kappa}}=0 \f]
+ *
+ \f[ \mathcal{\hat{K}}^{\hat{R}}_{\vec{\kappa}} =0  \f]
  */ 
 struct CurvatureKappaR: public aCloneableBinaryFunctor<CurvatureKappaR>
 {
     CurvatureKappaR( ){ }
     CurvatureKappaR( const TokamakMagneticField& mag){ }
-    /**
-     * @brief \f[ \mathcal{\hat{K}}^{\hat{R}}_{\vec{\kappa}} =0  \f]
-     */ 
-    double operator()( double R, double Z) const
+    private:
+    double do_compute( double R, double Z) const
     {
         return  0.;
     }
@@ -234,54 +225,51 @@ struct CurvatureKappaR: public aCloneableBinaryFunctor<CurvatureKappaR>
 
 /**
  * @brief \f[  \mathcal{\hat{K}}^{\hat{Z}}_{\vec{\kappa}}  \f]
+ *
+ * @brief \f[  \mathcal{\hat{K}}^{\hat{Z}}_{\vec{\kappa}} = - \frac{1}{\hat{R} \hat{B}} \f]
  */ 
 struct CurvatureKappaZ: public aCloneableBinaryFunctor<CurvatureKappaZ>
 {
     CurvatureKappaZ( const TokamakMagneticField& mag): invB_(mag) { }
-    /**
-     * @brief \f[  \mathcal{\hat{K}}^{\hat{Z}}_{\vec{\kappa}} = - \frac{1}{\hat{R} \hat{B}} \f]
-     */    
-    double operator()( double R, double Z) const
+    private:    
+    double do_compute( double R, double Z) const
     {
         return -invB_(R,Z)/R;
     }
-    private:    
     InvB invB_;
 };
 
 /**
  * @brief \f[  \vec{\hat{\nabla}}\cdot \mathcal{\hat{K}}_{\vec{\kappa}}  \f]
+
+     \f[  \vec{\hat{\nabla}}\cdot \mathcal{\hat{K}}_{\vec{\kappa}}  = \frac{1}{\hat{R}  \hat{B}^2 } \partial_{\hat{Z}} \hat{B}\f]
  */ 
 struct DivCurvatureKappa: public aCloneableBinaryFunctor<DivCurvatureKappa>
 {
     DivCurvatureKappa( const TokamakMagneticField& mag): invB_(mag), bZ_(mag){ }
-    /**
-     * @brief \f[  \vec{\hat{\nabla}}\cdot \mathcal{\hat{K}}_{\vec{\kappa}}  = \frac{1}{\hat{R}  \hat{B}^2 } \partial_{\hat{Z}} \hat{B}\f]
-     */    
-    double operator()( double R, double Z) const
+    private:    
+    double do_compute( double R, double Z) const
     {
         return bZ_(R,Z)*invB_(R,Z)*invB_(R,Z)/R;
     }
-    private:    
     InvB invB_;
     BZ bZ_;    
 };
 
 /**
  * @brief \f[  \hat{\nabla}_\parallel \ln{(\hat{B})} \f]
+ *
+ *    \f[  \hat{\nabla}_\parallel \ln{(\hat{B})} = \frac{1}{\hat{R}\hat{B}^2 } \left[ \hat{B}, \hat{\psi}_p\right]_{\hat{R}\hat{Z}} \f]
  */ 
 struct GradLnB: public aCloneableBinaryFunctor<GradLnB>
 {
     GradLnB( const TokamakMagneticField& mag): mag_(mag), invB_(mag), bR_(mag), bZ_(mag) { } 
-    /**
-     * @brief \f[  \hat{\nabla}_\parallel \ln{(\hat{B})} = \frac{1}{\hat{R}\hat{B}^2 } \left[ \hat{B}, \hat{\psi}_p\right]_{\hat{R}\hat{Z}} \f]
-     */ 
-    double operator()( double R, double Z) const
+    private:
+    double do_compute( double R, double Z) const
     {
         double invB = invB_(R,Z);
         return mag_.R0()*invB*invB*(bR_(R,Z)*mag_.psipZ()(R,Z)-bZ_(R,Z)*mag_.psipR()(R,Z))/R ;
     }
-    private:
     TokamakMagneticField mag_;
     InvB invB_;
     BR bR_;
@@ -291,15 +279,15 @@ struct GradLnB: public aCloneableBinaryFunctor<GradLnB>
 /**
  * @brief \f[ B_\varphi = R_0I/R^2\f]
 */
-struct FieldP: public aCloneableBinaryFunctor<LnB>
+struct FieldP: public aCloneableBinaryFunctor<FieldP>
 {
     FieldP( const TokamakMagneticField& mag): mag_(mag){}
-    double operator()( double R, double Z, double phi) const
+    private:
+    double do_compute( double R, double Z) const
     {
         return mag_.R0()*mag_.ipol()(R,Z)/R/R;
     }
     
-    private:
     TokamakMagneticField mag_;
 }; 
 
@@ -309,11 +297,11 @@ struct FieldP: public aCloneableBinaryFunctor<LnB>
 struct FieldR: public aCloneableBinaryFunctor<FieldR>
 {
     FieldR( const TokamakMagneticField& mag): mag_(mag){}
-    double operator()( double R, double Z) const
+    private:
+    double do_compute( double R, double Z) const
     {
         return  mag_.R0()/R*mag_.psipZ()(R,Z);
     }
-    private:
     TokamakMagneticField mag_;
    
 };
@@ -324,11 +312,11 @@ struct FieldR: public aCloneableBinaryFunctor<FieldR>
 struct FieldZ: public aCloneableBinaryFunctor<FieldZ>
 {
     FieldZ( const TokamakMagneticField& mag): mag_(mag){}
-    double operator()( double R, double Z) const
+    private:
+    double do_compute( double R, double Z) const
     {
         return -mag_.R0()/R*mag_.psipR()(R,Z);
     }
-    private:
     TokamakMagneticField mag_;
 };
 
@@ -344,12 +332,12 @@ struct FieldT: public aCloneableBinaryFunctor<FieldT>
      * B^R\partial_R\theta + B^Z\partial_Z\theta\f]
      * where \f$ \theta \f$ is the geometrical poloidal angle.
      */ 
-    double operator()(double R, double Z) const
+    private:
+    double do_compute(double R, double Z) const
     { 
         double r2 = (R-R_0_)*(R-R_0_) + Z*Z;
         return fieldR_(R,Z)*(-Z/r2) + fieldZ_(R,Z)*(R-R_0_)/r2;
     }
-    private:
     double R_0_;
     FieldR fieldR_;
     FieldZ fieldZ_;
@@ -361,11 +349,11 @@ struct FieldT: public aCloneableBinaryFunctor<FieldT>
 struct BHatR: public aCloneableBinaryFunctor<BHatR>
 {
     BHatR( const TokamakMagneticField& mag): mag_(mag), invB_(mag){ }
-    double operator()( double R, double Z) const
+    private:
+    double do_compute( double R, double Z) const
     {
         return  invB_(R,Z)*mag_.R0()/R*mag_.psipZ()(R,Z);
     }
-    private:
     TokamakMagneticField mag_;
     InvB invB_;
 
@@ -377,11 +365,11 @@ struct BHatR: public aCloneableBinaryFunctor<BHatR>
 struct BHatZ: public aCloneableBinaryFunctor<BHatZ>
 {
     BHatZ( const TokamakMagneticField& mag): mag_(mag), invB_(mag){ }
-    double operator()( double R, double Z) const
+    private:
+    double do_compute( double R, double Z) const
     {
         return  -invB_(R,Z)*mag_.R0()/R*mag_.psipR()(R,Z);
     }
-    private:
     TokamakMagneticField mag_;
     InvB invB_;
 };
@@ -392,12 +380,11 @@ struct BHatZ: public aCloneableBinaryFunctor<BHatZ>
 struct BHatP: public aCloneableBinaryFunctor<BHatP>
 {
     BHatP( const TokamakMagneticField& mag): mag_(mag), invB_(mag){ }
-    double operator()( double R, double Z) const
+    private:
+    double do_compute( double R, double Z) const
     {
         return invB_(R,Z)*mag_.R0()*mag_.ipol()(R,Z)/R/R;
     }
-    
-    private:
     TokamakMagneticField mag_;
     InvB invB_;
 }; 
diff --git a/inc/geometries/solovev.h b/inc/geometries/solovev.h
index 84615f60f..0a8c1b7c4 100644
--- a/inc/geometries/solovev.h
+++ b/inc/geometries/solovev.h
@@ -62,30 +62,11 @@ struct Psip: public aCloneableBinaryFunctor<Psip>
     /**
      * @brief Construct from given geometric parameters
      *
-     * @param gp useful geometric parameters
+     * @param gp geometric parameters
      */
     Psip( GeomParameters gp): R_0_(gp.R_0), A_(gp.A), c_(gp.c) {}
-/**
- * @brief \f$ \hat \psi_p(R,Z) \f$
-
-      @param R radius (cylindrical coordinates)
-      @param Z height (cylindrical coordinates)
-      @return \f$ \hat \psi_p(R,Z) \f$
- */
-    double operator()(double R, double Z) const
-    {    
-        return psi_alt( R, Z);
-    }
-    /**
-     * @brief Show parameters to std::cout
-     */
-    void display() const
-    {
-      std::cout << R_0_ <<"  " <<A_ <<"\n";
-      std::cout << c_[0] <<"\n";
-    }
   private:
-    double psi_alt(double R, double Z) const
+    double do_compute(double R, double Z) const
     {
         double Rn,Rn2,Rn4,Zn,Zn2,Zn3,Zn4,Zn5,Zn6,lgRn;
         Rn = R/R_0_; Rn2 = Rn*Rn; Rn4 = Rn2*Rn2;
@@ -156,34 +137,10 @@ struct Psip: public aCloneableBinaryFunctor<Psip>
  */ 
 struct PsipR: public aCloneableBinaryFunctor<PsipR>
 {
-    /**
-     * @brief Construct from given geometric parameters
-     *
-     * @param gp useful geometric parameters
-     */
+    ///@copydoc Psip::Psip()
     PsipR( GeomParameters gp): R_0_(gp.R_0), A_(gp.A), c_(gp.c) {}
-/**
- * @brief \f$ \frac{\partial  \hat{\psi}_p }{ \partial \hat{R}}(R,Z)  \f$
-
-      @param R radius (cylindrical coordinates)
-      @param Z height (cylindrical coordinates)
-    * @return \f$ \frac{\partial  \hat{\psi}_p}{ \partial \hat{R}}(R,Z)  \f$
- */ 
-    double operator()(double R, double Z) const
-    {    
-        return psipR_alt( R, Z);
-    }
-
-    /**
-     * @brief Print parameters to std::cout
-     */
-    void display() const
-    {
-      std::cout << R_0_ <<"  " <<A_ <<"\n";
-      std::cout << c_[0] <<"\n";
-    }
   private:
-    double psipR_alt(double R, double Z) const
+    double do_compute(double R, double Z) const
     {    
         double Rn,Rn2,Rn3,Rn5,Zn,Zn2,Zn3,Zn4,lgRn;
         Rn = R/R_0_; Rn2 = Rn*Rn; Rn3 = Rn2*Rn;  Rn5 = Rn3*Rn2; 
@@ -204,17 +161,8 @@ struct PsipR: public aCloneableBinaryFunctor<PsipR>
 };
 /**
  * @brief \f[ \frac{\partial^2  \hat{\psi}_p }{ \partial \hat{R}^2}\f]
- */ 
-struct PsipRR: public aCloneableBinaryFunctor<PsipRR>
-{
-    /**
-    * @brief Constructor
-    *
-    * @param gp geometric parameters
-    */
-    PsipRR( GeomParameters gp ): R_0_(gp.R_0), A_(gp.A), c_(gp.c) {}
-/**
- * @brief \f[ \frac{\partial^2  \hat{\psi}_p }{ \partial \hat{R}^2}=
+ *
+ * \f[ \frac{\partial^2  \hat{\psi}_p }{ \partial \hat{R}^2}=
      \hat{R}_0^{-1} \Bigg\{ 2 c_2 +(3 \hat{\bar{R}}^2 )/2+2 c_9  \bar{Z}+c_4 (12 \bar{R}^2 -8  \bar{Z}^2)+c_{11} 
       (36 \bar{R}^2  \bar{Z}-8  \bar{Z}^3)
       +c_6 (30 \bar{R}^4 -144 \bar{R}^2  \bar{Z}^2+16  \bar{Z}^4)+c_3 (-3 -2  \ln{(\bar{R} 
@@ -227,24 +175,13 @@ struct PsipRR: public aCloneableBinaryFunctor<PsipRR>
       -160  \bar{Z}^3 \ln{(\bar{R}   )})
       + c_7 (-165 \bar{R}^4 +2160 \bar{R}^2  \bar{Z}^2-640  \bar{Z}^4-450 \bar{R}^4  \ln{(\bar{R}   )}+2160 \bar{R}^2  \bar{Z}^2
       \ln{(\bar{R}   )}-240  \bar{Z}^4 \ln{(\bar{R}   )})\Bigg\}\f]
-      @param R radius (cylindrical coordinates)
-      @param Z height (cylindrical coordinates)
-      @return value
  */ 
-    double operator()(double R, double Z) const
-    {    
-        return psipRR_alt( R, Z);
-    }
-    /**
-    * @brief Display the internal parameters to std::cout
-    */
-    void display()
-    {
-      std::cout << R_0_ <<"  " <<A_ <<"\n";
-      std::cout << c_[0] <<"\n";
-    }
+struct PsipRR: public aCloneableBinaryFunctor<PsipRR>
+{
+    ///@copydoc Psip::Psip()
+    PsipRR( GeomParameters gp ): R_0_(gp.R_0), A_(gp.A), c_(gp.c) {}
   private:
-    double psipRR_alt(double R, double Z) const
+    double do_compute(double R, double Z) const
     {    
        double Rn,Rn2,Rn4,Zn,Zn2,Zn3,Zn4,lgRn;
        Rn = R/R_0_; Rn2 = Rn*Rn;  Rn4 = Rn2*Rn2;
@@ -263,12 +200,8 @@ struct PsipRR: public aCloneableBinaryFunctor<PsipRR>
 };
 /**
  * @brief \f[\frac{\partial \hat{\psi}_p }{ \partial \hat{Z}}\f]
- */ 
-struct PsipZ: public aCloneableBinaryFunctor<PsipZ>
-{
-    PsipZ( GeomParameters gp ): R_0_(gp.R_0), A_(gp.A), c_(gp.c) { }
-/**
- * @brief \f[\frac{\partial \hat{\psi}_p }{ \partial \hat{Z}}= 
+ *
+ * \f[\frac{\partial \hat{\psi}_p }{ \partial \hat{Z}}= 
       \Bigg\{c_8 +c_9 \bar{R}^2 +2 c_3  \bar{Z}-8 c_4 \bar{R}^2  \bar{Z}+c_{11} 
       (3 \bar{R}^4 -12 \bar{R}^2  \bar{Z}^2)+c_6 (-24 \bar{R}^4  \bar{Z}+32 \bar{R}^2  \bar{Z}^3)
       +c_{10} (3  \bar{Z}^2-3 \bar{R}^2 
@@ -279,7 +212,12 @@ struct PsipZ: public aCloneableBinaryFunctor<PsipZ>
       +c_7 (150 \bar{R}^4  \bar{Z}-560 \bar{R}^2  \bar{Z}^3+48  
       \bar{Z}^5+360 \bar{R}^4  \bar{Z} \ln{(\bar{R}   )}-480 \bar{R}^2  \bar{Z}^3 \ln{(\bar{R}   )})\Bigg\} \f]
  */ 
-    double psipZ_alt(double R, double Z) const
+struct PsipZ: public aCloneableBinaryFunctor<PsipZ>
+{
+    ///@copydoc Psip::Psip()
+    PsipZ( GeomParameters gp ): R_0_(gp.R_0), A_(gp.A), c_(gp.c) { }
+  private:
+    double do_compute(double R, double Z) const
     {
         double Rn,Rn2,Rn4,Zn,Zn2,Zn3,Zn4,Zn5,lgRn;
         Rn = R/R_0_; Rn2 = Rn*Rn;  Rn4 = Rn2*Rn2; 
@@ -298,45 +236,26 @@ struct PsipZ: public aCloneableBinaryFunctor<PsipZ>
             + ((-45.)*Rn4 + 40. *Zn4 + 60. *Rn4*lgRn -  240. *Rn2 *Zn2*lgRn)* c_[11]);
           
     }
-    double operator()(double R, double Z) const
-    {    
-        return psipZ_alt(R, Z);
-    }
-    void display() const
-    {
-        std::cout << R_0_ <<"  " <<A_ <<"\n";
-        std::cout << c_[0] <<"\n";
-    }
-  private:
     double R_0_, A_;
     std::vector<double> c_;
 };
 /**
  * @brief \f[ \frac{\partial^2  \hat{\psi}_p }{ \partial \hat{Z}^2}\f]
- */ 
-struct PsipZZ: public aCloneableBinaryFunctor<PsipZZ>
-{
-    PsipZZ( GeomParameters gp): R_0_(gp.R_0), A_(gp.A), c_(gp.c) { }
-  /**
-   * @brief \f[ \frac{\partial^2  \hat{\psi}_p }{ \partial \hat{Z}^2}=
+
+   \f[ \frac{\partial^2  \hat{\psi}_p }{ \partial \hat{Z}^2}=
       \hat{R}_0^{-1} \Bigg\{2 c_3 -8 c_4 \bar{R}^2 +6 c_{10}  \bar{Z}-24 c_{11}
       \bar{R}^2  \bar{Z}+c_6 (-24 \bar{R}^4 +96 \bar{R}^2  \bar{Z}^2)
       +c_5 (-18 \bar{R}^2 +24  \bar{Z}^2-24 \bar{R}^2  \ln{(\bar{R}   )})+
       c_{12} (160  \bar{Z}^3-480 \bar{R}^2  \bar{Z} \ln{(\bar{R}   )})
       +c_7 (150 \bar{R}^4 -1680 \bar{R}^2  \bar{Z}^2+240  \bar{Z}^4+360 \bar{R}^4 
       \ln{(\bar{R}   )}-1440 \bar{R}^2  \bar{Z}^2 \ln{(\bar{R}   )})\Bigg\} \f]
-    */ 
-    double operator()(double R, double Z) const
-    {    
-        return psipZZ_alt( R, Z);
-    }
-    void display() const
-    {
-        std::cout << R_0_ <<"  " <<A_ <<"\n";
-        std::cout << c_[0] <<"\n";
-    }
+ */ 
+struct PsipZZ: public aCloneableBinaryFunctor<PsipZZ>
+{
+    ///@copydoc Psip::Psip()
+    PsipZZ( GeomParameters gp): R_0_(gp.R_0), A_(gp.A), c_(gp.c) { }
   private:
-    double psipZZ_alt(double R, double Z) const
+    double do_compute(double R, double Z) const
     {
         double Rn,Rn2,Rn4,Zn,Zn2,Zn3,Zn4,lgRn;
         Rn = R/R_0_; Rn2 = Rn*Rn; Rn4 = Rn2*Rn2; 
@@ -350,12 +269,8 @@ struct PsipZZ: public aCloneableBinaryFunctor<PsipZZ>
 };
 /**
  * @brief  \f[\frac{\partial^2  \hat{\psi}_p }{ \partial \hat{R} \partial\hat{Z}}\f] 
- */ 
-struct PsipRZ: public aCloneableBinaryFunctor<PsipRZ>
-{
-    PsipRZ( GeomParameters gp ): R_0_(gp.R_0), A_(gp.A), c_(gp.c) { }
-/**
- * @brief  \f[\frac{\partial^2  \hat{\psi}_p }{ \partial \hat{R} \partial\hat{Z}}= 
+
+  \f[\frac{\partial^2  \hat{\psi}_p }{ \partial \hat{R} \partial\hat{Z}}= 
         \hat{R}_0^{-1} \Bigg\{2 c_9 \bar{R} -16 c_4 \bar{R}  \bar{Z}+c_{11} 
       (12 \bar{R}^3 -24 \bar{R}  \bar{Z}^2)+c_6 (-96 \bar{R}^3  \bar{Z}+64 \bar{R}  \bar{Z}^3)
       + c_{10} (-3 \bar{R} -6 \bar{R}  \ln{(\bar{R}   )})
@@ -365,17 +280,12 @@ struct PsipRZ: public aCloneableBinaryFunctor<PsipRZ>
       +c_7(960 \bar{R}^3  \bar{Z}-1600 \bar{R}  \bar{Z}^3+1440 \bar{R}^3  \bar{Z} \ln{(\bar{R} 
       )}-960 \bar{R}  \bar{Z}^3 \ln{(\bar{R}   )})\Bigg\} \f] 
  */ 
-    double operator()(double R, double Z) const
-    {    
-        return psipRZ_alt( R, Z);
-    }
-    void display() const
-    {
-        std::cout << R_0_ <<"  " <<A_ <<"\n";
-        std::cout << c_[0] <<"\n";
-    }
+struct PsipRZ: public aCloneableBinaryFunctor<PsipRZ>
+{
+    ///@copydoc Psip::Psip()
+    PsipRZ( GeomParameters gp ): R_0_(gp.R_0), A_(gp.A), c_(gp.c) { }
   private:
-    double psipRZ_alt(double R, double Z) const
+    double do_compute(double R, double Z) const
     {
         double Rn,Rn2,Rn3,Zn,Zn2,Zn3,lgRn;
         Rn = R/R_0_; Rn2 = Rn*Rn; Rn3 = Rn2*Rn; 
@@ -393,19 +303,19 @@ struct PsipRZ: public aCloneableBinaryFunctor<PsipRZ>
 
 /**
  * @brief \f[\hat{I}\f] 
+
+    \f[\hat{I}= \sqrt{-2 A \hat{\psi}_p / \hat{R}_0 +1}\f] 
  */ 
 struct Ipol: public aCloneableBinaryFunctor<Ipol>
 {
+    ///@copydoc Psip::Psip()
     Ipol(  GeomParameters gp ):  R_0_(gp.R_0), A_(gp.A), qampl_(gp.qampl), psip_(gp) { }
-    /**
-    * @brief \f[\hat{I}= \sqrt{-2 A \hat{\psi}_p / \hat{R}_0 +1}\f] 
-    */ 
-    double operator()(double R, double Z) const
+  private:
+    double do_compute(double R, double Z) const
     {    
         //sign before A changed to -
         return qampl_*sqrt(-2.*A_* psip_(R,Z) /R_0_ + 1.);
     }
-  private:
     double R_0_, A_,qampl_;
     Psip psip_;
 };
@@ -414,12 +324,13 @@ struct Ipol: public aCloneableBinaryFunctor<Ipol>
  */
 struct IpolR: public aCloneableBinaryFunctor<IpolR>
 {
+    ///@copydoc Psip::Psip()
     IpolR(  GeomParameters gp ):  R_0_(gp.R_0), A_(gp.A), qampl_(gp.qampl), psip_(gp), psipR_(gp) { }
-    double operator()(double R, double Z) const
+  private:
+    double do_compute(double R, double Z) const
     {    
         return -qampl_/sqrt(-2.*A_* psip_(R,Z) /R_0_ + 1.)*(A_*psipR_(R,Z)/R_0_);
     }
-  private:
     double R_0_, A_,qampl_;
     Psip psip_;
     PsipR psipR_;
@@ -429,12 +340,13 @@ struct IpolR: public aCloneableBinaryFunctor<IpolR>
  */
 struct IpolZ: public aCloneableBinaryFunctor<IpolZ>
 {
+    ///@copydoc Psip::Psip()
     IpolZ(  GeomParameters gp ):  R_0_(gp.R_0), A_(gp.A), qampl_(gp.qampl), psip_(gp), psipZ_(gp) { }
-    double operator()(double R, double Z) const
+  private:
+    double do_compute(double R, double Z) const
     {    
         return -qampl_/sqrt(-2.*A_* psip_(R,Z) /R_0_ + 1.)*(A_*psipZ_(R,Z)/R_0_);
     }
-  private:
     double R_0_, A_,qampl_;
     Psip psip_;
     PsipZ psipZ_;
@@ -470,14 +382,14 @@ struct Psip: public aCloneableBinaryFunctor<Psip>
         psipRR_X_ = psipRR_(R_X, Z_X);
     
     }
-    double operator()(double R, double Z) const
+    private:
+    double do_compute(double R, double Z) const
     {    
         double psip_RZ = psip_(R,Z);
         double Rbar = R - R_X, Zbar = Z - Z_X;
         double psip_2 =  0.5*(- psipZZ_X_*Rbar*Rbar + 2.*psipRZ_X_*Rbar*Zbar - psipRR_X_*Zbar*Zbar) - psip_RZ ; 
         return  psip_RZ + 0.5*psip_2*cauchy_(R,Z);
     }
-    private:
     double R_X, Z_X; 
     double psipZZ_X_, psipRZ_X_, psipRR_X_;
     solovev::Psip psip_;
@@ -495,7 +407,8 @@ struct PsipR: public aCloneableBinaryFunctor<PsipR>
         psipRZ_X_ = psipRZ_(R_X, Z_X);
         psipRR_X_ = psipRR_(R_X, Z_X);
     }
-    double operator()(double R, double Z) const
+    private:
+    double do_compute(double R, double Z) const
     {    
         double psipR_RZ = psipR_(R,Z);
         if( !cauchy_.inside(R,Z)) return psipR_RZ;
@@ -504,7 +417,6 @@ struct PsipR: public aCloneableBinaryFunctor<PsipR>
         double psip_2R =  - psipZZ_X_*Rbar + psipRZ_X_*Zbar - psipR_RZ;
         return psipR_RZ + 0.5*(psip_2R*cauchy_(R,Z) + psip_2*cauchy_.dx(R,Z)  );
     }
-    private:
     double R_X, Z_X; 
     double psipZZ_X_, psipRZ_X_, psipRR_X_;
     solovev::Psip psip_;
@@ -523,7 +435,8 @@ struct PsipZ: public aCloneableBinaryFunctor<PsipZ>
         psipRZ_X_ = psipRZ_(R_X, Z_X);
         psipRR_X_ = psipRR_(R_X, Z_X);
     }
-    double operator()(double R, double Z) const
+    private:
+    double do_compute(double R, double Z) const
     {    
         double psipZ_RZ = psipZ_(R,Z);
         if( !cauchy_.inside(R,Z)) return psipZ_RZ;
@@ -532,7 +445,6 @@ struct PsipZ: public aCloneableBinaryFunctor<PsipZ>
         double psip_2Z =  - psipRR_X_*Zbar + psipRZ_X_*Rbar - psipZ_RZ;
         return psipZ_RZ + 0.5*(psip_2Z*cauchy_(R,Z) + psip_2*cauchy_.dy(R,Z));
     }
-    private:
     double R_X, Z_X; 
     double psipZZ_X_, psipRZ_X_, psipRR_X_;
     solovev::Psip psip_;
@@ -552,7 +464,8 @@ struct PsipZZ: public aCloneableBinaryFunctor<PsipZZ>
         psipRZ_X_ = psipRZ_(R_X, Z_X);
         psipRR_X_ = psipRR_(R_X, Z_X);
     }
-    double operator()(double R, double Z) const
+    private:
+    double do_compute(double R, double Z) const
     {    
         double psipZZ_RZ = psipZZ_(R,Z);
         if( !cauchy_.inside(R, Z)) return psipZZ_RZ;
@@ -562,7 +475,6 @@ struct PsipZZ: public aCloneableBinaryFunctor<PsipZZ>
         double psip_2ZZ =  - psipRR_X_ - psipZZ_RZ;
         return psipZZ_RZ + 0.5*(psip_2ZZ*cauchy_(R,Z) + 2.*cauchy_.dy(R,Z)*psip_2Z +  psip_2*cauchy_.dyy(R,Z));
     }
-    private:
     double R_X, Z_X; 
     double psipZZ_X_, psipRZ_X_, psipRR_X_;
     solovev::Psip psip_;
@@ -581,7 +493,8 @@ struct PsipRR: public aCloneableBinaryFunctor<PsipRR>
         psipRZ_X_ = psipRZ_(R_X, Z_X);
         psipRR_X_ = psipRR_(R_X, Z_X);
     }
-    double operator()(double R, double Z) const
+    private:
+    double do_compute(double R, double Z) const
     {    
         double psipRR_RZ = psipRR_(R,Z);
         if( !cauchy_.inside(R,Z)) return psipRR_RZ;
@@ -591,7 +504,6 @@ struct PsipRR: public aCloneableBinaryFunctor<PsipRR>
         double psip_2RR =  - psipZZ_X_ - psipRR_RZ;
         return psipRR_RZ + 0.5*(psip_2RR*cauchy_(R,Z) + 2.*cauchy_.dx(R,Z)*psip_2R +  psip_2*cauchy_.dxx(R,Z));
     }
-    private:
     double R_X, Z_X; 
     double psipZZ_X_, psipRZ_X_, psipRR_X_;
     solovev::Psip psip_;
@@ -610,7 +522,8 @@ struct PsipRZ: public aCloneableBinaryFunctor<PsipRZ>
         psipRZ_X_ = psipRZ_(R_X, Z_X);
         psipRR_X_ = psipRR_(R_X, Z_X);
     }
-    double operator()(double R, double Z) const
+    private:
+    double do_compute(double R, double Z) const
     {    
         double psipRZ_RZ = psipRZ_(R,Z);
         if( !cauchy_.inside(R,Z)) return psipRZ_RZ;
@@ -621,7 +534,6 @@ struct PsipRZ: public aCloneableBinaryFunctor<PsipRZ>
         double psip_2RZ =  - psipRZ_X_ - psipRZ_RZ;
         return psipRZ_RZ + 0.5*(psip_2RZ*cauchy_(R,Z) + cauchy_.dx(R,Z)*psip_2Z + cauchy_.dy(R,Z)*psip_2R  +  psip_2*cauchy_.dxy(R,Z));
     }
-    private:
     double R_X, Z_X; 
     double psipZZ_X_, psipRZ_X_, psipRR_X_;
     solovev::Psip psip_;
diff --git a/inc/geometries/taylor.h b/inc/geometries/taylor.h
index 19c58d38a..870094d65 100644
--- a/inc/geometries/taylor.h
+++ b/inc/geometries/taylor.h
@@ -317,12 +317,12 @@ BinaryFunctorsLvl2 createPsip( solovev::GeomParameters gp)
 }
 BinaryFunctorsLvl1 createIpol( solovev::GeomParameters gp)
 {
-    BinaryFunctorsLvl1 ipol( new Ipol(gp), new IpolR(gp), new IpolZ(gp))
+    BinaryFunctorsLvl1 ipol( new Ipol(gp), new IpolR(gp), new IpolZ(gp));
     return ipol;
 }
-TokamakMagneticField createMagField( solovev::GeomParameters gp)
+dg::geo::TokamakMagneticField createMagField( solovev::GeomParameters gp)
 {
-    return MagneticField( gp.R_0, createPsip(gp), createIpol(gp));
+    return TokamakMagneticField( gp.R_0, dg::geo::taylor::createPsip(gp), dg::geo::taylor::createIpol(gp));
 }
 
 ///@}
-- 
GitLab


From a521299fb041f7b61a18473e9149332fe3ef1aa4 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Thu, 17 Aug 2017 07:03:54 -0700
Subject: [PATCH 175/453] changed const Functor& to Functor& and let the
 compiler decide when it needs const

---
 inc/dg/elliptic.h           |  2 +-
 inc/dg/geometry/transform.h | 20 ++++++++++----------
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index 5c6e66de8..4b0454806 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -649,7 +649,7 @@ struct TensorElliptic
      *
      */
     template<class ChiRR, class ChiRZ, class ChiZZ>
-    void set( const ChiRR& chiRR, const ChiRZ& chiRZ, const ChiZZ& chiZZ)
+    void set( ChiRR& chiRR, ChiRZ& chiRZ, ChiZZ& chiZZ)
     {
         typename GeometryTraits<Geometry>::host_vector chiXX, chiXY, chiYY;
         dg::pushForwardPerp( chiRR, chiRZ, chiZZ, chiXX, chiXY, chiYY, g_);
diff --git a/inc/dg/geometry/transform.h b/inc/dg/geometry/transform.h
index b6f563a0c..37d599072 100644
--- a/inc/dg/geometry/transform.h
+++ b/inc/dg/geometry/transform.h
@@ -39,7 +39,7 @@ struct GeometryTraits
  * @ingroup pullback
  */
 template< class Functor>
-thrust::host_vector<double> pullback( const Functor& f, const aGeometry2d& g)
+thrust::host_vector<double> pullback( Functor& f, const aGeometry2d& g)
 {
     std::vector<thrust::host_vector<double> > map = g.map();
     thrust::host_vector<double> vec( g.size());
@@ -48,10 +48,10 @@ thrust::host_vector<double> pullback( const Functor& f, const aGeometry2d& g)
     return vec;
 }
 
-///@copydoc pullback(Functor,const aGeometry2d&)
+///@copydoc pullback(Functor&,const aGeometry2d&)
 ///@ingroup pullback
 template< class Functor>
-thrust::host_vector<double> pullback( const Functor& f, const aGeometry3d& g)
+thrust::host_vector<double> pullback( Functor& f, const aGeometry3d& g)
 {
     std::vector<thrust::host_vector<double> > map = g.map();
     thrust::host_vector<double> vec( g.size());
@@ -70,10 +70,10 @@ struct MemoryTraits< MPITag>
 };
 ///@endcond
 
-///@copydoc pullback(Functor,const aGeometry2d&)
+///@copydoc pullback(Functor&,const aGeometry2d&)
 ///@ingroup pullback
 template< class Functor>
-MPI_Vector<thrust::host_vector<double> > pullback( const Functor& f, const aMPIGeometry2d& g)
+MPI_Vector<thrust::host_vector<double> > pullback( Functor& f, const aMPIGeometry2d& g)
 {
     std::vector<MPI_Vector<thrust::host_vector<double> > > map = g.map();
     thrust::host_vector<double> vec( g.size());
@@ -82,10 +82,10 @@ MPI_Vector<thrust::host_vector<double> > pullback( const Functor& f, const aMPIG
     return MPI_Vector<thrust::host_vector<double> >( vec, g.communicator());
 }
 
-///@copydoc pullback(Functor,const aGeometry2d&)
+///@copydoc pullback(Functor&,const aGeometry2d&)
 ///@ingroup pullback
 template< class Functor>
-MPI_Vector<thrust::host_vector<double> > pullback( const Functor& f, const aMPIGeometry3d& g)
+MPI_Vector<thrust::host_vector<double> > pullback( Functor& f, const aMPIGeometry3d& g)
 {
     std::vector<MPI_Vector<thrust::host_vector<double> > > map = g.map();
     thrust::host_vector<double> vec( g.size());
@@ -113,7 +113,7 @@ MPI_Vector<thrust::host_vector<double> > pullback( const Functor& f, const aMPIG
  * @ingroup pullback
  */
 template<class Functor1, class Functor2, class container, class Geometry> 
-void pushForwardPerp( const Functor1& vR, const Functor2& vZ, 
+void pushForwardPerp( Functor1& vR, Functor2& vZ, 
         container& vx, container& vy,
         const Geometry& g)
 {
@@ -144,7 +144,7 @@ void pushForwardPerp( const Functor1& vR, const Functor2& vZ,
  * @ingroup pullback
  */
 template<class Functor1, class Functor2, class Functor3, class container, class Geometry> 
-void pushForward( const Functor1& vR, const Functor2& vZ, const Functor3& vPhi,
+void pushForward( Functor1& vR, Functor2& vZ, Functor3& vPhi,
         container& vx, container& vy, container& vz,
         const Geometry& g)
 {
@@ -179,7 +179,7 @@ void pushForward( const Functor1& vR, const Functor2& vZ, const Functor3& vPhi,
  * @ingroup pullback
  */
 template<class FunctorRR, class FunctorRZ, class FunctorZZ, class container, class Geometry> 
-void pushForwardPerp( const FunctorRR& chiRR, const FunctorRZ& chiRZ, const FunctorZZ& chiZZ,
+void pushForwardPerp( FunctorRR& chiRR, FunctorRZ& chiRZ, FunctorZZ& chiZZ,
         container& chixx, container& chixy, container& chiyy,
         const Geometry& g)
 {
-- 
GitLab


From 1a81635ebab61e2f223ab190918605b3071ebe5c Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Thu, 17 Aug 2017 07:19:10 -0700
Subject: [PATCH 176/453] succesfully transitioned to do_compute function

---
 inc/geometries/adaption.h      |  46 ++++++------
 inc/geometries/fluxfunctions.h |   2 +-
 inc/geometries/guenther.h      |  34 ++++-----
 inc/geometries/init.h          | 128 +++++++--------------------------
 inc/geometries/taylor.h        |  77 ++++++++------------
 5 files changed, 95 insertions(+), 192 deletions(-)

diff --git a/inc/geometries/adaption.h b/inc/geometries/adaption.h
index 4ac8f4fe8..7168988f8 100644
--- a/inc/geometries/adaption.h
+++ b/inc/geometries/adaption.h
@@ -18,13 +18,13 @@ namespace detail{
 struct LaplaceAdaptPsi: public aCloneableBinaryFunctor<LaplaceAdaptPsi>
 {
     LaplaceAdaptPsi( const BinaryFunctorsLvl2& psi, const BinaryFunctorsLvl1& chi) : psi_(psi), chi_(chi){}
-    double operator()(double x, double y)const
+    private:
+    double do_compute(double x, double y)const
     {
         return  psi_.dfx()(x,y)*chi_.dfx()(x,y) +
                 psi_.dfy()(x,y)*chi_.dfy()(x,y) +
                 chi_.f()(x,y)*( psi_.dfxx()(x,y) + psi_.dfyy()(x,y));
     }
-    private:
     BinaryFunctorsLvl2 psi_;
     BinaryFunctorsLvl1 chi_;
 };
@@ -33,13 +33,13 @@ struct LaplaceChiPsi: public aCloneableBinaryFunctor<LaplaceChiPsi>
 {
     LaplaceChiPsi( const BinaryFunctorsLvl2& psi, const BinarySymmTensorLvl1& chi):
         psi_(psi), chi_(chi){}
-    double operator()(double x, double y)const
+    private:
+    double do_compute(double x, double y)const
     {
         return psi_.dfxx()(x,y)*chi_.xx()(x,y)+2.*psi_.dfxy()(x,y)*chi_.xy()(x,y)+psi_.dfyy()(x,y)*chi_.yy()(x,y)
             + chi_.divX()(x,y)*psi_.dfx()(x,y) + chi_.divY()(x,y)*psi_.dfy()(x,y);
     }
 
-    private:
     BinaryFunctorsLvl2 psi_;
     BinarySymmTensorLvl1 chi_;
 };
@@ -47,8 +47,8 @@ struct LaplaceChiPsi: public aCloneableBinaryFunctor<LaplaceChiPsi>
 struct LaplacePsi: public aCloneableBinaryFunctor<LaplacePsi>
 {
     LaplacePsi( const BinaryFunctorsLvl2& psi): psi_(psi){}
-    double operator()(double x, double y)const{return psi_.dfxx()(x,y)+psi_.dfyy()(x,y);}
     private:
+    double do_compute(double x, double y)const{return psi_.dfxx()(x,y)+psi_.dfyy()(x,y);}
     BinaryFunctorsLvl2 psi_;
 };
 
@@ -70,16 +70,12 @@ struct NablaPsiInv: public aCloneableBinaryFunctor<NablaPsiInv>
      * @param psi \f$ \psi(x,y)\f$ and its first derivatives
      */
     NablaPsiInv( const BinaryFunctorsLvl1& psi): psi_(psi){}
-    /**
-     * @brief  A weight function for the Hector algorithm
-     * \f[ |\nabla\psi|^{-1} = (\psi_x^2 + \psi_y^2)^{-1/2} \f]
-     */
-    virtual double operator()(double x, double y)const
+    private:
+    virtual double do_compute(double x, double y)const
     {
         double psiX = psi_.dfx()(x,y), psiY = psi_.dfy()(x,y);
         return 1./sqrt(psiX*psiX+psiY*psiY);
     }
-    private:
     BinaryFunctorsLvl1 psi_;
 };
 
@@ -90,7 +86,8 @@ struct NablaPsiInv: public aCloneableBinaryFunctor<NablaPsiInv>
 struct NablaPsiInvX: public aCloneableBinaryFunctor<NablaPsiInvX>
 {
     NablaPsiInvX( const BinaryFunctorsLvl2& psi):psi_(psi) {}
-    virtual double operator()(double x, double y)const
+    private:
+    virtual double do_compute(double x, double y)const
     {
         double psiX = psi_.dfx()(x,y), psiY = psi_.dfy()(x,y);
         double psiXX = psi_.dfxx()(x,y), psiXY = psi_.dfxy()(x,y);
@@ -98,7 +95,6 @@ struct NablaPsiInvX: public aCloneableBinaryFunctor<NablaPsiInvX>
         return -(psiX*psiXX+psiY*psiXY)/psip/psip/psip;
     }
     
-    private:
     BinaryFunctorsLvl2 psi_;
 };
 
@@ -109,7 +105,8 @@ struct NablaPsiInvX: public aCloneableBinaryFunctor<NablaPsiInvX>
 struct NablaPsiInvY: public aCloneableBinaryFunctor<NablaPsiInvY>
 {
     NablaPsiInvY( const BinaryFunctorsLvl2& psi):psi_(psi) {}
-    double operator()(double x, double y)const
+    private:
+    double do_compute(double x, double y)const
     {
         double psiX = psi_.dfx()(x,y), psiY = psi_.dfy()(x,y);
         double psiYY = psi_.dfyy()(x,y), psiXY = psi_.dfxy()(x,y);
@@ -117,7 +114,6 @@ struct NablaPsiInvY: public aCloneableBinaryFunctor<NablaPsiInvY>
         return -(psiX*psiXY+psiY*psiYY)/psip/psip/psip;
     }
     
-    private:
     BinaryFunctorsLvl2 psi_;
 };
 
@@ -142,7 +138,8 @@ BinaryFunctorsLvl1 make_NablaPsiInvCollective( const BinaryFunctorsLvl2& psi)
 struct Liseikin_XX: public aCloneableBinaryFunctor<Liseikin_XX>
 {
     Liseikin_XX(const BinaryFunctorsLvl1& psi, double k, double eps):k_(k), eps_(eps), psi_(psi){}
-    double operator()(double x, double y)const
+    private:
+    double do_compute(double x, double y)const
     {
         double psiX = psi_.dfx()(x,y), psiY = psi_.dfy()(x,y), k2 = k_*k_;
         double psip2 = psiX*psiX+psiY*psiY;
@@ -150,7 +147,6 @@ struct Liseikin_XX: public aCloneableBinaryFunctor<Liseikin_XX>
         return (psiY*psiY+k2*psiX*psiX + eps_)/sqrtG;
     }
 
-    private:
     double k_, eps_;
     BinaryFunctorsLvl1 psi_;
 };
@@ -165,7 +161,8 @@ struct Liseikin_XX: public aCloneableBinaryFunctor<Liseikin_XX>
 struct Liseikin_XY: public aCloneableBinaryFunctor<Liseikin_XY>
 {
     Liseikin_XY(const BinaryFunctorsLvl1& psi, double k, double eps):k_(k), eps_(eps), psi_(psi){}
-    double operator()(double x, double y)const
+    private:
+    double do_compute(double x, double y)const
     {
         double psiX = psi_.dfx()(x,y), psiY = psi_.dfy()(x,y), k2 = k_*k_;
         double psip2 = psiX*psiX+psiY*psiY;
@@ -173,7 +170,6 @@ struct Liseikin_XY: public aCloneableBinaryFunctor<Liseikin_XY>
         return (-psiX*psiY+k2*psiX*psiY)/sqrtG;
     }
 
-    private:
     double k_, eps_;
     BinaryFunctorsLvl1 psi_;
 };
@@ -188,7 +184,8 @@ struct Liseikin_XY: public aCloneableBinaryFunctor<Liseikin_XY>
 struct Liseikin_YY: public aCloneableBinaryFunctor<Liseikin_YY>
 {
     Liseikin_YY(const BinaryFunctorsLvl1& psi, double k, double eps):k_(k), eps_(eps), psi_(psi){}
-    double operator()(double x, double y)const
+    private:
+    double do_compute(double x, double y)const
     {
         double psiX = psi_.dfx()(x,y), psiY = psi_.dfy()(x,y), k2 = k_*k_;
         double psip2 = psiX*psiX+psiY*psiY;
@@ -196,7 +193,6 @@ struct Liseikin_YY: public aCloneableBinaryFunctor<Liseikin_YY>
         return (eps_+psiX*psiX+k2*psiY*psiY)/sqrtG;
     }
 
-    private:
     double k_, eps_;
     BinaryFunctorsLvl1 psi_;
 };
@@ -208,7 +204,8 @@ struct Liseikin_YY: public aCloneableBinaryFunctor<Liseikin_YY>
 struct DivLiseikinX: public aCloneableBinaryFunctor<DivLiseikinX>
 {
     DivLiseikinX(const BinaryFunctorsLvl2& psi, double k, double eps): k_(k), eps_(eps), psi_(psi){}
-    double operator()(double x, double y)const
+    private:
+    double do_compute(double x, double y)const
     {
         double psiX = psi_.dfx()(x,y), psiY = psi_.dfy()(x,y), k2 = k_*k_;
         double psiXX = psi_.dfxx()(x,y), psiXY = psi_.dfxy()(x,y), psiYY=psi_.dfyy()(x,y);
@@ -223,7 +220,6 @@ struct DivLiseikinX: public aCloneableBinaryFunctor<DivLiseikinX>
                     psiY3*(eps_*(1.+k2)*psiXY-(eps_+2.*k2*psiX2)*psiXY))/sqrtG/sqrtG/sqrtG;
     }
 
-    private:
     double k_, eps_;
     BinaryFunctorsLvl2 psi_;
 };
@@ -235,7 +231,8 @@ struct DivLiseikinX: public aCloneableBinaryFunctor<DivLiseikinX>
 struct DivLiseikinY : public aCloneableBinaryFunctor<DivLiseikinY>
 {
     DivLiseikinY(const BinaryFunctorsLvl2& psi, double k, double eps):k_(k), eps_(eps), psi_(psi){}
-    double operator()(double x, double y)const
+    private:
+    double do_compute(double x, double y)const
     {
         double psiX = psi_.dfx()(x,y), psiY = psi_.dfy()(x,y), k2 = k_*k_;
         double psiXX = psi_.dfxx()(x,y), psiXY = psi_.dfxy()(x,y), psiYY=psi_.dfyy()(x,y);
@@ -250,7 +247,6 @@ struct DivLiseikinY : public aCloneableBinaryFunctor<DivLiseikinY>
                 psiX*(-(eps_+2.*psiY2)*(eps_+k2*psiY2)*psiXY + (eps_*eps_-k2*psiY4)*psiXY))/sqrtG/sqrtG/sqrtG;
     }
 
-    private:
     double k_, eps_;
     BinaryFunctorsLvl2 psi_;
 };
diff --git a/inc/geometries/fluxfunctions.h b/inc/geometries/fluxfunctions.h
index 3ab6f5787..3e892ade3 100644
--- a/inc/geometries/fluxfunctions.h
+++ b/inc/geometries/fluxfunctions.h
@@ -58,7 +58,7 @@ struct aBinaryFunctor
     */
     aBinaryFunctor& operator=(const aBinaryFunctor&){return *this;}
     private:
-    double do_compute(double R, double Z) const=0;
+    virtual double do_compute(double R, double Z) const=0;
 };
 
 /**
diff --git a/inc/geometries/guenther.h b/inc/geometries/guenther.h
index b1e2ac1ab..70195f22f 100644
--- a/inc/geometries/guenther.h
+++ b/inc/geometries/guenther.h
@@ -34,11 +34,11 @@ namespace guenther
 struct Psip : public aCloneableBinaryFunctor<Psip>
 {
     Psip(double R_0 ):   R_0(R_0) {}
-    double operator()(double R, double Z) const
+  private:
+    double do_compute(double R, double Z) const
     {    
         return cos(M_PI*0.5*(R-R_0))*cos(M_PI*Z*0.5);
     }
-  private:
     double R_0;
 };
 /**
@@ -47,11 +47,11 @@ struct Psip : public aCloneableBinaryFunctor<Psip>
 struct PsipR : public aCloneableBinaryFunctor<PsipR>
 {
     PsipR(double R_0 ):   R_0(R_0) {}
-    double operator()(double R, double Z) const
+  private:
+    double do_compute(double R, double Z) const
     {    
         return -M_PI*0.5*sin(M_PI*0.5*(R-R_0))*cos(M_PI*Z*0.5);
     }
-  private:
     double R_0;
 };
 /**
@@ -60,11 +60,11 @@ struct PsipR : public aCloneableBinaryFunctor<PsipR>
 struct PsipRR : public aCloneableBinaryFunctor<PsipRR>
 {
     PsipRR(double R_0 ):   R_0(R_0) {}
-    double operator()(double R, double Z) const
+  private:
+    double do_compute(double R, double Z) const
     {    
         return -M_PI*M_PI*0.25*cos(M_PI*0.5*(R-R_0))*cos(M_PI*Z*0.5);
     }
-  private:
     double R_0;
 };
 /**
@@ -74,11 +74,11 @@ struct PsipZ : public aCloneableBinaryFunctor<PsipZ>
 
 {
     PsipZ(double R_0 ):   R_0(R_0) {}
-    double operator()(double R, double Z) const
+  private:
+    double do_compute(double R, double Z) const
     {    
         return -M_PI*0.5*cos(M_PI*0.5*(R-R_0))*sin(M_PI*Z*0.5);
     }
-  private:
     double R_0;
 };
 /**
@@ -87,11 +87,11 @@ struct PsipZ : public aCloneableBinaryFunctor<PsipZ>
 struct PsipZZ : public aCloneableBinaryFunctor<PsipZZ>
 {
     PsipZZ(double R_0 ):   R_0(R_0){}
-    double operator()(double R, double Z) const
+  private:
+    double do_compute(double R, double Z) const
     {    
         return -M_PI*M_PI*0.25*cos(M_PI*0.5*(R-R_0))*cos(M_PI*Z*0.5);
     }
-  private:
     double R_0;
 };
 /**
@@ -100,11 +100,11 @@ struct PsipZZ : public aCloneableBinaryFunctor<PsipZZ>
 struct PsipRZ : public aCloneableBinaryFunctor<PsipRZ>
 {
     PsipRZ(double R_0 ):   R_0(R_0) {}
-    double operator()(double R, double Z) const
+  private:
+    double do_compute(double R, double Z) const
     {    
         return M_PI*M_PI*0.25*sin(M_PI*0.5*(R-R_0))*sin(M_PI*Z*0.5);
     }
-  private:
     double R_0;
 };
 
@@ -114,8 +114,8 @@ struct PsipRZ : public aCloneableBinaryFunctor<PsipRZ>
 struct Ipol : public aCloneableBinaryFunctor<Ipol>
 {
     Ipol( double I_0):   I_0(I_0) {}
-    double operator()(double R, double Z) const { return I_0; }
-  private:
+    private:
+    double do_compute(double R, double Z) const { return I_0; }
     double I_0;
 };
 /**
@@ -124,7 +124,8 @@ struct Ipol : public aCloneableBinaryFunctor<Ipol>
 struct IpolR : public aCloneableBinaryFunctor<IpolR>
 {
     IpolR(  ) {}
-    double operator()(double R, double Z) const { return 0; }
+    private:
+    double do_compute(double R, double Z) const { return 0; }
 };
 /**
  * @brief \f[0\f]
@@ -132,7 +133,8 @@ struct IpolR : public aCloneableBinaryFunctor<IpolR>
 struct IpolZ : public aCloneableBinaryFunctor<IpolZ>
 {
     IpolZ(  ) {}
-    double operator()(double R, double Z) const { return 0; }
+    private:
+    double do_compute(double R, double Z) const { return 0; }
 };
 
 BinaryFunctorsLvl2 createPsip( GeomParameters gp)
diff --git a/inc/geometries/init.h b/inc/geometries/init.h
index fb3b714c7..fee6fabcf 100644
--- a/inc/geometries/init.h
+++ b/inc/geometries/init.h
@@ -13,6 +13,7 @@ namespace geo
 {
 ///@addtogroup profiles
 ///@{
+//
 /**
  * @brief Returns zero outside psipmax and inside psipmin, otherwise 1
      \f[ \begin{cases}
@@ -24,21 +25,13 @@ struct Iris : public aCloneableBinaryFunctor<Iris>
 {
     Iris( const aBinaryFunctor& psi, double psi_min, double psi_max ): 
         psip_(psi), psipmin_(psi_min), psipmax_(psi_max) { }
-
-    /**
-     * @brief 
-     \f[ \begin{cases}
-        1  \text{ if } \psi_{p,min} < \psi_p(R,Z) < \psi_{p,max}\\
-        0  \text{ else}
-     \end{cases}\f]
-     */
-    double operator( )(double R, double Z)const
+    private:
+    double do_compute(double R, double Z)const
     {
         if( psip_.get()(R,Z) > psipmax_) return 0.;
         if( psip_.get()(R,Z) < psipmin_) return 0.;
         return 1.;
     }
-    private:
     Handle<aBinaryFunctor> psip_;
     double psipmin_, psipmax_;
 };
@@ -53,19 +46,12 @@ struct Pupil : public aCloneableBinaryFunctor<Pupil>
 {
     Pupil( const aBinaryFunctor& psi, double psipmaxcut): 
         psip_(psi), psipmaxcut_(psipmaxcut) { }
-    /**
-     * @brief 
-     \f[ \begin{cases}
-        0  \text{ if } \psi_p(R,Z) > \psi_{p,maxcut} \\
-        1  \text{ else}
-     \end{cases}\f]
-     */
-    double operator( )(double R, double Z)const
+    private:
+    double do_compute(double R, double Z)const
     {
         if( psip_.get()(R,Z) > psipmaxcut_) return 0.;
         return 1.;
     }
-    private:
     Handle<aBinaryFunctor> psip_;
     double psipmaxcut_;
 };
@@ -80,19 +66,12 @@ struct PsiPupil : public aCloneableBinaryFunctor<PsiPupil>
 {
     PsiPupil(const aBinaryFunctor& psi, double psipmax): 
         psipmax_(psipmax), psip_(psi) { } 
-    /**
-     * @brief 
-     \f[ \begin{cases}
-        \psi_{p,max}  \text{ if } \psi_p(R,Z) > \psi_{p,max} \\
-        \psi_p(R,Z) \text{ else}
-     \end{cases}\f]
-     */
-    double operator( )(double R, double Z)const
+    private:
+    double do_compute(double R, double Z)const
     {
         if( psip_.get()(R,Z) > psipmax_) return psipmax_;
         return  psip_.get()(R,Z);
     }
-    private:
     double psipmax_;
     Handle<aBinaryFunctor> psip_;
 };
@@ -109,19 +88,12 @@ struct PsiLimiter : public aCloneableBinaryFunctor<PsiLimiter>
     PsiLimiter( const aBinaryFunctor& psi, double psipmaxlim): 
         psipmaxlim_(psipmaxlim), psip_(psi) { }
 
-    /**
-     * @brief 
-     \f[ \begin{cases}
-        1  \text{ if } \psi_p(R,Z) > \psi_{p,maxlim} \\
-        0  \text{ else}
-     \end{cases}\f]
-     */
-    double operator( )(double R, double Z)const
+    private:
+    double do_compute(double R, double Z)const
     {
         if( psip_.get()(R,Z) > psipmaxlim_) return 1.;
         return 0.;
     }
-    private:
     double psipmaxlim_;
     Handle<aBinaryFunctor> psip_;
 };
@@ -144,25 +116,17 @@ struct GaussianDamping : public aCloneableBinaryFunctor<GaussianDamping>
 {
     GaussianDamping( const aBinaryFunctor& psi, double psipmaxcut, double alpha):
         psip_(psi), psipmaxcut_(psipmaxcut), alpha_(alpha) { }
-    /**
-     * @brief 
-     \f[ \begin{cases}
- 1 \text{ if } \psi_p(R,Z) < \psi_{p,max,cut}\\
- 0 \text{ if } \psi_p(R,Z) > (\psi_{p,max,cut} + 4\alpha) \\
- \exp\left( - \frac{(\psi_p - \psi_{p,max,cut})^2}{2\alpha^2}\right), \text{ else}
- \end{cases}
-   \f]
-     */
-    double operator( )(double R, double Z)const
+    private:
+    double do_compute(double R, double Z)const
     {
         if( psip_.get()(R,Z) > psipmaxcut_ + 4.*alpha_) return 0.;
         if( psip_.get()(R,Z) < psipmaxcut_) return 1.;
         return exp( -( psip_.get()(R,Z)-psipmaxcut_)*( psip_.get()(R,Z)-psipmaxcut_)/2./alpha_/alpha_);
     }
-    private:
     Handle<aBinaryFunctor> psip_;
     double psipmaxcut_, alpha_;
 };
+
 /**
  * @brief Damps the inner boundary in a zone 
  * from psipmax to psipmax+ 4*alpha with a normal distribution
@@ -179,22 +143,13 @@ struct GaussianProfDamping : public aCloneableBinaryFunctor<GaussianProfDamping>
 {
     GaussianProfDamping( const aBinaryFunctor& psi, double psipmax, double alpha):
         psip_(psi), psipmax_(psipmax), alpha_(alpha) { }
-    /**
-     * @brief 
-     \f[ \begin{cases}
- 1 \text{ if } \psi_p(R,Z) < (\psi_{p,max} - 4\alpha)\\
- 0 \text{ if } \psi_p(R,Z) > \psi_{p,max} \\
- \exp\left( - \frac{(\psi_p - \psi_{p,max} + 4\alpha)^2}{2\alpha^2}\right), \text{ else}
- \end{cases}
-   \f]
-     */
-    double operator( )(double R, double Z)const
+    private:
+    double do_compute(double R, double Z)const
     {
         if( psip_.get()(R,Z) > psipmax_ ) return 0.;
         if( psip_.get()(R,Z) < (psipmax_-4.*alpha_)) return 1.;
         return exp( -( psip_.get()(R,Z)-(psipmax_-4.*alpha_))*( psip_.get()(R,Z)-(psipmax_-4.*alpha_))/2./alpha_/alpha_);
     }
-    private:
     Handle<aBinaryFunctor> psip_;
     double psipmax_, alpha_;
 };
@@ -215,24 +170,14 @@ struct GaussianProfXDamping : public aCloneableBinaryFunctor<GaussianProfXDampin
 {
     GaussianProfXDamping( const aBinaryFunctor& psi, dg::geo::solovev::GeomParameters gp):
         gp_(gp),
-        psip_(psi) {
-        }
-    /**
-     * @brief 
-     \f[ \begin{cases}
- 1 \text{ if } \psi_p(R,Z) < (\psi_{p,max} - 4\alpha) \\
- 0 \text{ if } \psi_p(R,Z) > \psi_{p,max} \text{ or } Z < -1.1\varepsilon a  \\
- \exp\left( - \frac{(\psi_p - \psi_{p,max})^2}{2\alpha^2}\right), \text{ else}
- \end{cases}
-   \f]
-     */
-    double operator( )(double R, double Z)const
+        psip_(psi) { }
+    private:
+    double do_compute(double R, double Z)const
     {
         if( psip_.get()(R,Z) > gp_.psipmax || Z<-1.1*gp_.elongation*gp_.a) return 0.;
         if( psip_.get()(R,Z) < (gp_.psipmax-4.*gp_.alpha)) return 1.;
         return exp( -( psip_.get()(R,Z)-(gp_.psipmax-4.*gp_.alpha))*( psip_.get()(R,Z)-(gp_.psipmax-4.*gp_.alpha))/2./gp_.alpha/gp_.alpha);
     }
-    private:
     dg::geo::solovev::GeomParameters gp_;
     Handle<aBinaryFunctor> psip_;
 };
@@ -246,15 +191,11 @@ struct TanhSource : public aCloneableBinaryFunctor<TanhSource>
 {
     TanhSource(const aBinaryFunctor& psi, double psipmin, double alpha):
             psipmin_(psipmin), alpha_(alpha), psip_(psi) { }
-    /**
-     * @brief \f[ 0.5\left( 1 + \tanh\left( -\frac{\psi_p(R,Z) - \psi_{p,min} + 3\alpha}{\alpha}\right)\right)
-   \f]
-     */
-    double operator( )(double R, double Z)const
+    private:
+    double do_compute(double R, double Z)const
     {
         return 0.5*(1.+tanh(-(psip_.get()(R,Z)-psipmin_ + 3.*alpha_)/alpha_) );
     }
-    private:
     double psipmin_, alpha_; 
     Handle<aBinaryFunctor> psip_;
 };
@@ -266,18 +207,13 @@ struct TanhSource : public aCloneableBinaryFunctor<TanhSource>
 //         gp_(gp),
 //         psip_(gp) {
 //     }
-//     double operator( )(double R, double Z)
+//     private:
+//     double do_compute(double R, double Z)
 //     {
 //         if( psip_.get()(R,Z) < (gp_.psipmin)) return p_.nprofileamp+p_.bgprofamp;
 //         if( psip_.get()(R,Z) < 0.) return p_.nprofileamp+p_.bgprofamp-(gp_.psipmin-psip_.get()(R,Z))*(p_.nprofileamp/gp_.psipmin);
 //         return p_.bgprofamp;
 //     }
-//     double operator( )(double R, double Z, double phi)const
-//     {
-//         return (*this)(R,Z);
-// 
-//     }
-//     private:
 //     eule::Parameters p_;
 //     GeomParameters gp_;
 //     Handle<aBinaryFunctor> psip_;
@@ -297,20 +233,13 @@ struct Nprofile : public aCloneableBinaryFunctor<Nprofile>
          bgamp(bgprofamp), namp( peakamp),
          gp_(gp),
          psip_(psi) { }
-    /**
-     * @brief \f[ N(R,Z)=\begin{cases}
- A_{bg} + A_{peak}\frac{\psi_p(R,Z)} {\psi_p(R_0, 0)} \text{ if }\psi_p < \psi_{p,max} \\
- A_{bg} \text{ else } 
- \end{cases}
-   \f]
-     */
-   double operator( )(double R, double Z)const
+    private:
+    double do_compute(double R, double Z)const
     {
         if (psip_.get()(R,Z)<gp_.psipmax) return bgamp +(psip_.get()(R,Z)/psip_.get()(gp_.R_0,0.0)*namp);
 	if( psip_.get()(R,Z) > gp_.psipmax || Z<-1.1*gp_.elongation*gp_.a) return bgamp;
         return bgamp;
     }
-    private:
     double bgamp, namp;
     dg::geo::solovev::GeomParameters gp_;
     Handle<aBinaryFunctor> psip_;
@@ -330,21 +259,14 @@ struct ZonalFlow : public aCloneableBinaryFunctor<ZonalFlow>
         amp_(amplitude), k_(k_psi),
         gp_(gp),
         psip_(psi) { }
-    /**
-     * @brief \f[ N(R,Z)=\begin{cases}
- A_{bg} |\cos(2\pi\psi_p(R,Z) k_\psi)| \text{ if }\psi_p < \psi_{p,max} \\
-  0 \text{ else } 
- \end{cases}
-   \f]
-     */
-    double operator() (double R, double Z)const
+    private:
+    double do_compute(double R, double Z)const
     {
       if (psip_.get()(R,Z)<gp_.psipmax) 
           return (amp_*fabs(cos(2.*M_PI*psip_.get()(R,Z)*k_)));
       return 0.;
 
     }
-    private:
     double amp_, k_;
     dg::geo::solovev::GeomParameters gp_;
     Handle<aBinaryFunctor> psip_;
diff --git a/inc/geometries/taylor.h b/inc/geometries/taylor.h
index 870094d65..34db3e8b9 100644
--- a/inc/geometries/taylor.h
+++ b/inc/geometries/taylor.h
@@ -45,19 +45,13 @@ struct Psip : public aCloneableBinaryFunctor<Psip>
 { /**
      * @brief Construct from given geometric parameters
      *
-     * @param gp useful geometric parameters
+     * @param gp geometric parameters
      */
     Psip( solovev::GeomParameters gp): R0_(gp.R_0), c_(gp.c) {
         cs_ = sqrt( c_[11]*c_[11]-c_[10]*c_[10]);
     }
-    /**
-     * @brief \f$ \hat \psi_p(R,Z) \f$
-     *
-      @param R radius (cylindrical coordinates)
-      @param Z height (cylindrical coordinates)
-      @return \f$ \hat \psi_p(R,Z) \f$
-     */
-    double operator()(double R, double Z) const
+  private:
+    double do_compute(double R, double Z) const
     {    
         double Rn = R/R0_, Zn = Z/R0_;
         double j1_c12 = boost::math::cyl_bessel_j( 1, c_[11]*Rn);
@@ -78,7 +72,6 @@ struct Psip : public aCloneableBinaryFunctor<Psip>
                + c_[9]*sin(c_[11]*Zn));
 
     }
-  private:
     double R0_, cs_;
     std::vector<double> c_;
 };
@@ -89,23 +82,13 @@ struct Psip : public aCloneableBinaryFunctor<Psip>
  */
 struct PsipR: public aCloneableBinaryFunctor<PsipR>
 {
-    /**
-     * @brief Construct from given geometric parameters
-     *
-     * @param gp useful geometric parameters
-     */
+    ///@copydoc Psip::Psip()
     PsipR( solovev::GeomParameters gp): R0_(gp.R_0), c_(gp.c) {
         cs_=sqrt(c_[11]*c_[11]-c_[10]*c_[10]);
     
     }
-    /**
-     * @brief \f$ \frac{\partial  \hat{\psi}_p }{ \partial \hat{R}}(R,Z)  \f$
-
-          @param R radius (boost::math::cylindrical coordinates)
-          @param Z height (boost::math::cylindrical coordinates)
-        * @return \f$ \frac{\partial  \hat{\psi}_p}{ \partial \hat{R}}(R,Z)  \f$
-     */ 
-    double operator()(double R, double Z) const
+    private:
+    double do_compute(double R, double Z) const
     {    
         double Rn=R/R0_, Zn=Z/R0_;
         double j1_c12R = boost::math::cyl_bessel_j(1, c_[11]*Rn) + c_[11]/2.*Rn*(
@@ -129,7 +112,6 @@ struct PsipR: public aCloneableBinaryFunctor<PsipR>
                + c_[7]*j1_csR*sin(c_[10]*Zn)
                + c_[8]*y1_csR*sin(c_[10]*Zn) );
     }
-  private:
     double R0_, cs_;
     std::vector<double> c_;
 };
@@ -138,15 +120,12 @@ struct PsipR: public aCloneableBinaryFunctor<PsipR>
  */ 
 struct PsipRR: public aCloneableBinaryFunctor<PsipRR>
 {
-    /**
-    * @brief Constructor
-    *
-    * @param gp geometric parameters
-    */
+    ///@copydoc Psip::Psip()
     PsipRR( solovev::GeomParameters gp ): R0_(gp.R_0), c_(gp.c) {
         cs_ = sqrt( c_[11]*c_[11]-c_[10]*c_[10]);
     }
-    double operator()(double R, double Z) const
+  private:
+    double do_compute(double R, double Z) const
     {    
         double Rn=R/R0_, Zn=Z/R0_;
         double j1_c12R = c_[11]*(boost::math::cyl_bessel_j(0, c_[11]*Rn) - Rn*c_[11]*boost::math::cyl_bessel_j(1, c_[11]*Rn));
@@ -166,7 +145,6 @@ struct PsipRR: public aCloneableBinaryFunctor<PsipRR>
                + c_[7]*j1_csR*sin(c_[10]*Zn)
                + c_[8]*y1_csR*sin(c_[10]*Zn) );
     }
-  private:
     double R0_, cs_;
     std::vector<double> c_;
 };
@@ -175,10 +153,12 @@ struct PsipRR: public aCloneableBinaryFunctor<PsipRR>
  */ 
 struct PsipZ: public aCloneableBinaryFunctor<PsipZ>
 {
+    ///@copydoc Psip::Psip()
     PsipZ( solovev::GeomParameters gp ): R0_(gp.R_0), c_(gp.c) { 
         cs_ = sqrt( c_[11]*c_[11]-c_[10]*c_[10]);
     }
-    double operator()(double R, double Z) const
+    private:
+    double do_compute(double R, double Z) const
     {    
         double Rn = R/R0_, Zn = Z/R0_;
         double j1_c12 = boost::math::cyl_bessel_j( 1, c_[11]*Rn);
@@ -196,7 +176,6 @@ struct PsipZ: public aCloneableBinaryFunctor<PsipZ>
                + c_[8]*Rn*y1_cs*c_[10]*cos(c_[10]*Zn)
                + c_[9]*c_[11]*cos(c_[11]*Zn));
     }
-  private:
     double R0_,cs_; 
     std::vector<double> c_;
 };
@@ -205,10 +184,12 @@ struct PsipZ: public aCloneableBinaryFunctor<PsipZ>
  */ 
 struct PsipZZ: public aCloneableBinaryFunctor<PsipZZ>
 {
-  PsipZZ( solovev::GeomParameters gp): R0_(gp.R_0), c_(gp.c) { 
+    ///@copydoc Psip::Psip()
+    PsipZZ( solovev::GeomParameters gp): R0_(gp.R_0), c_(gp.c) { 
         cs_ = sqrt( c_[11]*c_[11]-c_[10]*c_[10]);
     }
-    double operator()(double R, double Z) const
+    private:
+    double do_compute(double R, double Z) const
     {    
         double Rn = R/R0_, Zn = Z/R0_;
         double j1_cs = boost::math::cyl_bessel_j( 1, cs_*Rn);
@@ -224,7 +205,6 @@ struct PsipZZ: public aCloneableBinaryFunctor<PsipZZ>
                - c_[8]*Rn*y1_cs*c_[10]*c_[10]*sin(c_[10]*Zn)
                - c_[9]*c_[11]*c_[11]*sin(c_[11]*Zn));
     }
-  private:
     double R0_, cs_;
     std::vector<double> c_;
 };
@@ -233,10 +213,12 @@ struct PsipZZ: public aCloneableBinaryFunctor<PsipZZ>
  */ 
 struct PsipRZ: public aCloneableBinaryFunctor<PsipRZ>
 {
+    ///@copydoc Psip::Psip()
     PsipRZ( solovev::GeomParameters gp ): R0_(gp.R_0), c_(gp.c) { 
         cs_ = sqrt( c_[11]*c_[11]-c_[10]*c_[10]);
     }
-    double operator()(double R, double Z) const
+  private:
+    double do_compute(double R, double Z) const
     {    
         double Rn=R/R0_, Zn=Z/R0_;
         double j1_c12R = boost::math::cyl_bessel_j(1, c_[11]*Rn) + c_[11]/2.*Rn*(
@@ -258,26 +240,25 @@ struct PsipRZ: public aCloneableBinaryFunctor<PsipRZ>
                + c_[7]*j1_csR*c_[10]*cos(c_[10]*Zn)
                + c_[8]*y1_csR*c_[10]*cos(c_[10]*Zn) );
     }
-  private:
     double R0_, cs_;
     std::vector<double> c_;
 };
 
 /**
  * @brief \f[\hat{I} = c_{12}\psi\f] 
+ *
+   \f[\hat{I}= \sqrt{-2 A \hat{\psi}_p / \hat{R}_0 +1}\f] 
  */ 
 struct Ipol: public aCloneableBinaryFunctor<Ipol>
 {
+    ///@copydoc Psip::Psip()
     Ipol(  solovev::GeomParameters gp ): c12_(gp.c[11]), psip_(gp) { }
-    /**
-    * @brief \f[\hat{I}= \sqrt{-2 A \hat{\psi}_p / \hat{R}_0 +1}\f] 
-    */ 
-    double operator()(double R, double Z) const
+  private:
+    double do_compute(double R, double Z) const
     {    
         return c12_*psip_(R,Z);
         
     }
-  private:
     double c12_;
     Psip psip_;
 };
@@ -286,12 +267,13 @@ struct Ipol: public aCloneableBinaryFunctor<Ipol>
  */
 struct IpolR: public aCloneableBinaryFunctor<IpolR>
 {
+    ///@copydoc Psip::Psip()
     IpolR(  solovev::GeomParameters gp ): c12_(gp.c[11]), psipR_(gp) { }
-    double operator()(double R, double Z) const
+  private:
+    double do_compute(double R, double Z) const
     {    
         return c12_*psipR_(R,Z);
     }
-  private:
     double c12_;
     PsipR psipR_;
 };
@@ -300,12 +282,13 @@ struct IpolR: public aCloneableBinaryFunctor<IpolR>
  */
 struct IpolZ: public aCloneableBinaryFunctor<IpolZ>
 {
+    ///@copydoc Psip::Psip()
     IpolZ(  solovev::GeomParameters gp ): c12_(gp.c[11]), psipZ_(gp) { }
-    double operator()(double R, double Z) const
+  private:
+    double do_compute(double R, double Z) const
     {    
         return c12_*psipZ_(R,Z);
     }
-  private:
     double c12_;
     PsipZ psipZ_;
 };
-- 
GitLab


From d90c748e8a49c94630ae60f8b043dfab141d575d Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Thu, 17 Aug 2017 07:40:39 -0700
Subject: [PATCH 177/453] call TensorElliptic set function transform_and_set

---
 inc/dg/elliptic.h       | 2 +-
 inc/geometries/hector.h | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index 4b0454806..f7880abc0 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -649,7 +649,7 @@ struct TensorElliptic
      *
      */
     template<class ChiRR, class ChiRZ, class ChiZZ>
-    void set( ChiRR& chiRR, ChiRZ& chiRZ, ChiZZ& chiZZ)
+    void transform_and_set( ChiRR& chiRR, ChiRZ& chiRZ, ChiZZ& chiZZ)
     {
         typename GeometryTraits<Geometry>::host_vector chiXX, chiXY, chiYY;
         dg::pushForwardPerp( chiRR, chiRZ, chiZZ, chiXX, chiXY, chiYY, g_);
diff --git a/inc/geometries/hector.h b/inc/geometries/hector.h
index 83d6548b7..ef0840c6c 100644
--- a/inc/geometries/hector.h
+++ b/inc/geometries/hector.h
@@ -406,7 +406,7 @@ struct Hector : public aGenerator2d
         double eps = 1e10, eps_old = 2e10;
         dg::CurvilinearGrid2d g2d_old = g2d_;
         dg::TensorElliptic<dg::CurvilinearGrid2d, Matrix, container> ellipticD_old( g2d_old, dg::DIR, dg::PER, dg::not_normed, dg::centered);
-        ellipticD_old.set( chi.xx(), chi.xy(), chi.yy());
+        ellipticD_old.transform_and_set( chi.xx(), chi.xy(), chi.yy());
 
         container u_old = dg::evaluate( dg::zero, g2d_old), u(u_old);
         container lapu = dg::pullback( lapChiPsi, g2d_old);
@@ -418,7 +418,7 @@ struct Hector : public aGenerator2d
             g2d_.multiplyCellNumber(2,2);
             if(verbose)std::cout << "Nx "<<Nx<<" Ny "<<Ny<<std::flush;
             dg::TensorElliptic<dg::CurvilinearGrid2d, Matrix, container> ellipticD( g2d_, dg::DIR, dg::PER, dg::not_normed, dg::centered);
-            ellipticD.set( chi.xx(), chi.xy(), chi.yy() );
+            ellipticD.transform_and_set( chi.xx(), chi.xy(), chi.yy() );
             lapu = dg::pullback( lapChiPsi, g2d_);
             const container vol2d = dg::create::weights( g2d_);
             const IMatrix Q = dg::create::interpolation( g2d_, g2d_old);
-- 
GitLab


From e688ed8f21d6e828e0ab0724e59126e6a147352b Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 17 Aug 2017 20:37:39 +0200
Subject: [PATCH 178/453] made const References for evaluate and pullback
 functions again and introduced const in functors.h

---
 inc/dg/backend/evaluation.cuh   | 10 ++--
 inc/dg/backend/evaluationX.cuh  |  4 +-
 inc/dg/backend/mpi_evaluation.h |  8 +--
 inc/dg/dg_doc.h                 | 10 ++++
 inc/dg/elliptic.h               |  3 +-
 inc/dg/functors.h               | 92 ++++++++++++++++-----------------
 inc/dg/geometry/transform.h     | 14 ++---
 7 files changed, 75 insertions(+), 66 deletions(-)

diff --git a/inc/dg/backend/evaluation.cuh b/inc/dg/backend/evaluation.cuh
index cad9ab4a3..1cdcf62b4 100644
--- a/inc/dg/backend/evaluation.cuh
+++ b/inc/dg/backend/evaluation.cuh
@@ -29,7 +29,7 @@ namespace dg
  * @return  A DG Host Vector with values
  */
 template< class UnaryOp>
-thrust::host_vector<double> evaluate( UnaryOp& f, const Grid1d& g)
+thrust::host_vector<double> evaluate( const UnaryOp& f, const Grid1d& g)
 {
     thrust::host_vector<double> abs = create::abscissas( g);
     for( unsigned i=0; i<g.size(); i++)
@@ -49,7 +49,7 @@ thrust::host_vector<double> evaluate( double (f)(double), const Grid1d& g)
  * @brief Evaluate a function on gaussian abscissas
  *
  * Evaluates f(x) on the given grid
- * @tparam BinaryOp Model of Binary Function
+ * @copydoc hide_binary
  * @param f The function to evaluate: f = f(x,y)
  * @param g The 2d grid on which to evaluate f
  *
@@ -57,7 +57,7 @@ thrust::host_vector<double> evaluate( double (f)(double), const Grid1d& g)
  * @note if you don't like to copy f then just pass a (const) reference then the type should adapt
  */
 template< class BinaryOp>
-thrust::host_vector<double> evaluate( BinaryOp& f, const aTopology2d& g)
+thrust::host_vector<double> evaluate( const BinaryOp& f, const aTopology2d& g)
 {
     unsigned n= g.n();
     Grid1d gx(g.x0(), g.x1(), g.n(), g.Nx());
@@ -87,7 +87,7 @@ thrust::host_vector<double> evaluate( double(f)(double, double), const aTopology
  * @brief Evaluate a function on gaussian abscissas
  *
  * Evaluates f(x,y,z) on the given grid
- * @tparam TernaryOp Model of Ternary Function
+ * @copydoc hide_ternary
  * @param f The function to evaluate: f = f(x,y,z)
  * @param g The 3d grid on which to evaluate f
  *
@@ -95,7 +95,7 @@ thrust::host_vector<double> evaluate( double(f)(double, double), const aTopology
  * @note if you don't like to copy f then just pass a (const) reference then the type should adapt
  */
 template< class TernaryOp>
-thrust::host_vector<double> evaluate( TernaryOp& f, const aTopology3d& g)
+thrust::host_vector<double> evaluate( const TernaryOp& f, const aTopology3d& g)
 {
     unsigned n= g.n();
     Grid1d gx(g.x0(), g.x1(), g.n(), g.Nx());
diff --git a/inc/dg/backend/evaluationX.cuh b/inc/dg/backend/evaluationX.cuh
index 67af562b3..5de2afabb 100644
--- a/inc/dg/backend/evaluationX.cuh
+++ b/inc/dg/backend/evaluationX.cuh
@@ -41,7 +41,7 @@ thrust::host_vector<double> evaluate( double (f)(double), const GridX1d& g)
  * @brief Evaluate a function on gaussian abscissas
  *
  * Evaluates f(x) on the given grid
- * @tparam BinaryOp Model of Binary Function
+ * @copydoc hide_binary
  * @param f The function to evaluate: f = f(x,y)
  * @param g The 2d grid on which to evaluate f
  *
@@ -65,7 +65,7 @@ thrust::host_vector<double> evaluate( double(f)(double, double), const aTopology
  * @brief Evaluate a function on gaussian abscissas
  *
  * Evaluates f(x,y,z) on the given grid
- * @tparam TernaryOp Model of Ternary Function
+ * @copydoc hide_ternary
  * @param f The function to evaluate: f = f(x,y,z)
  * @param g The 3d grid on which to evaluate f
  *
diff --git a/inc/dg/backend/mpi_evaluation.h b/inc/dg/backend/mpi_evaluation.h
index a89b09800..6e0b51811 100644
--- a/inc/dg/backend/mpi_evaluation.h
+++ b/inc/dg/backend/mpi_evaluation.h
@@ -19,7 +19,7 @@ namespace dg
  * @brief Evaluate a function on gaussian abscissas
  *
  * Evaluates f(x) on the given grid
- * @tparam Function Model of Binary Function
+ * @copydoc hide_binary
  * @param f The function to evaluate: f = f(x,y)
  * @param g The 2d grid on which to evaluate f
  *
@@ -28,7 +28,7 @@ namespace dg
             may be constructed during function call.
  */
 template< class BinaryOp>
-MPI_Vector<thrust::host_vector<double> > evaluate( BinaryOp& f, const aMPITopology2d& g)
+MPI_Vector<thrust::host_vector<double> > evaluate( const BinaryOp& f, const aMPITopology2d& g)
 {
     thrust::host_vector<double> w = evaluate( f, g.local());
     MPI_Vector<thrust::host_vector<double> > v( w, g.communicator());
@@ -46,7 +46,7 @@ MPI_Vector<thrust::host_vector<double> > evaluate( double(f)(double, double), co
  * @brief Evaluate a function on gaussian abscissas
  *
  * Evaluates f(x,y,z) on the given grid
- * @tparam Function Model of Ternary Function
+ * @copydoc hide_ternary
  * @param f The function to evaluate: f = f(x,y,z)
  * @param g The 3d grid on which to evaluate f
  *
@@ -55,7 +55,7 @@ MPI_Vector<thrust::host_vector<double> > evaluate( double(f)(double, double), co
             may be constructed during function call.
  */
 template< class TernaryOp>
-MPI_Vector<thrust::host_vector<double> > evaluate( TernaryOp& f, const aMPITopology3d& g)
+MPI_Vector<thrust::host_vector<double> > evaluate( const TernaryOp& f, const aMPITopology3d& g)
 {
     thrust::host_vector<double> w = evaluate( f, g.local());
     MPI_Vector<thrust::host_vector<double> > v( w, g.communicator());
diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index 62e31e9ed..f0e441a4f 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -101,6 +101,16 @@
  * 
  */
 
+/**
+  * @class hide_binary
+  * @tparam BinaryOp A class or function type with a member/signature equivalent to
+  *  - double operator()(double, double) const
+  */
+/**
+  * @class hide_ternary
+  * @tparam TernaryOp A class or function type with a member/signature equivalent to
+  *  - double operator()(double, double, double) const
+  */
  /**
   * @class hide_container_lvl1
   * @tparam container A data container class for which the blas1 functionality is overloaded. Currently this is one of 
diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index f7880abc0..3dd92ac26 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -646,10 +646,9 @@ struct TensorElliptic
 
     /**
      * @brief Transform components to the current coordinate system
-     *
      */
     template<class ChiRR, class ChiRZ, class ChiZZ>
-    void transform_and_set( ChiRR& chiRR, ChiRZ& chiRZ, ChiZZ& chiZZ)
+    void transform_and_set( const ChiRR& chiRR, const ChiRZ& chiRZ, const ChiZZ& chiZZ)
     {
         typename GeometryTraits<Geometry>::host_vector chiXX, chiXY, chiYY;
         dg::pushForwardPerp( chiRR, chiRZ, chiZZ, chiXX, chiXY, chiYY, g_);
diff --git a/inc/dg/functors.h b/inc/dg/functors.h
index a1006d442..8e903c739 100644
--- a/inc/dg/functors.h
+++ b/inc/dg/functors.h
@@ -38,7 +38,7 @@ struct AbsMax
 #ifdef __CUDACC__
     __host__ __device__
 #endif
-    T operator() (const T& x, const T& y)
+    T operator() (const T& x, const T& y) const
     {
         T absx = x>0 ? x : -x;
         T absy = y>0 ? y : -y;
@@ -65,7 +65,7 @@ struct AbsMin
 #ifdef __CUDACC__
     __host__ __device__
 #endif
-    T operator() (const T& x, const T& y)
+    T operator() (const T& x, const T& y) const
     {
         T absx = x<0 ? -x : x;
         T absy = y<0 ? -y : y;
@@ -104,7 +104,7 @@ struct Gaussian
      *
      * @return gaussian
      */
-    double operator()(double x, double y)
+    double operator()(double x, double y) const
     {
         return  amplitude*
                    exp( -((x-x00)*(x-x00)/2./sigma_x/sigma_x +
@@ -121,7 +121,7 @@ struct Gaussian
      *
      * @return gaussian
      */
-    double operator()(double x, double y, double z)
+    double operator()(double x, double y, double z) const
     {
         return  amplitude*cos(kz_*z)*
                    exp( -((x-x00)*(x-x00)/2./sigma_x/sigma_x +
@@ -236,7 +236,7 @@ struct Gaussian3d
      *
      * @return gaussian
      */
-    double operator()(double x, double y)
+    double operator()(double x, double y) const
     {
         return  amplitude*
                    exp( -((x-x00)*(x-x00)/2./sigma_x/sigma_x +
@@ -254,7 +254,7 @@ struct Gaussian3d
      *
      * @return gaussian
      */
-    double operator()(double x, double y, double z)
+    double operator()(double x, double y, double z) const
     {
 //         if (z== z00)
 //         {
@@ -299,7 +299,7 @@ struct GaussianX
      *
      * @return gaussian
      */
-    double operator()(double x, double y)
+    double operator()(double x, double y) const
     {
         return  amplitude* exp( -((x-x00)*(x-x00)/2./sigma_x/sigma_x ));
     }
@@ -335,7 +335,7 @@ struct GaussianY
      *
      * @return gaussian
      */
-    double operator()(double x, double y)
+    double operator()(double x, double y) const
     {
         return  amplitude*exp( -((y-y00)*(y-y00)/2./sigma_y/sigma_y) );
     }
@@ -370,7 +370,7 @@ struct GaussianZ
      *
      * @return gaussian
      */
-    double operator()( double z)
+    double operator()( double z) const
     {
         return  amplitude*exp( -((z-z00)*(z-z00)/2./sigma_z/sigma_z) );
     }
@@ -386,7 +386,7 @@ struct GaussianZ
      *
      * @return gaussian
      */
-    double operator()(double x, double y, double z)
+    double operator()(double x, double y, double z) const
     {
         return  amplitude*exp( -((z-z00)*(z-z00)/2./sigma_z/sigma_z) );
     }
@@ -417,7 +417,7 @@ struct SinXSinY
      
      * @return \f$ f(x,y)\f$
      */
-    double operator()( double x, double y){ return bamp_+amp_*sin(x*kx_)*sin(y*ky_);}
+    double operator()( double x, double y)const{ return bamp_+amp_*sin(x*kx_)*sin(y*ky_);}
   private:
     double amp_,bamp_,kx_,ky_;
 };
@@ -443,7 +443,7 @@ struct SinX
      
      * @return \f$ f(x,y)\f$
      */
-    double operator()( double x, double y){ return bamp_+amp_*sin(x*kx_);}
+    double operator()( double x, double y)const{ return bamp_+amp_*sin(x*kx_);}
   private:
     double amp_,bamp_,kx_;
 };
@@ -470,7 +470,7 @@ struct SinProfX
      
      * @return \f$ f(x,y)\f$
      */
-    double operator()( double x, double y){ return bamp_+amp_*(1.-sin(x*kx_));}
+    double operator()( double x, double y)const{ return bamp_+amp_*(1.-sin(x*kx_));}
   private:
     double amp_,bamp_,kx_;
 };
@@ -496,7 +496,7 @@ struct ExpProfX
      
      * @return result
      */
-    double operator()( double x, double y){ return bamp_+amp_*exp(-x/ln_);}
+    double operator()( double x, double y)const{ return bamp_+amp_*exp(-x/ln_);}
   private:
     double amp_,bamp_,ln_;
 };
@@ -522,7 +522,7 @@ struct LinearX
      
      * @return result
      */
-   double operator()( double x, double y, double z){ return a_*x+b_;}
+   double operator()( double x, double y, double z)const { return a_*x+b_;}
     /**
      * @brief Return linear polynomial in x 
      *
@@ -531,7 +531,7 @@ struct LinearX
      
      * @return result
      */
-   double operator()( double x, double y){ return a_*x+b_;}
+   double operator()( double x, double y)const{ return a_*x+b_;}
     /**
      * @brief Return linear polynomial in x 
      *
@@ -539,7 +539,7 @@ struct LinearX
      
      * @return result
      */
-   double operator()(double x){ return a_*x+b_;}
+   double operator()(double x)const{ return a_*x+b_;}
    private:
     double a_,b_;
 };
@@ -565,7 +565,7 @@ struct LinearY
      
      * @return result
      */
-    double operator()( double x, double y, double z){ return a_*y+b_;}
+    double operator()( double x, double y, double z)const { return a_*y+b_;}
     /**
      * @brief Return linear polynomial in x 
      *
@@ -574,7 +574,7 @@ struct LinearY
      
      * @return result
      */
-    double operator()( double x, double y){ return a_*y+b_;}
+    double operator()( double x, double y)const{ return a_*y+b_;}
   private:
     double a_,b_;
 };
@@ -600,7 +600,7 @@ struct LinearZ
      
      * @return result
      */
-    double operator()( double x, double y, double z){ return a_*z+b_;}
+    double operator()( double x, double y, double z)const{ return a_*z+b_;}
   private:
     double a_,b_;
 };
@@ -630,7 +630,7 @@ struct TanhProfX {
 
      * @return result
      */
-    double operator() (double x, double y)
+    double operator() (double x, double y)const
     {
         return profa_*0.5*(1.+s_*tanh((x-xb_)/w_))+bga_; 
     }
@@ -679,7 +679,7 @@ struct Lamb
      *
      * @return Lamb
      */
-    double operator() (double x, double y)
+    double operator() (double x, double y)const
     {
         double radius = sqrt( (x-x0_)*(x-x0_) + (y-y0_)*(y-y0_));
         double theta = atan2( (y-y0_),(x-x0_));
@@ -766,7 +766,7 @@ struct Vortex
      *
      * @return the above function value
      */
-    double operator()( double x, double y)
+    double operator()( double x, double y)const
     {
         double r = sqrt( (x-x0_)*(x-x0_)+(y-y0_)*(y-y0_));
         double theta = atan2( y-y0_, x-x0_);
@@ -797,7 +797,7 @@ struct Vortex
      *
      * @return the above function value
      */
-    double operator()( double x, double y, double z)
+    double operator()( double x, double y, double z)const
     {
         return this->operator()(x,y)*cos(kz_*z);
     }
@@ -913,7 +913,7 @@ struct BathRZ{
      * @param Z Z - coordinate
      *
      */
-    double operator()(double R, double Z)
+    double operator()(double R, double Z)const
     { 
         double f, RZphasecos, RR, ZZ;
         RR=R-R_min_;
@@ -938,7 +938,7 @@ struct BathRZ{
      * @param phi phi - coordinate
      *
      */
-    double operator()(double R, double Z, double phi) { 
+    double operator()(double R, double Z, double phi)const { 
         double f, RZphasecos;
         double  RR, ZZ;
         RR=R-R_min_;
@@ -1008,7 +1008,7 @@ struct EXP
 #ifdef __CUDACC__
     __host__ __device__
 #endif
-    T operator() (const T& x) 
+    T operator() (const T& x) const
     { 
         return amp_*exp(lambda_*x);
     }
@@ -1034,7 +1034,7 @@ struct LN
 #ifdef __CUDACC__
     __host__ __device__
 #endif
-    T operator() (const T& x) 
+    T operator() (const T& x) const
     { 
         return log(x);
     }
@@ -1059,7 +1059,7 @@ struct SQRT
 #ifdef __CUDACC__
     __host__ __device__
 #endif
-    T operator() (const T& x) 
+    T operator() (const T& x) const
     { 
         return sqrt(x);
     }
@@ -1087,7 +1087,7 @@ struct MinMod
 #ifdef __CUDACC__
     __host__ __device__
 #endif
-    T operator() ( T a1, T a2, T a3)
+    T operator() ( T a1, T a2, T a3)const
     {
         if( a1*a2 > 0) 
             if( a1*a3 > 0)
@@ -1142,7 +1142,7 @@ struct PLUS
 #ifdef __CUDACC__
     __host__ __device__
 #endif
-        T operator()(const T& x){ return x + x_;}
+        T operator()(const T& x)const{ return x + x_;}
     private:
     T x_;
 };
@@ -1165,7 +1165,7 @@ struct INVERT
 #ifdef __CUDACC__
     __host__ __device__
 #endif
-        T operator()(const T& x){ return 1./x;}
+        T operator()(const T& x)const{ return 1./x;}
 };
 
 /**
@@ -1194,7 +1194,7 @@ struct MOD
 #ifdef __CUDACC__
     __host__ __device__
 #endif
-        T operator()(const T& x){
+        T operator()(const T& x)const{
             return (fmod(x,x_) < 0 ) ? (x_ + fmod(x,x_)) : fmod(x,x_);
         }
     private:
@@ -1220,7 +1220,7 @@ struct ABS
 #ifdef __CUDACC__
     __host__ __device__
 #endif
-        T operator()(const T& x){ return fabs(x);}
+        T operator()(const T& x)const{ return fabs(x);}
 };
 /**
  * @brief returns positive values
@@ -1241,7 +1241,7 @@ struct POSVALUE
 #ifdef __CUDACC__
     __host__ __device__
 #endif
-        T operator()(const T& x){
+        T operator()(const T& x)const{
             if (x >= 0.0) return x;
             return 0.0;
             }
@@ -1272,7 +1272,7 @@ struct CONSTANT
 #ifdef __CUDACC__
     __host__ __device__
 #endif
-    double operator()(double x){return value_;}
+    double operator()(double x)const{return value_;}
     /**
      * @brief constant
      *
@@ -1284,7 +1284,7 @@ struct CONSTANT
 #ifdef __CUDACC__
     __host__ __device__
 #endif
-    double operator()(double x, double y){return value_;}
+    double operator()(double x, double y)const{return value_;}
     /**
      * @brief constant
      *
@@ -1297,7 +1297,7 @@ struct CONSTANT
 #ifdef __CUDACC__
     __host__ __device__
 #endif
-    double operator()(double x, double y, double z){return value_;}
+    double operator()(double x, double y, double z)const{return value_;}
     private:
     double value_;
 };
@@ -1312,15 +1312,15 @@ struct ONE
 #ifdef __CUDACC__
     __host__ __device__
 #endif
-    double operator()(double x){return 1.;}
+    double operator()(double x)const{return 1.;}
 #ifdef __CUDACC__
     __host__ __device__
 #endif
-    double operator()(double x, double y){return 1.;}
+    double operator()(double x, double y)const{return 1.;}
 #ifdef __CUDACC__
     __host__ __device__
 #endif
-    double operator()(double x, double y, double z){return 1.;}
+    double operator()(double x, double y, double z)const{return 1.;}
 };
 /**
  * @brief Return zero
@@ -1332,15 +1332,15 @@ struct ZERO
 #ifdef __CUDACC__
     __host__ __device__
 #endif
-    double operator()(double x){return 0.;}
+    double operator()(double x)const{return 0.;}
 #ifdef __CUDACC__
     __host__ __device__
 #endif
-    double operator()(double x, double y){return 0.;}
+    double operator()(double x, double y)const{return 0.;}
 #ifdef __CUDACC__
     __host__ __device__
 #endif
-    double operator()(double x, double y, double z){return 0.;}
+    double operator()(double x, double y, double z)const{return 0.;}
 };
 
 /**
@@ -1387,7 +1387,7 @@ struct Histogram
      *
      * @return 
      */
-    double operator()(double x)
+    double operator()(double x)const
     {    
         unsigned bin = floor((x-g1d_.x0())/binwidth_+0.5);
         bin = std::max(bin,(unsigned) 0);
@@ -1450,7 +1450,7 @@ struct Histogram2D
      *
      * @return 
      */
-    double operator()(double x, double y)
+    double operator()(double x, double y)const
     {
         unsigned binx = floor((x-g2d_.x0())/binwidthx_+0.5) ;
         binx = std::max(binx,(unsigned) 0);
diff --git a/inc/dg/geometry/transform.h b/inc/dg/geometry/transform.h
index 37d599072..2f4a09339 100644
--- a/inc/dg/geometry/transform.h
+++ b/inc/dg/geometry/transform.h
@@ -39,7 +39,7 @@ struct GeometryTraits
  * @ingroup pullback
  */
 template< class Functor>
-thrust::host_vector<double> pullback( Functor& f, const aGeometry2d& g)
+thrust::host_vector<double> pullback( const Functor& f, const aGeometry2d& g)
 {
     std::vector<thrust::host_vector<double> > map = g.map();
     thrust::host_vector<double> vec( g.size());
@@ -51,7 +51,7 @@ thrust::host_vector<double> pullback( Functor& f, const aGeometry2d& g)
 ///@copydoc pullback(Functor&,const aGeometry2d&)
 ///@ingroup pullback
 template< class Functor>
-thrust::host_vector<double> pullback( Functor& f, const aGeometry3d& g)
+thrust::host_vector<double> pullback( const Functor& f, const aGeometry3d& g)
 {
     std::vector<thrust::host_vector<double> > map = g.map();
     thrust::host_vector<double> vec( g.size());
@@ -73,7 +73,7 @@ struct MemoryTraits< MPITag>
 ///@copydoc pullback(Functor&,const aGeometry2d&)
 ///@ingroup pullback
 template< class Functor>
-MPI_Vector<thrust::host_vector<double> > pullback( Functor& f, const aMPIGeometry2d& g)
+MPI_Vector<thrust::host_vector<double> > pullback( const Functor& f, const aMPIGeometry2d& g)
 {
     std::vector<MPI_Vector<thrust::host_vector<double> > > map = g.map();
     thrust::host_vector<double> vec( g.size());
@@ -85,7 +85,7 @@ MPI_Vector<thrust::host_vector<double> > pullback( Functor& f, const aMPIGeometr
 ///@copydoc pullback(Functor&,const aGeometry2d&)
 ///@ingroup pullback
 template< class Functor>
-MPI_Vector<thrust::host_vector<double> > pullback( Functor& f, const aMPIGeometry3d& g)
+MPI_Vector<thrust::host_vector<double> > pullback( const Functor& f, const aMPIGeometry3d& g)
 {
     std::vector<MPI_Vector<thrust::host_vector<double> > > map = g.map();
     thrust::host_vector<double> vec( g.size());
@@ -113,7 +113,7 @@ MPI_Vector<thrust::host_vector<double> > pullback( Functor& f, const aMPIGeometr
  * @ingroup pullback
  */
 template<class Functor1, class Functor2, class container, class Geometry> 
-void pushForwardPerp( Functor1& vR, Functor2& vZ, 
+void pushForwardPerp( const Functor1& vR, const Functor2& vZ, 
         container& vx, container& vy,
         const Geometry& g)
 {
@@ -144,7 +144,7 @@ void pushForwardPerp( Functor1& vR, Functor2& vZ,
  * @ingroup pullback
  */
 template<class Functor1, class Functor2, class Functor3, class container, class Geometry> 
-void pushForward( Functor1& vR, Functor2& vZ, Functor3& vPhi,
+void pushForward( const Functor1& vR, const Functor2& vZ, const Functor3& vPhi,
         container& vx, container& vy, container& vz,
         const Geometry& g)
 {
@@ -179,7 +179,7 @@ void pushForward( Functor1& vR, Functor2& vZ, Functor3& vPhi,
  * @ingroup pullback
  */
 template<class FunctorRR, class FunctorRZ, class FunctorZZ, class container, class Geometry> 
-void pushForwardPerp( FunctorRR& chiRR, FunctorRZ& chiRZ, FunctorZZ& chiZZ,
+void pushForwardPerp( const FunctorRR& chiRR, const FunctorRZ& chiRZ, const FunctorZZ& chiZZ,
         container& chixx, container& chixy, container& chiyy,
         const Geometry& g)
 {
-- 
GitLab


From 7ff88264891e8f160a1d4b94acc3a696f6e934e6 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 17 Aug 2017 22:24:30 +0200
Subject: [PATCH 179/453] debugging geometry and geometries

---
 inc/dg/backend/evaluation.cuh             |  2 +-
 inc/dg/backend/evaluationX.cuh            |  2 +-
 inc/dg/backend/weightsX.cuh               |  6 ++
 inc/dg/functors.h                         |  6 +-
 inc/dg/geometry/base_geometryX.h          | 31 +++++++---
 inc/dg/geometry/curvilinearX.h            | 11 ++--
 inc/dg/geometry/generatorX.h              | 22 +++----
 inc/dg/geometry/geometry_mpit.cu          |  9 +--
 inc/dg/geometry/refined_curvilinearX.h    | 30 +++++-----
 inc/dg/geometry/refined_gridX.h           | 73 ++++++++++++-----------
 inc/dg/geometry/transform.h               |  6 +-
 inc/geometries/fluxfunctions.h            |  2 +-
 inc/geometries/ribeiroX.h                 | 12 ++--
 inc/geometries/separatrix_orthogonal.h    | 35 +++++------
 inc/geometries/separatrix_orthogonal_t.cu | 66 ++++++++++----------
 inc/geometries/utilitiesX.h               |  8 +--
 16 files changed, 172 insertions(+), 149 deletions(-)

diff --git a/inc/dg/backend/evaluation.cuh b/inc/dg/backend/evaluation.cuh
index 1cdcf62b4..2d7db0cf6 100644
--- a/inc/dg/backend/evaluation.cuh
+++ b/inc/dg/backend/evaluation.cuh
@@ -29,7 +29,7 @@ namespace dg
  * @return  A DG Host Vector with values
  */
 template< class UnaryOp>
-thrust::host_vector<double> evaluate( const UnaryOp& f, const Grid1d& g)
+thrust::host_vector<double> evaluate( UnaryOp f, const Grid1d& g)
 {
     thrust::host_vector<double> abs = create::abscissas( g);
     for( unsigned i=0; i<g.size(); i++)
diff --git a/inc/dg/backend/evaluationX.cuh b/inc/dg/backend/evaluationX.cuh
index 5de2afabb..7983e981b 100644
--- a/inc/dg/backend/evaluationX.cuh
+++ b/inc/dg/backend/evaluationX.cuh
@@ -26,7 +26,7 @@ namespace dg
  * @return  A DG Host Vector with values
  */
 template< class UnaryOp>
-thrust::host_vector<double> evaluate( const UnaryOp& f, const GridX1d& g)
+thrust::host_vector<double> evaluate( UnaryOp f, const GridX1d& g)
 {
     return evaluate( f, g.grid());
 };
diff --git a/inc/dg/backend/weightsX.cuh b/inc/dg/backend/weightsX.cuh
index 92e7a31e9..d92b5c186 100644
--- a/inc/dg/backend/weightsX.cuh
+++ b/inc/dg/backend/weightsX.cuh
@@ -11,6 +11,12 @@
 
 namespace dg{
 namespace create{
+///@cond
+thrust::host_vector<double> abscissas( const GridX1d& g)
+{
+    return abscissas(g.grid());
+}
+///@endcond
     
 ///@addtogroup highlevel
 ///@{
diff --git a/inc/dg/functors.h b/inc/dg/functors.h
index 8e903c739..c75af9fdd 100644
--- a/inc/dg/functors.h
+++ b/inc/dg/functors.h
@@ -803,7 +803,7 @@ struct Vortex
     }
     private:
     // Returns the modified Bessel function K1(x) for positive real x.
-    double bessk1(double x)
+    double bessk1(double x)const
     { 
         double y,ans;
         if (x <= 2.0) 
@@ -823,7 +823,7 @@ struct Vortex
         return ans; 
     }
     //Returns the modified Bessel function I1(x) for any real x. 
-    double bessi1(double x) 
+    double bessi1(double x) const
     {   
         double ax,ans; 
         double y; 
@@ -1105,7 +1105,7 @@ struct MinMod
 #ifdef __CUDACC__
     __host__ __device__
 #endif
-    T min( T a1, T a2, T a3, T sign)
+    T min( T a1, T a2, T a3, T sign)const
     {
         T temp = sign*a1;
         if( sign*a2 < temp)
diff --git a/inc/dg/geometry/base_geometryX.h b/inc/dg/geometry/base_geometryX.h
index 82ed83b09..6c612cf83 100644
--- a/inc/dg/geometry/base_geometryX.h
+++ b/inc/dg/geometry/base_geometryX.h
@@ -118,10 +118,6 @@ struct CartesianGridX2d: public dg::aGeometryX2d
      */
     CartesianGridX2d( const dg::GridX2d& g):dg::aGeometryX2d(g.x0(),g.x1(),g.y0(),g.y1(),g.fx(),g.fy(),g.n(),g.Nx(),g.Ny(),g.bcx(),g.bcy()){}
     virtual CartesianGridX2d* clone()const{return new CartesianGridX2d(*this);}
-    private:
-    virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny){
-        aTopologyX2d::do_set(new_n,new_Nx,new_Ny);
-    }
 };
 
 /**
@@ -137,12 +133,31 @@ struct CartesianGridX3d: public dg::aGeometryX3d
      */
     CartesianGridX3d( const dg::GridX3d& g):dg::aGeometryX3d(g.x0(), g.x1(), g.y0(), g.y1(), g.z0(), g.z1(),g.fx(),g.fy(),g.n(),g.Nx(),g.Ny(),g.Nz(),g.bcx(),g.bcy(),g.bcz()){}
     virtual CartesianGridX3d* clone()const{return new CartesianGridX3d(*this);}
-    private:
-    virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz){
-        aTopologyX3d::do_set(new_n,new_Nx,new_Ny,new_Nz);
-    }
 };
 
 ///@}
 
+///@copydoc pullback(const Functor&,const aGeometry2d&)
+///@ingroup pullback
+template< class Functor>
+thrust::host_vector<double> pullback( const Functor& f, const aGeometryX2d& g)
+{
+    std::vector<thrust::host_vector<double> > map = g.map();
+    thrust::host_vector<double> vec( g.size());
+    for( unsigned i=0; i<g.size(); i++)
+        vec[i] = f( map[0][i], map[1][i]);
+    return vec;
+}
+
+///@copydoc pullback(const Functor&,const aGeometry2d&)
+///@ingroup pullback
+template< class Functor>
+thrust::host_vector<double> pullback( const Functor& f, const aGeometryX3d& g)
+{
+    std::vector<thrust::host_vector<double> > map = g.map();
+    thrust::host_vector<double> vec( g.size());
+    for( unsigned i=0; i<g.size(); i++)
+        vec[i] = f( map[0][i], map[1][i], map[2][i]);
+    return vec;
+}
 } //namespace dg
diff --git a/inc/dg/geometry/curvilinearX.h b/inc/dg/geometry/curvilinearX.h
index 5d4b1bc82..8d9adbe34 100644
--- a/inc/dg/geometry/curvilinearX.h
+++ b/inc/dg/geometry/curvilinearX.h
@@ -4,7 +4,8 @@
 #include "dg/backend/evaluationX.cuh"
 #include "dg/backend/functions.h"
 #include "dg/blas1.h"
-#include "dg/geometry/geometry_traits.h"
+#include "base_geometryX.h"
+#include "generatorX.h"
 
 namespace dg
 {
@@ -33,7 +34,7 @@ struct CurvilinearProductGridX3d : public dg::aGeometryX3d
      * @param bcz boundary condition in z
      */
     CurvilinearProductGridX3d( const aGeneratorX2d& generator, 
-        double fx, double fy, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, double fx, double fy, bc bcx=dg::DIR, bc bcy=dg::PER, bc bcz=dg::PER):
+        double fx, double fy, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx=dg::DIR, bc bcy=dg::PER, bc bcz=dg::PER):
         dg::aGeometryX3d( generator.zeta0(fx), generator.zeta1(fx), generator.eta0(fy), generator.eta1(fy), 0., 2.*M_PI, fx,fy,n, Nx, Ny, Nz, bcx, bcy, bcz)
     { 
         map_.resize(3);
@@ -125,15 +126,15 @@ struct CurvilinearGridX2d : public dg::aGeometryX2d
     CurvilinearGridX2d( const aGeneratorX2d& generator, double fx, double fy, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx=dg::DIR, bc bcy=dg::PER):
         dg::aGeometryX2d( generator.zeta0(fx), generator.zeta1(fx), generator.eta0(fy), generator.eta1(fy),fx,fy, n, Nx, Ny, bcx, bcy), handle_(generator)
     {
-        construct( n,Nx,Ny);
+        construct(fx,fy, n,Nx,Ny);
     }
 
     const aGeneratorX2d& generator() const{return handle_.get();}
     virtual CurvilinearGridX2d* clone()const{return new CurvilinearGridX2d(*this);}
     private:
-    void construct( unsigned n, unsigned Nx, unsigned Ny)
+    void construct( double fx, double fy, unsigned n, unsigned Nx, unsigned Ny)
     {
-        CurvilinearProductGridX3d g( handle_.get(), n,Nx,Ny,1,bcx());
+        CurvilinearProductGridX3d g( handle_.get(),fx,fy,n,Nx,Ny,1,bcx());
         jac_=g.jacobian();
         map_=g.map();
         metric_=g.metric();
diff --git a/inc/dg/geometry/generatorX.h b/inc/dg/geometry/generatorX.h
index fa9aa6245..221484a99 100644
--- a/inc/dg/geometry/generatorX.h
+++ b/inc/dg/geometry/generatorX.h
@@ -14,10 +14,10 @@ is a product space.
 */
 struct aGeneratorX2d
 {
-    unsigned zeta0(double fx) const{return do_zeta0(fx);}
-    unsigned zeta1(double fx) const{return do_zeta1(fx);}
-    unsigned eta0(double fy) const{return do_eta0(fy);}
-    unsigned eta1(double fy) const{return do_eta1(fy);}
+    double zeta0(double fx) const{return do_zeta0(fx);}
+    double zeta1(double fx) const{return do_zeta1(fx);}
+    double eta0(double fy) const{return do_eta0(fy);}
+    double eta1(double fy) const{return do_eta1(fy);}
     ///@brief sparsity pattern for metric
     bool isOrthogonal() const { return do_isOrthogonal(); }
 
@@ -40,7 +40,7 @@ struct aGeneratorX2d
     void generate( 
          const thrust::host_vector<double>& zeta1d, 
          const thrust::host_vector<double>& eta1d, 
-         const unsigned nodeX0, const unsigned nodeX1, 
+         unsigned nodeX0, unsigned nodeX1, 
          thrust::host_vector<double>& x, 
          thrust::host_vector<double>& y, 
          thrust::host_vector<double>& zetaX, 
@@ -51,7 +51,7 @@ struct aGeneratorX2d
         unsigned size = zeta1d.size()*eta1d.size();
         x.resize(size), y.resize(size);
         zetaX = zetaY = etaX = etaY =x ;
-        do_generate( zeta1d, eta1d, x,y,zetaX,zetaY,etaX,etaY);
+        do_generate( zeta1d, eta1d,nodeX0,nodeX1,x,y,zetaX,zetaY,etaX,etaY);
     }
 
     /**
@@ -72,7 +72,7 @@ struct aGeneratorX2d
     virtual void do_generate(
          const thrust::host_vector<double>& zeta1d, 
          const thrust::host_vector<double>& eta1d, 
-         const unsigned nodeX0, const unsigned nodeX1, 
+         unsigned nodeX0, unsigned nodeX1, 
          thrust::host_vector<double>& x, 
          thrust::host_vector<double>& y, 
          thrust::host_vector<double>& zetaX, 
@@ -80,10 +80,10 @@ struct aGeneratorX2d
          thrust::host_vector<double>& etaX, 
          thrust::host_vector<double>& etaY) const = 0;
     virtual bool do_isOrthogonal()const{return false;}
-    virtual unsigned do_zeta0(double fx) const=0;
-    virtual unsigned do_zeta1(double fx) const=0;
-    virtual unsigned do_eta0(double fy) const=0;
-    virtual unsigned do_eta1(double fy) const=0;
+    virtual double do_zeta0(double fx) const=0;
+    virtual double do_zeta1(double fx) const=0;
+    virtual double do_eta0(double fy) const=0;
+    virtual double do_eta1(double fy) const=0;
 
 
 };
diff --git a/inc/dg/geometry/geometry_mpit.cu b/inc/dg/geometry/geometry_mpit.cu
index 9440afe6e..4b1fa100e 100644
--- a/inc/dg/geometry/geometry_mpit.cu
+++ b/inc/dg/geometry/geometry_mpit.cu
@@ -3,8 +3,8 @@
 #include <cusp/print.h>
 
 #include <mpi.h>
-#include "blas2.h"
 #include "geometry.h"
+#include "../blas2.h"
 
 
 double R_0 = 4.*M_PI;
@@ -43,7 +43,8 @@ int main( int argc, char* argv[] )
 
     MPI_Comm comm;
     MPI_Cart_create( MPI_COMM_WORLD, 3, np, periods, true, &comm);
-    dg::CylindricalMPIGrid3d<dg::MDVec> grid( R_0 , R_0+ 2.*M_PI, 0.,2.*M_PI, 0., 2.*M_PI,  3,32,24,16, dg::PER, dg::PER, dg::PER, comm);
+    dg::CylindricalMPIGrid3d grid( R_0 , R_0+ 2.*M_PI, 0.,2.*M_PI, 0., 2.*M_PI,  3,32,24,16, dg::PER, dg::PER, dg::PER, comm);
+    dg::SparseElement<dg::MDVec> vol = dg::tensor::volume(grid.metric());
 
     dg::MDVec b = dg::evaluate( sine, grid);
     dg::MDVec vol3d = dg::create::volume( grid);
@@ -52,7 +53,7 @@ int main( int argc, char* argv[] )
     if(rank==0)std::cout << "Test of volume:         "<<test<< " sol = "<<sol<<"\t";
     if(rank==0)std::cout << "rel diff = " <<( test -  sol)/ sol<<"\n";
     dg::MDVec temp = dg::create::weights( grid);
-    dg::geo::multiplyVolume( temp, grid);
+    dg::tensor::pointwiseDot( temp, vol, temp);
     test = dg::blas2::dot( b, temp, b);
     if(rank==0)std::cout << "Test of multiplyVolume: "<<test<< " sol = "<<sol<<"\t";
     if(rank==0)std::cout << "rel diff = " <<( test -  sol)/ sol<<"\n";
@@ -63,7 +64,7 @@ int main( int argc, char* argv[] )
     if(rank==0)std::cout << "Test of inv_volume:     "<<test<< " sol = "<<sol<<"\t";
     if(rank==0)std::cout << "rel diff = " <<( test -  sol)/ sol<<"\n";
     temp = dg::create::inv_weights( grid);
-    dg::geo::divideVolume( temp, grid);
+    dg::tensor::pointwiseDivide(temp, vol, temp );
     test = dg::blas2::dot( b, temp, b);
     if(rank==0)std::cout << "Test of divideVolume:   "<<test<< " sol = "<<sol<<"\t";
     if(rank==0)std::cout << "rel diff = " <<( test -  sol)/ sol<<"\n";
diff --git a/inc/dg/geometry/refined_curvilinearX.h b/inc/dg/geometry/refined_curvilinearX.h
index 46b117dc2..958769ba9 100644
--- a/inc/dg/geometry/refined_curvilinearX.h
+++ b/inc/dg/geometry/refined_curvilinearX.h
@@ -13,7 +13,7 @@ namespace dg
  * 
  * The base coordinate system is the cylindrical coordinate system R,Z,phi
  */
-struct CurvilinearProductRefinedGridX3d : public dg::aGeometryX3d
+struct CurvilinearRefinedProductGridX3d : public dg::aGeometryX3d
 {
     /*!@brief Constructor
     
@@ -30,8 +30,8 @@ struct CurvilinearProductRefinedGridX3d : public dg::aGeometryX3d
      * @param bcy boundary condition in y
      * @param bcz boundary condition in z
      */
-    CurvilinearProductRefinedGridX3d( const aRefinementX2d& ref, const aGeneratorX2d& generator, 
-        double fx, double fy, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, double fx, double fy, bc bcx=dg::DIR, bc bcy=dg::PER, bc bcz=dg::PER):
+    CurvilinearRefinedProductGridX3d( const aRefinementX2d& ref, const aGeneratorX2d& generator, 
+        double fx, double fy, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx=dg::DIR, bc bcy=dg::PER, bc bcz=dg::PER):
         dg::aGeometryX3d( generator.zeta0(fx), generator.zeta1(fx), generator.eta0(fy), generator.eta1(fy), 0., 2.*M_PI, ref.fx_new(Nx,fx),ref.fy_new(Ny,fy),n, ref.nx_new(Nx,fx), ref.ny_new(Ny,fy), Nz, bcx, bcy, bcz), map_(3)
     { 
         handle_ = generator;
@@ -41,7 +41,7 @@ struct CurvilinearProductRefinedGridX3d : public dg::aGeometryX3d
     }
 
     const aGeneratorX2d & generator() const{return handle_.get();}
-    virtual CurvilinearProductRefinedGridX3d* clone()const{return new CurvilinearProductRefinedGridX3d(*this);}
+    virtual CurvilinearRefinedProductGridX3d* clone()const{return new CurvilinearRefinedProductGridX3d(*this);}
     private:
     //construct phi and lift rest to 3d
     void constructParallel(unsigned Nz)
@@ -68,19 +68,19 @@ struct CurvilinearProductRefinedGridX3d : public dg::aGeometryX3d
     void constructPerp( double fx, double fy, unsigned n, unsigned Nx, unsigned Ny)
     {
         std::vector<thrust::host_vector<double> > w(2),abs(2);
-        GridX2d g( x0(),x1(),y0(),y1(),fx,fy,n,Nx,Ny,bcx,bcy);
-        ref.generate(g,w[0],w[1],abs[0],abs[1]);
-        thrust::host_vector<double> x_vec(n()*Nx()), y_vec(n()*Ny());
+        GridX2d g( x0(),x1(),y0(),y1(),fx,fy,n,Nx,Ny,bcx(),bcy());
+        ref_.get().generate(g,w[0],w[1],abs[0],abs[1]);
+        thrust::host_vector<double> x_vec(this->n()*this->Nx()), y_vec(this->n()*this->Ny());
         for( unsigned i=0; i<x_vec.size(); i++)
             x_vec[i] = abs[0][i];
         for( unsigned i=0; i<y_vec.size(); i++)
             x_vec[i] = abs[1][i*x_vec.size()];
-        handle_.get().generate( x_vec, y_vec, n()*outer_Ny(), n()*(inner_Ny()+outer_Ny()), map_[0], map_[1], jac_.value(0), jac_.value(1), jac_.value(2), jac_.value(3));
+        handle_.get().generate( x_vec, y_vec, this->n()*outer_Ny(), this->n()*(inner_Ny()+outer_Ny()), map_[0], map_[1], jac_.value(0), jac_.value(1), jac_.value(2), jac_.value(3));
         //multiply by weights
-        dg::blas1::pointwiseDot( jac.value(0), w[0], jac.value(0));
-        dg::blas1::pointwiseDot( jac.value(1), w[0], jac.value(1));
-        dg::blas1::pointwiseDot( jac.value(2), w[1], jac.value(2));
-        dg::blas1::pointwiseDot( jac.value(3), w[1], jac.value(3));
+        dg::blas1::pointwiseDot( jac_.value(0), w[0], jac_.value(0));
+        dg::blas1::pointwiseDot( jac_.value(1), w[0], jac_.value(1));
+        dg::blas1::pointwiseDot( jac_.value(2), w[1], jac_.value(2));
+        dg::blas1::pointwiseDot( jac_.value(3), w[1], jac_.value(3));
         jac_.idx(0,0) = 0, jac_.idx(0,1) = 1, jac_.idx(1,0)=2, jac_.idx(1,1) = 3;
     }
     virtual SparseTensor<thrust::host_vector<double> > do_compute_jacobian( ) const {
@@ -136,15 +136,15 @@ struct CurvilinearRefinedGridX2d : public dg::aGeometryX2d
     {
         handle_ = generator;
         ref_=ref;
-        construct( double fx,double fy,n,Nx,Ny);
+        construct( fx,fy,n,Nx,Ny);
     }
 
     const aGeneratorX2d& generator() const{return handle_.get();}
-    virtual CurvilinearGridX2d* clone()const{return new CurvilinearGridX2d(*this);}
+    virtual CurvilinearRefinedGridX2d* clone()const{return new CurvilinearRefinedGridX2d(*this);}
     private:
     void construct(double fx, double fy, unsigned n, unsigned Nx, unsigned Ny)
     {
-        CurvilinearProductRefinedGridX3d g( ref_.get(), handle_.get(),fx,fy,n,Nx,Ny,1,bcx());
+        CurvilinearRefinedProductGridX3d g( ref_.get(), handle_.get(),fx,fy,n,Nx,Ny,1,bcx());
         jac_=g.jacobian();
         map_=g.map();
         metric_=g.metric();
diff --git a/inc/dg/geometry/refined_gridX.h b/inc/dg/geometry/refined_gridX.h
index d58cb8bc5..767a466a3 100644
--- a/inc/dg/geometry/refined_gridX.h
+++ b/inc/dg/geometry/refined_gridX.h
@@ -2,6 +2,7 @@
 
 #include "cusp/transpose.h"
 #include "dg/backend/interpolation.cuh"
+#include "dg/backend/evaluationX.cuh"
 #include "dg/backend/weightsX.cuh"
 #include "dg/backend/gridX.h"
 #include "refined_grid.h"
@@ -29,8 +30,9 @@ struct aRefinementX2d
         do_generateX(gx,g_old.inner_Nx(), wx,ax);
         GridX1d gy( g_old.y0(), g_old.y1(), g_old.fy(), g_old.n(), g_old.Ny(), g_old.bcy());
         do_generateY(gy,wy,ay);
-        weightsX.resize(size()), weightsY.resize(size()); 
-        abscissasX.resize(size()), abscissasY.resize(size());
+        unsigned size=wx.size()*wy.size();
+        weightsX.resize(size), weightsY.resize(size); 
+        abscissasX.resize(size), abscissasY.resize(size);
         //now make product space
         for( unsigned i=0; i<wy.size(); i++)
             for( unsigned j=0; j<wx.size(); j++)
@@ -53,11 +55,11 @@ struct aRefinementX2d
     {
         return do_Ny_new(Ny_old, fy_old);
     }
-    unsigned fx_new( unsigned Nx_old, double fx_old) const
+    double fx_new( unsigned Nx_old, double fx_old) const
     {
         return do_fx_new(Nx_old, fx_old);
     }
-    unsigned fy_new( unsigned Ny_old, double fy_old) const
+    double fy_new( unsigned Ny_old, double fy_old) const
     {
         return do_fy_new(Ny_old, fy_old);
     }
@@ -74,8 +76,8 @@ struct aRefinementX2d
     virtual void do_generateY( const GridX1d& gy, thrust::host_vector<double>& weights, thrust::host_vector<double>& abscissas) const =0;
     virtual unsigned do_Nx_new( unsigned Nx_old, double fx) const =0;
     virtual unsigned do_Ny_new( unsigned Ny_old, double fy) const =0;
-    virtual unsigned do_fx_new( unsigned Nx_old, double fx) const =0;
-    virtual unsigned do_fy_new( unsigned Ny_old, double fy) const =0;
+    virtual double do_fx_new( unsigned Nx_old, double fx) const =0;
+    virtual double do_fy_new( unsigned Ny_old, double fy) const =0;
 };
 
 /**
@@ -83,20 +85,20 @@ struct aRefinementX2d
 */
 struct IdentityXRefinement : public aRefinementX2d
 {
-    IdentityXRefinement* clone()const{return new IdentityRefinementX();}
+    IdentityXRefinement* clone()const{return new IdentityXRefinement(*this);}
     private:
-    virtual void do_generateX( const GridX1d& g, unsigned nodeXX, thrust::host_vector<double>& weights, thrust::host_vector<double>& abscissas) const {
+    virtual void do_generateX( const Grid1d& g, unsigned nodeXX, thrust::host_vector<double>& weights, thrust::host_vector<double>& abscissas) const {
         weights=dg::create::weights(g);
         abscissas=dg::create::abscissas(g);
     }
-    virtual void do_generateY( const Grid1d& g, thrust::host_vector<double>& weights, thrust::host_vector<double>& abscissas) const {
+    virtual void do_generateY( const GridX1d& g, thrust::host_vector<double>& weights, thrust::host_vector<double>& abscissas) const {
         weights=dg::create::weights(g);
         abscissas=dg::create::abscissas(g);
     }
     virtual unsigned do_Nx_new( unsigned Nx_old, double fx) const { return Nx_old; }
     virtual unsigned do_Ny_new( unsigned Ny_old, double fy) const { return Ny_old; }
-    virtual unsigned do_fx_new( unsigned Nx_old, double fx) const { return fx; }
-    virtual unsigned do_fy_new( unsigned Ny_old, double fy) const { return fy; }
+    virtual double do_fx_new( unsigned Nx_old, double fx) const { return fx; }
+    virtual double do_fy_new( unsigned Ny_old, double fy) const { return fy; }
 };
 /**
  * @brief Equidistant cell refinement around the X-point
@@ -106,21 +108,21 @@ struct EquidistXRefinement : public aRefinementX2d
     EquidistXRefinement( unsigned add_x, unsigned add_y, unsigned howmanyX=1, unsigned howmanyY=1): add_x_(add_x), howm_x_(howmanyX), add_y_(add_y), howm_y_(howmanyY){ }
     EquidistXRefinement* clone()const{return new EquidistXRefinement(*this);}
     private:
-    unsigned add_x_, howm_x_, addy_, howm_y_;
+    unsigned add_x_, howm_x_, add_y_, howm_y_;
     virtual void do_generateX( const Grid1d& gx, unsigned nodeXX, thrust::host_vector<double>& weights, thrust::host_vector<double>& abscissas) const 
     {
-        EquidistRefinement equi(add_x, nodeXX, howm_x_);
-        equi(gx,weights,abscissas);
+        EquidistRefinement equi(add_x_, nodeXX, howm_x_);
+        equi.generate(gx,weights,abscissas);
     }
     virtual void do_generateY( const GridX1d& g, thrust::host_vector<double>& weights, thrust::host_vector<double>& abscissas) const 
     {
         EquidistRefinement equi0(add_y_,0,howm_y_);
         if( add_y_ == 0 || howm_y_ == 0) { 
-            equi0(g.grid(), weights, abscissas);
+            equi0.generate(g.grid(), weights, abscissas);
             return;
         }
         if( g.f() == 0) { 
-            equi0( Grid1d( g.x0(), g.x1(), g.n(), g.N(), dg::PER), weights, abscissas); 
+            equi0.generate( Grid1d( g.x0(), g.x1(), g.n(), g.N(), dg::PER), weights, abscissas); 
             return;
         }
         thrust::host_vector<double> w1, w2, w3;
@@ -149,13 +151,13 @@ struct EquidistXRefinement : public aRefinementX2d
         if( fx==0 ) return Nx + add_x_; 
         return Nx + 2*add_x_;
     }
-    virtual unsigned do_fx_new( unsigned Nx, double fx) const {
+    virtual double do_fx_new( unsigned Nx, double fx) const {
         if( fx==0 ) return 0; 
-        return (fx*(double)Nx + (double)add_x)/(double)(Nx+2.*add_x);
+        return (fx*(double)Nx + (double)add_x_)/(double)(Nx+2.*add_x_);
     }
-    virtual unsigned do_fy_new( unsigned Ny, double fy) const { 
+    virtual double do_fy_new( unsigned Ny, double fy) const { 
         if( fy==0 ) return 0; 
-        return (fy*(double)Ny + (double)add_y)/(double)(Ny+4.*add_y);
+        return (fy*(double)Ny + (double)add_y_)/(double)(Ny+4.*add_y_);
     }
 };
 
@@ -167,15 +169,15 @@ struct ExponentialXRefinement : public aRefinementX2d
     ExponentialXRefinement( unsigned add_x, unsigned add_y, unsigned howmanyX=1, unsigned howmanyY=1): add_x_(add_x), howm_x_(howmanyX), add_y_(add_y), howm_y_(howmanyY){ }
     ExponentialXRefinement* clone()const{return new ExponentialXRefinement(*this);}
     private:
-    unsigned add_x_, howm_x_, addy_, howm_y_;
+    unsigned add_x_, howm_x_, add_y_, howm_y_;
     virtual void do_generateX( const Grid1d& gx, unsigned nodeXX, thrust::host_vector<double>& weights, thrust::host_vector<double>& abscissas) const 
     {
-        EquidistRefinement equi(add_x, nodeXX, howm_x_);
-        equi(gx,weights,abscissas);
+        EquidistRefinement equi(add_x_, nodeXX, howm_x_);
+        equi.generate(gx,weights,abscissas);
     }
     virtual void do_generateY( const GridX1d& g, thrust::host_vector<double>& weights, thrust::host_vector<double>& abscissas) const 
     {
-        ExponentialRefinement expo0( add_x, 0);
+        ExponentialRefinement expo0( add_x_, 0);
         if( add_y_ == 0) { return expo0.generate( g.grid(), weights, abscissas); }
         if( g.f()  == 0) { return expo0.generate( Grid1d( g.x0(), g.x1(), g.n(), g.N(), dg::PER), weights, abscissas); }
         thrust::host_vector<double> w1, w2, w3;
@@ -186,7 +188,7 @@ struct ExponentialXRefinement : public aRefinementX2d
         expo0.generate( Grid1d(g.x0(),g.x1(),g.n(), g.inner_N(), dg::PER), w2,a2); //inner side
         expo0.generate( Grid1d(g.x0(),g.x1(),g.n(), g.outer_N(), dg::DIR), w3,a3); //right side
         //now combine unnormalized weights
-        thrust::host_vector<double> wtot( g.size() + 4*g.n()*add_x);
+        thrust::host_vector<double> wtot( g.size() + 4*g.n()*add_x_);
         for( unsigned i=0; i<w1.size() ; i++)
             wtot[i] = w1[i];
         for( unsigned i=0; i<w2.size(); i++)
@@ -195,7 +197,7 @@ struct ExponentialXRefinement : public aRefinementX2d
             wtot[w1.size()+w2.size()+i] = w3[i];
         weights = wtot;
 
-        abscissas = normalize_weights_and_compute_abscissas( g.grid(), weights);
+        abscissas = detail::normalize_weights_and_compute_abscissas( g.grid(), weights);
     }
     virtual unsigned do_Ny_new( unsigned Ny, double fy) const {
         if( fy==0 ) return Ny + 2*add_y_; 
@@ -205,13 +207,13 @@ struct ExponentialXRefinement : public aRefinementX2d
         if( fx==0 ) return Nx + add_x_; 
         return Nx + 2*add_x_;
     }
-    virtual unsigned do_fx_new( unsigned Nx, double fx) const {
+    virtual double do_fx_new( unsigned Nx, double fx) const {
         if( fx==0 ) return 0; 
-        return (fx*(double)Nx + (double)add_x)/(double)(Nx+2.*add_x);
+        return (fx*(double)Nx + (double)add_x_)/(double)(Nx+2.*add_x_);
     }
-    virtual unsigned do_fy_new( unsigned Ny, double fy) const { 
+    virtual double do_fy_new( unsigned Ny, double fy) const { 
         if( fy==0 ) return 0; 
-        return (fy*(double)Ny + (double)add_y)/(double)(Ny+4.*add_y);
+        return (fy*(double)Ny + (double)add_y_)/(double)(Ny+4.*add_y_);
     }
 };
 ///@}
@@ -228,7 +230,7 @@ struct CartesianRefinedGridX2d : public dg::aGeometryX2d
             double fx, double fy, 
             unsigned n, unsigned Nx, unsigned Ny, 
             bc bcx = dg::PER, bc bcy = dg::PER) : dg::aGeometryX2d( x0, x1, y0, y1, 
-                ref.fx_new(Nx, fx), ref.fy_new(Ny, fy), n, ref.nx_new(Nx, fx), ny_new(Ny, fy), bcx, bcy), w_(2), abs_(2)
+                ref.fx_new(Nx, fx), ref.fy_new(Ny, fy), n, ref.nx_new(Nx, fx), ref.ny_new(Ny, fy), bcx, bcy), w_(2), abs_(2)
     {
         GridX2d g( x0,x1,y0,y1,fx,fy,n,Nx,Ny,bcx,bcy);
         ref.generate(g,w_[0],w_[1],abs_[0],abs_[1]);
@@ -264,14 +266,17 @@ struct CartesianRefinedGridX3d : public dg::aGeometryX3d
             double x0, double x1, double y0, double y1, double z0, double z1,
             double fx, double fy, 
             unsigned n, unsigned Nx, unsigned Ny, unsigned Nz,
-            bc bcx = dg::PER, bc bcy = dg::PER) : dg::aGeometryX3d( x0, x1, y0, y1,z0,z1,
-                ref.fx_new(Nx, fx), ref.fy_new(Ny, fy), n, ref.nx_new(Nx, fx), ny_new(Ny, fy),Nz, bcx, bcy,bcz), w_(2), abs_(2)
+            bc bcx = dg::PER, bc bcy = dg::PER, bc bcz = dg::PER) : dg::aGeometryX3d( 
+                x0, x1, y0, y1,z0,z1,
+                ref.fx_new(Nx, fx), ref.fy_new(Ny, fy), 
+                n, ref.nx_new(Nx, fx), ref.ny_new(Ny, fy), Nz, 
+                bcx, bcy, bcz), w_(2), abs_(2)
     {
         GridX2d g( x0,x1,y0,y1,fx,fy,n,Nx,Ny,bcx,bcy);
         ref.generate(g,w_[0],w_[1],abs_[0],abs_[1]);
         //lift to 3d
         w_[0].resize(size()), w_[1].resize(size()), abs_[0].resize(size()), abs_[1].resize(size());
-        unsigned size2d=n()*n()*Nx()*Ny();
+        unsigned size2d=this->n()*this->n()*this->Nx()*this->Ny();
         for( unsigned i=1; i<Nz; i++)
             for(unsigned k=0; k<size2d; k++)
             {
diff --git a/inc/dg/geometry/transform.h b/inc/dg/geometry/transform.h
index 2f4a09339..d030ad551 100644
--- a/inc/dg/geometry/transform.h
+++ b/inc/dg/geometry/transform.h
@@ -48,7 +48,7 @@ thrust::host_vector<double> pullback( const Functor& f, const aGeometry2d& g)
     return vec;
 }
 
-///@copydoc pullback(Functor&,const aGeometry2d&)
+///@copydoc pullback(const Functor&,const aGeometry2d&)
 ///@ingroup pullback
 template< class Functor>
 thrust::host_vector<double> pullback( const Functor& f, const aGeometry3d& g)
@@ -70,7 +70,7 @@ struct MemoryTraits< MPITag>
 };
 ///@endcond
 
-///@copydoc pullback(Functor&,const aGeometry2d&)
+///@copydoc pullback(const Functor&,const aGeometry2d&)
 ///@ingroup pullback
 template< class Functor>
 MPI_Vector<thrust::host_vector<double> > pullback( const Functor& f, const aMPIGeometry2d& g)
@@ -82,7 +82,7 @@ MPI_Vector<thrust::host_vector<double> > pullback( const Functor& f, const aMPIG
     return MPI_Vector<thrust::host_vector<double> >( vec, g.communicator());
 }
 
-///@copydoc pullback(Functor&,const aGeometry2d&)
+///@copydoc pullback(const Functor&,const aGeometry2d&)
 ///@ingroup pullback
 template< class Functor>
 MPI_Vector<thrust::host_vector<double> > pullback( const Functor& f, const aMPIGeometry3d& g)
diff --git a/inc/geometries/fluxfunctions.h b/inc/geometries/fluxfunctions.h
index 3e892ade3..d965b5825 100644
--- a/inc/geometries/fluxfunctions.h
+++ b/inc/geometries/fluxfunctions.h
@@ -157,7 +157,7 @@ struct BinaryFunctorsLvl2
     ///the access functions are undefined as long as the class remains empty
     BinaryFunctorsLvl2(){}
     /**
-    * @copydoc BinaryFunctorsLvl1
+    * @copydoc BinaryFunctorsLvl1::BinaryFunctorsLvl1(aBinaryFunctor*,aBinaryFunctor*,aBinaryFunctor*)
     * @param fxx \f$ \partial^2 f / \partial x^2\f$ second derivative in first coordinate
     * @param fxy \f$ \partial^2 f / \partial x \partial y\f$ second mixed derivative 
     * @param fyy \f$ \partial^2 f / \partial y^2\f$ second derivative in second coordinate
diff --git a/inc/geometries/ribeiroX.h b/inc/geometries/ribeiroX.h
index 0383a6a79..526c48697 100644
--- a/inc/geometries/ribeiroX.h
+++ b/inc/geometries/ribeiroX.h
@@ -266,13 +266,13 @@ struct RibeiroX : public aGeneratorX2d
     void do_generate( 
          const thrust::host_vector<double>& zeta1d, 
          const thrust::host_vector<double>& eta1d, 
-         const unsigned nodeX0, const unsigned nodeX1, 
+         unsigned nodeX0, unsigned nodeX1, 
          thrust::host_vector<double>& x, 
          thrust::host_vector<double>& y, 
          thrust::host_vector<double>& zetaX, 
          thrust::host_vector<double>& zetaY, 
          thrust::host_vector<double>& etaX, 
-         thrust::host_vector<double>& etaY) 
+         thrust::host_vector<double>& etaY) const
     {
         //compute psi(x) for a grid on x and call construct_rzy for all psi
         unsigned inside=0;
@@ -303,10 +303,10 @@ struct RibeiroX : public aGeneratorX2d
         }
     }
 
-    virtual unsigned do_zeta0(double fx) const { return zeta0_; }
-    virtual unsigned do_zeta1(double fx) const { return zeta1_;}
-    virtual unsigned do_eta0(double fy) const { return -2.*M_PI*fy/(1.-2.*fy); }
-    virtual unsigned do_eta1(double fy) const { return 2.*M_PI*(1.+fy/(1.-2.*fy));}
+    virtual double do_zeta0(double fx) const { return zeta0_; }
+    virtual double do_zeta1(double fx) const { return zeta1_;}
+    virtual double do_eta0(double fy) const { return -2.*M_PI*fy/(1.-2.*fy); }
+    virtual double do_eta1(double fy) const { return 2.*M_PI*(1.+fy/(1.-2.*fy));}
     private:
     BinaryFunctorsLvl2 psi_;
     dg::geo::ribeiro::detail::XFieldFinv fpsiMinv_; 
diff --git a/inc/geometries/separatrix_orthogonal.h b/inc/geometries/separatrix_orthogonal.h
index af4187353..867d3ebab 100644
--- a/inc/geometries/separatrix_orthogonal.h
+++ b/inc/geometries/separatrix_orthogonal.h
@@ -30,7 +30,7 @@ void computeX_rzy( const BinaryFunctorsLvl1& psi,
         const unsigned nodeX0, const unsigned nodeX1,
         thrust::host_vector<double>& r, //output r - values
         thrust::host_vector<double>& z, //output z - values
-        double* R_init, double* Z_init,  //2 input coords on perp line
+        const double* R_init, const double* Z_init,  //2 input coords on perp line
         double f_psi,  //input f
         int mode ) 
 {
@@ -39,8 +39,8 @@ void computeX_rzy( const BinaryFunctorsLvl1& psi,
     r.resize( y_vec.size()), z.resize(y_vec.size());
     thrust::host_vector<double> begin( 2, 0), end(begin), temp(begin);
     begin[0] = R_init[0], begin[1] = Z_init[0];
-    dg::geo::ribeiro::FieldRZY fieldRZYconf(psi.dfx(), psi.dfy());
-    dg::geo::equalarc::FieldRZY fieldRZYequi(psi.dfx(), psi.dfy());
+    dg::geo::ribeiro::FieldRZY fieldRZYconf(psi);
+    dg::geo::equalarc::FieldRZY fieldRZYequi(psi);
     fieldRZYconf.set_f(f_psi);
     fieldRZYequi.set_f(f_psi);
     unsigned steps = 1; double eps = 1e10, eps_old=2e10;
@@ -133,6 +133,7 @@ struct SimpleOrthogonalX : public aGeneratorX2d
         dg::geo::orthogonal::detail::InitialX initX(psi_, xX, yX);
         initX.find_initial(psi_0, R0_, Z0_);
     }
+    SimpleOrthogonalX* clone()const{return new SimpleOrthogonalX(*this);}
     private:
     bool isConformal()const{return false;}
     bool do_isOrthogonal()const{return true;}
@@ -140,13 +141,13 @@ struct SimpleOrthogonalX : public aGeneratorX2d
     virtual void do_generate( //this one doesn't know if the separatrix comes to lie on a cell boundary or not
          const thrust::host_vector<double>& zeta1d, 
          const thrust::host_vector<double>& eta1d, 
-         const unsigned nodeX0, const unsigned nodeX1,
+         unsigned nodeX0, unsigned nodeX1,
          thrust::host_vector<double>& x, 
          thrust::host_vector<double>& y, 
          thrust::host_vector<double>& zetaX, 
          thrust::host_vector<double>& zetaY, 
          thrust::host_vector<double>& etaX, 
-         thrust::host_vector<double>& etaY) 
+         thrust::host_vector<double>& etaY) const
     {
 
         thrust::host_vector<double> r_init, z_init;
@@ -167,10 +168,10 @@ struct SimpleOrthogonalX : public aGeneratorX2d
             etaY[idx] = +h[idx]*psipR;
         }
     }
-    unsigned do_zeta0(double fx) const { return zeta0_; }
-    unsigned do_zeta1(double fx) const { return -fx/(1.-fx)*zeta0_;}
-    unsigned do_eta0(double fy) const { return -2.*M_PI*fy/(1.-2.*fy); }
-    unsigned do_eta1(double fy) const { return 2.*M_PI*(1.+fy/(1.-2.*fy));}
+    double do_zeta0(double fx) const { return zeta0_; }
+    double do_zeta1(double fx) const { return -fx/(1.-fx)*zeta0_;}
+    double do_eta0(double fy) const { return -2.*M_PI*fy/(1.-2.*fy); }
+    double do_eta1(double fy) const { return 2.*M_PI*(1.+fy/(1.-2.*fy));}
     BinaryFunctorsLvl2 psi_;
     double R0_[2], Z0_[2];
     double zeta0_, f0_;
@@ -184,7 +185,6 @@ struct SimpleOrthogonalX : public aGeneratorX2d
  */
 struct SeparatrixOrthogonal : public aGeneratorX2d
 {
-    typedef dg::OrthogonalTag metric_category;
     /**
      * @brief Construct 
      *
@@ -198,13 +198,14 @@ struct SeparatrixOrthogonal : public aGeneratorX2d
      */
     SeparatrixOrthogonal( const BinaryFunctorsLvl2& psi, double psi_0, //psi_0 must be the closed surface, 0 the separatrix
             double xX, double yX, double x0, double y0, int firstline ):
-        psi_(psi)
+        psi_(psi),
         sep_( psi, xX, yX, x0, y0, firstline)
     {
         firstline_ = firstline;
         f0_ = sep_.get_f();
         psi_0_=psi_0;
     }
+    SeparatrixOrthogonal* clone()const{return new SeparatrixOrthogonal(*this);}
     private:
     bool isConformal()const{return false;}
     bool do_isOrthogonal()const{return true;}
@@ -212,13 +213,13 @@ struct SeparatrixOrthogonal : public aGeneratorX2d
     virtual void do_generate(  //this one doesn't know if the separatrix comes to lie on a cell boundary or not
          const thrust::host_vector<double>& zeta1d, 
          const thrust::host_vector<double>& eta1d, 
-         const unsigned nodeX0, const unsigned nodeX1, 
+         unsigned nodeX0, unsigned nodeX1, 
          thrust::host_vector<double>& x, 
          thrust::host_vector<double>& y, 
          thrust::host_vector<double>& zetaX, 
          thrust::host_vector<double>& zetaY, 
          thrust::host_vector<double>& etaX, 
-         thrust::host_vector<double>& etaY) 
+         thrust::host_vector<double>& etaY) const
     {
 
         thrust::host_vector<double> r_init, z_init;
@@ -323,10 +324,10 @@ struct SeparatrixOrthogonal : public aGeneratorX2d
             etaY[idx] = +h[idx]*psipX;
         }
     }
-    virtual unsigned do_zeta0(double fx) const { return f0_*psi_0_; }
-    virtual unsigned do_zeta1(double fx) const { return -fx/(1.-fx)*f0_*psi_0_;}
-    virtual unsigned do_eta0(double fy) const { return -2.*M_PI*fy/(1.-2.*fy); }
-    virtual unsigned do_eta1(double fy) const { return 2.*M_PI*(1.+fy/(1.-2.*fy));}
+    virtual double do_zeta0(double fx) const { return f0_*psi_0_; }
+    virtual double do_zeta1(double fx) const { return -fx/(1.-fx)*f0_*psi_0_;}
+    virtual double do_eta0(double fy) const { return -2.*M_PI*fy/(1.-2.*fy); }
+    virtual double do_eta1(double fy) const { return 2.*M_PI*(1.+fy/(1.-2.*fy));}
     private:
     double R0_[2], Z0_[2];
     double f0_, psi_0_;
diff --git a/inc/geometries/separatrix_orthogonal_t.cu b/inc/geometries/separatrix_orthogonal_t.cu
index bf298bd73..0d104f0ea 100644
--- a/inc/geometries/separatrix_orthogonal_t.cu
+++ b/inc/geometries/separatrix_orthogonal_t.cu
@@ -12,10 +12,10 @@
 #include "solovev.h"
 #include "taylor.h"
 //#include "guenther.h"
-#include "curilinearX.h"
-#include "refined_orthogonalX.h"
+#include "dg/geometry/transform.h"
+#include "dg/geometry/curvilinearX.h"
+#include "dg/geometry/refined_curvilinearX.h"
 #include "separatrix_orthogonal.h"
-#include "dg/ds.h"
 #include "init.h"
 
 #include "file/nc_utilities.h"
@@ -34,7 +34,7 @@ struct ZCutter
 };
 double sine( double x) {return sin(x);}
 double cosine( double x) {return cos(x);}
-typedef dg::FieldAligned< dg::CurvilinearGridX3d<dg::HVec> , dg::IHMatrix, dg::HVec> HFA;
+//typedef dg::FieldAligned< dg::CurvilinearGridX3d<dg::HVec> , dg::IHMatrix, dg::HVec> HFA;
 
 //using namespace dg::geo::solovev;
 using namespace dg::geo::taylor;
@@ -116,23 +116,24 @@ int main( int argc, char* argv[])
     gp.display( std::cout);
     std::cout << "Constructing orthogonal grid ... \n";
     t.tic();
-    MagneticField c(gp);
-    std::cout << "Psi min "<<c.psip(gp.R_0, 0)<<"\n";
+    dg::geo::TokamakMagneticField c = dg::geo::taylor::createMagField(gp);
+    std::cout << "Psi min "<<c.psip()(gp.R_0, 0)<<"\n";
     double R_X = gp.R_0-1.1*gp.triangularity*gp.a;
     double Z_X = -1.1*gp.elongation*gp.a;
-    dg::geo::findXpoint( c.psipR, c.psipZ, c.psipRR, c.psipRZ, c.psipZZ, R_X, Z_X);
+    dg::geo::findXpoint( c.get_psip(), R_X, Z_X);
 
     //solovev::Psip psip( gp); 
     //std::cout << "Psi min "<<psip(gp.R_0, 0)<<"\n";
     //solovev::PsipR psipR(gp); solovev::PsipZ psipZ(gp);
     //solovev::LaplacePsip laplacePsip(gp); 
     double R0 = gp.R_0, Z0 = 0;
-    dg::geo::SeparatrixOrthogonal<Psip,PsipR,PsipZ,LaplacePsip> generator(c.psip, c.psipR, c.psipZ, c.laplacePsip, psi_0, R_X,Z_X, R0, Z0,0);
-    //dg::geo::SimpleOrthogonalX<Psip,PsipR,PsipZ,LaplacePsip> generator(c.psip, c.psipR, c.psipZ, c.laplacePsip, psi_0, R_X,Z_X, R0, Z0,0);
-    //dg::OrthogonalGridX3d<dg::HVec> g3d(generator, psi_0, fx_0, fy_0, n, Nx, Ny,Nz, dg::DIR, dg::NEU);
-    //dg::CurvilinearGridX2d<dg::HVec> g2d = g3d.perp_grid();
-    dg::CurvilinearRefinedGridX3d<dg::HVec> g3d(add_x, add_y, 1,1, generator, psi_0, fx_0, fy_0, n, n, Nx, Ny,Nz, dg::DIR, dg::NEU);
-    dg::CurvilinearRefinedGridX2d<dg::HVec> g2d = g3d.perp_grid();
+    dg::geo::SeparatrixOrthogonal generator(c.get_psip(), psi_0, R_X,Z_X, R0, Z0,0);
+    //dg::geo::SimpleOrthogonalX generator(c.get_psip(), psi_0, R_X,Z_X, R0, Z0,0);
+    //dg::OrthogonalGridX3d g3d(generator, psi_0, fx_0, fy_0, n, Nx, Ny,Nz, dg::DIR, dg::NEU);
+    //dg::CurvilinearGridX2d g2d = g3d.perp_grid();
+    dg::EquidistXRefinement equi(add_x, add_y, 1,1);
+    dg::CurvilinearRefinedProductGridX3d g3d(equi, generator, fx_0, fy_0, n, Nx, Ny,Nz, dg::DIR, dg::NEU);
+    dg::CurvilinearRefinedGridX2d g2d = g3d.perp_grid();
     t.toc();
     dg::GridX3d g3d_periodic(g3d.x0(), g3d.x1(), g3d.y0(), g3d.y1(), g3d.z0(), g3d.z1(), g3d.fx(), g3d.fy(), g3d.n(), g3d.Nx(), g3d.Ny(), 2); 
     std::cout << "Construction took "<<t.diff()<<"s"<<std::endl;
@@ -164,15 +165,15 @@ int main( int argc, char* argv[])
     err = nc_def_var( ncid, "volume", NC_DOUBLE, 3, dim3d, &volID);
     err = nc_def_var( ncid, "divB", NC_DOUBLE, 3, dim3d, &divBID);
 
-    thrust::host_vector<double> psi_p = dg::pullback( c.psip, g2d);
+    thrust::host_vector<double> psi_p = dg::pullback( c.psip(), g2d);
     g2d.display();
     err = nc_put_var_double( ncid, onesID, periodify(psi_p, g3d_periodic).data());
     //err = nc_put_var_double( ncid, onesID, periodify(g2d.g(), g3d_periodic).data());
     dg::HVec X( g2d.size()), Y(X); //P = dg::pullback( dg::coo3, g);
     for( unsigned i=0; i<g2d.size(); i++)
     {
-        X[i] = g2d.r()[i];
-        Y[i] = g2d.z()[i];
+        X[i] = g2d.map()[0][i];
+        Y[i] = g2d.map()[1][i];
     }
 
     dg::HVec ones = dg::evaluate( dg::one, g2d);
@@ -191,35 +192,32 @@ int main( int argc, char* argv[])
     //err = nc_put_var_double( ncid, coord1D[4], g3d.f_x().data());
     //err = nc_put_var_double( ncid, coordsID[2], g.z().data());
 
-    dg::blas1::pointwiseDivide( g2d.g_yy(), g2d.g_xx(), temp0);
+    dg::SparseTensor<dg::HVec> metric = g2d.metric();
+    dg::HVec g_xx = metric.value(0,0), g_yy=metric.value(1,1);
+    dg::SparseElement<dg::HVec> vol_ = dg::tensor::volume(metric);
+    dg::HVec vol = vol_.value();
+    dg::blas1::pointwiseDivide( g_yy, g_xx, temp0);
     dg::blas1::axpby( 1., ones, -1., temp0, temp0);
     dg::blas1::transfer( temp0, X);
     err = nc_put_var_double( ncid, defID, periodify(X, g3d_periodic).data());
     //err = nc_put_var_double( ncid, defID, X.data());
-    dg::blas1::transfer( g2d.vol(), X);
-    dg::blas1::transfer( g2d.g_yy(),Y);
+    dg::blas1::transfer( vol, X);
+    dg::blas1::transfer( g_yy, Y);
     dg::blas1::pointwiseDot( Y, X, X);
     err = nc_put_var_double( ncid, volID, periodify(X, g3d_periodic).data());
     //err = nc_put_var_double( ncid, volID, X.data());
 
     std::cout << "Construction successful!\n";
 
-    //compute error in volume element
-    dg::blas1::pointwiseDot( g2d.g_xy(), g2d.g_xy(), temp1);
-    double error = sqrt( dg::blas2::dot( temp1, w2d, temp1));
-   // double error = sqrt( dg::blas1::dot( temp1, temp1));
-    std::cout<< "    Error in Off-diagonal is "<<error<<"\n";
-
     //compare determinant vs volume form
-    dg::blas1::pointwiseDot( g2d.g_xx(), g2d.g_yy(), temp0);
-    dg::blas1::pointwiseDot( g2d.g_xy(), g2d.g_xy(), temp1);
+    dg::blas1::pointwiseDot( g_xx, g_yy, temp0);
     dg::blas1::axpby( 1., temp0, -1., temp1, temp0);
     dg::blas1::transform( temp0, temp0, dg::SQRT<double>());
     dg::blas1::pointwiseDivide( ones, temp0, temp0);
     dg::blas1::transfer( temp0, X);
     err = nc_put_var_double( ncid, volID, periodify(X, g3d_periodic).data());
-    dg::blas1::axpby( 1., temp0, -1., g2d.vol(), temp0);
-    error = sqrt(dg::blas2::dot( temp0, w2d, temp0)/dg::blas2::dot( g2d.vol(), w2d, g2d.vol()));
+    dg::blas1::axpby( 1., temp0, -1., vol, temp0);
+    double error = sqrt(dg::blas2::dot( temp0, w2d, temp0)/dg::blas2::dot( vol, w2d, vol));
     //error = sqrt(dg::blas1::dot( temp0, temp0)/dg::blas1::dot( g2d.vol(), g2d.vol()));
     std::cout << "Rel Consistency  of volume is "<<error<<"\n";
 
@@ -265,18 +263,18 @@ int main( int argc, char* argv[])
     std::cout << "TEST VOLUME IS:\n";
     dg::CartesianGrid2d g2dC( gp.R_0 -1.2*gp.a, gp.R_0 + 1.2*gp.a, Z_X, 1.2*gp.a*gp.elongation, 1, 5e3, 5e3, dg::PER, dg::PER);
     gp.psipmax = 0., gp.psipmin = psi_0;
-    dg::geo::Iris<Psip> iris( c.psip, gp.psipmin, gp.psipmax);
+    dg::geo::Iris iris( c.psip(), gp.psipmin, gp.psipmax);
     dg::HVec vec  = dg::evaluate( iris, g2dC);
     dg::HVec g2d_weights = dg::create::volume( g2dC);
     double volumeRZP = dg::blas1::dot( vec, g2d_weights);
 
-    dg::HVec cutter = dg::pullback( iris, g2d), vol( cutter);
+    dg::HVec cutter = dg::pullback( iris, g2d), vol_cut( cutter);
     ZCutter cut(Z_X);
     dg::HVec zcutter = dg::pullback( cut, g2d); 
     w2d = dg::create::weights( g2d);//make weights w/o refined weights
-    dg::blas1::pointwiseDot(cutter, w2d, vol);
-    dg::blas1::pointwiseDot(zcutter, vol, vol);
-    double volume = dg::blas1::dot( g2d.vol(), vol);
+    dg::blas1::pointwiseDot(cutter, w2d, vol_cut);
+    dg::blas1::pointwiseDot(zcutter, vol_cut, vol_cut);
+    double volume = dg::blas1::dot( vol, vol_cut);
     std::cout << "volumeXYP is "<< volume<<std::endl;
     std::cout << "volumeRZP is "<< volumeRZP<<std::endl;
     std::cout << "relative difference in volume is "<<fabs(volumeRZP - volume)/volume<<std::endl;
diff --git a/inc/geometries/utilitiesX.h b/inc/geometries/utilitiesX.h
index ad2dd22be..081e1b5d5 100644
--- a/inc/geometries/utilitiesX.h
+++ b/inc/geometries/utilitiesX.h
@@ -10,11 +10,7 @@ namespace geo
  * @brief This function finds the X-point via Newton iteration applied to the gradient of psi, 
  *
  * The inverse of the Hessian matrix is computed analytically
-    @param psiR \f$ \partial_R \psi(R,Z)\f$, where R, Z are cylindrical coordinates
-    @param psiZ \f$ \partial_Z \psi(R,Z)\f$, where R, Z are cylindrical coordinates
-    @param psiRR \f$ \partial_R\partial_R \psi(R,Z)\f$, where R, Z are cylindrical coordinates
-    @param psiRZ \f$ \partial_R\partial_Z \psi(R,Z)\f$, where R, Z are cylindrical coordinates
-    @param psiZZ \f$ \partial_Z\partial_Z \psi(R,Z)\f$, where R, Z are cylindrical coordinates
+    @param psi \f$ \psi(R,Z)\f$, where R, Z are cylindrical coordinates
  * @param R_X start value on input, X-point on output
  * @param Z_X start value on input, X-point on output
  * @ingroup misc
@@ -345,7 +341,7 @@ struct SeparatriX
     void compute_rzy( const thrust::host_vector<double>& y_vec, 
             const unsigned nodeX0, const unsigned nodeX1,
             thrust::host_vector<double>& r, //same size as y_vec on output
-            thrust::host_vector<double>& z ) 
+            thrust::host_vector<double>& z ) const
     {
         ///////////////////////////find y coordinate line//////////////
         thrust::host_vector<double> begin( 2, 0), end(begin), temp(begin), end_old(end);
-- 
GitLab


From 31547a81f8181c9e22f3003134f6db39af644a1d Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 17 Aug 2017 22:50:04 +0200
Subject: [PATCH 180/453] hide_geometry in documentation and made poisson
 compile

---
 inc/dg/arakawa.h                |  7 +----
 inc/dg/dg_doc.h                 |  8 +++++
 inc/dg/elliptic.h               | 43 +++------------------------
 inc/dg/general_elliptic_b.cu    |  4 +--
 inc/dg/geometry/refined_grid.h  |  2 +-
 inc/dg/geometry/refined_gridX.h | 13 +++++----
 inc/dg/geometry/tensor.h        |  9 ++----
 inc/dg/helmholtz.h              |  4 +--
 inc/dg/helmholtz_t.cu           |  2 +-
 inc/dg/poisson.h                | 52 +++++++++++++++++----------------
 10 files changed, 57 insertions(+), 87 deletions(-)

diff --git a/inc/dg/arakawa.h b/inc/dg/arakawa.h
index e5a827d30..040325afe 100644
--- a/inc/dg/arakawa.h
+++ b/inc/dg/arakawa.h
@@ -22,6 +22,7 @@ namespace dg
  * @brief X-space generalized version of Arakawa's scheme
  *
  * @copydoc hide_matrix_container
+ * @copydoc hide_geometry
  * @ingroup arakawa
  */
 template< class Geometry, class Matrix, class container >
@@ -29,17 +30,11 @@ struct ArakawaX
 {
     /**
      * @brief Create Arakawa on a grid
-     *
-     * @tparam Grid The Grid class. The functions dg::create::dx( g, bcx) and
-     * dg::create::dy( g, bcy) must be callable and return an instance of the Matrix class. Furthermore dg::evaluate( one, g) must return an instance of the container class.
      * @param g The grid
      */
     ArakawaX( Geometry g);
     /**
      * @brief Create Arakawa on a grid using different boundary conditions
-     *
-     * @tparam Grid The Grid class. The functions dg::create::dx( g, bcx) and
-     * dg::create::dy( g, bcy) must be callable and return an instance of the Matrix class. Furthermore dg::evaluate( one, g) must return an instance of the container class.
      * @param g The grid
      * @param bcx The boundary condition in x
      * @param bcy The boundary condition in y
diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index f0e441a4f..df296a172 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -111,6 +111,14 @@
   * @tparam TernaryOp A class or function type with a member/signature equivalent to
   *  - double operator()(double, double, double) const
   */
+ /**
+  * @class hide_geometry
+  * @tparam Geometry One of the geometry classes. The functions dg::create::dx( g, bcx) and
+  * dg::create::dy( g, bcy) must be callable and return an instance of the Matrix class. 
+  * Furthermore dg::evaluate( one, g) must return an instance of the container class.
+     as do calls to dg::create::weights(g) and dg::create::inv_weights(g)
+  */
+
  /**
   * @class hide_container_lvl1
   * @tparam container A data container class for which the blas1 functionality is overloaded. Currently this is one of 
diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index 3dd92ac26..c292612d7 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -38,7 +38,7 @@ namespace dg
  numerical diffusion, i.e. for low values numerical oscillations may appear. 
  Also note that a forward discretization has more diffusion than a centered discretization.
 
- * @tparam Geometry The geometry sets the metric of the grid
+ * @copydoc hide_geometry
  * @copydoc hide_matrix_container
  * This class has the SelfMadeMatrixTag so it can be used in blas2::symv functions 
  * and thus in a conjugate gradient solver. 
@@ -56,10 +56,6 @@ class Elliptic
     /**
      * @brief Construct from Grid
      *
-     * @tparam Geometry The Grid class. A call to dg::evaluate( one, g) must return an instance of the container class, 
-     * a call to dg::create::weights(g) and dg::create::inv_weights(g)
-     * must return instances of the container class and 
-     * calls to dg::create::dx( g, no, backward) and jump2d( g, bcx, bcy, no) are made.
      * @param g The Grid, boundary conditions are taken from here
      * @param no Not normed for elliptic equations, normed else
      * @param dir Direction of the right first derivative
@@ -75,11 +71,6 @@ class Elliptic
 
     /**
      * @brief Construct from grid and boundary conditions
-     *
-     * @tparam Geometry The Grid class. A call to dg::evaluate( one, g) must return an instance of the container class, 
-     * a call to dg::create::weights(g) and dg::create::inv_weights(g)
-     * must return instances of the container class and 
-     * calls to dg::create::dx( g, no, backward) and jump2d( g, bcx, bcy, no) are made.
      * @param g The Grid
      * @param bcx boundary condition in x
      * @param bcy boundary contition in y
@@ -230,7 +221,7 @@ class Elliptic
  *  \end{align}
  *  \f] 
  * is discretized, with \f$ b^i\f$ being the contravariant components of \f$\mathbf b\f$ . 
- * @tparam Geometry The Geometry class to use
+ * @copydoc geometry
  * @copydoc hide_matrix_container
  * This class has the SelfMadeMatrixTag so it can be used in blas2::symv functions 
  * and thus in a conjugate gradient solver. 
@@ -243,10 +234,6 @@ struct GeneralElliptic
     /**
      * @brief Construct from Grid
      *
-     * @tparam Geometry The Grid class. A call to dg::evaluate( one, g) must return an instance of the container class, 
-     * a call to dg::create::weights(g) and dg::create::inv_weights(g)
-     * must return instances of the container class and 
-     * calls to dg::create::dx( g, no, backward) and jump2d( g, bcx, bcy, no) are made.
      * @param g The Grid, boundary conditions are taken from here
      * @param no Not normed for elliptic equations, normed else
      * @param dir Direction of the right first derivative
@@ -272,10 +259,6 @@ struct GeneralElliptic
     /**
      * @brief Construct from Grid and bc 
      *
-     * @tparam Grid The Grid class. A call to dg::evaluate( one, g) must return an instance of the container class, 
-     * a call to dg::create::weights(g) and dg::create::inv_weights(g)
-     * must return instances of the container class and 
-     * calls to dg::create::dx( g, no, backward) and jump2d( g, bcx, bcy, no) are made.
      * @param g The Grid
      * @param bcx boundary condition in x
      * @param bcy boundary contition in y
@@ -441,7 +424,7 @@ struct GeneralElliptic
  *  \end{align}
  *  \f] 
  * is discretized, with \f$ b^i\f$ being the contravariant components of \f$\mathbf b\f$ . 
- * @tparam Geometry The Geometry class to use
+ * @copydoc hide_geometry
  * @copydoc hide_matrix_container
  * This class has the SelfMadeMatrixTag so it can be used in blas2::symv functions 
  * and thus in a conjugate gradient solver. 
@@ -454,10 +437,6 @@ struct GeneralEllipticSym
     /**
      * @brief Construct from Grid
      *
-     * @tparam Grid The Grid class. A call to dg::evaluate( one, g) must return an instance of the container class, 
-     * a call to dg::create::weights(g) and dg::create::inv_weights(g)
-     * must return instances of the container class and 
-     * calls to dg::create::dx( g, no, backward) and jump2d( g, bcx, bcy, no) are made.
      * @param g The Grid, boundary conditions are taken from here
      * @param no Not normed for elliptic equations, normed else
      * @param dir Direction of the right first derivative
@@ -471,10 +450,6 @@ struct GeneralEllipticSym
         /**
      * @brief Construct from Grid and bc
      *
-     * @tparam Grid The Grid class. A call to dg::evaluate( one, g) must return an instance of the container class, 
-     * a call to dg::create::weights(g) and dg::create::inv_weights(g)
-     * must return instances of the container class and 
-     * calls to dg::create::dx( g, no, backward) and jump2d( g, bcx, bcy, no) are made.
      * @param g The Grid
      * @param bcx boundary condition in x
      * @param bcy boundary contition in y
@@ -584,7 +559,7 @@ struct GeneralEllipticSym
  * -\frac{1}{\sqrt{g}} \left(\partial_x(\sqrt{g} v^x ) + \partial_y(\sqrt{g} v^y) \right)
  *  \end{align}
  *  \f] 
- * @tparam Geometry The Geometry class to use
+ * @copydoc hide_geometry
  * @copydoc hide_matrix_container
  * This class has the SelfMadeMatrixTag so it can be used in blas2::symv functions 
  * and thus in a conjugate gradient solver. 
@@ -596,11 +571,6 @@ struct TensorElliptic
 {
     /**
      * @brief Construct from Grid
-     *
-     * @tparam Geometry The Grid class. A call to dg::evaluate( one, g) must return an instance of the container class, 
-     * a call to dg::create::weights(g) and dg::create::inv_weights(g)
-     * must return instances of the container class and 
-     * calls to dg::create::dx( g, no, backward) and jump2d( g, bcx, bcy, no) are made.
      * @param g The Grid, boundary conditions are taken from here
      * @param no Not normed for elliptic equations, normed else
      * @param dir Direction of the right first derivative
@@ -612,11 +582,6 @@ struct TensorElliptic
     }
     /**
      * @brief Construct from Grid and bc 
-     *
-     * @tparam Grid The Grid class. A call to dg::evaluate( one, g) must return an instance of the container class, 
-     * a call to dg::create::weights(g) and dg::create::inv_weights(g)
-     * must return instances of the container class and 
-     * calls to dg::create::dx( g, no, backward) and jump2d( g, bcx, bcy, no) are made.
      * @param g The Grid
      * @param bcx boundary condition in x
      * @param bcy boundary contition in y
diff --git a/inc/dg/general_elliptic_b.cu b/inc/dg/general_elliptic_b.cu
index 2545ccd08..345885f83 100644
--- a/inc/dg/general_elliptic_b.cu
+++ b/inc/dg/general_elliptic_b.cu
@@ -31,14 +31,14 @@ int main()
     double eps;
     std::cout << "Type epsilon! \n";
     std::cin >> eps;
-    dg::CylindricalGrid3d<dg::DVec> grid( R_0, R_0+lx, 0, ly, 0,lz, n, Nx, Ny,Nz, bcx, dg::PER, dg::PER);
+    dg::CylindricalGrid3d grid( R_0, R_0+lx, 0, ly, 0,lz, n, Nx, Ny,Nz, bcx, dg::PER, dg::PER);
     dg::DVec w3d = dg::create::weights( grid);
     dg::DVec v3d = dg::create::inv_weights( grid);
     dg::DVec x = dg::evaluate( initial, grid);
 
     std::cout << "Create Laplacian\n";
     t.tic();
-    dg::GeneralElliptic<dg::CylindricalGrid3d<dg::DVec>, dg::DMatrix, dg::DVec> laplace(grid, dg::not_normed, dg::centered);
+    dg::GeneralElliptic<dg::CylindricalGrid3d, dg::DMatrix, dg::DVec> laplace(grid, dg::not_normed, dg::centered);
     dg::DMatrix DX = dg::create::dx( grid);
     t.toc();
     std::cout<< "Creation took "<<t.diff()<<"s\n";
diff --git a/inc/dg/geometry/refined_grid.h b/inc/dg/geometry/refined_grid.h
index 1def6a112..51cd41991 100644
--- a/inc/dg/geometry/refined_grid.h
+++ b/inc/dg/geometry/refined_grid.h
@@ -59,7 +59,7 @@ struct aRefinement1d
 {
     /*! @brief Generate the grid transformation
      *  
-     * @param g The 1d grid to refine
+     * @param g_old The 1d grid to refine
      * @param weights A 1d vector of size N_new. These represent the Jacobian of the transformation \f[\frac{\partial \zeta}{\partial x} \f]. The new metric element has thus to be multiplied by weights^2 and the volume by 1/weights
      * @param abscissas A 1d vector of size N_new. These are the new abscissas \f$ x(\zeta) \f$ of the grid. 
     */
diff --git a/inc/dg/geometry/refined_gridX.h b/inc/dg/geometry/refined_gridX.h
index 767a466a3..23e87e2a5 100644
--- a/inc/dg/geometry/refined_gridX.h
+++ b/inc/dg/geometry/refined_gridX.h
@@ -19,9 +19,11 @@ struct aRefinementX2d
 {
     /*! @brief Generate the grid transformation
      *  
-     * @param g The 1d grid to refine
-     * @param weights A 1d vector of size N_new. These represent the Jacobian of the transformation \f[\frac{\partial \zeta}{\partial x} \f]. The new metric element has thus to be multiplied by weights^2 and the volume by 1/weights
-     * @param abscissas A 1d vector of size N_new. These are the new abscissas \f$ x(\zeta) \f$ of the grid. 
+     * @param g_old The 1d grid to refine
+     * @param weightsX A 2d vector of size nx_new()*ny_new(). These represent the Jacobian of the transformation \f[\frac{\partial \zeta}{\partial x} \f]. The new metric element has thus to be multiplied by weights^2 and the volume by 1/weights
+     * @param weightsY A 2d vector of size nx_new()*ny_new(). These represent the Jacobian of the transformation \f[\frac{\partial \zeta}{\partial x} \f]. The new metric element has thus to be multiplied by weights^2 and the volume by 1/weights
+     * @param abscissasX A 2d vector of size nx_new()*ny_new(). These are the new abscissas \f$ x(\zeta) \f$ of the grid. 
+     * @param abscissasY A 2d vector of size nx_new()*ny_new(). These are the new abscissas \f$ x(\zeta) \f$ of the grid. 
     */
     void generate( const GridX2d& g_old, thrust::host_vector<double>& weightsX, thrust::host_vector<double>& weightsY, thrust::host_vector<double>& abscissasX, thrust::host_vector<double>& abscissasY) const
     {
@@ -43,9 +45,10 @@ struct aRefinementX2d
                 abscissasY[i*wx.size()+j] = ay[i];
             }
     }
+
     /*! @brief the new number of cells
-     * @param N_old the old number of cells
-     * @param fx Factor to partition x-axis
+     * @param Nx_old the old number of cells
+     * @param fx_old Factor to partition x-axis
      */
     unsigned nx_new( unsigned Nx_old, double fx_old) const
     {
diff --git a/inc/dg/geometry/tensor.h b/inc/dg/geometry/tensor.h
index a75e8eb7b..1d76f113b 100644
--- a/inc/dg/geometry/tensor.h
+++ b/inc/dg/geometry/tensor.h
@@ -331,19 +331,16 @@ struct CholeskyTensor
      *
      * @param in must be symmetric and positive definite
      */
-    CholeskyTensor( const SparseTensor<container>& in)
-    {
+    CholeskyTensor( const SparseTensor<container>& in) {
         decompose(in);
     }
     /**
      * @brief Type conversion from other value types
      * @tparam OtherContainer dg::blas1::transfer must be callable for container and OtherContainer
-     * @param src the source matrix to convert
+     * @param in the source matrix to convert
      */
     template<class OtherContainer>
-    CholeskyTensor( const CholeskyTensor<OtherContainer>& in):q_(in.lower()),diag_(in.diagonal()),upper_(in.upper())
-    { 
-        }
+    CholeskyTensor( const CholeskyTensor<OtherContainer>& in):q_(in.lower()),diag_(in.diagonal()),upper_(in.upper()) { }
 
     /**
      * @brief decompose given tensor
diff --git a/inc/dg/helmholtz.h b/inc/dg/helmholtz.h
index 7d9a48df9..4cfb5d777 100644
--- a/inc/dg/helmholtz.h
+++ b/inc/dg/helmholtz.h
@@ -19,7 +19,7 @@ namespace dg{
  * Unnormed discretization of \f[ (\chi+\alpha\Delta) \f]
  * where \f$ \chi\f$ is a function and \f$\alpha\f$ a scalar.
  * Can be used by the Invert class
- * @tparam container The container class you want to use
+ * @copydoc hide_geometry
  * @copydoc hide_matrix_container
  * @attention The Laplacian in this formula is positive as opposed to the negative sign in the Elliptic operator
  */
@@ -133,7 +133,7 @@ struct Helmholtz
  * \f[ \left[ \chi +2 \alpha\Delta +  \alpha^2\Delta \left(\chi^{-1}\Delta \right)\right] \f] 
  * where \f$ \chi\f$ is a function and \f$\alpha\f$ a scalar.
  * Can be used by the Invert class
- * @tparam Geometry The geometry class you want to use
+ * @copydoc hide_geometry
  * @copydoc hide_matrix_container
  * @attention The Laplacian in this formula is positive as opposed to the negative sign in the Elliptic operator
  * @attention It might be better to solve the normal Helmholtz operator twice
diff --git a/inc/dg/helmholtz_t.cu b/inc/dg/helmholtz_t.cu
index ffc16d3e5..ec5d80ef7 100644
--- a/inc/dg/helmholtz_t.cu
+++ b/inc/dg/helmholtz_t.cu
@@ -92,7 +92,7 @@ int main()
     std::cout << "error2 " << sqrt( dg::blas2::dot( w2d, x_))<<std::endl;
     //std::cout << "error3 " << sqrt( dg::blas2::dot( w2d, x__))<<std::endl;
     std::cout << "Test 3d cylincdrical norm:\n";
-    dg::CylindricalGrid3d<dg::DVec> g3d( R_0, R_0+lx, 0, ly, 0,lz, n, Nx, Ny,Nz, bcx, dg::PER, dg::PER);
+    dg::CylindricalGrid3d g3d( R_0, R_0+lx, 0, ly, 0,lz, n, Nx, Ny,Nz, bcx, dg::PER, dg::PER);
     dg::DVec fct_ = dg::evaluate(fct, g3d );
     dg::DVec laplace_fct_ = dg::evaluate( laplace_fct, g3d);
     dg::DVec helmholtz_fct_ = dg::evaluate( helmholtz_fct, g3d);
diff --git a/inc/dg/poisson.h b/inc/dg/poisson.h
index d6fa7ad2d..8dfb7decf 100644
--- a/inc/dg/poisson.h
+++ b/inc/dg/poisson.h
@@ -2,7 +2,7 @@
 #define _DG_POISSON_CUH
 
 #include "blas.h"
-#include "geometry.h"
+#include "geometry/geometry.h"
 #include "enums.h"
 #include "backend/evaluation.cuh"
 #include "backend/derivatives.h"
@@ -23,25 +23,19 @@ namespace dg
  *
  * Equal to the Arakawa class except for the possitility to use mixed boundary conditions
  * @ingroup arakawa
- * @tparam Matrix The Matrix class to use
- * @tparam container The vector class on which to operate on. The blas2 function symv( m, x, y) must be callable and may not change x. 
+ * @copydoc hide_geometry
+ * @copydoc hide_matrix_container
  */
 template< class Geometry, class Matrix, class container >
 struct Poisson
 {
     /**
      * @brief Create Poisson on a grid
-     *
-     * @tparam Grid The Grid class. The functions dg::create::dx( g, bcx) and
-     * dg::create::dy( g, bcy) must be callable and return an instance of the Matrix class. Furthermore dg::evaluate( one, g) must return an instance of the container class.
      * @param g The grid
      */
     Poisson( Geometry g);
     /**
      * @brief Create Poisson on a grid using different boundary conditions
-     *
-     * @tparam Grid The Grid class. The functions dg::create::dx( g, bcx) and
-     * dg::create::dy( g, bcy) must be callable and return an instance of the Matrix class. Furthermore dg::evaluate( one, g) must return an instance of the container class.
      * @param g The grid
      * @param bcx The boundary condition in x
      * @param bcy The boundary condition in y
@@ -49,9 +43,6 @@ struct Poisson
     Poisson( Geometry g, bc bcx, bc bcy);
     /**
      * @brief Create Poisson on a grid using different boundary conditions
-     *
-     * @tparam Grid The Grid class. The functions dg::create::dx( g, bcx) and
-     * dg::create::dy( g, bcy) must be callable and return an instance of the Matrix class. Furthermore dg::evaluate( one, g) must return an instance of the container class.
      * @param g The grid
      * @param bcxlhs The lhs boundary condition in x
      * @param bcxrhs The rhs boundary condition in x
@@ -110,17 +101,16 @@ struct Poisson
     {
         blas2::symv( dxrhs_, phi, dxrhsrhs_);
         blas2::symv( dyrhs_, phi, dyrhsrhs_);
-        blas1::copy( dxrhsrhs_, dxlhslhs_);//save results
-        blas1::copy( dyrhsrhs_, dylhslhs_);
-        geo::raisePerpIndex( dxlhslhs_, dylhslhs_, varphi, helper_, g_); //input gets destroyed
+        tensor::multiply2d( metric_, dxrhsrhs_, dyrhsrhs_, varphi, helper_);
         blas1::pointwiseDot( varphi, dxrhsrhs_, varphi);
         blas1::pointwiseDot( 1., helper_, dyrhsrhs_,1., varphi );
     }
 
   private:
-    container dxlhslhs_,dxrhsrhs_,dylhslhs_,dyrhsrhs_,helper_;
-    Matrix dxlhs_, dylhs_,dxrhs_,dyrhs_;
-    Geometry g_;
+    container dxlhslhs_, dxrhsrhs_, dylhslhs_, dyrhsrhs_, helper_;
+    Matrix dxlhs_, dylhs_, dxrhs_, dyrhs_;
+    SparseElement<container> perp_vol_inv_;
+    SparseTensor<container> metric_;
 };
 
 //idea: backward transform lhs and rhs and then use bdxf and bdyf , then forward transform
@@ -131,8 +121,12 @@ Poisson<Geometry, Matrix, container>::Poisson( Geometry g ):
     dxlhs_(dg::create::dx( g, g.bcx(),dg::centered)),
     dylhs_(dg::create::dy( g, g.bcy(),dg::centered)),
     dxrhs_(dg::create::dx( g, g.bcx(),dg::centered)),
-    dyrhs_(dg::create::dy( g, g.bcy(),dg::centered)),g_(g)
-{ }
+    dyrhs_(dg::create::dy( g, g.bcy(),dg::centered))
+{
+    metric_=g.metric().perp();
+    perp_vol_inv_ = dg::tensor::determinant(metric_);
+    dg::tensor::sqrt(perp_vol_inv_);
+}
 
 template< class Geometry, class Matrix, class container>
 Poisson<Geometry, Matrix, container>::Poisson( Geometry g, bc bcx, bc bcy): 
@@ -140,8 +134,12 @@ Poisson<Geometry, Matrix, container>::Poisson( Geometry g, bc bcx, bc bcy):
     dxlhs_(dg::create::dx( g, bcx,dg::centered)),
     dylhs_(dg::create::dy( g, bcy,dg::centered)),
     dxrhs_(dg::create::dx( g, bcx,dg::centered)),
-    dyrhs_(dg::create::dy( g, bcy,dg::centered)),g_(g)
-{ }
+    dyrhs_(dg::create::dy( g, bcy,dg::centered))
+{ 
+    metric_=g.metric().perp();
+    perp_vol_inv_ = dg::tensor::determinant(metric_);
+    dg::tensor::sqrt(perp_vol_inv_);
+}
 
 template< class Geometry, class Matrix, class container>
 Poisson<Geometry, Matrix, container>::Poisson(  Geometry g, bc bcxlhs, bc bcylhs, bc bcxrhs, bc bcyrhs): 
@@ -149,8 +147,12 @@ Poisson<Geometry, Matrix, container>::Poisson(  Geometry g, bc bcxlhs, bc bcylhs
     dxlhs_(dg::create::dx( g, bcxlhs,dg::centered)),
     dylhs_(dg::create::dy( g, bcylhs,dg::centered)),
     dxrhs_(dg::create::dx( g, bcxrhs,dg::centered)),
-    dyrhs_(dg::create::dy( g, bcyrhs,dg::centered)),g_(g)
-{ }
+    dyrhs_(dg::create::dy( g, bcyrhs,dg::centered))
+{ 
+    metric_=g.metric().perp();
+    perp_vol_inv_ = dg::tensor::determinant(metric_);
+    dg::tensor::sqrt(perp_vol_inv_);
+}
 
 template< class Geometry, class Matrix, class container>
 void Poisson< Geometry, Matrix, container>::operator()( const container& lhs, const container& rhs, container& result)
@@ -163,7 +165,7 @@ void Poisson< Geometry, Matrix, container>::operator()( const container& lhs, co
     blas1::pointwiseDot( dxlhslhs_, dyrhsrhs_, result);   //dx_lhs lhs * dy_rhs rhs
     blas1::pointwiseDot( -1., dylhslhs_, dxrhsrhs_, 1., result);    //- dy_lhs lhs * dx_rhs rhs
 
-    geo::dividePerpVolume( result, g_);
+    tensor::pointwiseDot( perp_vol_inv_, result, result);
 }
 
 }//namespace dg
-- 
GitLab


From 332d4198444e1284722861eb1ecc332d485c7236 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Fri, 18 Aug 2017 02:00:50 -0700
Subject: [PATCH 181/453] refined tests on memory.h (ld manage.h

---
 inc/dg/backend/{manage.h => memory.h}       | 23 +++++++++----
 inc/dg/backend/{manage_t.cu => memory_t.cu} | 36 ++++++++++++---------
 2 files changed, 37 insertions(+), 22 deletions(-)
 rename inc/dg/backend/{manage.h => memory.h} (84%)
 rename inc/dg/backend/{manage_t.cu => memory_t.cu} (53%)

diff --git a/inc/dg/backend/manage.h b/inc/dg/backend/memory.h
similarity index 84%
rename from inc/dg/backend/manage.h
rename to inc/dg/backend/memory.h
index 434c39a17..d116421b0 100644
--- a/inc/dg/backend/manage.h
+++ b/inc/dg/backend/memory.h
@@ -6,10 +6,11 @@ namespace dg
 //there is probably a better class in boost...
 /*!@brief a manager class that invokes the clone() method on the managed ptr when copied
 *
-*when copied invokes a deep copy using the clone() method 
-* this class is most useful when a class needs to hold a polymorphic, cloneable oject as a variable. 
-@tparam cloneable a type that has the clone() method 
-@ingroup misc
+*When copied invokes a deep copy using the clone() method.
+* This class is most useful when a class needs to hold a polymorphic, cloneable oject as a variable. 
+@tparam cloneable a type that may be uncopyable/unassignable but provides the clone() method with signature
+ - cloneable* clone() const;
+@ingroup lowlevel
 */
 template<class cloneable>
 struct Handle
@@ -42,12 +43,19 @@ struct Handle
         this->swap( src );
         return *this;
     }
-    ///delete managed pointer if not empty
-    ~Handle(){ if(ptr_!=0) delete ptr_; }
+    ///delete managed pointer if not NULL
+    ~Handle(){ clear();}
+
+    ///delete managed pointer if not NULL
+    void clear(){
+        if(ptr_!=0) delete ptr_; 
+        ptr_=0;
+    }
 
     /**
     * @brief Get a constant reference to the object on the heap
     * @return a reference to the cloneable object
+    * @note undefined if the Handle manages a NULL pointer
     */
     const cloneable& get()const {return *ptr_;}
 
@@ -55,6 +63,7 @@ struct Handle
     /**
     * @brief Non constant access to the object on the heap
     * @return a non-const reference to the cloneable object
+    * @note undefined if the Handle manages a NULL pointer
     */
     cloneable& get() {return *ptr_;}
 
@@ -93,7 +102,7 @@ struct Handle
 * some workspace to fulfill their task but do otherwise not change their state. A buffer object
 can be declared const while the data it holds are still writeable.
 * @tparam T must be default constructible and copyable
-* @ingroup misc
+* @ingroup lowlevel
 */
 template< class T>
 struct Buffer
diff --git a/inc/dg/backend/manage_t.cu b/inc/dg/backend/memory_t.cu
similarity index 53%
rename from inc/dg/backend/manage_t.cu
rename to inc/dg/backend/memory_t.cu
index fd26213ea..70446f206 100644
--- a/inc/dg/backend/manage_t.cu
+++ b/inc/dg/backend/memory_t.cu
@@ -1,6 +1,6 @@
 #include <iostream>
 
-#include "manage.h"
+#include "memory.h"
 
 struct aAnimal
 {
@@ -14,13 +14,15 @@ struct aAnimal
 //Yes pure virtual functions can have a definition
 void aAnimal::speak()const{ std::cout << "I am ";}
 
-struct Dog: public aAnimal
+struct Mouse: public aAnimal
 {
     virtual void speak()const{
         aAnimal::speak();
-        std::cout << " a dog!\n";
+        std::cout << " a mouse!\n";
     }
-    virtual Dog* clone()const{return new Dog(*this);}
+    virtual Mouse* clone()const{
+        std::cout << "Mouse is cloned!\n";
+        return new Mouse(*this);}
 };
 
 struct Cat : public aAnimal
@@ -29,33 +31,37 @@ struct Cat : public aAnimal
         aAnimal::speak();
         std::cout << " a cat!\n";
     }
-    virtual Cat* clone()const{return new Cat(*this);}
+    virtual Cat* clone()const{
+        std::cout << "Cat is cloned!\n";
+        return new Cat(*this);}
 };
 
 
 int main()
 {
     {
-        dg::Handle<aAnimal> h0, h1(new Dog());
+        std::cout << "Test correct behaviour of handle: cat and mouse\n";
+        dg::Handle<aAnimal> h0, h1(new Mouse()); //default and pointer constructor
+        dg::Handle<aAnimal> h2(h1.get()); //reference constructor
         aAnimal* ptr = new Cat();
-        h0.reset(ptr);
-        std::cout << "Test correct behaviour of handle: cat and dog\n";
+        h0.reset(ptr); //pointer reset
+        h1.reset( h2.get()); //reference reset
+        h1.get()=h2.get();//reference test
+        h1.swap(h0); //swap test
         h0.get().speak();
         h1.get().speak();
-        std::cout << "Now copy handle: cat \n";
+        h2.get().speak();
         h1 = h0;
         h1.get().speak();
+        h1.clear();
     }
     {
-        dg::Buffer<Dog> buffer;
-        std::cout << "Test correct behaviour of buffer class with dog\n";
+        std::cout << "Test correct behaviour of buffer class with mouse\n";
+        dg::Buffer<Mouse> buffer;
         buffer.data().speak();
-        dg::Buffer<Dog> buffer2 = buffer;
+        dg::Buffer<Mouse> buffer2 = buffer;
         buffer2.data().speak();
     }
 
-
-
-
     return 0;
 }
-- 
GitLab


From c26093d107bb0821775112c0da745d2402ee9cd7 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Fri, 18 Aug 2017 02:42:47 -0700
Subject: [PATCH 182/453] corrected but in conversion from MPIGrid to
 CartesianMPIGrid

---
 inc/dg/algorithm.h                |  1 -
 inc/dg/backend/dlt.h              |  2 +-
 inc/dg/{ => backend}/exceptions.h |  3 +--
 inc/dg/backend/mpi_collective.h   |  2 +-
 inc/dg/backend/mpi_vector.h       |  2 +-
 inc/dg/backend/sparseblockmat.h   | 18 ++++++++++++++----
 inc/dg/enums.h                    |  2 +-
 inc/dg/geometry/curvilinear.h     |  2 +-
 inc/dg/geometry/mpi_base.h        |  4 ++--
 inc/dg/geometry/refined_grid.h    |  2 +-
 inc/dg/helmholtzg2_b.cu           |  2 +-
 inc/dg/nullstelle.h               |  2 +-
 inc/dg/runge_kutta.h              |  2 +-
 inc/geometries/fluxfunctions.h    |  2 +-
 14 files changed, 27 insertions(+), 19 deletions(-)
 rename inc/dg/{ => backend}/exceptions.h (97%)

diff --git a/inc/dg/algorithm.h b/inc/dg/algorithm.h
index fb9f57eee..80e592dfe 100644
--- a/inc/dg/algorithm.h
+++ b/inc/dg/algorithm.h
@@ -10,7 +10,6 @@
 #include "arakawa.h"
 #include "helmholtz.h"
 #include "cg.h"
-#include "exceptions.h"
 #include "functors.h"
 #include "multistep.h"
 #include "elliptic.h"
diff --git a/inc/dg/backend/dlt.h b/inc/dg/backend/dlt.h
index a270995b2..66a1fb88d 100644
--- a/inc/dg/backend/dlt.h
+++ b/inc/dg/backend/dlt.h
@@ -5,7 +5,7 @@
 #include <sstream>
 #include <stdexcept>
 #include <vector>
-#include "../exceptions.h"
+#include "exceptions.h"
 
 
 namespace dg{
diff --git a/inc/dg/exceptions.h b/inc/dg/backend/exceptions.h
similarity index 97%
rename from inc/dg/exceptions.h
rename to inc/dg/backend/exceptions.h
index 399c706e4..e08c20378 100644
--- a/inc/dg/exceptions.h
+++ b/inc/dg/backend/exceptions.h
@@ -5,7 +5,6 @@
  */
 #pragma once
 
-
 #include <exception>
 #include <iostream>
 #include <sstream>
@@ -67,7 +66,7 @@ class Message
  * The objects of this class store a message (that describes the error when thrown)
  * that can then be displayed in a catch block
  * \code
- * try{ throw Error(Message()<<"This is error number "<<number, _ping_);}
+ * try{ throw Error(Message(_ping_)<<"This is error number "<<number);}
  * catch( Error& m) {std::cerr << m.what();}
  * \endcode
  * @ingroup misc
diff --git a/inc/dg/backend/mpi_collective.h b/inc/dg/backend/mpi_collective.h
index f3ae52ba9..271a2f08e 100644
--- a/inc/dg/backend/mpi_collective.h
+++ b/inc/dg/backend/mpi_collective.h
@@ -10,7 +10,7 @@
 #include <thrust/device_vector.h>
 #include "thrust_vector_blas.cuh"
 #include "dg/blas1.h"
-#include "manage.h"
+#include "memory.h"
 
 namespace dg{
 
diff --git a/inc/dg/backend/mpi_vector.h b/inc/dg/backend/mpi_vector.h
index 8478946f7..f38c3030b 100644
--- a/inc/dg/backend/mpi_vector.h
+++ b/inc/dg/backend/mpi_vector.h
@@ -5,7 +5,7 @@
 #include <thrust/gather.h>
 #include "vector_traits.h"
 #include "thrust_vector_blas.cuh"
-#include "manage.h"
+#include "memory.h"
 
 namespace dg
 {
diff --git a/inc/dg/backend/sparseblockmat.h b/inc/dg/backend/sparseblockmat.h
index 83d57c7a8..048e180fa 100644
--- a/inc/dg/backend/sparseblockmat.h
+++ b/inc/dg/backend/sparseblockmat.h
@@ -1,6 +1,7 @@
 #pragma once
 
 #include <thrust/host_vector.h>
+#include "exceptions.h"
 #include "matrix_traits.h"
 
 namespace dg
@@ -184,8 +185,13 @@ struct CooSparseBlockMat
 template<class value_type>
 void EllSparseBlockMat<value_type>::symv(const thrust::host_vector<value_type>& x, thrust::host_vector<value_type>& y) const
 {
-    assert( y.size() == (unsigned)num_rows*n*left_size*right_size);
-    assert( x.size() == (unsigned)num_cols*n*left_size*right_size);
+    if( y.size() != (unsigned)num_rows*n*left_size*right_size) {
+        throw Error( Message(_ping_)<<"y has the wrong size "<<y.size()<<" and not "<<(unsigned)num_rows*n*left_size*right_size);
+    }
+    if( x.size() != (unsigned)num_cols*n*left_size*right_size) {
+        throw Error( Message(_ping_)<<"x has the wrong size "<<x.size()<<" and not "<<(unsigned)num_cols*n*left_size*right_size);
+    }
+
 
     //simplest implementation
     for( int s=0; s<left_size; s++)
@@ -257,8 +263,12 @@ void CooSparseBlockMat<value_type>::display( std::ostream& os) const
 template<class value_type>
 void CooSparseBlockMat<value_type>::symv( value_type alpha, const thrust::host_vector<value_type>& x, value_type beta, thrust::host_vector<value_type>& y) const
 {
-    assert( y.size() == (unsigned)num_rows*n*left_size*right_size);
-    assert( x.size() == (unsigned)num_cols*n*left_size*right_size);
+    if( y.size() != (unsigned)num_rows*n*left_size*right_size) {
+        throw Error( Message(_ping_)<<"y has the wrong size "<<y.size()<<" and not "<<(unsigned)num_rows*n*left_size*right_size);
+    }
+    if( x.size() != (unsigned)num_cols*n*left_size*right_size) {
+        throw Error( Message(_ping_)<<"x has the wrong size "<<x.size()<<" and not "<<(unsigned)num_cols*n*left_size*right_size);
+    }
     assert( beta == 1);
 
     //simplest implementation
diff --git a/inc/dg/enums.h b/inc/dg/enums.h
index 891338525..8fdb0e74a 100644
--- a/inc/dg/enums.h
+++ b/inc/dg/enums.h
@@ -1,6 +1,6 @@
 #pragma once
 #include <string>
-#include "exceptions.h"
+#include "backend/exceptions.h"
 
 /*! @file 
   
diff --git a/inc/dg/geometry/curvilinear.h b/inc/dg/geometry/curvilinear.h
index 892cfa7e5..fc0cc50e6 100644
--- a/inc/dg/geometry/curvilinear.h
+++ b/inc/dg/geometry/curvilinear.h
@@ -1,6 +1,6 @@
 #pragma once
 
-#include "dg/backend/manage.h"
+#include "dg/backend/memory.h"
 #include "dg/blas1.h"
 #include "base_geometry.h"
 #include "generator.h"
diff --git a/inc/dg/geometry/mpi_base.h b/inc/dg/geometry/mpi_base.h
index 2d0269ea9..c9b2b8eb6 100644
--- a/inc/dg/geometry/mpi_base.h
+++ b/inc/dg/geometry/mpi_base.h
@@ -118,7 +118,7 @@ struct CartesianMPIGrid2d : public aMPIGeometry2d
      * @note the paramateres given in the constructor are global parameters 
      */
     CartesianMPIGrid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx, bc bcy, MPI_Comm comm):dg::aMPIGeometry2d( x0, x1, y0, y1, n, Nx, Ny,bcx, bcy, comm){}
-    CartesianMPIGrid2d( const dg::MPIGrid2d& g): aMPIGeometry2d( g.x0(),g.x1(),g.y0(),g.y1(),g.n(),g.Nx(),g.Ny(),g.bcx(),g.bcy(),g.communicator()){}
+    CartesianMPIGrid2d( const dg::MPIGrid2d& g): aMPIGeometry2d( g.global().x0(),g.global().x1(),g.global().y0(),g.global().y1(),g.global().n(),g.global().Nx(),g.global().Ny(),g.global().bcx(),g.global().bcy(),g.communicator()){}
     virtual CartesianMPIGrid2d* clone()const{return new CartesianMPIGrid2d(*this);}
     private:
     virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny){
@@ -146,7 +146,7 @@ struct CartesianMPIGrid3d : public aMPIGeometry3d
      */
     CartesianMPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz, MPI_Comm comm):aMPIGeometry3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, bcx, bcy, bcz, comm){}
 
-    CartesianMPIGrid3d( const dg::MPIGrid3d& g): aMPIGeometry3d( g.x0(),g.x1(),g.y0(),g.y1(),g.z0(),g.z1(),g.n(),g.Nx(),g.Ny(),g.Nz(),g.bcx(),g.bcy(),g.bcz(),g.communicator()){}
+    CartesianMPIGrid3d( const dg::MPIGrid3d& g): aMPIGeometry3d( g.global().x0(),g.global().x1(),g.global().y0(),g.global().y1(),g.global().z0(),g.global().z1(),g.global().n(),g.global().Nx(),g.global().Ny(),g.global().Nz(),g.global().bcx(),g.global().bcy(),g.global().bcz(),g.communicator()){}
     virtual CartesianMPIGrid3d* clone()const{return new CartesianMPIGrid3d(*this);}
     private:
     virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz){
diff --git a/inc/dg/geometry/refined_grid.h b/inc/dg/geometry/refined_grid.h
index 51cd41991..1ea7d3ce3 100644
--- a/inc/dg/geometry/refined_grid.h
+++ b/inc/dg/geometry/refined_grid.h
@@ -1,7 +1,7 @@
 #pragma once
 
 #include "cusp/transpose.h"
-#include "dg/backend/manage.h"
+#include "dg/backend/memory.h"
 #include "dg/backend/grid.h"
 #include "dg/backend/weights.cuh"
 #include "dg/backend/interpolation.cuh"
diff --git a/inc/dg/helmholtzg2_b.cu b/inc/dg/helmholtzg2_b.cu
index a09b1114a..75a1cf898 100644
--- a/inc/dg/helmholtzg2_b.cu
+++ b/inc/dg/helmholtzg2_b.cu
@@ -3,9 +3,9 @@
 #include "blas.h"
 #include "backend/timer.cuh"
 #include "backend/typedefs.cuh"
+#include "backend/exceptions.h"
 
 #include "helmholtz.h"
-#include "exceptions.h"
 #include "cg.h"
 #include "functors.h"
 
diff --git a/inc/dg/nullstelle.h b/inc/dg/nullstelle.h
index 68a91ef44..40c5a45cb 100644
--- a/inc/dg/nullstelle.h
+++ b/inc/dg/nullstelle.h
@@ -8,7 +8,7 @@
 
 #include <exception>
 #include <math.h>
-#include "exceptions.h"
+#include "backend/exceptions.h"
 namespace dg{
 
 /*! @brief Exception class, that stores boundaries for 1D root finding
diff --git a/inc/dg/runge_kutta.h b/inc/dg/runge_kutta.h
index 3025577d6..251dbbec9 100644
--- a/inc/dg/runge_kutta.h
+++ b/inc/dg/runge_kutta.h
@@ -4,7 +4,7 @@
 #include <cassert>
 #include <vector>
 
-#include "exceptions.h"
+#include "backend/exceptions.h"
 #include "blas1.h"
 
 
diff --git a/inc/geometries/fluxfunctions.h b/inc/geometries/fluxfunctions.h
index d965b5825..f8b0f930c 100644
--- a/inc/geometries/fluxfunctions.h
+++ b/inc/geometries/fluxfunctions.h
@@ -1,5 +1,5 @@
 #pragma once
-#include "dg/backend/manage.h"
+#include "dg/backend/memory.h"
 
 namespace dg
 {
-- 
GitLab


From e7203210edfaf9a2b51a3ae021d7727aa18b3494 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Fri, 18 Aug 2017 03:09:18 -0700
Subject: [PATCH 183/453] try to debug refined elliptic_b

---
 inc/dg/ellipticX2d_b.cu        |  1 +
 inc/dg/elliptic_mpib.cu        |  4 +-
 inc/dg/refined_elliptic.h      | 67 ++++++++++++++++------------------
 inc/dg/refined_elliptic2d_b.cu | 45 ++++++++++++-----------
 4 files changed, 58 insertions(+), 59 deletions(-)

diff --git a/inc/dg/ellipticX2d_b.cu b/inc/dg/ellipticX2d_b.cu
index ea5e0a70d..8a01b376a 100644
--- a/inc/dg/ellipticX2d_b.cu
+++ b/inc/dg/ellipticX2d_b.cu
@@ -9,6 +9,7 @@
 #include "backend/derivativesX.h"
 #include "backend/gridX.h"
 #include "backend/evaluationX.cuh"
+#include "geometry/base_geometryX.h"
 #include "elliptic.h"
 #include "cg.h"
 #include "backend/timer.cuh"
diff --git a/inc/dg/elliptic_mpib.cu b/inc/dg/elliptic_mpib.cu
index 2704058a5..0a0525570 100644
--- a/inc/dg/elliptic_mpib.cu
+++ b/inc/dg/elliptic_mpib.cu
@@ -28,7 +28,7 @@ int main( int argc, char* argv[])
     MPI_Comm comm;
     mpi_init3d( bcx, dg::PER, dg::PER, n, Nx, Ny, Nz, comm);
 
-    dg::CylindricalMPIGrid3d<dg::MDVec> grid( R_0, R_0+lx, 0, ly, 0,lz, n, Nx, Ny,Nz, bcx, dg::PER, dg::PER, comm);
+    dg::CylindricalMPIGrid3d grid( R_0, R_0+lx, 0, ly, 0,lz, n, Nx, Ny,Nz, bcx, dg::PER, dg::PER, comm);
     const dg::MDVec w3d = dg::create::volume( grid);
     const dg::MDVec v3d = dg::create::inv_volume( grid);
     int rank;
@@ -44,7 +44,7 @@ int main( int argc, char* argv[])
 
     if(rank==0)std::cout << "Create Laplacian\n";
     t.tic();
-    dg::Elliptic<dg::CylindricalMPIGrid3d<dg::MDVec>, dg::MDMatrix, dg::MDVec> laplace(grid, dg::not_normed, dg::centered);
+    dg::Elliptic<dg::CylindricalMPIGrid3d, dg::MDMatrix, dg::MDVec> laplace(grid, dg::not_normed, dg::centered);
     dg::MDMatrix DX = dg::create::dx( grid);
     t.toc();
     if(rank==0)std::cout<< "Creation took "<<t.diff()<<"s\n";
diff --git a/inc/dg/refined_elliptic.h b/inc/dg/refined_elliptic.h
index ece70c466..67cb7bf5f 100644
--- a/inc/dg/refined_elliptic.h
+++ b/inc/dg/refined_elliptic.h
@@ -1,5 +1,7 @@
 #pragma once
 
+#include "backend/interpolation.cuh"
+#include "backend/projection.cuh"
 #include "elliptic.h"
 #include "geometry/refined_grid.h"
 
@@ -10,44 +12,36 @@
 namespace dg
 {
 
-template < class Geometry,class IMatrix, class Matrix, class Vector>
+template < class Geometry,class IMatrix, class Matrix, class container>
 class RefinedElliptic
 {
     public:
     /**
      * @brief Construct from Grid
      *
-     * @tparam Geometry The Grid class. A call to dg::evaluate( one, g) must return an instance of the Vector class, 
-     * a call to dg::create::weights(g) and dg::create::inv_weights(g)
-     * must return instances of the Vector class and 
-     * calls to dg::create::dx( g, no, backward) and jump2d( g, bcx, bcy, no) are made.
      * @param g The Grid, boundary conditions are taken from here
      * @param no Not normed for elliptic equations, normed else
      * @param dir Direction of the right first derivative
      */
-    RefinedElliptic( Geometry g, norm no = not_normed, direction dir = forward): 
-        no_(no), elliptic_( g, no, dir)
+    RefinedElliptic( const Geometry& g_coarse, const Geometry& g_fine, norm no = not_normed, direction dir = forward): 
+        no_(no), elliptic_( g_fine, no, dir)
     { 
-        construct( g, g.bcx(), g.bcy(), dir);
+        construct( g_coarse, g_fine, g_fine.bcx(), g_fine.bcy(), dir);
     }
 
     /**
      * @brief Construct from grid and boundary conditions
      *
-     * @tparam Geometry The Grid class. A call to dg::evaluate( one, g) must return an instance of the Vector class, 
-     * a call to dg::create::weights(g) and dg::create::inv_weights(g)
-     * must return instances of the Vector class and 
-     * calls to dg::create::dx( g, no, backward) and jump2d( g, bcx, bcy, no) are made.
      * @param g The Grid
      * @param bcx boundary condition in x
      * @param bcy boundary contition in y
      * @param no Not normed for elliptic equations, normed else
      * @param dir Direction of the right first derivative (i.e. forward, backward or centered)
      */
-    RefinedElliptic( Geometry g, bc bcx, bc bcy, norm no = not_normed, direction dir = forward): 
-        no_(no), elliptic_( g, bcx, bcy, no, dir)
+    RefinedElliptic( const Geometry& g_coarse, const Geometry& g_fine, bc bcx, bc bcy, norm no = not_normed, direction dir = forward): 
+        no_(no), elliptic_( g_fine, bcx, bcy, no, dir)
     { 
-        construct( g, bcx, bcy, dir);
+        construct( g_coarse, g_fine, bcx, bcy, dir);
     }
 
     /**
@@ -55,7 +49,7 @@ class RefinedElliptic
      *
      * @param chi The new chi
      */
-    void set_chi( const Vector& chi)
+    void set_chi( const container& chi)
     {
         //dg::blas2::gemv( Q_, chi, temp1_);
         //elliptic_.set_chi( temp1_);
@@ -67,14 +61,14 @@ class RefinedElliptic
      *
      * @return weights
      */
-    const Vector& weights()const {return weights_;}
+    const container& weights()const {return weights_;}
     /**
      * @brief Returns the preconditioner to use in conjugate gradient
      *
      * In this case inverse weights are the best choice
      * @return inverse weights
      */
-    const Vector& precond()const {return inv_weights_;}
+    const container& precond()const {return inv_weights_;}
 
     /**
      * @brief Computes the polarisation term
@@ -82,7 +76,7 @@ class RefinedElliptic
      * @param x left-hand-side
      * @param y result
      */
-    void symv( const Vector& x, Vector& y) 
+    void symv( const container& x, container& y) 
     {
         dg::blas2::gemv( Q_, x, temp1_); 
         elliptic_.symv( temp1_, temp2_);
@@ -105,34 +99,35 @@ class RefinedElliptic
      * @param rhs the original right hand side
      * @param rhs_mod the modified right hand side of the same size (may equal rhs)
      */
-    void compute_rhs( const Vector& rhs, Vector& rhs_mod )
+    void compute_rhs( const container& rhs, container& rhs_mod )
     {
         //dg::blas2::gemv( Q_, rhs, temp1_);
         //dg::blas1::pointwiseDot( vol_, temp1_, temp1_);
-        dg::blas1::pointwiseDot( vol_, rhs, temp1_);
+        dg::tensor::pointwiseDot( vol_, rhs, temp1_);
         dg::blas2::gemv( QT_, temp1_, rhs_mod);
         dg::blas2::symv( inv_weights_, rhs_mod, rhs_mod);
     }
     private:
-    void construct( Geometry g, bc bcx, bc bcy, direction dir)
+    void construct( const Geometry& g_coarse, const Geometry& g_fine, bc bcx, bc bcy, direction dir)
     {
-        dg::blas2::transfer( dg::create::interpolation( g), Q_);
-        dg::blas2::transfer( dg::create::interpolationT( g), QT_);
-        dg::blas2::transfer( dg::create::projection( g), P_);
-
-        dg::blas1::transfer( dg::evaluate( dg::one, g), temp1_);
-        dg::blas1::transfer( dg::evaluate( dg::one, g), temp2_);
-        dg::blas1::transfer( dg::create::weights( g.associated()), weights_);
-        dg::blas1::transfer( dg::create::inv_weights( g.associated()), inv_weights_);
-        dg::blas1::transfer( dg::create::volume( g), vol_);
-        dg::blas1::transfer( dg::create::inv_volume( g), inv_vol_);
+        dg::blas2::transfer( dg::create::interpolation( g_fine, g_coarse), Q_);
+        dg::blas2::transfer( dg::create::interpolationT( g_fine, g_coarse), QT_);
+        dg::blas2::transfer( dg::create::projection( g_coarse, g_fine), P_);
+
+        dg::blas1::transfer( dg::evaluate( dg::one, g_fine), temp1_);
+        dg::blas1::transfer( dg::evaluate( dg::one, g_fine), temp2_);
+        dg::blas1::transfer( dg::create::weights( g_coarse), weights_);
+        dg::blas1::transfer( dg::create::inv_weights( g_coarse), inv_weights_);
+        inv_vol_ = vol_ = dg::tensor::volume( g_fine.metric());
+        dg::tensor::invert( inv_vol_);
+
     }
     norm no_;
     IMatrix P_, Q_, QT_;
-    Elliptic<Geometry, Matrix, Vector> elliptic_;
-    Vector temp1_, temp2_;
-    Vector weights_, inv_weights_;
-    Vector vol_, inv_vol_;
+    Elliptic<Geometry, Matrix, container> elliptic_;
+    container temp1_, temp2_;
+    container weights_, inv_weights_;
+    dg::SparseElement<container> vol_, inv_vol_;
 };
 
 
diff --git a/inc/dg/refined_elliptic2d_b.cu b/inc/dg/refined_elliptic2d_b.cu
index 2a912f5e7..e2724d4ca 100644
--- a/inc/dg/refined_elliptic2d_b.cu
+++ b/inc/dg/refined_elliptic2d_b.cu
@@ -46,24 +46,27 @@ int main()
     std::cin >> n_ref >> multiple_x >> multiple_y; //more N means less iterations for same error
     std::cout << "Computation on: "<< n <<" x "<<Nx<<" x "<<Ny<<std::endl;
     //std::cout << "# of 2d cells                 "<< Nx*Ny <<std::endl;
-    dg::CartesianRefinedGrid2d<dg::DVec> grid( multiple_x, multiple_y, 0, lx, 0, ly, n_ref, n, Nx, Ny, bcx, bcy);
+    dg::LinearRefinement equiX( multiple_x), equiY(multiple_y);
+    dg::IdentityRefinement id;
+    dg::CartesianRefinedGrid2d grid_fine( equiX, equiY, 0, lx, 0, ly, n_ref, Nx, Ny, bcx, bcy);
+    dg::CartesianRefinedGrid2d grid_coarse( id,id, 0, lx, 0, ly, n, Nx, Ny, bcx, bcy);
     //evaluate on fine grid
-    dg::DVec w2dFINE = dg::create::volume( grid);
-    dg::DVec v2dFINE = dg::create::inv_weights( grid);
+    dg::DVec w2dFINE = dg::create::volume( grid_fine);
+    dg::DVec v2dFINE = dg::create::inv_weights( grid_fine);
     //evaluate on coarse grid
-    dg::DVec w2d = dg::create::volume( grid.associated());
-    dg::DVec v2d = dg::create::inv_weights( grid.associated());
+    dg::DVec w2d = dg::create::volume( grid_coarse);
+    dg::DVec v2d = dg::create::inv_weights( grid_coarse);
     //create interpolation and projection matrices
-    dg::IDMatrix Q = dg::create::interpolation( grid);
-    dg::IDMatrix P = dg::create::projection( grid);
+    dg::IDMatrix Q = dg::create::interpolation( grid_fine, grid_coarse);
+    dg::IDMatrix P = dg::create::projection( grid_coarse, grid_fine);
     //create functions A(chi) x = b
-    dg::DVec x =        dg::evaluate( initial, grid.associated());
-    dg::DVec b =        dg::evaluate( rhs, grid.associated());
-    dg::DVec bFINE =    dg::evaluate( rhs, grid);
-    dg::DVec xFINE =    dg::evaluate( rhs, grid);
+    dg::DVec x =        dg::evaluate( initial, grid_coarse);
+    dg::DVec b =        dg::evaluate( rhs, grid_coarse);
+    dg::DVec bFINE =    dg::evaluate( rhs, grid_fine);
+    dg::DVec xFINE =    dg::evaluate( rhs, grid_fine);
     dg::blas2::gemv( P, bFINE, b);
-    dg::DVec chi     =  dg::evaluate( pol, grid.associated());
-    dg::DVec chiFINE =  dg::evaluate( pol, grid);
+    dg::DVec chi     =  dg::evaluate( pol, grid_coarse);
+    dg::DVec chiFINE =  dg::evaluate( pol, grid_fine);
     dg::blas2::gemv( Q, chi, chiFINE);
     dg::DVec temp = x;
 
@@ -71,7 +74,7 @@ int main()
     std::cout << "Create Polarisation object and set chi!\n";
     t.tic();
     {
-    dg::RefinedElliptic<dg::CartesianRefinedGrid2d<dg::DVec>, dg::IDMatrix, dg::DMatrix, dg::DVec> pol( grid, dg::not_normed, dg::centered);
+    dg::RefinedElliptic<dg::CartesianRefinedGrid2d, dg::IDMatrix, dg::DMatrix, dg::DVec> pol( grid_coarse, grid_fine, dg::not_normed, dg::centered);
     pol.set_chi( chiFINE);
     t.toc();
     std::cout << "Creation of polarisation object took: "<<t.diff()<<"s\n";
@@ -87,11 +90,11 @@ int main()
     }
 
     //compute errorFINE
-    const dg::DVec solutionFINE = dg::evaluate( sol, grid);
-    const dg::DVec derivatiFINE = dg::evaluate( der, grid);
+    const dg::DVec solutionFINE = dg::evaluate( sol, grid_fine);
+    const dg::DVec derivatiFINE = dg::evaluate( der, grid_fine);
     dg::DVec errorFINE( solutionFINE);
-    const dg::DVec solution = dg::evaluate( sol, grid.associated());
-    const dg::DVec derivati = dg::evaluate( der, grid.associated());
+    const dg::DVec solution = dg::evaluate( sol, grid_coarse);
+    const dg::DVec derivati = dg::evaluate( der, grid_coarse);
     dg::DVec error( solution);
 
     dg::blas2::gemv( Q, x, xFINE);
@@ -102,7 +105,7 @@ int main()
     const double norm = dg::blas2::dot( w2dFINE, solutionFINE);
     std::cout << " "<<sqrt( err/norm) << " " <<sqrt(errFINE/norm);
     {
-    dg::RefinedElliptic<dg::CartesianRefinedGrid2d<dg::DVec>, dg::IDMatrix, dg::DMatrix, dg::DVec> pol_forward( grid, dg::not_normed, dg::forward);
+    dg::RefinedElliptic<dg::CartesianRefinedGrid2d, dg::IDMatrix, dg::DMatrix, dg::DVec> pol_forward( grid_coarse, grid_fine, dg::not_normed, dg::forward);
     pol_forward.set_chi( chiFINE);
     x = temp;
     dg::Invert<dg::DVec > invert_fw( x, n*n*Nx*Ny, eps);
@@ -116,7 +119,7 @@ int main()
     }
 
     {
-    dg::RefinedElliptic<dg::CartesianRefinedGrid2d<dg::DVec>, dg::IDMatrix, dg::DMatrix, dg::DVec> pol_backward( grid, dg::not_normed, dg::backward);
+    dg::RefinedElliptic<dg::CartesianRefinedGrid2d, dg::IDMatrix, dg::DMatrix, dg::DVec> pol_backward( grid_coarse, grid_fine, dg::not_normed, dg::backward);
     pol_backward.set_chi( chiFINE);
     x = temp;
     dg::Invert<dg::DVec > invert_bw( x, n*n*Nx*Ny, eps);
@@ -128,7 +131,7 @@ int main()
     }
 
 
-    dg::DMatrix DX = dg::create::dx( grid);
+    dg::DMatrix DX = dg::create::dx( grid_fine);
     dg::blas2::symv( DX, xFINE, errorFINE);
     dg::blas1::axpby( 1.,derivatiFINE,-1., errorFINE);
     err = dg::blas2::dot( w2dFINE, errorFINE);
-- 
GitLab


From cb60f3f1839c30c289c67e73f201594334f0d671 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sat, 19 Aug 2017 12:14:45 +0200
Subject: [PATCH 184/453] debugged refined_elliptic2d_b.cu

---
 inc/dg/geometry/refined_grid.h | 22 +++++--------
 inc/dg/refined_elliptic.h      |  2 +-
 inc/dg/refined_elliptic2d_b.cu | 60 +++++++++++++++++-----------------
 3 files changed, 39 insertions(+), 45 deletions(-)

diff --git a/inc/dg/geometry/refined_grid.h b/inc/dg/geometry/refined_grid.h
index 1ea7d3ce3..12de67c6a 100644
--- a/inc/dg/geometry/refined_grid.h
+++ b/inc/dg/geometry/refined_grid.h
@@ -115,29 +115,23 @@ struct LinearRefinement : public aRefinement1d
      * @brief Refine every cell in the grid by an integer number of new cells
      * @param multiple multiply every cell
      */
-    LinearRefinement( unsigned multiple): m_(multiple){}
+    LinearRefinement( unsigned multiple): m_(multiple){
+        assert( multiple>= 1);
+    }
     LinearRefinement* clone()const{return new LinearRefinement(*this);}
     private:
     unsigned m_;
     virtual void do_generate( const Grid1d& g, thrust::host_vector<double>& weights, thrust::host_vector<double>& abscissas) const 
     {
-        weights = linear_ref( m_, g.n(), g.N(), g.bcx());
+        thrust::host_vector< double> left( g.n()*g.N()*m_, 1);
+        for( unsigned k=0; k<left.size(); k++)
+            left[k] = (double)m_;
+        weights = left;
         abscissas = dg::detail::normalize_weights_and_compute_abscissas( g, weights);
     }
-    virtual unsigned do_N_new( unsigned N_old, bc bcx) const
-    {
+    virtual unsigned do_N_new( unsigned N_old, bc bcx) const {
         return N_old*m_;
     }
-    thrust::host_vector<double> linear_ref( unsigned multiple_x, unsigned n, unsigned N, dg::bc bcx) const
-    {
-        assert( multiple_x >= 1);
-        //there are add_x+1 finer cells per refined cell ...
-        thrust::host_vector< double> left( n*N*multiple_x, 1);
-        for( unsigned k=0; k<left.size(); k++)
-            left[k] = (double)multiple_x;
-        return left;
-    }
-
 };
 
 /**
diff --git a/inc/dg/refined_elliptic.h b/inc/dg/refined_elliptic.h
index 67cb7bf5f..f58e2ebec 100644
--- a/inc/dg/refined_elliptic.h
+++ b/inc/dg/refined_elliptic.h
@@ -111,7 +111,7 @@ class RefinedElliptic
     void construct( const Geometry& g_coarse, const Geometry& g_fine, bc bcx, bc bcy, direction dir)
     {
         dg::blas2::transfer( dg::create::interpolation( g_fine, g_coarse), Q_);
-        dg::blas2::transfer( dg::create::interpolationT( g_fine, g_coarse), QT_);
+        dg::blas2::transfer( dg::create::interpolationT( g_coarse, g_fine), QT_);
         dg::blas2::transfer( dg::create::projection( g_coarse, g_fine), P_);
 
         dg::blas1::transfer( dg::evaluate( dg::one, g_fine), temp1_);
diff --git a/inc/dg/refined_elliptic2d_b.cu b/inc/dg/refined_elliptic2d_b.cu
index e2724d4ca..58cbd21b1 100644
--- a/inc/dg/refined_elliptic2d_b.cu
+++ b/inc/dg/refined_elliptic2d_b.cu
@@ -74,19 +74,19 @@ int main()
     std::cout << "Create Polarisation object and set chi!\n";
     t.tic();
     {
-    dg::RefinedElliptic<dg::CartesianRefinedGrid2d, dg::IDMatrix, dg::DMatrix, dg::DVec> pol( grid_coarse, grid_fine, dg::not_normed, dg::centered);
-    pol.set_chi( chiFINE);
-    t.toc();
-    std::cout << "Creation of polarisation object took: "<<t.diff()<<"s\n";
+        dg::RefinedElliptic<dg::CartesianRefinedGrid2d, dg::IDMatrix, dg::DMatrix, dg::DVec> pol( grid_coarse, grid_fine, dg::not_normed, dg::centered);
+        pol.set_chi( chiFINE);
+        t.toc();
+        std::cout << "Creation of polarisation object took: "<<t.diff()<<"s\n";
 
-    dg::Invert<dg::DVec > invert( x, n*n*Nx*Ny, eps);
+        dg::Invert<dg::DVec > invert( x, n*n*Nx*Ny, eps);
 
 
-    std::cout << eps<<" ";
-    t.tic();
-    std::cout << " "<< invert( pol, x, b, w2d,v2d);
-    t.toc();
-    //std::cout << "Took "<<t.diff()<<"s\n";
+        std::cout << eps<<" ";
+        t.tic();
+        std::cout << " "<< invert( pol, x, b);
+        t.toc();
+        //std::cout << "Took "<<t.diff()<<"s\n";
     }
 
     //compute errorFINE
@@ -105,29 +105,29 @@ int main()
     const double norm = dg::blas2::dot( w2dFINE, solutionFINE);
     std::cout << " "<<sqrt( err/norm) << " " <<sqrt(errFINE/norm);
     {
-    dg::RefinedElliptic<dg::CartesianRefinedGrid2d, dg::IDMatrix, dg::DMatrix, dg::DVec> pol_forward( grid_coarse, grid_fine, dg::not_normed, dg::forward);
-    pol_forward.set_chi( chiFINE);
-    x = temp;
-    dg::Invert<dg::DVec > invert_fw( x, n*n*Nx*Ny, eps);
-    std::cout << " "<< invert_fw( pol_forward, x, b);
-    dg::blas2::gemv( Q, x, xFINE);
-    dg::blas1::axpby( 1.,xFINE,-1., solutionFINE, errorFINE);
-    errFINE = dg::blas2::dot( w2dFINE, errorFINE);
-    dg::blas1::axpby( 1.,x,-1., solution, error);
-    err = dg::blas2::dot( w2d, error);
-    std::cout << " "<<sqrt( err/norm) << " " <<sqrt(errFINE/norm);
+        dg::RefinedElliptic<dg::CartesianRefinedGrid2d, dg::IDMatrix, dg::DMatrix, dg::DVec> pol_forward( grid_coarse, grid_fine, dg::not_normed, dg::forward);
+        pol_forward.set_chi( chiFINE);
+        x = temp;
+        dg::Invert<dg::DVec > invert_fw( x, n*n*Nx*Ny, eps);
+        std::cout << " "<< invert_fw( pol_forward, x, b);
+        dg::blas2::gemv( Q, x, xFINE);
+        dg::blas1::axpby( 1.,xFINE,-1., solutionFINE, errorFINE);
+        errFINE = dg::blas2::dot( w2dFINE, errorFINE);
+        dg::blas1::axpby( 1.,x,-1., solution, error);
+        err = dg::blas2::dot( w2d, error);
+        std::cout << " "<<sqrt( err/norm) << " " <<sqrt(errFINE/norm);
     }
 
     {
-    dg::RefinedElliptic<dg::CartesianRefinedGrid2d, dg::IDMatrix, dg::DMatrix, dg::DVec> pol_backward( grid_coarse, grid_fine, dg::not_normed, dg::backward);
-    pol_backward.set_chi( chiFINE);
-    x = temp;
-    dg::Invert<dg::DVec > invert_bw( x, n*n*Nx*Ny, eps);
-    std::cout << " "<< invert_bw( pol_backward, x, b);
-    dg::blas2::gemv( Q, x, xFINE);
-    dg::blas1::axpby( 1.,xFINE,-1., solutionFINE, errorFINE);
-    err = dg::blas2::dot( w2dFINE, errorFINE);
-    std::cout << " "<<sqrt( err/norm)<<std::endl;
+        dg::RefinedElliptic<dg::CartesianRefinedGrid2d, dg::IDMatrix, dg::DMatrix, dg::DVec> pol_backward( grid_coarse, grid_fine, dg::not_normed, dg::backward);
+        pol_backward.set_chi( chiFINE);
+        x = temp;
+        dg::Invert<dg::DVec > invert_bw( x, n*n*Nx*Ny, eps);
+        std::cout << " "<< invert_bw( pol_backward, x, b);
+        dg::blas2::gemv( Q, x, xFINE);
+        dg::blas1::axpby( 1.,xFINE,-1., solutionFINE, errorFINE);
+        err = dg::blas2::dot( w2dFINE, errorFINE);
+        std::cout << " "<<sqrt( err/norm)<<std::endl;
     }
 
 
-- 
GitLab


From acb85dada2c13e34a810eafc6e2eeeeec3a384ba Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sat, 19 Aug 2017 13:28:22 +0200
Subject: [PATCH 185/453] work on documentation

---
 inc/dg/algorithm.h     |   3 +-
 inc/dg/cg.h            |  94 ++++++++++++---------------
 inc/dg/cluster_mpib.cu |   7 +--
 inc/dg/dg_doc.h        |  26 ++++----
 inc/dg/helmholtz_t.cu  |   4 +-
 inc/dg/multistep.h     | 121 ++++++++++++++++++-----------------
 inc/dg/runge_kutta.h   | 140 ++++++++++++++++++++---------------------
 7 files changed, 196 insertions(+), 199 deletions(-)

diff --git a/inc/dg/algorithm.h b/inc/dg/algorithm.h
index 80e592dfe..a8b1ffac1 100644
--- a/inc/dg/algorithm.h
+++ b/inc/dg/algorithm.h
@@ -6,7 +6,7 @@
  * @note include <mpi.h> before this header to activate mpi support
  */
 #include "blas.h"
-#include "geometry.h"
+#include "geometry/geometry.h"
 #include "arakawa.h"
 #include "helmholtz.h"
 #include "cg.h"
@@ -14,4 +14,3 @@
 #include "multistep.h"
 #include "elliptic.h"
 #include "runge_kutta.h"
-#include "ds.h"
diff --git a/inc/dg/cg.h b/inc/dg/cg.h
index 7d469c0cd..0e82f1de0 100644
--- a/inc/dg/cg.h
+++ b/inc/dg/cg.h
@@ -23,23 +23,23 @@ namespace dg{
 * \f[ Ax=b\f]
 *
  @ingroup invert
- @tparam Vector The Vector class: needs to model Assignable 
+ @copydoc hide_container_lvl1
 
  The following 3 pseudo - BLAS routines need to be callable 
- \li value_type dot = dg::blas1::dot( const Vector&, const Vector&); 
- \li dg::blas1::axpby();  with the Vector type
- \li dg::blas2::symv(Matrix& m, Vector1& x, Vector2& y ); with the Matrix type
+ \li value_type dot = dg::blas1::dot( const container&, const container&); 
+ \li dg::blas1::axpby();  with the container type
+ \li dg::blas2::symv(SymmetricOp& m, container1& x, container2& y ); with the SymmetricOp type
  \li value_type dot = dg::blas2::dot( );  with the Preconditioner type
  \li dg::blas2::symv( ); with the Preconditioner type
 
  @note Conjugate gradients might become unstable for positive semidefinite
  matrices arising e.g. in the discretization of the periodic laplacian
 */
-template< class Vector>
+template< class container>
 class CG
 {
   public:
-    typedef typename VectorTraits<Vector>::value_type value_type;//!< value type of the Vector class
+    typedef typename VectorTraits<container>::value_type value_type;//!< value type of the container class
     /**
      * @brief Allocate nothing, 
      */
@@ -47,10 +47,10 @@ class CG
       /**
        * @brief Reserve memory for the pcg method
        *
-       * @param copyable A Vector must be copy-constructible from this
+       * @param copyable A container must be copy-constructible from this
        * @param max_iter Maximum number of iterations to be used
        */
-    CG( const Vector& copyable, unsigned max_iter):r(copyable), p(r), ap(r), max_iter(max_iter){}
+    CG( const container& copyable, unsigned max_iter):r(copyable), p(r), ap(r), max_iter(max_iter){}
     /**
      * @brief Set the maximum number of iterations 
      *
@@ -70,7 +70,7 @@ class CG
      * @param copyable
      * @param max_iterations
      */
-    void construct( const Vector& copyable, unsigned max_iterations) { 
+    void construct( const container& copyable, unsigned max_iterations) { 
         ap = p = r = copyable;
         max_iter = max_iterations;
     }
@@ -79,40 +79,31 @@ class CG
      *
      * The iteration stops if \f$ ||Ax|| < \epsilon( ||b|| + C) \f$ where \f$C\f$ is 
      * a correction factor to the absolute error
-     @tparam Matrix The matrix class: no requirements except for the 
-            BLAS routines
-     @tparam Preconditioner no requirements except for the blas routines. Thus far the dg library
-        provides only diagonal preconditioners, which should be enough if the result is extrapolated from
-        previous timesteps.
-     * In every iteration the following BLAS functions are called: \n
-       symv 1x, dot 1x, axpby 2x, Prec. dot 1x, Prec. symv 1x
+     * @copydoc hide_symmetric_op
+     * @tparam Preconditioner A class for which the blas2::symv() and 
+     blas2::dot( const Matrix&, const Vector&) functions are callable. This can for example be one of the container classes (diagonal preconditioner), 
+     which should be enough if the initial guess is extrapolated from previous timesteps.
      * @param A A symmetric positive definit matrix
      * @param x Contains an initial value on input and the solution on output.
      * @param b The right hand side vector. x and b may be the same vector.
      * @param P The preconditioner to be used
      * @param eps The relative error to be respected
      * @param nrmb_correction Correction factor C for norm of b
-     * @attention This versions uses the Preconditioner to compute the norm for the error condition (this safes one scalar product)
+     * @attention This version uses the Preconditioner to compute the norm for the error condition (this safes one scalar product)
      *
      * @return Number of iterations used to achieve desired precision
      */
-    template< class Matrix, class Preconditioner >
-    unsigned operator()( Matrix& A, Vector& x, const Vector& b, Preconditioner& P , value_type eps = 1e-12, value_type nrmb_correction = 1);
+    template< class SymmetricOp, class Preconditioner >
+    unsigned operator()( SymmetricOp& A, container& x, const container& b, Preconditioner& P , value_type eps = 1e-12, value_type nrmb_correction = 1);
     //version of CG where Preconditioner is not trivial
     /**
      * @brief Solve the system A*x = b using a preconditioned conjugate gradient method
      *
      * The iteration stops if \f$ ||Ax||_S < \epsilon( ||b||_S + C) \f$ where \f$C\f$ is 
      * a correction factor to the absolute error and \f$ S \f$ defines a square norm
-     @tparam Matrix The matrix class: no requirements except for the 
-            BLAS routines
-     @tparam Preconditioner no requirements except for the blas routines. Thus far the dg library
-        provides only diagonal preconditioners, which should be enough if the result is extrapolated from
-        previous timesteps.
-     @tparam SquareNorm  (usually is the same as the container class)
-
-     * In every iteration the following BLAS functions are called: \n
-       symv 1x, dot 1x, axpby 2x, Prec. dot 1x, Prec. symv 1x
+     * @copydoc hide_symmetric_op
+     * @tparam Preconditioner A type for which the blas2::symv(Matrix&, Vector1&, Vector2&) function is callable. 
+     * @tparam SquareNorm A type for which the blas2::dot( const Matrix&, const Vector&) function is callable. This can e.g. be one of the container types.
      * @param A A symmetric positive definit matrix
      * @param x Contains an initial value on input and the solution on output.
      * @param b The right hand side vector. x and b may be the same vector.
@@ -123,10 +114,10 @@ class CG
      *
      * @return Number of iterations used to achieve desired precision
      */
-    template< class Matrix, class Preconditioner, class SquareNorm >
-    unsigned operator()( Matrix& A, Vector& x, const Vector& b, Preconditioner& P, SquareNorm& S, value_type eps = 1e-12, value_type nrmb_correction = 1);
+    template< class SymmetricOp, class Preconditioner, class SquareNorm >
+    unsigned operator()( SymmetricOp& A, container& x, const container& b, Preconditioner& P, SquareNorm& S, value_type eps = 1e-12, value_type nrmb_correction = 1);
   private:
-    Vector r, p, ap; 
+    container r, p, ap; 
     unsigned max_iter;
 };
 
@@ -146,9 +137,9 @@ class CG
     NOTE: the same comparison hold for A with the result that A contains 
     significantly more elements than z whence ddot(r,A,r) is far slower than ddot(r,z)
 */
-template< class Vector>
+template< class container>
 template< class Matrix, class Preconditioner>
-unsigned CG< Vector>::operator()( Matrix& A, Vector& x, const Vector& b, Preconditioner& P, value_type eps, value_type nrmb_correction)
+unsigned CG< container>::operator()( Matrix& A, container& x, const container& b, Preconditioner& P, value_type eps, value_type nrmb_correction)
 {
     value_type nrmb = sqrt( blas2::dot( P, b));
 #ifdef DG_DEBUG
@@ -209,9 +200,9 @@ unsigned CG< Vector>::operator()( Matrix& A, Vector& x, const Vector& b, Precond
     return max_iter;
 }
 
-template< class Vector>
+template< class container>
 template< class Matrix, class Preconditioner, class SquareNorm>
-unsigned CG< Vector>::operator()( Matrix& A, Vector& x, const Vector& b, Preconditioner& P, SquareNorm& S, value_type eps, value_type nrmb_correction)
+unsigned CG< container>::operator()( Matrix& A, container& x, const container& b, Preconditioner& P, SquareNorm& S, value_type eps, value_type nrmb_correction)
 {
     value_type nrmb = sqrt( blas2::dot( S, b));
 #ifdef DG_DEBUG
@@ -277,7 +268,7 @@ unsigned CG< Vector>::operator()( Matrix& A, Vector& x, const Vector& b, Precond
  * by appropriate weights \f$W\f$ (s. comment below). 
  * It uses solutions from the last two calls to 
  * extrapolate a solution for the current call.
- * @tparam container The Vector class to be used
+ * @copydoc hide_container_lvl1
  * @note A note on weights, inverse weights and preconditioning. 
  * A normalized DG-discretized derivative or operator is normally not symmetric. 
  * The diagonal coefficient matrix that is used to make the operator 
@@ -400,10 +391,7 @@ struct Invert
      * Solves the Equation \f[ \hat O \phi = W\rho \f] using a preconditioned 
      * conjugate gradient method. The initial guess comes from an extrapolation 
      * of the last solutions
-     * @tparam SymmetricOp Symmetric operator with the SelfMadeMatrixTag
-        The functions weights() and precond() need to be callable and return
-        weights and the preconditioner for the conjugate gradient method.
-        The Operator is assumed to be symmetric!
+     * @copydoc hide_symmetric_op
      * @param op selfmade symmetric Matrix operator class
      * @param phi solution (write only)
      * @param rho right-hand-side
@@ -425,9 +413,9 @@ struct Invert
      * Solves the Equation \f[ \hat O \phi = W\rho \f] using a preconditioned 
      * conjugate gradient method. The initial guess comes from an extrapolation 
      * of the last solutions.
-     * @tparam SymmetricOp Symmetric matrix or operator (with the selfmade tag)
-     * @tparam Preconditioner class of the Preconditioner
-     * @param op selfmade symmetric Matrix operator class
+     * @copydoc hide_symmetric_op
+     * @tparam Preconditioner A type for which the blas2::symv(Matrix&, Vector1&, Vector2&) function is callable. 
+     * @param op symmetric Matrix operator class
      * @param phi solution (write only)
      * @param rho right-hand-side
      * @param w The weights that made the operator symmetric
@@ -450,9 +438,9 @@ struct Invert
      * Solves the Equation \f[ \hat O \phi = W\rho \f] using a preconditioned 
      * conjugate gradient method. The initial guess comes from an extrapolation 
      * of the last solutions.
-     * @tparam SymmetricOp Symmetric matrix or operator (with the selfmade tag)
-     * @tparam Preconditioner class of the Preconditioner
-     * @param op selfmade symmetric Matrix operator class
+     * @copydoc hide_symmetric_op
+     * @tparam Preconditioner A type for which the blas2::symv(Matrix&, Vector1&, Vector2&) function is callable. 
+     * @param op symmetric Matrix operator
      * @param phi solution (write only)
      * @param rho right-hand-side
      * @param w The weights that made the operator symmetric
@@ -517,8 +505,8 @@ struct Invert
  *
  * The inverse is computed with a conjugate gradient method
  * @ingroup invert
- * @tparam SymmetricOp A symmetric Matrix type
- * @tparam container The container type to use
+ * @copydoc hide_symmetric_op
+ * @copydoc hide_container_lvl1
  */
 template<class SymmetricOp, class container>
 struct Inverse
@@ -562,7 +550,7 @@ struct MatrixTraits< Inverse< M, V > >
  *
  * @ingroup invert 
  * @tparam Matrix Matrix type
- * @tparam Vector Vector type (must be constructible from given size)
+ * @copydoc hide_container_lvl1
  * @tparam Preconditioner Preconditioner type
  * @param A Matrix 
  * @param x contains initial guess on input and solution on output
@@ -574,10 +562,10 @@ struct MatrixTraits< Inverse< M, V > >
  * @return number of iterations
  */
 /*
-template< class Matrix, class Vector, class Preconditioner>
-unsigned cg( Matrix& A, Vector& x, const Vector& b, const Preconditioner& P, typename VectorTraits<Vector>::value_type eps, unsigned max_iter)
+template< class Matrix, class container, class Preconditioner>
+unsigned cg( Matrix& A, container& x, const container& b, const Preconditioner& P, typename VectorTraits<container>::value_type eps, unsigned max_iter)
 {
-    typedef typename VectorTraits<Vector>::value_type value_type;
+    typedef typename VectorTraits<container>::value_type value_type;
     value_type nrmb = sqrt( blas2::dot( P, b)); //norm of b
 #ifdef DG_DEBUG
 #ifdef MPI_VERSION
@@ -595,7 +583,7 @@ unsigned cg( Matrix& A, Vector& x, const Vector& b, const Preconditioner& P, typ
         blas1::axpby( 1., b, 0., x); //x=b
         return 0;
     }
-    Vector r(x.size()), p(x.size()), ap(x.size()); //1% time at 20 iterations
+    container r(x.size()), p(x.size()), ap(x.size()); //1% time at 20 iterations
     //r = b; blas2::symv( -1., A, x, 1.,r); //compute r_0 
     blas2::symv( A,x,r); //r=A x
     blas1::axpby( 1., b, -1., r); //r=b-Ax
diff --git a/inc/dg/cluster_mpib.cu b/inc/dg/cluster_mpib.cu
index 7be1a67ef..a0d5c4f0c 100644
--- a/inc/dg/cluster_mpib.cu
+++ b/inc/dg/cluster_mpib.cu
@@ -6,7 +6,6 @@
 #include <omp.h>
 #endif//_OPENMP
 #include "algorithm.h"
-#include "geometry.h"
 
 #include "backend/timer.cuh"
 #include "../geometries/guenther.h"
@@ -157,10 +156,10 @@ int main(int argc, char* argv[])
     periods[0] = false, periods[1] = false;
     MPI_Comm commEll;
     MPI_Cart_create( MPI_COMM_WORLD, 3, np, periods, true, &commEll);
-    dg::CylindricalMPIGrid3d<Vector> gridEll( R_0, R_0+lx, 0., ly, 0.,lz, n, Nx, Ny,Nz, dg::DIR, dg::DIR, dg::PER, commEll);
+    dg::CylindricalMPIGrid3d gridEll( R_0, R_0+lx, 0., ly, 0.,lz, n, Nx, Ny,Nz, dg::DIR, dg::DIR, dg::PER, commEll);
     const Vector ellw3d = dg::create::volume(gridEll);
     const Vector ellv3d = dg::create::inv_volume(gridEll);
-    dg::Elliptic<dg::CylindricalMPIGrid3d<Vector>, Matrix, Vector> laplace(gridEll, dg::not_normed, dg::centered);
+    dg::Elliptic<dg::CylindricalMPIGrid3d, Matrix, Vector> laplace(gridEll, dg::not_normed, dg::centered);
     const Vector solution = dg::evaluate ( fct, gridEll);
     const Vector deriv = dg::evaluate( derivative, gridEll);
     Vector x = dg::evaluate( initial, gridEll);
@@ -181,7 +180,7 @@ int main(int argc, char* argv[])
         double Zmin=-1.0*gpa*1.00;
         double Rmax=gpR0+1.0*gpa; 
         double Zmax=1.0*gpa*1.00;
-        dg::CylindricalMPIGrid3d<Vector> g3d( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, n, Nx ,Ny, Nz,dg::DIR, dg::DIR, dg::PER,commEll);
+        dg::CylindricalMPIGrid3d g3d( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, n, Nx ,Ny, Nz,dg::DIR, dg::DIR, dg::PER,commEll);
         dg::geo::guenther::MagneticField magfield(gpR0, gpI0);
         dg::geo::Field<dg::geo::guenther::MagneticField> field(magfield, gpR0);
         dg::MDDS::FieldAligned dsFA( field, g3d, 1e-4, dg::DefaultLimiter(), dg::DIR);
diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index df296a172..61ea5ab27 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -101,31 +101,26 @@
  * 
  */
 
-/**
-  * @class hide_binary
+/** @class hide_binary
   * @tparam BinaryOp A class or function type with a member/signature equivalent to
   *  - double operator()(double, double) const
   */
-/**
-  * @class hide_ternary
+/** @class hide_ternary
   * @tparam TernaryOp A class or function type with a member/signature equivalent to
   *  - double operator()(double, double, double) const
   */
- /**
-  * @class hide_geometry
+ /** @class hide_geometry
   * @tparam Geometry One of the geometry classes. The functions dg::create::dx( g, bcx) and
   * dg::create::dy( g, bcy) must be callable and return an instance of the Matrix class. 
   * Furthermore dg::evaluate( one, g) must return an instance of the container class.
      as do calls to dg::create::weights(g) and dg::create::inv_weights(g)
   */
 
- /**
-  * @class hide_container_lvl1
-  * @tparam container A data container class for which the blas1 functionality is overloaded. Currently this is one of 
+ /** @class hide_container_lvl1
+  * @tparam container A data container class for which the blas1 functionality is overloaded. Also we assume that the type is copyable/assignable. Currently this is one of 
   *   dg::HVec, dg::DVec, dg::MHVec or dg::MDVec
   */
- /**
-  * @class hide_matrix_container
+ /** @class hide_matrix_container
   * @tparam Matrix A class for which the blas2 functions are callable in connection with the container class
   *  - dg::HMatrix with dg::HVec
   *  - dg::DMatrix with dg::DVec
@@ -136,6 +131,15 @@
   *   dg::HVec, dg::DVec, dg::MHVec or dg::MDVec
   */
 
+ /** @class hide_symmetric_op
+ * @tparam SymmetricOp A class for which the blas2::symv(Matrix&, Vector1&, Vector2&) function is callable 
+ with the container type as argument. Also, The functions %weights() and %precond() 
+ need to be callable and return weights and the preconditioner for the conjugate 
+ gradient method. The Operator is assumed to be linear and symmetric!
+ @note you can make your own SymmetricOp by providing the member function void symv(const container&, container&);
+  and specializing MatrixTraits with the SelfMadeMatrixTag as the matrix_category
+  */
+
 /**
  * @brief Struct that performs collective scatter and gather operations across processes
  * on distributed vectors using mpi
diff --git a/inc/dg/helmholtz_t.cu b/inc/dg/helmholtz_t.cu
index ec5d80ef7..2d8a41263 100644
--- a/inc/dg/helmholtz_t.cu
+++ b/inc/dg/helmholtz_t.cu
@@ -97,8 +97,8 @@ int main()
     dg::DVec laplace_fct_ = dg::evaluate( laplace_fct, g3d);
     dg::DVec helmholtz_fct_ = dg::evaluate( helmholtz_fct, g3d);
     dg::DVec temp_(fct_);
-    dg::Elliptic< dg::CylindricalGrid3d<dg::DVec>, dg::DMatrix, dg::DVec > laplaceM( g3d, dg::normed);
-    dg::Helmholtz< dg::CylindricalGrid3d<dg::DVec>, dg::DMatrix, dg::DVec > helmholtz( g3d, alpha);
+    dg::Elliptic< dg::CylindricalGrid3d, dg::DMatrix, dg::DVec > laplaceM( g3d, dg::normed);
+    dg::Helmholtz< dg::CylindricalGrid3d, dg::DMatrix, dg::DVec > helmholtz( g3d, alpha);
     dg::blas2::symv( laplaceM, fct_, temp_);
     dg::blas1::axpby( 1., laplace_fct_, -1., temp_);
     std::cout << "error Laplace " << sqrt( dg::blas2::dot( laplaceM.weights(), temp_))<<" (Note the supraconvergence!)"<<std::endl;
diff --git a/inc/dg/multistep.h b/inc/dg/multistep.h
index 6b6b6f1fc..5b11bc303 100644
--- a/inc/dg/multistep.h
+++ b/inc/dg/multistep.h
@@ -35,18 +35,18 @@ const double ab_coeff<5>::b[5] = {1901./720., -1387./360., 109./30., -637./360.,
 * Uses only blas1::axpby routines to integrate one step
 * and only one right-hand-side evaluation per step.
 * @tparam k Order of the method (Currently one of 1, 2, 3, 4 or 5)
-* @tparam Vector The Argument type used in the Functor class
+* @copydoc hide_container_lvl1
 */
-template< size_t k, class Vector>
+template< size_t k, class container>
 struct AB
 {
     /**
     * @brief Reserve memory for the integration
     *
-    * @param copyable Vector of size which is used in integration. 
-    * A Vector object must be copy-constructible from copyable.
+    * @param copyable container of size which is used in integration. 
+    * A container object must be copy-constructible from copyable.
     */
-    AB( const Vector& copyable): f_(k, Vector(copyable)), u_(copyable){ }
+    AB( const container& copyable): f_(k, container(copyable)), u_(copyable){ }
    
     /**
      * @brief Init with initial value
@@ -55,7 +55,7 @@ struct AB
      * backwards with a Euler method. This routine has to be called
      * before the first timestep is made and with the same initial value as the first timestep.
      * @tparam Functor models BinaryFunction with no return type (subroutine).
-        Its arguments both have to be of type Vector.
+        Its arguments both have to be of type container.
         The first argument is the actual argument, the second contains
         the return value, i.e. y' = f(y) translates to f( y, y').
      * @param f The rhs functor
@@ -64,12 +64,12 @@ struct AB
      * @note The class allows Functor to change its first (input) argument, i.e. the first argument need not be const
      */
     template< class Functor>
-    void init( Functor& f, const Vector& u0, double dt);
+    void init( Functor& f, const container& u0, double dt);
     /**
     * @brief Advance u0 one timestep
     *
     * @tparam Functor models BinaryFunction with no return type (subroutine)
-        Its arguments both have to be of type Vector.
+        Its arguments both have to be of type container.
         The first argument is the actual argument, The second contains
         the return value, i.e. y' = f(y) translates to f( y, y').
     * @param f right hand side function or functor
@@ -77,19 +77,19 @@ struct AB
      * @note The class allows Functor to change its first (input) argument, i.e. the first argument need not be const
     */
     template< class Functor>
-    void operator()( Functor& f, Vector& u);
+    void operator()( Functor& f, container& u);
   private:
     double dt_;
-    std::vector<Vector> f_; //TODO std::array is more natural here (but unfortunately not available)
-    Vector u_;
+    std::vector<container> f_; //TODO std::array is more natural here (but unfortunately not available)
+    container u_;
 };
 
-template< size_t k, class Vector>
+template< size_t k, class container>
 template< class Functor>
-void AB<k, Vector>::init( Functor& f, const Vector& u0,  double dt)
+void AB<k, container>::init( Functor& f, const container& u0,  double dt)
 {
     dt_ = dt;
-    Vector u1(u0), u2(u0);
+    container u1(u0), u2(u0);
     blas1::axpby( 1., u0, 0, u_);
     f( u1, f_[0]);
     for( unsigned i=1; i<k; i++)
@@ -100,9 +100,9 @@ void AB<k, Vector>::init( Functor& f, const Vector& u0,  double dt)
     }
 }
 
-template< size_t k, class Vector>
+template< size_t k, class container>
 template< class Functor>
-void AB<k, Vector>::operator()( Functor& f, Vector& u)
+void AB<k, container>::operator()( Functor& f, container& u)
 {
     blas1::axpby( 1., u_, 0, u);
     f( u, f_[0]);
@@ -116,15 +116,15 @@ void AB<k, Vector>::operator()( Functor& f, Vector& u)
 
 ///@cond
 //Euler specialisation
-template < class Vector>
-struct AB<1, Vector>
+template < class container>
+struct AB<1, container>
 {
     AB(){}
-    AB( const Vector& copyable):temp_(2, copyable){}
+    AB( const container& copyable):temp_(2, copyable){}
     template < class Functor>
-    void init( Functor& f, const Vector& u0, double dt){ dt_=dt;}
+    void init( Functor& f, const container& u0, double dt){ dt_=dt;}
     template < class Functor>
-    void operator()( Functor& f, Vector& u)
+    void operator()( Functor& f, container& u)
     {
         blas1::axpby( 1., u, 0, temp_[0]);
         f( u, temp_[1]);
@@ -132,7 +132,7 @@ struct AB<1, Vector>
     }
     private:
     double dt_;
-    std::vector<Vector> temp_;
+    std::vector<container> temp_;
 };
 ///@endcond
 ///@cond
@@ -184,26 +184,33 @@ struct MatrixTraits< detail::Implicit<M, V> >
     \left( 1  - \frac{\Delta t}{\gamma_0}  \hat L\right)  v^{n+1} &= {\bar v}^n  
     \end{align}
     \f]
+
+    with
+    \f[
+    \alpha_0 = \frac{18}{11}\ \alpha_1 = -\frac{9}{11}\ \alpha_2 = \frac{2}{11} \\
+    \beta_0 = \frac{18}{11}\ \beta_1 = -\frac{18}{11}\ \beta_2 = \frac{6}{11} \\
+    \gamma_0 = \frac{11}{6} 
+\f]
 *
 * Uses blas1::axpby routines to integrate one step
 * and only one right-hand-side evaluation per step. 
 * Uses a conjugate gradient method for the implicit operator  
 * @ingroup time
-* @tparam Vector The Argument type used in the Functor class
+* @copydoc hide_container_lvl1
 */
-template<class Vector>
+template<class container>
 struct Karniadakis
 {
 
     /**
     * @brief Reserve memory for the integration
     *
-    * @param copyable Vector of size which is used in integration. 
+    * @param copyable container of size which is used in integration. 
     * @param max_iter parameter for cg
     * @param eps  parameter for cg
-    * A Vector object must be copy-constructible from copyable.
+    * A container object must be copy-constructible from copyable.
     */
-    Karniadakis( const Vector& copyable, unsigned max_iter, double eps): u_(3, Vector(copyable)), f_(3, Vector(copyable)), pcg( copyable, max_iter), eps_(eps){
+    Karniadakis( const container& copyable, unsigned max_iter, double eps): u_(3, container(copyable)), f_(3, container(copyable)), pcg( copyable, max_iter), eps_(eps){
         //a[0] =  1.908535476882378;  b[0] =  1.502575553858997;
         //a[1] = -1.334951446162515;  b[1] = -1.654746338401493;
         //a[2] =  0.426415969280137;  b[2] =  0.670051276940255;
@@ -216,11 +223,11 @@ struct Karniadakis
      * @brief Initialize with initial value
      *
      * @tparam Functor models BinaryFunction with no return type (subroutine)
-        Its arguments both have to be of type Vector.
+        Its arguments both have to be of type container.
         The first argument is the actual argument, The second contains
         the return value, i.e. y' = f(y) translates to f( y, y').
      * @tparam LinearOp models BinaryFunction with no return type (subroutine)
-        Its arguments both have to be of type Vector. 
+        Its arguments both have to be of type container. 
         The first argument is the actual argument, The second contains
         the return value, i.e. y' = L(y) translates to diff( y, y').
         Furthermore the routines weights() and precond() must be callable
@@ -232,17 +239,17 @@ struct Karniadakis
      * @note Both Functor and LinearOp may change their first (input) argument, i.e. the first argument need not be const
      */
     template< class Functor, class LinearOp>
-    void init( Functor& f, LinearOp& diff, const Vector& u0, double dt);
+    void init( Functor& f, LinearOp& diff, const container& u0, double dt);
 
     /**
     * @brief Advance u for one timestep
     *
     * @tparam Functor models BinaryFunction with no return type (subroutine)
-        Its arguments both have to be of type Vector.
+        Its arguments both have to be of type container.
         The first argument is the actual argument, The second contains
         the return value, i.e. y' = f(y) translates to f( y, y').
     * @tparam LinearOp models BinaryFunction with no return type (subroutine)
-        Its arguments both have to be of type Vector.
+        Its arguments both have to be of type container.
         The first argument is the actual argument, The second contains
         the return value, i.e. y' = L(y) translates to diff( y, y').
         Furthermore the routines weights() and precond() must be callable
@@ -254,7 +261,7 @@ struct Karniadakis
      * @note Both Functor and LinearOp may change their first (input) argument, i.e. the first argument need not be const
     */
     template< class Functor, class LinearOp>
-    void operator()( Functor& f, LinearOp& diff, Vector& u);
+    void operator()( Functor& f, LinearOp& diff, container& u);
 
 
     /**
@@ -262,16 +269,16 @@ struct Karniadakis
      *
      * @return current head
      */
-    const Vector& head()const{return u_[0];}
+    const container& head()const{return u_[0];}
     /**
      * @brief return the last vector for which f was called
      *
      * @return current head^
      */
-    const Vector& last()const{return u_[1];}
+    const container& last()const{return u_[1];}
   private:
-    std::vector<Vector> u_, f_; 
-    CG< Vector> pcg;
+    std::vector<container> u_, f_; 
+    CG< container> pcg;
     double eps_;
     double dt_;
     double a[3];
@@ -280,13 +287,13 @@ struct Karniadakis
 };
 
 ///@cond
-template< class Vector>
+template< class container>
 template< class Functor, class Diffusion>
-void Karniadakis<Vector>::init( Functor& f, Diffusion& diff,  const Vector& u0,  double dt)
+void Karniadakis<container>::init( Functor& f, Diffusion& diff,  const container& u0,  double dt)
 {
     dt_ = dt;
-    Vector temp_(u0);
-    detail::Implicit<Diffusion, Vector> implicit( -dt, diff, temp_);
+    container temp_(u0);
+    detail::Implicit<Diffusion, container> implicit( -dt, diff, temp_);
     blas1::axpby( 1., u0, 0, temp_); //copy u0
     f( temp_, f_[0]);
     blas1::axpby( 1., u0, 0, u_[0]); 
@@ -300,9 +307,9 @@ void Karniadakis<Vector>::init( Functor& f, Diffusion& diff,  const Vector& u0,
     f( temp_, f_[2]);
 }
 
-template<class Vector>
+template<class container>
 template< class Functor, class Diffusion>
-void Karniadakis<Vector>::operator()( Functor& f, Diffusion& diff, Vector& u)
+void Karniadakis<container>::operator()( Functor& f, Diffusion& diff, container& u)
 {
 
     blas1::axpby( 1., u_[0], 0, u); //save u_[0]
@@ -323,7 +330,7 @@ void Karniadakis<Vector>::operator()( Functor& f, Diffusion& diff, Vector& u)
     //double alpha[2] = {1., 0.};
     blas1::axpby( alpha[0], u_[1], alpha[1],  u_[2], u_[0]); //extrapolate previous solutions
     blas2::symv( diff.weights(), u, u);
-    detail::Implicit<Diffusion, Vector> implicit( -dt_/11.*6., diff, f_[0]);
+    detail::Implicit<Diffusion, container> implicit( -dt_/11.*6., diff, f_[0]);
 #ifdef DG_BENCHMARK
 #ifdef MPI_VERSION
     int rank;
@@ -351,19 +358,19 @@ void Karniadakis<Vector>::operator()( Functor& f, Diffusion& diff, Vector& u)
  * @brief Semi implicit Runge Kutta method after Yoh and Zhong (AIAA 42, 2004)
  *
  * @ingroup time
- * @tparam Vector Vector class to use
+ * @copydoc hide_container_lvl1
  */
-template <class Vector>
+template <class container>
 struct SIRK
 {
     /**
-     * @brief Construct from copyable Vector
+     * @brief Construct from copyable container
      *
-     * @param copyable Vector of right size
+     * @param copyable container of right size
      * @param max_iter maximum iterations for conjugate gradient
      * @param eps error for conjugate gradient
      */
-    SIRK(const Vector& copyable, unsigned max_iter, double eps): k_(3, copyable), f_(copyable), g_(copyable), rhs( f_), pcg( copyable, max_iter), eps_(eps)
+    SIRK(const container& copyable, unsigned max_iter, double eps): k_(3, copyable), f_(copyable), g_(copyable), rhs( f_), pcg( copyable, max_iter), eps_(eps)
     {
         w[0] = 1./8., w[1] = 1./8., w[2] = 3./4.;
         b[1][0] = 8./7., b[2][0] = 71./252., b[2][1] = 7./36.;
@@ -382,10 +389,10 @@ struct SIRK
      * @param dt timestep
      */
     template <class Explicit, class Imp>
-    void operator()( Explicit& f, Imp& g, const Vector& u0, Vector& u1, double dt)
+    void operator()( Explicit& f, Imp& g, const container& u0, container& u1, double dt)
     {
-        Vector u0_ = u0;
-        detail::Implicit<Imp, Vector> implicit( -dt*d[0], g, f_);
+        container u0_ = u0;
+        detail::Implicit<Imp, container> implicit( -dt*d[0], g, f_);
         f(u0_, f_);
         u0_ = u0;
         g(u0_, g_);
@@ -434,9 +441,9 @@ struct SIRK
      * @param tolerance tolerable error
      */
     template <class Explicit, class Imp>
-    void adaptive_step( Explicit& f, Imp& g, const Vector& u0, Vector& u1, double& dt, double tolerance)
+    void adaptive_step( Explicit& f, Imp& g, const container& u0, container& u1, double& dt, double tolerance)
     {
-        Vector temp = u0;
+        container temp = u0;
         this->operator()( f, g, u0, u1, dt/2.);
         this->operator()( f, g, u1, temp, dt/2.);
         this->operator()( f, g, u0, u1, dt);
@@ -449,13 +456,13 @@ struct SIRK
         if( dt < 0.75*dt_old) dt = 0.75*dt_old;
     }
     private:
-    std::vector<Vector> k_;
-    Vector f_, g_, rhs;
+    std::vector<container> k_;
+    container f_, g_, rhs;
     double w[3];
     double b[3][3];
     double d[3];
     double c[3][3];
-    CG<Vector> pcg; 
+    CG<container> pcg; 
     double eps_;
 };
 
diff --git a/inc/dg/runge_kutta.h b/inc/dg/runge_kutta.h
index 251dbbec9..4584878d1 100644
--- a/inc/dg/runge_kutta.h
+++ b/inc/dg/runge_kutta.h
@@ -305,10 +305,10 @@ const double rk_classic<17>::b[17] = {
 0.0333333333333333333333333333333333333333333333333333333333333
 };
 ///@endcond
-//RHS contains Information about Vector type it uses
+//RHS contains Information about container type it uses
 //k is the order of the method
-// Vector f( const Vector& v)
-// Vector should probably be rvalue assignable
+// container f( const container& v)
+// container should probably be rvalue assignable
 
 /**
 * @brief Struct for Runge-Kutta explicit time-integration
@@ -334,23 +334,23 @@ const double rk_classic<17>::b[17] = {
  *  by ones on the left and \f$ D\f$ its
  *  diagonal part. 
 * @tparam k Order of the method (1, 2, 3 or 4)
-* @tparam Vector The argument type used in the Functor class
+* @copydoc hide_container_lvl1
 */
-template< size_t k, class Vector>
+template< size_t k, class container>
 struct RK
 {
     /**
     * @brief Reserve memory for the integration
     *
-    * @param copyable Vector of size which is used in integration. 
-    * A Vector object must be copy-constructible from copyable.
+    * @param copyable container of size which is used in integration. 
+    * A container object must be copy-constructible from copyable.
     */
-    RK( const Vector& copyable): u_(k-1, Vector(copyable)){ }
+    RK( const container& copyable): u_(k-1, container(copyable)){ }
     /**
     * @brief Advance u0 one timestep
     *
     * @tparam Functor models BinaryFunction with no return type (subroutine)
-        Its arguments both have to be of type Vector.
+        Its arguments both have to be of type container.
         The first argument is the actual argument, The second contains
         the return value, i.e. y' = f(y) translates to f( y, y').
     * @param f right hand side function
@@ -359,17 +359,17 @@ struct RK
     * @param dt The timestep.
     */
     template< class Functor>
-    void operator()( Functor& f, const Vector& u0, Vector& u1, double dt);
+    void operator()( Functor& f, const container& u0, container& u1, double dt);
   private:
-    std::vector<Vector> u_; //TODO std::array is more natural here (but unfortunately not available)
+    std::vector<container> u_; //TODO std::array is more natural here (but unfortunately not available)
 };
 
 //u0 and u1 may not be the same vector
 //TO DO: this might be cured by adding u0 first to u_[0] in the last step
 //f( y, yp) where y is const and yp contains the result
-template< size_t k, class Vector>
+template< size_t k, class container>
 template< class Functor>
-void RK<k, Vector>::operator()( Functor& f, const Vector& u0, Vector& u1, double dt)
+void RK<k, container>::operator()( Functor& f, const container& u0, container& u1, double dt)
 {
     assert( &u0 != &u1);
     u1 = u0;
@@ -409,23 +409,23 @@ void RK<k, Vector>::operator()( Functor& f, const Vector& u0, Vector& u1, double
 * The coefficients are chosen in the classic form given by Runge and Kutta. 
 * Needs more calls for axpby than our RK class but we implemented higher orders
 * @tparam s Order of the method (1, 2, 3, 4, 6, 17)
-* @tparam Vector The argument type used in the Functor class
+* @copydoc hide_container_lvl1
 */
-template< size_t s, class Vector>
+template< size_t s, class container>
 struct RK_classic
 {
     /**
     * @brief Reserve memory for the integration
     *
-    * @param copyable Vector of size which is used in integration. 
-    * A Vector object must be copy-constructible from copyable.
+    * @param copyable container of size which is used in integration. 
+    * A container object must be copy-constructible from copyable.
     */
-    RK_classic( const Vector& copyable): k_(s, Vector(copyable)), u_(copyable){ }
+    RK_classic( const container& copyable): k_(s, container(copyable)), u_(copyable){ }
     /**
     * @brief Advance u0 one timestep
     *
     * @tparam Functor models BinaryFunction with no return type (subroutine)
-        Its arguments both have to be of type Vector.
+        Its arguments both have to be of type container.
         The first argument is the actual argument, The second contains
         the return value, i.e. y' = f(y) translates to f( y, y').
     * @param f right hand side function
@@ -434,18 +434,18 @@ struct RK_classic
     * @param dt The timestep.
     */
     template< class Functor>
-    void operator()( Functor& f, const Vector& u0, Vector& u1, double dt);
+    void operator()( Functor& f, const container& u0, container& u1, double dt);
   private:
-    std::vector<Vector> k_;
-    Vector u_;
+    std::vector<container> k_;
+    container u_;
 };
 
 //u0 and u1 may not be the same vector
 //TO DO: this might be cured by adding u0 first to u_[0] in the last step
 //f( y, yp) where y is const and yp contains the result
-template< size_t s, class Vector>
+template< size_t s, class container>
 template< class Functor>
-void RK_classic<s, Vector>::operator()( Functor& f, const Vector& u0, Vector& u1, double dt)
+void RK_classic<s, container>::operator()( Functor& f, const container& u0, container& u1, double dt)
 {
     assert( &u0 != &u1);
     u1 = u0; //to let u0 const and later for summation
@@ -486,7 +486,7 @@ struct NotANumber : public std::exception
  *
  * @ingroup time 
  * @tparam RHS The right-hand side class
- * @tparam Vector Vector-class (needs to be copyable)
+ * @copydoc hide_container_lvl1
  * @tparam s # of stages (1, 2, 3, 4, 6, 17)
  * @param rhs The right-hand-side
  * @param begin initial condition (size 3)
@@ -495,11 +495,11 @@ struct NotANumber : public std::exception
  * @param T_max final time
  * @param N # of steps to use
  */
-template< class RHS, class Vector, unsigned s>
-void stepperRK(RHS& rhs, const Vector& begin, Vector& end, double T_min, double T_max, unsigned N )
+template< class RHS, class container, unsigned s>
+void stepperRK(RHS& rhs, const container& begin, container& end, double T_min, double T_max, unsigned N )
 {
-    RK_classic<s, Vector > rk( begin); 
-    Vector temp(begin);
+    RK_classic<s, container > rk( begin); 
+    container temp(begin);
     if( T_max == T_min){ end = begin; return;}
     double dt = (T_max-T_min)/(double)N;
     end = begin;
@@ -510,21 +510,21 @@ void stepperRK(RHS& rhs, const Vector& begin, Vector& end, double T_min, double
     }
 }
 
-template< class RHS, class Vector>
-void stepperRK1(RHS& rhs, const Vector& begin, Vector& end, double T_min, double T_max, unsigned N )
+template< class RHS, class container>
+void stepperRK1(RHS& rhs, const container& begin, container& end, double T_min, double T_max, unsigned N )
 {
-    stepperRK<RHS, Vector, 1>( rhs, begin, end, T_min, T_max, N);
+    stepperRK<RHS, container, 1>( rhs, begin, end, T_min, T_max, N);
 }
-template< class RHS, class Vector>
-void stepperRK2(RHS& rhs, const Vector& begin, Vector& end, double T_min, double T_max, unsigned N )
+template< class RHS, class container>
+void stepperRK2(RHS& rhs, const container& begin, container& end, double T_min, double T_max, unsigned N )
 {
-    stepperRK<RHS, Vector, 2>( rhs, begin, end, T_min, T_max, N);
+    stepperRK<RHS, container, 2>( rhs, begin, end, T_min, T_max, N);
 }
 
-template< class RHS, class Vector>
-void stepperRK3(RHS& rhs, const Vector& begin, Vector& end, double T_min, double T_max, unsigned N )
+template< class RHS, class container>
+void stepperRK3(RHS& rhs, const container& begin, container& end, double T_min, double T_max, unsigned N )
 {
-    stepperRK<RHS, Vector, 3>( rhs, begin, end, T_min, T_max, N);
+    stepperRK<RHS, container, 3>( rhs, begin, end, T_min, T_max, N);
 }
 
 /**
@@ -532,7 +532,7 @@ void stepperRK3(RHS& rhs, const Vector& begin, Vector& end, double T_min, double
  *
  * @ingroup time 
  * @tparam RHS The right-hand side class
- * @tparam Vector Vector-class (needs to be copyable)
+ * @copydoc hide_container_lvl1
  * @param rhs The right-hand-side
  * @param begin initial condition 
  * @param end (write-only) contains solution on output
@@ -540,17 +540,17 @@ void stepperRK3(RHS& rhs, const Vector& begin, Vector& end, double T_min, double
  * @param T_max final time
  * @param N number of steps 
  */
-template< class RHS, class Vector>
-void stepperRK4(RHS& rhs, const Vector& begin, Vector& end, double T_min, double T_max, unsigned N )
+template< class RHS, class container>
+void stepperRK4(RHS& rhs, const container& begin, container& end, double T_min, double T_max, unsigned N )
 {
-    stepperRK<RHS, Vector, 4>( rhs, begin, end, T_min, T_max, N);
+    stepperRK<RHS, container, 4>( rhs, begin, end, T_min, T_max, N);
 }
 /**
  * @brief Integrates the differential equation using RK 6 and a fixed number of steps
  *
  * @ingroup time 
  * @tparam RHS The right-hand side class
- * @tparam Vector Vector-class (needs to be copyable)
+ * @copydoc hide_container_lvl1
  * @param rhs The right-hand-side
  * @param begin initial condition 
  * @param end (write-only) contains solution on output
@@ -558,17 +558,17 @@ void stepperRK4(RHS& rhs, const Vector& begin, Vector& end, double T_min, double
  * @param T_max final time
  * @param N number of steps 
  */
-template< class RHS, class Vector>
-void stepperRK6(RHS& rhs, const Vector& begin, Vector& end, double T_min, double T_max, unsigned N )
+template< class RHS, class container>
+void stepperRK6(RHS& rhs, const container& begin, container& end, double T_min, double T_max, unsigned N )
 {
-    stepperRK<RHS, Vector, 6>( rhs, begin, end, T_min, T_max, N);
+    stepperRK<RHS, container, 6>( rhs, begin, end, T_min, T_max, N);
 }
 /**
  * @brief Integrates the differential equation using RK 17 and a fixed number of steps
  *
  * @ingroup time 
  * @tparam RHS The right-hand side class
- * @tparam Vector Vector-class (needs to be copyable)
+ * @copydoc hide_container_lvl1
  * @param rhs The right-hand-side
  * @param begin initial condition 
  * @param end (write-only) contains solution on output
@@ -576,10 +576,10 @@ void stepperRK6(RHS& rhs, const Vector& begin, Vector& end, double T_min, double
  * @param T_max final time
  * @param N number of steps 
  */
-template< class RHS, class Vector>
-void stepperRK17(RHS& rhs, const Vector& begin, Vector& end, double T_min, double T_max, unsigned N )
+template< class RHS, class container>
+void stepperRK17(RHS& rhs, const container& begin, container& end, double T_min, double T_max, unsigned N )
 {
-    stepperRK<RHS, Vector, 17>( rhs, begin, end, T_min, T_max, N);
+    stepperRK<RHS, container, 17>( rhs, begin, end, T_min, T_max, N);
 }
 
 
@@ -590,20 +590,20 @@ void stepperRK17(RHS& rhs, const Vector& begin, Vector& end, double T_min, doubl
  *
  * Doubles the number of timesteps until the desired accuracy is reached
  *
- * @tparam RHS The right-hand side class. There must be the function bool monitor( const Vector& end); available which is called after every step. Return true if everything is ok and false if the integrator certainly fails.
- * The other function is the double error( const Vector& end0, const Vector& end1); which computes the error norm in which the integrator should converge. 
- * @tparam Vector Vector-class (needs to be copyable)
+ * @tparam RHS The right-hand side class. There must be the function bool monitor( const container& end); available which is called after every step. Return true if everything is ok and false if the integrator certainly fails.
+ * The other function is the double error( const container& end0, const container& end1); which computes the error norm in which the integrator should converge. 
+ * @copydoc hide_container_lvl1
  * @param rhs The right-hand-side
  * @param begin initial condition (size 3)
  * @param end (write-only) contains solution on output
  * @param T_max time difference
  * @param eps_abs desired absolute accuracy
  */
-template< class RHS, class Vector, unsigned s>
-int integrateRK(RHS& rhs, const Vector& begin, Vector& end, double T_max, double eps_abs )
+template< class RHS, class container, unsigned s>
+int integrateRK(RHS& rhs, const container& begin, container& end, double T_max, double eps_abs )
 {
-    RK_classic<s, Vector > rk( begin); 
-    Vector old_end(begin), temp(begin);
+    RK_classic<s, container > rk( begin); 
+    container old_end(begin), temp(begin);
     end = begin;
     if( T_max == 0) return 0;
     double dt = T_max/10;
@@ -657,34 +657,34 @@ int integrateRK(RHS& rhs, const Vector& begin, Vector& end, double T_max, double
 
 }
 
-template< class RHS, class Vector>
-int integrateRK4(RHS& rhs, const Vector& begin, Vector& end, double T_max, double eps_abs )
+template< class RHS, class container>
+int integrateRK4(RHS& rhs, const container& begin, container& end, double T_max, double eps_abs )
 {
-    return integrateRK<RHS, Vector, 4>( rhs, begin, end, T_max, eps_abs);
+    return integrateRK<RHS, container, 4>( rhs, begin, end, T_max, eps_abs);
 }
 
-template< class RHS, class Vector>
-int integrateRK6(RHS& rhs, const Vector& begin, Vector& end, double T_max, double eps_abs )
+template< class RHS, class container>
+int integrateRK6(RHS& rhs, const container& begin, container& end, double T_max, double eps_abs )
 {
-    return integrateRK<RHS, Vector, 6>( rhs, begin, end, T_max, eps_abs);
+    return integrateRK<RHS, container, 6>( rhs, begin, end, T_max, eps_abs);
 }
-template< class RHS, class Vector>
-int integrateRK17(RHS& rhs, const Vector& begin, Vector& end, double T_max, double eps_abs )
+template< class RHS, class container>
+int integrateRK17(RHS& rhs, const container& begin, container& end, double T_max, double eps_abs )
 {
-    return integrateRK<RHS, Vector, 17>( rhs, begin, end, T_max, eps_abs);
+    return integrateRK<RHS, container, 17>( rhs, begin, end, T_max, eps_abs);
 }
 
 ///@}
 //
 ///@cond
 //Euler specialisation
-template < class Vector>
-struct RK<1, Vector>
+template < class container>
+struct RK<1, container>
 {
     RK(){}
-    RK( const Vector& copyable){}
+    RK( const container& copyable){}
     template < class Functor>
-    void operator()( Functor& f, const Vector& u0, Vector& u1, double dt)
+    void operator()( Functor& f, const container& u0, container& u1, double dt)
     {
         f( u0, u1);
         blas1::axpby( 1., u0, dt, u1);
-- 
GitLab


From 27805935852863405b6bdf2f1488bf073e6e6730 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sat, 19 Aug 2017 13:37:44 +0200
Subject: [PATCH 186/453] small corrections in guenther.h

---
 inc/dg/cluster_mpib.cu    |  2 +-
 inc/geometries/guenther.h | 12 ++++++------
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/inc/dg/cluster_mpib.cu b/inc/dg/cluster_mpib.cu
index a0d5c4f0c..3d8e3a906 100644
--- a/inc/dg/cluster_mpib.cu
+++ b/inc/dg/cluster_mpib.cu
@@ -181,7 +181,7 @@ int main(int argc, char* argv[])
         double Rmax=gpR0+1.0*gpa; 
         double Zmax=1.0*gpa*1.00;
         dg::CylindricalMPIGrid3d g3d( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, n, Nx ,Ny, Nz,dg::DIR, dg::DIR, dg::PER,commEll);
-        dg::geo::guenther::MagneticField magfield(gpR0, gpI0);
+        dg::geo::TokamakMagneticField magfield = dg::geo::guenther::createMagField(gpR0, gpI0);
         dg::geo::Field<dg::geo::guenther::MagneticField> field(magfield, gpR0);
         dg::MDDS::FieldAligned dsFA( field, g3d, 1e-4, dg::DefaultLimiter(), dg::DIR);
         dg::MDDS ds ( dsFA, field, dg::not_normed, dg::centered);
diff --git a/inc/geometries/guenther.h b/inc/geometries/guenther.h
index 70195f22f..54a7ee7ca 100644
--- a/inc/geometries/guenther.h
+++ b/inc/geometries/guenther.h
@@ -137,19 +137,19 @@ struct IpolZ : public aCloneableBinaryFunctor<IpolZ>
     double do_compute(double R, double Z) const { return 0; }
 };
 
-BinaryFunctorsLvl2 createPsip( GeomParameters gp)
+BinaryFunctorsLvl2 createPsip( double R_0)
 {
-    BinaryFunctorsLvl2 psip( new Psip(gp), new PsipR(gp), new PsipZ(gp),new PsipRR(gp), new PsipRZ(gp), new PsipZZ(gp));
+    BinaryFunctorsLvl2 psip( new Psip(R_0), new PsipR(R_0), new PsipZ(R_0),new PsipRR(R_0), new PsipRZ(R_0), new PsipZZ(R_0));
     return psip;
 }
-BinaryFunctorsLvl1 createIpol( GeomParameters gp)
+BinaryFunctorsLvl1 createIpol( double I_0)
 {
-    BinaryFunctorsLvl1 ipol( new Ipol(gp), new IpolR(gp), new IpolZ(gp))
+    BinaryFunctorsLvl1 ipol( new Ipol(I_0), new IpolR(), new IpolZ());
     return ipol;
 }
-TokamakMagneticField createMagField( GeomParameters gp)
+TokamakMagneticField createMagField( double R_0, double I_0)
 {
-    return TokamakMagneticField( gp.R_0, createPsip(gp), createIpol(gp));
+    return TokamakMagneticField( R_0, createPsip(R_0), createIpol(I_0));
 }
 ///@}
 
-- 
GitLab


From 6b59c3d81859be6fc680ca90223b823556aca4ec Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sat, 19 Aug 2017 14:27:23 +0200
Subject: [PATCH 187/453] geometry_elliptic_b works again

---
 inc/file/nc_utilities.h               | 10 +--
 inc/geometries/geometry_elliptic_b.cu | 88 +++++++++------------------
 2 files changed, 34 insertions(+), 64 deletions(-)

diff --git a/inc/file/nc_utilities.h b/inc/file/nc_utilities.h
index 1a8347958..c2955acb3 100644
--- a/inc/file/nc_utilities.h
+++ b/inc/file/nc_utilities.h
@@ -182,7 +182,7 @@ int define_dimensions( int ncid, int* dimsIDs, int* tvarID, const dg::Grid1d& g)
  * @return if anything goes wrong it returns the netcdf code, else SUCCESS
  * @note File stays in define mode
  */
-int define_dimensions( int ncid, int* dimsIDs, const dg::Grid2d& g)
+int define_dimensions( int ncid, int* dimsIDs, const dg::aTopology2d& g)
 {
     dg::Grid1d gx( g.x0(), g.x1(), g.n(), g.Nx());
     dg::Grid1d gy( g.y0(), g.y1(), g.n(), g.Ny());
@@ -204,7 +204,7 @@ int define_dimensions( int ncid, int* dimsIDs, const dg::Grid2d& g)
  * @return if anything goes wrong it returns the netcdf code, else SUCCESS
  * @note File stays in define mode
  */
-int define_dimensions( int ncid, int* dimsIDs, int* tvarID, const dg::Grid2d& g)
+int define_dimensions( int ncid, int* dimsIDs, int* tvarID, const dg::aTopology2d& g)
 {
     dg::Grid1d gx( g.x0(), g.x1(), g.n(), g.Nx());
     dg::Grid1d gy( g.y0(), g.y1(), g.n(), g.Ny());
@@ -229,7 +229,7 @@ int define_dimensions( int ncid, int* dimsIDs, int* tvarID, const dg::Grid2d& g)
  * @return if anything goes wrong it returns the netcdf code, else SUCCESS
  * @note File stays in define mode
  */
-int define_limtime_xy( int ncid, int* dimsIDs, int size, int* tvarID, const dg::Grid2d& g)
+int define_limtime_xy( int ncid, int* dimsIDs, int size, int* tvarID, const dg::aTopology2d& g)
 {
     dg::Grid1d gx( g.x0(), g.x1(), g.n(), g.Nx());
     dg::Grid1d gy( g.y0(), g.y1(), g.n(), g.Ny());
@@ -251,7 +251,7 @@ int define_limtime_xy( int ncid, int* dimsIDs, int size, int* tvarID, const dg::
  * @return if anything goes wrong it returns the netcdf code, else SUCCESS
  * @note File stays in define mode
  */
-int define_dimensions( int ncid, int* dimsIDs, const dg::Grid3d& g)
+int define_dimensions( int ncid, int* dimsIDs, const dg::aTopology3d& g)
 {
     dg::Grid1d gx( g.x0(), g.x1(), g.n(), g.Nx());
     dg::Grid1d gy( g.y0(), g.y1(), g.n(), g.Ny());
@@ -275,7 +275,7 @@ int define_dimensions( int ncid, int* dimsIDs, const dg::Grid3d& g)
  * @return if anything goes wrong it returns the netcdf code, else SUCCESS
  * @note File stays in define mode
  */
-int define_dimensions( int ncid, int* dimsIDs, int* tvarID, const dg::Grid3d& g)
+int define_dimensions( int ncid, int* dimsIDs, int* tvarID, const dg::aTopology3d& g)
 {
     int retval;
     if( (retval = define_dimensions( ncid, &dimsIDs[1], g)) ){ return retval;}
diff --git a/inc/geometries/geometry_elliptic_b.cu b/inc/geometries/geometry_elliptic_b.cu
index 52bb5d904..4eab5637c 100644
--- a/inc/geometries/geometry_elliptic_b.cu
+++ b/inc/geometries/geometry_elliptic_b.cu
@@ -2,19 +2,14 @@
 
 #include "file/nc_utilities.h"
 
-#include "dg/geometry/refined_grid.h"
 #include "dg/backend/timer.cuh"
 #include "dg/backend/grid.h"
+#include "dg/geometry/geometry.h"
 #include "dg/elliptic.h"
-#include "dg/refined_elliptic.h"
 #include "dg/cg.h"
 
 #include "solovev.h"
-//#include "guenther.h"
-#include "conformal.h"
-#include "orthogonal.h"
-#include "refined_curvilinear.h"
-#include "refined_orthogonal.h"
+#include "guenther.h"
 #include "simple_orthogonal.h"
 #include "testfunctors.h"
 
@@ -29,9 +24,6 @@ int main(int argc, char**argv)
     std::cout << "Type psi_0 and psi_1\n";
     double psi_0, psi_1;
     std::cin >> psi_0>> psi_1;
-    std::cout << "Type new_n, multiple_x and multiple_y \n";
-    double n_ref, multiple_x, multiple_y;
-    std::cin >> n_ref>>multiple_x >> multiple_y;
     Json::Reader reader;
     Json::Value js;
     if( argc==1)
@@ -45,23 +37,17 @@ int main(int argc, char**argv)
         reader.parse(is,js,false);
     }
     //write parameters from file into variables
-    GeomParameters gp(js);
-    MagneticField c(gp);
+    dg::geo::solovev::GeomParameters gp(js);
+    dg::geo::TokamakMagneticField c = dg::geo::solovev::createMagField(gp);
     gp.display( std::cout);
     dg::Timer t;
-    Psip psip( gp); 
-    std::cout << "Psi min "<<psip(gp.R_0, 0)<<"\n";
+    std::cout << "Psi min "<<c.psip()(gp.R_0, 0)<<"\n";
     std::cout << "Constructing grid ... \n";
     t.tic();
-//     ConformalGrid3d<dg::DVec> g3d(gp, psi_0, psi_1, n, Nx, Ny,Nz, dg::DIR);
-//     ConformalGrid2d<dg::DVec> g2d = g3d.perp_grid();
-//     dg::Elliptic<ConformalGrid3d<dg::DVec>, dg::DMatrix, dg::DVec> pol( g3d, dg::not_normed, dg::centered);
-    dg::geo::SimpleOrthogonal<Psip, PsipR, PsipZ, LaplacePsip> 
-        generator( c.psip, c.psipR, c.psipZ, c.laplacePsip, psi_0, psi_1, gp.R_0, 0., 1);
-    dg::OrthogonalRefinedGrid3d<dg::DVec> g3d(multiple_x, multiple_y, generator, n_ref, n, Nx, Ny,Nz, dg::DIR);
-    dg::OrthogonalRefinedGrid2d<dg::DVec> g2d = g3d.perp_grid();
-    dg::Elliptic<dg::OrthogonalRefinedGrid2d<dg::DVec>, dg::DMatrix, dg::DVec> pol( g2d, dg::not_normed, dg::forward);
-    dg::RefinedElliptic<dg::OrthogonalRefinedGrid2d<dg::DVec>, dg::IDMatrix, dg::DMatrix, dg::DVec> pol_refined( g2d, dg::not_normed, dg::forward);
+    dg::geo::SimpleOrthogonal generator( c.get_psip(), psi_0, psi_1, gp.R_0, 0., 1);
+    dg::CurvilinearProductGrid3d g3d( generator, n, Nx, Ny,Nz, dg::DIR);
+    dg::CurvilinearGrid2d g2d = g3d.perp_grid();
+    dg::Elliptic<dg::CurvilinearGrid2d, dg::DMatrix, dg::DVec> pol( g2d, dg::not_normed, dg::forward);
     t.toc();
     std::cout << "Construction took "<<t.diff()<<"s\n";
     ///////////////////////////////////////////////////////////////////////////
@@ -69,7 +55,7 @@ int main(int argc, char**argv)
     file::NC_Error_Handle ncerr;
     ncerr = nc_create( "testE.nc", NC_NETCDF4|NC_CLOBBER, &ncid);
     int dim2d[2];
-    ncerr = file::define_dimensions(  ncid, dim2d, g2d.associated());
+    ncerr = file::define_dimensions(  ncid, dim2d, g2d);
     int coordsID[2], psiID, functionID, function2ID;
     ncerr = nc_def_var( ncid, "x_XYP", NC_DOUBLE, 2, dim2d, &coordsID[0]);
     ncerr = nc_def_var( ncid, "y_XYP", NC_DOUBLE, 2, dim2d, &coordsID[1]);
@@ -80,38 +66,25 @@ int main(int argc, char**argv)
     dg::HVec X( g2d.size()), Y(X); //P = dg::pullback( dg::coo3, g);
     for( unsigned i=0; i<g2d.size(); i++)
     {
-        X[i] = g2d.associated().r()[i];
-        Y[i] = g2d.associated().z()[i];
+        X[i] = g2d.map()[0][i];
+        Y[i] = g2d.map()[1][i];
     }
     ncerr = nc_put_var_double( ncid, coordsID[0], X.data());
     ncerr = nc_put_var_double( ncid, coordsID[1], Y.data());
     ///////////////////////////////////////////////////////////////////////////
-    dg::DVec x =    dg::evaluate( dg::zero, g2d.associated());
-    dg::DVec x_fine =    dg::evaluate( dg::zero, g2d);
-    //const dg::DVec b =    dg::pullback( dg::geo::EllipticDirNeuM<MagneticField>(c, gp.R_0, psi_0, psi_1, 440, -220, 40., 1), g2d.associated());
-    //const dg::DVec chi =  dg::pullback( dg::geo::BmodTheta<MagneticField>(c, gp.R_0), g2d.associated());
-    //const dg::DVec solution = dg::pullback( dg::geo::FuncDirNeu<MagneticField>(c,psi_0, psi_1, 440, -220, 40.,1 ), g2d.associated());
-    const dg::DVec b =    dg::pullback( dg::geo::EllipticDirPerM<MagneticField>(c, gp.R_0, psi_0, psi_1, 4), g2d.associated());
-    const dg::DVec chi =  dg::pullback( dg::geo::Bmodule<MagneticField>(c, gp.R_0), g2d.associated());
-    const dg::DVec solution = dg::pullback( dg::geo::FuncDirPer<MagneticField>(c, gp.R_0, psi_0, psi_1, 4), g2d.associated());
-    //const dg::DVec b =        dg::pullback( dg::geo::LaplacePsi(gp), g2d.associated());
-    //const dg::DVec bFINE =    dg::pullback( dg::geo::LaplacePsi(gp), g2d);
-    //const dg::DVec chi =      dg::pullback( dg::one, g2d.associated());
-    //const dg::DVec chiFINE =  dg::pullback( dg::one, g2d);
-    //const dg::DVec solution =     dg::pullback( psip, g2d.associated());
-    //const dg::DVec solutionFINE = dg::pullback( psip, g2d);
-    const dg::DVec vol3dFINE = dg::create::volume( g2d);
-    dg::HVec inv_vol3dFINE = dg::create::inv_weights( g2d);
-    const dg::DVec vol3d = dg::create::volume( g2d.associated());
-    const dg::DVec v3dFINE( inv_vol3dFINE);
-    const dg::IDMatrix Q = dg::create::interpolation( g2d);
-    const dg::IDMatrix P = dg::create::projection( g2d);
-    dg::DVec chi_fine = dg::evaluate( dg::zero, g2d), b_fine(chi_fine);
-    dg::blas2::gemv( Q, chi, chi_fine);
-    dg::blas2::gemv( Q, b, b_fine);
-    //pol.set_chi( chi);
-    pol.set_chi( chi_fine);
-    pol_refined.set_chi( chi_fine);
+    dg::DVec x = dg::evaluate( dg::zero, g2d);
+    //const dg::DVec b =    dg::pullback( dg::geo::EllipticDirNeuM(c, psi_0, psi_1, 440, -220, 40., 1), g2d);
+    //const dg::DVec chi =  dg::pullback( dg::geo::BmodTheta(c), g2d);
+    //const dg::DVec solution = dg::pullback( dg::geo::FuncDirNeu(c,psi_0, psi_1, 440, -220, 40.,1 ), g2d);
+    const dg::DVec b =    dg::pullback( dg::geo::EllipticDirPerM(c, psi_0, psi_1, 4), g2d);
+    const dg::DVec chi =  dg::pullback( dg::geo::Bmodule(c), g2d);
+    const dg::DVec solution = dg::pullback( dg::geo::FuncDirPer(c, psi_0, psi_1, 4), g2d);
+    //const dg::DVec b =        dg::pullback( dg::geo::LaplacePsi(gp), g2d);
+    //const dg::DVec chi =      dg::pullback( dg::one, g2d);
+    //const dg::DVec solution =     dg::pullback( psip, g2d);
+
+    const dg::DVec vol3d = dg::create::volume( g2d);
+    pol.set_chi( chi);
     //compute error
     dg::DVec error( solution);
     const double eps = 1e-10;
@@ -119,19 +92,16 @@ int main(int argc, char**argv)
     std::cout << eps<<"\t";
     t.tic();
     dg::Invert<dg::DVec > invert( x, n*n*Nx*Ny*Nz, eps);
-    dg::DVec bmod(b);
-    pol_refined.compute_rhs( b_fine, bmod);
-    unsigned number = invert(pol_refined, x,bmod);// vol3d, v3d );
-    //dg::Invert<dg::DVec > invert( x_fine, x_fine.size(), eps);
-    //unsigned number = invert(pol, x_fine ,b_fine, vol3dFINE, v3dFINE );
-    //dg::blas2::gemv( P, x_fine, x);
+    unsigned number = invert(pol, x,b);// vol3d, v3d );
     std::cout <<number<<"\t";
     t.toc();
     dg::blas1::axpby( 1.,x,-1., solution, error);
     double err = dg::blas2::dot( vol3d, error);
     const double norm = dg::blas2::dot( vol3d, solution);
     std::cout << sqrt( err/norm) << "\t";
-    dg::DVec gyy = g2d.g_xx(), gxx=g2d.g_yy(), vol = g2d.vol();
+
+    dg::SparseTensor<dg::DVec> metric = g2d.metric();
+    dg::DVec gyy = metric.value(1,1), gxx=metric.value(0,0), vol = dg::tensor::volume(metric).value();
     dg::blas1::transform( gxx, gxx, dg::SQRT<double>());
     dg::blas1::transform( gyy, gyy, dg::SQRT<double>());
     dg::blas1::pointwiseDot( gxx, vol, gxx);
-- 
GitLab


From 36024faf77c1c7219feaacb7c38060d4c54056c3 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sat, 19 Aug 2017 14:38:34 +0200
Subject: [PATCH 188/453] debugging flux_t

---
 inc/geometries/flux_t.cu                 | 25 +----------------
 inc/geometries/geometry_elliptic_b.cu    |  2 --
 inc/geometries/geometry_elliptic_mpib.cu | 35 ++++++++++--------------
 3 files changed, 16 insertions(+), 46 deletions(-)

diff --git a/inc/geometries/flux_t.cu b/inc/geometries/flux_t.cu
index 668ca6058..a414bc70f 100644
--- a/inc/geometries/flux_t.cu
+++ b/inc/geometries/flux_t.cu
@@ -76,8 +76,6 @@ int main( int argc, char* argv[])
     dg::geo::FluxGenerator flux( c.get_psip(), c.get_ipol(), psi_0, psi_1, gp.R_0, 0., 1);
     dg::CurvilinearProductGrid3d g3d(flux, n, Nx, Ny,Nz, dg::DIR);
     dg::CurvilinearGrid2d g2d = g3d.perp_grid();
-    //OrthogonalGrid3d g3d(gp, psi_0, psi_1, n, Nx, Ny,Nz, dg::DIR);
-    //OrthogonalGrid2d g2d = g3d.perp_grid();
     dg::Grid2d g2d_periodic(g2d.x0(), g2d.x1(), g2d.y0(), g2d.y1(), g2d.n(), g2d.Nx(), g2d.Ny()+1); 
     t.toc();
     std::cout << "Construction took "<<t.diff()<<"s"<<std::endl;
@@ -128,21 +126,6 @@ int main( int argc, char* argv[])
 
     std::cout << "Construction successful!\n";
 
-    ////compute error in volume element
-    //const dg::HVec f_ = g2d.f();
-    //dg::blas1::pointwiseDot( g2d.g_xx(), g2d.g_yy(), temp0);
-    //dg::blas1::pointwiseDot( g2d.g_xy(), g2d.g_xy(), temp1);
-    //dg::blas1::axpby( 1., temp0, -1., temp1, temp0); //temp0=1/g = g^xx g^yy - g^xy^2
-    //solovev::flux::FieldY fieldY(gp);
-    //dg::HVec fby = dg::pullback( fieldY, g2d);//?
-    //dg::blas1::pointwiseDot( f_, fby,fby);
-//  //       dg::blas1::scal( fby, 2.*M_PI);
-
-    //dg::blas1::pointwiseDot( fby, fby, temp1);
-    //dg::blas1::axpby( 1., temp1, -1., temp0, temp0); ////temp0= g_xx g_yy - g_xy^2 - g
-    //double error = sqrt( dg::blas2::dot( temp0, w3d, temp0)/dg::blas2::dot( temp1, w3d, temp1));
-    //std::cout<< "Rel Error in Determinant is "<<error<<"\n";
-
     dg::blas1::pointwiseDot( g_xx, g_yy, temp0);
     dg::blas1::pointwiseDot( g_xy, g_xy, temp1);
     dg::blas1::axpby( 1., temp0, -1., temp1, temp0);
@@ -154,11 +137,6 @@ int main( int argc, char* argv[])
     double error = sqrt(dg::blas2::dot( temp0, w3d, temp0)/dg::blas2::dot(vol_.value(), w3d, vol_.value()));
     std::cout << "Rel Consistency  of volume is "<<error<<"\n";
 
-    //dg::blas1::pointwiseDivide(ones,fby,temp1); //=sqrt(g)
-    //dg::blas1::axpby( 1., temp1, -1., vol_, temp0);
-    //error=sqrt(dg::blas2::dot( temp0, w3d, temp0))/sqrt( dg::blas2::dot(vol_, w3d, vol_));
-    //std::cout << "Rel Error of volume form is "<<error<<"\n";
-
     const dg::HVec vol = dg::create::volume( g3d);
     dg::HVec ones3d = dg::evaluate( dg::one, g3d);
     double volume = dg::blas1::dot( vol, ones3d);
@@ -167,13 +145,12 @@ int main( int argc, char* argv[])
     if( psi_0 < psi_1) gp.psipmax = psi_1, gp.psipmin = psi_0;
     else               gp.psipmax = psi_0, gp.psipmin = psi_1;
     dg::geo::Iris iris( c.psip(), gp.psipmin, gp.psipmax);
-    //dg::CylindricalGrid3d<dg::HVec> g3d( gp.R_0 -2.*gp.a, gp.R_0 + 2*gp.a, -2*gp.a, 2*gp.a, 0, 2*M_PI, 3, 2200, 2200, 1, dg::PER, dg::PER, dg::PER);
     dg::CartesianGrid2d g2dC( gp.R_0 -2.0*gp.a, gp.R_0 + 2.0*gp.a, -2.0*gp.a,2.0*gp.a,1, 2e3, 2e3, dg::PER, dg::PER);
     dg::HVec vec  = dg::evaluate( iris, g2dC);
     dg::HVec R  = dg::evaluate( dg::cooX2d, g2dC);
     dg::HVec g2d_weights = dg::create::volume( g2dC);
     double volumeRZP = 2.*M_PI*dg::blas2::dot( vec, g2d_weights, R);
-    std::cout << "volumeXYP is "<< volume<<std::endl;
+    std::cout << "volumeXYP is "<< volume   <<std::endl;
     std::cout << "volumeRZP is "<< volumeRZP<<std::endl;
     std::cout << "relative difference in volume is "<<fabs(volumeRZP - volume)/volume<<std::endl;
     std::cout << "Note that the error might also come from the volume in RZP!\n"; //since integration of jacobian is fairly good probably
diff --git a/inc/geometries/geometry_elliptic_b.cu b/inc/geometries/geometry_elliptic_b.cu
index 4eab5637c..7b857940d 100644
--- a/inc/geometries/geometry_elliptic_b.cu
+++ b/inc/geometries/geometry_elliptic_b.cu
@@ -14,8 +14,6 @@
 #include "testfunctors.h"
 
 
-using namespace dg::geo::solovev;
-
 int main(int argc, char**argv)
 {
     std::cout << "Type n, Nx, Ny, Nz\n";
diff --git a/inc/geometries/geometry_elliptic_mpib.cu b/inc/geometries/geometry_elliptic_mpib.cu
index 832bae736..9d521f947 100644
--- a/inc/geometries/geometry_elliptic_mpib.cu
+++ b/inc/geometries/geometry_elliptic_mpib.cu
@@ -19,8 +19,6 @@
 #include "testfunctors.h"
 
 
-using namespace dg::geo::solovev;
-
 int main(int argc, char**argv)
 {
     MPI_Init( &argc, &argv);
@@ -41,10 +39,9 @@ int main(int argc, char**argv)
         std::ifstream is(argv[1]);
         reader.parse(is,js,false);
     }
-    GeomParameters gp(js);
-    MagneticField c(gp);
-    Psip psip( gp); 
-    if(rank==0)std::cout << "Psi min "<<psip(gp.R_0, 0)<<"\n";
+    dg::geo::solovev::GeomParameters gp(js);
+    dg::geo::TokamakMagneticField c = dg::geo::solovev::createMagField(gp);
+    if(rank==0)std::cout << "Psi min "<<c.psip()(gp.R_0, 0)<<"\n";
     if(rank==0)std::cout << "Type psi_0 and psi_1\n";
     double psi_0, psi_1;
     if(rank==0)std::cin >> psi_0>> psi_1;
@@ -54,14 +51,10 @@ int main(int argc, char**argv)
     if(rank==0)std::cout << "Constructing grid ... \n";
     dg::Timer t;
     t.tic();
-    //ConformalMPIGrid3d<dg::DVec> g3d(gp, psi_0, psi_1, n, Nx, Ny,Nz, dg::DIR, comm);
-    //ConformalMPIGrid2d<dg::DVec> g2d = g3d.perp_grid();
-    //dg::Elliptic<ConformalMPIGrid3d<dg::DVec>, dg::MDMatrix, dg::MDVec> pol( g3d, dg::not_normed, dg::centered);
-    dg::geo::SimpleOrthogonal<Psip, PsipR, PsipZ, LaplacePsip> 
-        generator( c.psip, c.psipR, c.psipZ, c.laplacePsip, psi_0, psi_1, gp.R_0, 0., 1);
-    dg::OrthogonalMPIGrid3d<dg::DVec> g3d(generator, n, Nx, Ny,Nz,dg::DIR, comm); 
-    dg::OrthogonalMPIGrid2d<dg::DVec> g2d = g3d.perp_grid();
-    dg::Elliptic<dg::OrthogonalMPIGrid2d<dg::DVec>, dg::MDMatrix, dg::MDVec> pol( g2d, dg::not_normed, dg::forward); //something is wrong if g3d is used here
+    dg::geo::SimpleOrthogonal generator( c.get_psip(), psi_0, psi_1, gp.R_0, 0., 1);
+    dg::CurvilinearProductMPIGrid3d g3d( generator, n, Nx, Ny,Nz, dg::DIR, dg::PER, dg::PER, comm);
+    dg::CurvilinearMPIGrid2d g2d = g3d.perp_grid();
+    dg::Elliptic<dg::CurvilinearMPIGrid2d, dg::MDMatrix, dg::MDVec> pol( g2d, dg::not_normed, dg::forward);
     t.toc();
     if(rank==0)std::cout << "Construction took "<<t.diff()<<"s\n";
     ///////////////////////////////////////////////////////////////////////////
@@ -92,16 +85,16 @@ int main(int argc, char**argv)
     dg::HVec X( g2d.size()), Y(X); //P = dg::pullback( dg::coo3, g);
     for( unsigned i=0; i<g2d.size(); i++)
     {
-        X[i] = g2d.r()[i];
-        Y[i] = g2d.z()[i];
+        X[i] = g2d.map()[0].data().[i];
+        Y[i] = g2d.map()[1].data().[i];
     }
     ncerr = nc_put_vara_double( ncid, coordsID[0], start, count, X.data());
     ncerr = nc_put_vara_double( ncid, coordsID[1], start, count, Y.data());
     ///////////////////////////////////////////////////////////////////////////
     dg::MDVec x =    dg::evaluate( dg::zero, g2d);
-    const dg::MDVec b =    dg::pullback( dg::geo::EllipticDirPerM<MagneticField>(c, gp.R_0, psi_0, psi_1, 4), g2d);
-    const dg::MDVec chi =  dg::pullback( dg::geo::Bmodule<MagneticField>(c, gp.R_0), g2d);
-    const dg::MDVec solution = dg::pullback( dg::geo::FuncDirPer<MagneticField>(c, gp.R_0, psi_0, psi_1, 4), g2d);
+    const dg::MDVec b =    dg::pullback( dg::geo::EllipticDirPerM(c, psi_0, psi_1, 4), g2d);
+    const dg::MDVec chi =  dg::pullback( dg::geo::Bmodule(c), g2d);
+    const dg::MDVec solution = dg::pullback( dg::geo::FuncDirPer(c, psi_0, psi_1, 4), g2d);
     const dg::MDVec vol3d = dg::create::volume( g2d);
     pol.set_chi( chi);
     //compute error
@@ -118,7 +111,9 @@ int main(int argc, char**argv)
     double err = dg::blas2::dot( vol3d, error);
     const double norm = dg::blas2::dot( vol3d, solution);
     if(rank==0)std::cout << sqrt( err/norm) << "\t";
-    dg::MDVec gyy = g2d.g_xx(), gxx=g2d.g_yy(), vol = g2d.vol();
+
+    dg::SparseTensor<dg::MDVec> metric = g2d.metric();
+    dg::MDVec gyy = metric.value(1,1), gxx=metric.value(0,0), vol = dg::tensor::volume(metric).value();
     dg::blas1::transform( gxx, gxx, dg::SQRT<double>());
     dg::blas1::transform( gyy, gyy, dg::SQRT<double>());
     dg::blas1::pointwiseDot( gxx, vol, gxx);
-- 
GitLab


From c36fd7886c318cee0fa635ccb53902cead1f08ef Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sat, 19 Aug 2017 15:05:49 +0200
Subject: [PATCH 189/453] documented jacobian, metric and map in geometry
 classes

---
 inc/dg/geometry/base_geometry.h  | 63 ++++++++++++++++++++++++++++++++
 inc/dg/geometry/base_geometryX.h |  6 +++
 inc/dg/geometry/mpi_base.h       |  6 +++
 3 files changed, 75 insertions(+)

diff --git a/inc/dg/geometry/base_geometry.h b/inc/dg/geometry/base_geometry.h
index 48ff268a9..e97c68fca 100644
--- a/inc/dg/geometry/base_geometry.h
+++ b/inc/dg/geometry/base_geometry.h
@@ -14,12 +14,41 @@ namespace dg
  */
 struct aGeometry2d : public aTopology2d
 {
+    /**
+    * @brief The Jacobian of the coordinate transformation from physical to computational space
+    *
+    *  The elements of the Tensor are (if x,y are the coordinates in computational space and R,Z are the physical space coordinates)
+    \f[
+    J = \begin{pmatrix} x_R(x,y) & x_Z(x,y) \\ y_R(x,y) & y_Z(x,y) 
+    \end{pmatrix}
+    \f]
+    * @return Jacobian 
+    */
     SparseTensor<thrust::host_vector<double> > jacobian()const{
         return do_compute_jacobian();
     }
+    /**
+    * @brief The Metric tensor of the coordinate system 
+    *
+    *  The elements are the contravariant elements (if x,y are the coordinates) 
+    \f[
+    g = \begin{pmatrix} g^{xx}(x,y) & g^{xy}(x,y) \\  & g^{yy}(x,y) \end{pmatrix}
+    \f]
+    * @return symmetric tensor
+    */
     SparseTensor<thrust::host_vector<double> > metric()const { 
         return do_compute_metric(); 
     }
+    /**
+    * @brief The coordinate map from computational to physical space
+    *
+    *  The elements of the map are (if x,y are the coordinates in computational space and R,Z are the physical space coordinates)
+    \f[
+    R(x,y) \\
+    Z(x,y)
+    \f]
+    * @return a vector of size 2 
+    */
     std::vector<thrust::host_vector<double> > map()const{
         return do_compute_map();
     }
@@ -60,12 +89,46 @@ struct aGeometry2d : public aTopology2d
  */
 struct aGeometry3d : public aTopology3d
 {
+    /**
+    * @brief The Jacobian of the coordinate transformation from physical to computational space
+    *
+    *  The elements of the Tensor are (if x,y,z are the coordinates in computational space and R,Z,P are the physical space coordinates)
+    \f[
+    J = \begin{pmatrix} x_R(x,y,z) & x_Z(x,y,z) & x_\varphi(x,y,z) \\ 
+    y_R(x,y,z) & y_Z(x,y,z) & y_\varphi(x,y,z) \\
+    z_R(x,y,z) & z_Z(x,y,z) & z_\varphi(x,y,z)
+    \end{pmatrix}
+    \f]
+    * @return Jacobian 
+    */
     SparseTensor<thrust::host_vector<double> > jacobian()const{
         return do_compute_jacobian();
     }
+    /**
+    * @brief The Metric tensor of the coordinate system 
+    *
+    *  The elements are the contravariant elements (if x,y,z are the coordinates) 
+    \f[
+    g = \begin{pmatrix} g^{xx}(x,y,z) & g^{xy}(x,y,z) & g^{zz}(x,y,z)\\
+      & g^{yy}(x,y,z) & g^{yz}(x,y,z) \\
+      & & g^{zz}(x,y,z)\end{pmatrix}
+    \f]
+    * @return symmetric tensor
+    */
     SparseTensor<thrust::host_vector<double> > metric()const { 
         return do_compute_metric(); 
     }
+    /**
+    * @brief The coordinate map from computational to physical space
+    *
+    *  The elements of the map are (if x,y,z are the coordinates in computational space and R,Z,P are the physical space coordinates)
+    \f[
+    R(x,y,z) \\
+    Z(x,y,z) \\
+    \varphi(x,y,z)
+    \f]
+    * @return a vector of size 3 
+    */
     std::vector<thrust::host_vector<double> > map()const{
         return do_compute_map();
     }
diff --git a/inc/dg/geometry/base_geometryX.h b/inc/dg/geometry/base_geometryX.h
index 6c612cf83..573aa5ce4 100644
--- a/inc/dg/geometry/base_geometryX.h
+++ b/inc/dg/geometry/base_geometryX.h
@@ -14,12 +14,15 @@ namespace dg
  */
 struct aGeometryX2d : public aTopologyX2d
 {
+    ///@copydoc aGeometry2d::jacobian()
     SparseTensor<thrust::host_vector<double> > jacobian()const{
         return do_compute_jacobian();
     }
+    ///@copydoc aGeometry2d::metric()
     SparseTensor<thrust::host_vector<double> > metric()const { 
         return do_compute_metric(); 
     }
+    ///@copydoc aGeometry2d::map()
     std::vector<thrust::host_vector<double> > map()const{
         return do_compute_map();
     }
@@ -60,12 +63,15 @@ struct aGeometryX2d : public aTopologyX2d
  */
 struct aGeometryX3d : public aTopologyX3d
 {
+    ///@copydoc aGeometry3d::jacobian()
     SparseTensor<thrust::host_vector<double> > jacobian()const{
         return do_compute_jacobian();
     }
+    ///@copydoc aGeometry3d::metric()
     SparseTensor<thrust::host_vector<double> > metric()const { 
         return do_compute_metric(); 
     }
+    ///@copydoc aGeometry3d::map()
     std::vector<thrust::host_vector<double> > map()const{
         return do_compute_map();
     }
diff --git a/inc/dg/geometry/mpi_base.h b/inc/dg/geometry/mpi_base.h
index c9b2b8eb6..2050d82a5 100644
--- a/inc/dg/geometry/mpi_base.h
+++ b/inc/dg/geometry/mpi_base.h
@@ -15,12 +15,15 @@ namespace dg
 struct aMPIGeometry2d : public aMPITopology2d
 {
     typedef MPI_Vector<thrust::host_vector<double> > host_vector;
+    ///@copydoc aGeometry2d::jacobian()
     SparseTensor<host_vector > jacobian()const {
         return do_compute_jacobian();
     }
+    ///@copydoc aGeometry2d::metric()
     SparseTensor<host_vector > metric()const {
         return do_compute_metric();
     }
+    ///@copydoc aGeometry2d::map()
     std::vector<host_vector > map()const{
         return do_compute_map();
     }
@@ -58,12 +61,15 @@ struct aMPIGeometry2d : public aMPITopology2d
 struct aMPIGeometry3d : public aMPITopology3d
 {
     typedef MPI_Vector<thrust::host_vector<double> > host_vector;
+    ///@copydoc aGeometry3d::jacobian()
     SparseTensor<host_vector > jacobian()const{
         return do_compute_jacobian();
     }
+    ///@copydoc aGeometry3d::metric()
     SparseTensor<host_vector > metric()const { 
         return do_compute_metric(); 
     }
+    ///@copydoc aGeometry3d::map()
     std::vector<host_vector > map()const{
         return do_compute_map();
     }
-- 
GitLab


From 65697719c1f867b28dd81b4bb52a7c1d03303c7c Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sun, 20 Aug 2017 00:26:07 +0200
Subject: [PATCH 190/453] new graphs in doxygen

---
 inc/dg/Doxyfile          |  8 ++++----
 inc/geometries/flux_t.cu | 25 +++++++++++++++----------
 2 files changed, 19 insertions(+), 14 deletions(-)

diff --git a/inc/dg/Doxyfile b/inc/dg/Doxyfile
index 18ce2c8ba..cb293e7c1 100644
--- a/inc/dg/Doxyfile
+++ b/inc/dg/Doxyfile
@@ -783,7 +783,7 @@ WARN_LOGFILE           =
 
 INPUT                  = . \
                          ./backend \
-                         ./geometry 
+                         ./geometry
 
 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
@@ -2134,7 +2134,7 @@ PERL_PATH              = /usr/bin/perl
 # powerful graphs.
 # The default value is: YES.
 
-CLASS_DIAGRAMS         = YES
+CLASS_DIAGRAMS         = NO
 
 # You can define message sequence charts within doxygen comments using the \msc
 # command. Doxygen will then run the mscgen tool (see:
@@ -2165,7 +2165,7 @@ HIDE_UNDOC_RELATIONS   = YES
 # set to NO
 # The default value is: YES.
 
-HAVE_DOT               = NO
+HAVE_DOT               = YES
 
 # The DOT_NUM_THREADS specifies the number of dot invocations doxygen is allowed
 # to run in parallel. When set to 0 doxygen will base this on the number of
@@ -2216,7 +2216,7 @@ CLASS_GRAPH            = YES
 # The default value is: YES.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
-COLLABORATION_GRAPH    = YES
+COLLABORATION_GRAPH    = NO
 
 # If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for
 # groups, showing the direct groups dependencies.
diff --git a/inc/geometries/flux_t.cu b/inc/geometries/flux_t.cu
index a414bc70f..1fc95ea2b 100644
--- a/inc/geometries/flux_t.cu
+++ b/inc/geometries/flux_t.cu
@@ -82,17 +82,22 @@ int main( int argc, char* argv[])
     int ncid;
     file::NC_Error_Handle err;
     err = nc_create( "flux.nc", NC_NETCDF4|NC_CLOBBER, &ncid);
-    int dim3d[2];
-    err = file::define_dimensions(  ncid, dim3d, g2d_periodic);
+    int dim2d[2];
+    err = file::define_dimensions(  ncid, dim2d, g2d_periodic);
     int coordsID[2], onesID, defID, confID, volID, divBID;
-    err = nc_def_var( ncid, "x_XYP", NC_DOUBLE, 2, dim3d, &coordsID[0]);
-    err = nc_def_var( ncid, "y_XYP", NC_DOUBLE, 2, dim3d, &coordsID[1]);
-    //err = nc_def_var( ncid, "z_XYP", NC_DOUBLE, 3, dim3d, &coordsID[2]);
-    err = nc_def_var( ncid, "psi", NC_DOUBLE, 2, dim3d, &onesID);
-    err = nc_def_var( ncid, "d",    NC_DOUBLE, 2, dim3d, &defID);
-    err = nc_def_var( ncid, "R", NC_DOUBLE, 2, dim3d, &confID);
-    err = nc_def_var( ncid, "vol", NC_DOUBLE, 2, dim3d, &volID);
-    err = nc_def_var( ncid, "divB", NC_DOUBLE, 2, dim3d, &divBID);
+    err = nc_def_var( ncid, "x_XYP", NC_DOUBLE, 2, dim2d, &coordsID[0]);
+    err = nc_def_var( ncid, "y_XYP", NC_DOUBLE, 2, dim2d, &coordsID[1]);
+    err = nc_def_var( ncid, "psi", NC_DOUBLE, 2, dim2d, &onesID);
+    err = nc_def_var( ncid, "d",    NC_DOUBLE, 2, dim2d, &defID);
+    err = nc_def_var( ncid, "R", NC_DOUBLE, 2, dim2d, &confID);
+    err = nc_def_var( ncid, "vol", NC_DOUBLE, 2, dim2d, &volID);
+    err = nc_def_var( ncid, "divB", NC_DOUBLE, 2, dim2d, &divBID);
+    std::string coords = "x_XYP y_XYP";
+    err = nc_put_att_text(ncid, onesID, "coordinates", coords.size(), coords.data());
+    err = nc_put_att_text(ncid, defID, "coordinates", coords.size(), coords.data());
+    err = nc_put_att_text(ncid, confID, "coordinates", coords.size(), coords.data());
+    err = nc_put_att_text(ncid, volID, "coordinates", coords.size(), coords.data());
+    err = nc_put_att_text(ncid, divBID, "coordinates", coords.size(), coords.data());
 
     thrust::host_vector<double> psi_p = dg::pullback( psip, g2d);
     //g.display();
-- 
GitLab


From df989839a3b4c21fce94dcaebaba2628483ba4c7 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sun, 20 Aug 2017 21:00:58 +0200
Subject: [PATCH 191/453] worked on file documentation and cleaned up flux_t.cu
 a bit

---
 inc/file/Doxyfile        | 12 +++++------
 inc/file/nc_utilities.h  | 38 +++++++++++++++++++++------------
 inc/geometries/flux_t.cu | 46 ++++++++++++++++------------------------
 3 files changed, 48 insertions(+), 48 deletions(-)

diff --git a/inc/file/Doxyfile b/inc/file/Doxyfile
index 12597e9fd..744d136ad 100644
--- a/inc/file/Doxyfile
+++ b/inc/file/Doxyfile
@@ -32,7 +32,7 @@ DOXYFILE_ENCODING      = UTF-8
 # title of most generated pages and in a few other places.
 # The default value is: My Project.
 
-PROJECT_NAME           = "Input and Output utilities"
+PROJECT_NAME           = "Netcdf output utilities"
 
 # The PROJECT_NUMBER tag can be used to enter a project or revision number. This
 # could be handy for archiving the generated documentation or if some version
@@ -44,7 +44,7 @@ PROJECT_NUMBER         =
 # for a project that appears at the top of each page and should give viewer a
 # quick idea about the purpose of the project. Keep the description short.
 
-PROJECT_BRIEF          =
+PROJECT_BRIEF          = "Facilitate the creation of coordinate and time variables partly following CF conventions"
 
 # With the PROJECT_LOGO tag one can specify a logo or an icon that is included
 # in the documentation. The maximum height of the logo should not exceed 55
@@ -1046,7 +1046,7 @@ CLANG_OPTIONS          =
 # classes, structs, unions or interfaces.
 # The default value is: YES.
 
-ALPHABETICAL_INDEX     = YES
+ALPHABETICAL_INDEX     = NO
 
 # The COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns in
 # which the alphabetical index list will be split.
@@ -2131,7 +2131,7 @@ PERL_PATH              = /usr/bin/perl
 # powerful graphs.
 # The default value is: YES.
 
-CLASS_DIAGRAMS         = YES
+CLASS_DIAGRAMS         = NO
 
 # You can define message sequence charts within doxygen comments using the \msc
 # command. Doxygen will then run the mscgen tool (see:
@@ -2162,7 +2162,7 @@ HIDE_UNDOC_RELATIONS   = YES
 # set to NO
 # The default value is: YES.
 
-HAVE_DOT               = NO
+HAVE_DOT               = YES
 
 # The DOT_NUM_THREADS specifies the number of dot invocations doxygen is allowed
 # to run in parallel. When set to 0 doxygen will base this on the number of
@@ -2279,7 +2279,7 @@ INCLUDED_BY_GRAPH      = YES
 # The default value is: NO.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
-CALL_GRAPH             = NO
+CALL_GRAPH             = YES
 
 # If the CALLER_GRAPH tag is set to YES then doxygen will generate a caller
 # dependency graph for every global function or class method.
diff --git a/inc/file/nc_utilities.h b/inc/file/nc_utilities.h
index c2955acb3..26e185f43 100644
--- a/inc/file/nc_utilities.h
+++ b/inc/file/nc_utilities.h
@@ -6,18 +6,22 @@
 
 #include "dg/backend/grid.h"
 #include "dg/backend/weights.cuh"
+
 /*!@file
  *
  * Contains Error handling class and the define_dimensions functions
  */
 
+
+/**
+* @brief Namespace for netcdf output related classes and functions following the
+ <a href="http://cfconventions.org/Data/cf-conventions/cf-conventions-1.7/cf-conventions.html">CF-conventions</a>
+*/
 namespace file
 {
 
 /**
  * @brief Class thrown by the NC_ErrorHandle
- *
- * @ingroup utilities
  */
 struct NC_Error : public std::exception
 {
@@ -74,10 +78,12 @@ struct NC_Error_Handle
     }
 };
 /**
- * @brief Define an unlimited time variable 
+ * @brief Define an unlimited time dimension and variable following 
+  <a href="http://cfconventions.org/Data/cf-conventions/cf-conventions-1.7/cf-conventions.html">CF-conventions</a>
  *
+ * The conventions dictate that the units attribute must be defined for a time variable: we give it the value "time since start"
  * @param ncid file ID
- * @param name Name of time variable
+ * @param name Name of time variable (variable names are not standardized)
  * @param dimID time-dimension ID
  * @param tvarID time-variable ID
  *
@@ -94,8 +100,10 @@ int define_time( int ncid, const char* name, int* dimID, int* tvarID)
 }
 
 /**
- * @brief Define a limited time variable
+ * @brief Define a limited time dimension and variable following
+  <a href="http://cfconventions.org/Data/cf-conventions/cf-conventions-1.7/cf-conventions.html">CF-conventions</a>
  *
+ * The conventions dictate that the units attribute must be defined for a time variable: we give it the value "time since start"
  * @param ncid file ID
  * @param name Name of the time variable (usually "time")
  * @param size The number of timesteps 
@@ -115,13 +123,14 @@ int define_limited_time( int ncid, const char* name, int size, int* dimID, int*
 }
 
 /**
- * @brief Define a 1d dimension variable together with its data points
+ * @brief Define a 1d dimension and create a coordinate variable together with its data points in a netcdf file
  *
+ * By netcdf conventions a variable with the same name as a dimension is called a coordinate variable. 
  * @param ncid file ID
- * @param name Name of dimension
- * @param dimID dimension ID
- * @param points pointer to data
- * @param size size of data points
+ * @param name Name of dimension (input)
+ * @param dimID dimension ID (output)
+ * @param points pointer to data (input)
+ * @param size size of data points (input)
  *
  * @return netcdf error code if any
  */
@@ -137,12 +146,13 @@ int define_dimension( int ncid, const char* name, int* dimID, const double * poi
     return retval;
 }
 /**
- * @brief Define a 1d dimension variable together with its data points
+ * @brief Define a 1d dimension and create a coordinate variable together with its data points in a netcdf file
  *
+ * By netcdf conventions a variable with the same name as a dimension is called a coordinate variable. 
  * @param ncid file ID
- * @param name Name of dimension
- * @param dimID dimension ID
- * @param g The 1d DG grid from which data points are generated
+ * @param name Name of dimension (input)
+ * @param dimID dimension ID (output)
+ * @param g The 1d DG grid from which data points are generated (input)
  *
  * @return netcdf error code if any
  */
diff --git a/inc/geometries/flux_t.cu b/inc/geometries/flux_t.cu
index 1fc95ea2b..a41ac461b 100644
--- a/inc/geometries/flux_t.cu
+++ b/inc/geometries/flux_t.cu
@@ -79,43 +79,33 @@ int main( int argc, char* argv[])
     dg::Grid2d g2d_periodic(g2d.x0(), g2d.x1(), g2d.y0(), g2d.y1(), g2d.n(), g2d.Nx(), g2d.Ny()+1); 
     t.toc();
     std::cout << "Construction took "<<t.diff()<<"s"<<std::endl;
+    //////////////////////////////setup netcdf//////////////////
     int ncid;
     file::NC_Error_Handle err;
     err = nc_create( "flux.nc", NC_NETCDF4|NC_CLOBBER, &ncid);
     int dim2d[2];
     err = file::define_dimensions(  ncid, dim2d, g2d_periodic);
-    int coordsID[2], onesID, defID, confID, volID, divBID;
+    int coordsID[2];
     err = nc_def_var( ncid, "x_XYP", NC_DOUBLE, 2, dim2d, &coordsID[0]);
     err = nc_def_var( ncid, "y_XYP", NC_DOUBLE, 2, dim2d, &coordsID[1]);
-    err = nc_def_var( ncid, "psi", NC_DOUBLE, 2, dim2d, &onesID);
-    err = nc_def_var( ncid, "d",    NC_DOUBLE, 2, dim2d, &defID);
-    err = nc_def_var( ncid, "R", NC_DOUBLE, 2, dim2d, &confID);
-    err = nc_def_var( ncid, "vol", NC_DOUBLE, 2, dim2d, &volID);
-    err = nc_def_var( ncid, "divB", NC_DOUBLE, 2, dim2d, &divBID);
-    std::string coords = "x_XYP y_XYP";
-    err = nc_put_att_text(ncid, onesID, "coordinates", coords.size(), coords.data());
-    err = nc_put_att_text(ncid, defID, "coordinates", coords.size(), coords.data());
-    err = nc_put_att_text(ncid, confID, "coordinates", coords.size(), coords.data());
-    err = nc_put_att_text(ncid, volID, "coordinates", coords.size(), coords.data());
-    err = nc_put_att_text(ncid, divBID, "coordinates", coords.size(), coords.data());
+    dg::HVec X=dg::pullback(dg::cooX2d, g2d), Y=dg::pullback(dg::cooY2d, g2d); //P = dg::pullback( dg::coo3, g);
+    err = nc_put_var_double( ncid, coordsID[0], periodify(X, g2d_periodic).data());
+    err = nc_put_var_double( ncid, coordsID[1], periodify(Y, g2d_periodic).data());
+    //err = nc_put_var_double( ncid, coordsID[2], g.z().data());
+
+    std::string names[] = {"psi", "d", "R", "vol", "divB"};
+    unsigned size=5;
+    int varID[size];
+    for( unsigned i=0; i<size; i++)
+        err = nc_def_var( ncid, names[i].data(), NC_DOUBLE, 2, dim2d, &varID[i]);
+    ///////////////////////now fill variables///////////////////
 
     thrust::host_vector<double> psi_p = dg::pullback( psip, g2d);
     //g.display();
-    err = nc_put_var_double( ncid, onesID, periodify(psi_p, g2d_periodic).data());
-    dg::HVec X( g2d.size()), Y(X); //P = dg::pullback( dg::coo3, g);
-    for( unsigned i=0; i<g2d.size(); i++)
-    {
-        X[i] = g2d.map()[0][i];
-        Y[i] = g2d.map()[1][i];
-    }
-
+    err = nc_put_var_double( ncid, varID[0], periodify(psi_p, g2d_periodic).data());
     dg::HVec temp0( g2d.size()), temp1(temp0);
     dg::HVec w3d = dg::create::weights( g2d);
 
-    err = nc_put_var_double( ncid, coordsID[0], periodify(X, g2d_periodic).data());
-    err = nc_put_var_double( ncid, coordsID[1], periodify(Y, g2d_periodic).data());
-    //err = nc_put_var_double( ncid, coordsID[2], g.z().data());
-
     //compute and write deformation into netcdf
     dg::SparseTensor<dg::HVec> metric = g2d.metric();
     dg::HVec g_xx = metric.value(0,0), g_xy = metric.value(0,1), g_yy=metric.value(1,1);
@@ -123,11 +113,11 @@ int main( int argc, char* argv[])
     dg::blas1::pointwiseDivide( g_xy, g_xx, temp0);
     const dg::HVec ones = dg::evaluate( dg::one, g2d);
     X=g_yy;
-    err = nc_put_var_double( ncid, defID, periodify(X, g2d_periodic).data());
+    err = nc_put_var_double( ncid, varID[1], periodify(X, g2d_periodic).data());
     //compute and write conformalratio into netcdf
     dg::blas1::pointwiseDivide( g_yy, g_xx, temp0);
     X=g_xx;
-    err = nc_put_var_double( ncid, confID, periodify(X, g2d_periodic).data());
+    err = nc_put_var_double( ncid, varID[2], periodify(X, g2d_periodic).data());
 
     std::cout << "Construction successful!\n";
 
@@ -137,7 +127,7 @@ int main( int argc, char* argv[])
     dg::blas1::transform( temp0, temp0, dg::SQRT<double>()); //temp0=1/sqrt(g) = sqrt(g^xx g^yy - g^xy^2)
     dg::blas1::pointwiseDivide( ones, temp0, temp0); //temp0=sqrt(g)
     X=temp0;
-    err = nc_put_var_double( ncid, volID, periodify(X, g2d_periodic).data());
+    err = nc_put_var_double( ncid, varID[3], periodify(X, g2d_periodic).data());
     dg::blas1::axpby( 1., temp0, -1., vol_.value(), temp0); //temp0 = sqrt(g)-vol
     double error = sqrt(dg::blas2::dot( temp0, w3d, temp0)/dg::blas2::dot(vol_.value(), w3d, vol_.value()));
     std::cout << "Rel Consistency  of volume is "<<error<<"\n";
@@ -187,7 +177,7 @@ int main( int argc, char* argv[])
     //std::cout << "ana. norm of gradLnB is "<<norm<<"\n";
     //dg::blas1::axpby( 1., gradB, -1., gradLnB, gradLnB);
     //X = divB;
-    //err = nc_put_var_double( ncid, divBID, periodify(X, g2d_periodic).data());
+    //err = nc_put_var_double( ncid, varID[4], periodify(X, g2d_periodic).data());
     //double norm2 = sqrt(dg::blas2::dot(gradLnB, vol3d,gradLnB));
     //std::cout << "rel. error of lnB is    "<<norm2/norm<<"\n";
     err = nc_close( ncid);
-- 
GitLab


From 5a001be48fa4b3f51fe045420e790ae1fe6291fe Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sun, 20 Aug 2017 22:53:47 +0200
Subject: [PATCH 192/453] found and corrected bug in computation of 3d
 curvilinear metric

---
 inc/dg/geometry/curvilinear.h          | 39 +++++++++++++++-----------
 inc/dg/geometry/curvilinearX.h         | 18 ++----------
 inc/dg/geometry/mpi_curvilinear.h      |  2 +-
 inc/dg/geometry/refined_curvilinearX.h | 18 ++----------
 4 files changed, 28 insertions(+), 49 deletions(-)

diff --git a/inc/dg/geometry/curvilinear.h b/inc/dg/geometry/curvilinear.h
index fc0cc50e6..be2f222ac 100644
--- a/inc/dg/geometry/curvilinear.h
+++ b/inc/dg/geometry/curvilinear.h
@@ -12,6 +12,28 @@ namespace dg
 
 ///@cond
 struct CurvilinearGrid2d; 
+namespace detail
+{
+void square( const dg::SparseTensor<thrust::host_vector<double> >& jac, const thrust::host_vector<double>& R, dg::SparseTensor<thrust::host_vector<double> >& metric, bool orthogonal)
+{
+    thrust::host_vector<double> tempxx( R.size()), tempxy(R.size()), tempyy(R.size()), temppp(R.size());
+    for( unsigned i=0; i<R.size(); i++)
+    {
+        tempxx[i] = (jac.value(0,0)[i]*jac.value(0,0)[i]+jac.value(0,1)[i]*jac.value(0,1)[i]);
+        tempxy[i] = (jac.value(0,0)[i]*jac.value(1,0)[i]+jac.value(0,1)[i]*jac.value(1,1)[i]);
+        tempyy[i] = (jac.value(1,0)[i]*jac.value(1,0)[i]+jac.value(1,1)[i]*jac.value(1,1)[i]);
+        temppp[i] = 1./R[i]/R[i]; //1/R^2
+    }
+    metric.idx(0,0) = 0; metric.value(0) = tempxx;
+    metric.idx(1,1) = 1; metric.value(1) = tempyy;
+    metric.idx(2,2) = 2; metric.value(2) = temppp;
+    if( !orthogonal)
+    {
+        metric.idx(0,1) = metric.idx(1,0) = 3; 
+        metric.value(3) = tempxy;
+    }
+}
+}//namespace detail
 ///@endcond
 
 //when we make a 3d grid with eta and phi swapped the metric structure and the transformation changes 
@@ -95,23 +117,8 @@ struct CurvilinearProductGrid3d : public dg::aGeometry3d
     }
     virtual SparseTensor<thrust::host_vector<double> > do_compute_metric( ) const
     {
-        thrust::host_vector<double> tempxx( size()), tempxy(size()), tempyy(size()), temppp(size());
-        for( unsigned i=0; i<size(); i++)
-        {
-            tempxx[i] = (jac_.value(0,0)[i]*jac_.value(0,0)[i]+jac_.value(0,1)[i]*jac_.value(0,1)[i]);
-            tempxy[i] = (jac_.value(0,0)[i]*jac_.value(1,0)[i]+jac_.value(0,1)[i]*jac_.value(1,1)[i]);
-            tempyy[i] = (jac_.value(1,0)[i]*jac_.value(1,0)[i]+jac_.value(1,1)[i]*jac_.value(1,1)[i]);
-            temppp[i] = 1./map_[2][i]/map_[2][i]; //1/R^2
-        }
         SparseTensor<thrust::host_vector<double> > metric;
-        metric.idx(0,0) = 0; metric.value(0) = tempxx;
-        metric.idx(1,1) = 1; metric.value(1) = tempyy;
-        metric.idx(2,2) = 2; metric.value(2) = temppp;
-        if( !handle_.get().isOrthogonal())
-        {
-            metric.idx(0,1) = metric.idx(1,0) = 3; 
-            metric.value(3) = tempxy;
-        }
+        detail::square( jac_, map_[0], metric, handle_.get().isOrthogonal());
         return metric;
     }
     virtual std::vector<thrust::host_vector<double> > do_compute_map()const{return map_;}
diff --git a/inc/dg/geometry/curvilinearX.h b/inc/dg/geometry/curvilinearX.h
index 8d9adbe34..73fc3af22 100644
--- a/inc/dg/geometry/curvilinearX.h
+++ b/inc/dg/geometry/curvilinearX.h
@@ -6,6 +6,7 @@
 #include "dg/blas1.h"
 #include "base_geometryX.h"
 #include "generatorX.h"
+#include "curvilinear.h"
 
 namespace dg
 {
@@ -82,23 +83,8 @@ struct CurvilinearProductGridX3d : public dg::aGeometryX3d
     }
     virtual SparseTensor<thrust::host_vector<double> > do_compute_metric( ) const
     {
-        thrust::host_vector<double> tempxx( size()), tempxy(size()), tempyy(size()), temppp(size());
-        for( unsigned i=0; i<size(); i++)
-        {
-            tempxx[i] = (jac_.value(0,0)[i]*jac_.value(0,0)[i]+jac_.value(0,1)[i]*jac_.value(0,1)[i]);
-            tempxy[i] = (jac_.value(0,0)[i]*jac_.value(1,0)[i]+jac_.value(0,1)[i]*jac_.value(1,1)[i]);
-            tempyy[i] = (jac_.value(1,0)[i]*jac_.value(1,0)[i]+jac_.value(1,1)[i]*jac_.value(1,1)[i]);
-            temppp[i] = 1./map_[2][i]/map_[2][i]; //1/R^2
-        }
         SparseTensor<thrust::host_vector<double> > metric;
-        metric.idx(0,0) = 0; metric.value(0) = tempxx;
-        metric.idx(1,1) = 1; metric.value(1) = tempyy;
-        metric.idx(2,2) = 2; metric.value(2) = temppp;
-        if( !handle_.get().isOrthogonal())
-        {
-            metric.idx(0,1) = metric.idx(1,0) = 3; 
-            metric.value(3) = tempxy;
-        }
+        detail::square( jac_, map_[0], metric, handle_.get().isOrthogonal());
         return metric;
     }
     virtual std::vector<thrust::host_vector<double> > do_compute_map()const{return map_;}
diff --git a/inc/dg/geometry/mpi_curvilinear.h b/inc/dg/geometry/mpi_curvilinear.h
index f00469c01..777da1957 100644
--- a/inc/dg/geometry/mpi_curvilinear.h
+++ b/inc/dg/geometry/mpi_curvilinear.h
@@ -157,7 +157,7 @@ struct CurvilinearProductMPIGrid3d : public dg::aMPIGeometry3d
             tempxx[i] = (jac_.value(0,0).data()[i]*jac_.value(0,0).data()[i]+jac_.value(0,1).data()[i]*jac_.value(0,1).data()[i]);
             tempxy[i] = (jac_.value(0,0).data()[i]*jac_.value(1,0).data()[i]+jac_.value(0,1).data()[i]*jac_.value(1,1).data()[i]);
             tempyy[i] = (jac_.value(1,0).data()[i]*jac_.value(1,0).data()[i]+jac_.value(1,1).data()[i]*jac_.value(1,1).data()[i]);
-            temppp[i] = 1./map_[2][i]/map_[2][i]; //1/R^2
+            temppp[i] = 1./map_[0].data()[i]/map_[0].data()[i]; //1/R^2
         }
         SparseTensor<host_vector > metric;
         metric.idx(0,0) = 0; metric.value(0) = host_vector(tempxx, communicator());
diff --git a/inc/dg/geometry/refined_curvilinearX.h b/inc/dg/geometry/refined_curvilinearX.h
index 958769ba9..308854798 100644
--- a/inc/dg/geometry/refined_curvilinearX.h
+++ b/inc/dg/geometry/refined_curvilinearX.h
@@ -2,6 +2,7 @@
 
 #include "generatorX.h"
 #include "refined_gridX.h"
+#include "curvilinear.h"
 
 namespace dg
 {
@@ -88,23 +89,8 @@ struct CurvilinearRefinedProductGridX3d : public dg::aGeometryX3d
     }
     virtual SparseTensor<thrust::host_vector<double> > do_compute_metric( ) const
     {
-        thrust::host_vector<double> tempxx( size()), tempxy(size()), tempyy(size()), temppp(size());
-        for( unsigned i=0; i<size(); i++)
-        {
-            tempxx[i] = (jac_.value(0,0)[i]*jac_.value(0,0)[i]+jac_.value(0,1)[i]*jac_.value(0,1)[i]);
-            tempxy[i] = (jac_.value(0,0)[i]*jac_.value(1,0)[i]+jac_.value(0,1)[i]*jac_.value(1,1)[i]);
-            tempyy[i] = (jac_.value(1,0)[i]*jac_.value(1,0)[i]+jac_.value(1,1)[i]*jac_.value(1,1)[i]);
-            temppp[i] = 1./map_[2][i]/map_[2][i]; //1/R^2
-        }
         SparseTensor<thrust::host_vector<double> > metric;
-        metric.idx(0,0) = 0; metric.value(0) = tempxx;
-        metric.idx(1,1) = 1; metric.value(1) = tempyy;
-        metric.idx(2,2) = 2; metric.value(2) = temppp;
-        if( !handle_.get().isOrthogonal())
-        {
-            metric.idx(0,1) = metric.idx(1,0) = 3; 
-            metric.value(3) = tempxy;
-        }
+        detail::square( jac_, map_[0], metric, handle_.get().isOrthogonal());
         return metric;
     }
     virtual std::vector<thrust::host_vector<double> > do_compute_map()const{return map_;}
-- 
GitLab


From 8c4648029c10bf77688ac14c0a1031e6f8135982 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sun, 20 Aug 2017 23:20:18 +0200
Subject: [PATCH 193/453] adapting some further geometries files

---
 inc/dg/geometry/refined_gridX.h               |  1 +
 inc/geometries/geometryX_elliptic_b.cu        | 30 +++++++++----------
 .../geometryX_refined_elliptic_b.cu           | 30 +++++++++----------
 inc/geometries/separatrix_orthogonal_t.cu     |  8 +----
 inc/geometries/simple_orthogonal_t.cu         | 13 ++------
 5 files changed, 33 insertions(+), 49 deletions(-)

diff --git a/inc/dg/geometry/refined_gridX.h b/inc/dg/geometry/refined_gridX.h
index 23e87e2a5..d93152278 100644
--- a/inc/dg/geometry/refined_gridX.h
+++ b/inc/dg/geometry/refined_gridX.h
@@ -5,6 +5,7 @@
 #include "dg/backend/evaluationX.cuh"
 #include "dg/backend/weightsX.cuh"
 #include "dg/backend/gridX.h"
+#include "base_geometryX.h"
 #include "refined_grid.h"
 
 
diff --git a/inc/geometries/geometryX_elliptic_b.cu b/inc/geometries/geometryX_elliptic_b.cu
index f13fb2f70..bf5e05333 100644
--- a/inc/geometries/geometryX_elliptic_b.cu
+++ b/inc/geometries/geometryX_elliptic_b.cu
@@ -14,7 +14,7 @@
 #include "solovev.h"
 #include "taylor.h"
 //#include "ribeiroX.h"
-#include "orthogonalX.h"
+#include "dg/geometry/curvilinearX.h"
 #include "separatrix_orthogonal.h"
 #include "testfunctors.h"
 
@@ -51,18 +51,17 @@ int main(int argc, char**argv)
     t.tic();
 
     ////////////////construct Generator////////////////////////////////////
-    MagneticField c(gp);
-    std::cout << "Psi min "<<c.psip(gp.R_0, 0)<<"\n";
+    dg::geo::TokamakMagneticField c = dg::geo::taylor::createMagField(gp);
+    std::cout << "Psi min "<<c.psip()(gp.R_0, 0)<<"\n";
     double R0 = gp.R_0, Z0 = 0;
     //double R_X = gp.R_0-1.4*gp.triangularity*gp.a;
     //double Z_X = -1.0*gp.elongation*gp.a;
     double R_X = gp.R_0-1.1*gp.triangularity*gp.a;
     double Z_X = -1.1*gp.elongation*gp.a;
     std::cout << "X-point at "<<R_X <<" "<<Z_X<<"\n";
-    dg::geo::SeparatrixOrthogonal<Psip,PsipR,PsipZ,LaplacePsip> generator(c.psip, c.psipR, c.psipZ, c.laplacePsip, psi_0, R_X,Z_X, R0, Z0,0);
-    //dg::geo::SimpleOrthogonalX<Psip,PsipR,PsipZ,LaplacePsip> generator(c.psip, c.psipR, c.psipZ, c.laplacePsip, psi_0, R_X,Z_X, R0, Z0,0);
-    dg::OrthogonalGridX2d<dg::DVec> g2d( generator, psi_0, 0.25, 1./22., n, Nx, Ny, dg::DIR, dg::NEU);
-    dg::Elliptic<dg::OrthogonalGridX2d<dg::DVec>, dg::Composite<dg::DMatrix>, dg::DVec> pol( g2d, dg::not_normed, dg::forward);
+    dg::geo::SeparatrixOrthogonal generator(c.get_psip(), psi_0, R_X,Z_X, R0, Z0,0);
+    dg::CurvilinearGridX2d g2d( generator, 0.25, 1./22., n, Nx, Ny, dg::DIR, dg::NEU);
+    dg::Elliptic<dg::CurvilinearGridX2d, dg::Composite<dg::DMatrix>, dg::DVec> pol( g2d, dg::not_normed, dg::forward);
     double fx = 0.25;
     psi_1 = -fx/(1.-fx)*psi_0;
     std::cout << "psi 1 is          "<<psi_1<<"\n";
@@ -86,8 +85,8 @@ int main(int argc, char**argv)
     dg::HVec X( g2d.size()), Y(X); //P = dg::pullback( dg::coo3, g);
     for( unsigned i=0; i<g2d.size(); i++)
     {
-        X[i] = g2d.r()[i];
-        Y[i] = g2d.z()[i];
+        X[i] = g2d.map()[0][i];
+        Y[i] = g2d.map()[1][i];
     }
     ncerr = nc_put_var_double( ncid, coordsID[0], X.data());
     ncerr = nc_put_var_double( ncid, coordsID[1], Y.data());
@@ -105,11 +104,11 @@ int main(int argc, char**argv)
     //const dg::DVec chi =      dg::evaluate( dg::one, g2d);
     //const dg::DVec solution =     dg::pullback( c.psip, g2d);
     /////////////////////////////Dir/////FIELALIGNED SIN///////////////////
-    const dg::DVec b =    dg::pullback( dg::geo::EllipticXDirNeuM<MagneticField>(c, gp.R_0, psi_0, psi_1), g2d);
-    dg::DVec chi  =  dg::pullback( dg::geo::Bmodule<MagneticField>(c, gp.R_0), g2d);
+    const dg::DVec b =    dg::pullback( dg::geo::EllipticXDirNeuM(c, psi_0, psi_1), g2d);
+    dg::DVec chi  =  dg::pullback( dg::geo::Bmodule(c), g2d);
     dg::blas1::plus( chi, 1e4);
     //const dg::DVec chi =  dg::pullback( dg::ONE(), g2d);
-    const dg::DVec solution = dg::pullback( dg::geo::FuncXDirNeu<MagneticField>(c, psi_0, psi_1 ), g2d);
+    const dg::DVec solution = dg::pullback( dg::geo::FuncXDirNeu(c, psi_0, psi_1 ), g2d);
     ////////////////////////////////////////////////////////////////////////////
 
     const dg::DVec vol2d = dg::create::volume( g2d);
@@ -131,10 +130,9 @@ int main(int argc, char**argv)
     double err = dg::blas2::dot( vol2d, error);
     const double norm = dg::blas2::dot( vol2d, solution);
     std::cout << sqrt( err/norm) << "\t";
-    dg::HVec gyy, gxx, vol; 
-    dg::blas1::transfer( g2d.g_xx(), gyy);
-    dg::blas1::transfer( g2d.g_yy(), gxx); 
-    dg::blas1::transfer( g2d.vol() , vol);
+    ///////////////////////////////////metric//////////////////////
+    dg::SparseTensor<dg::DVec> metric = g2d.metric();
+    dg::DVec gyy = metric.value(1,1), gxx = metric.value(0,0), vol = dg::tensor::volume(metric).value(); 
     dg::blas1::transform( gxx, gxx, dg::SQRT<double>());
     dg::blas1::transform( gyy, gyy, dg::SQRT<double>());
     dg::blas1::pointwiseDot( gxx, vol, gxx);
diff --git a/inc/geometries/geometryX_refined_elliptic_b.cu b/inc/geometries/geometryX_refined_elliptic_b.cu
index 39f3b5e79..aa8b4e691 100644
--- a/inc/geometries/geometryX_refined_elliptic_b.cu
+++ b/inc/geometries/geometryX_refined_elliptic_b.cu
@@ -60,19 +60,20 @@ int main(int argc, char**argv)
     std::cin >> howmanyX >> howmanyY;
 
     ////////////////construct Generator////////////////////////////////////
-    MagneticField c(gp);
-    std::cout << "Psi min "<<c.psip(gp.R_0, 0)<<"\n";
+    dg::geo::TokamakMagneticField c = dg::geo::taylor::createMagField(gp);
+    std::cout << "Psi min "<<c.psip()(gp.R_0, 0)<<"\n";
     double R0 = gp.R_0, Z0 = 0;
     //double R_X = gp.R_0-1.4*gp.triangularity*gp.a;
     //double Z_X = -1.0*gp.elongation*gp.a;
     double R_X = gp.R_0-1.1*gp.triangularity*gp.a;
     double Z_X = -1.1*gp.elongation*gp.a;
-    dg::geo::SeparatrixOrthogonal<Psip,PsipR,PsipZ,LaplacePsip> generator(c.psip, c.psipR, c.psipZ, c.laplacePsip, psi_0, R_X,Z_X, R0, Z0,0);
-    //dg::geo::SimpleOrthogonalX<Psip,PsipR,PsipZ,LaplacePsip> generator(c.psip, c.psipR, c.psipZ, c.laplacePsip, psi_0, R_X,Z_X, R0, Z0,0);
-    //dg::OrthogonalGridX2d<dg::DVec> g2d( generator, psi_0, 0.25, 1./22., n, Nx, Ny, dg::DIR, dg::NEU);
-    dg::OrthogonalRefinedGridX2d<dg::DVec> g2d( add_x, add_y, howmanyX, howmanyY, generator, psi_0, 0.25, 1./22., n_ref, n, Nx, Ny, dg::DIR, dg::NEU);
-    dg::Elliptic<dg::OrthogonalRefinedGridX2d<dg::DVec>, dg::Composite<dg::DMatrix>, dg::DVec> pol( g2d, dg::not_normed, dg::forward);
-    dg::RefinedElliptic<dg::OrthogonalRefinedGridX2d<dg::DVec>, dg::IDMatrix, dg::Composite<dg::DMatrix>, dg::DVec> pol_refined( g2d, dg::not_normed, dg::forward);
+    dg::geo::SeparatrixOrthogonal generator(c.get_psip(), psi_0, R_X,Z_X, R0, Z0,0);
+    //dg::geo::SimpleOrthogonalX generator(c.get_psip(), psi_0, R_X,Z_X, R0, Z0,0);
+    //dg::CurvilinearGridX2d g2d( generator, 0.25, 1./22., n, Nx, Ny, dg::DIR, dg::NEU);
+    dg::EquidistRefinementX equi(add_x, add_y, howmanyX, howmanyY)
+    dg::OrthogonalRefinedGridX2d g2d( equi, generator, 0.25, 1./22., n_ref, n, Nx, Ny, dg::DIR, dg::NEU);
+    dg::Elliptic<dg::OrthogonalRefinedGridX2d, dg::Composite<dg::DMatrix>, dg::DVec> pol( g2d, dg::not_normed, dg::forward);
+    dg::RefinedElliptic<dg::OrthogonalRefinedGridX2d, dg::IDMatrix, dg::Composite<dg::DMatrix>, dg::DVec> pol_refined( g2d, dg::not_normed, dg::forward);
     double fx = 0.25;
     psi_1 = -fx/(1.-fx)*psi_0;
     std::cout << "psi 1 is          "<<psi_1<<"\n";
@@ -96,8 +97,8 @@ int main(int argc, char**argv)
     dg::HVec X( g2d.size()), Y(X); //P = dg::pullback( dg::coo3, g);
     for( unsigned i=0; i<g2d.size(); i++)
     {
-        X[i] = g2d.r()[i];
-        Y[i] = g2d.z()[i];
+        X[i] = g2d.map()[0][i];
+        Y[i] = g2d.map()[1][i];
     }
     ncerr = nc_put_var_double( ncid, coordsID[0], X.data());
     ncerr = nc_put_var_double( ncid, coordsID[1], Y.data());
@@ -185,10 +186,9 @@ int main(int argc, char**argv)
     std::cout << sqrt( err/norm) << "\t";//<<sqrt( errFINE/normFINE)<<"\t";
     err = dg::blas2::dot( vol3dFINE, error_direct);
     std::cout << sqrt( err/norm) << "\t";//<<sqrt( errFINE/normFINE)<<"\t";
-    dg::HVec gyy, gxx, vol; 
-    dg::blas1::transfer( g2d.g_xx(), gyy);
-    dg::blas1::transfer( g2d.g_yy(), gxx); 
-    dg::blas1::transfer( g2d.vol() , vol);
+    ///////////////////////////////////metric//////////////////////
+    dg::SparseTensor<dg::DVec> metric = g2d.metric();
+    dg::DVec gyy = metric.value(1,1), gxx = metric.value(0,0), vol = dg::tensor::volume(metric).value(); 
     dg::blas1::transform( gxx, gxx, dg::SQRT<double>());
     dg::blas1::transform( gyy, gyy, dg::SQRT<double>());
     dg::blas1::pointwiseDot( gxx, vol, gxx);
@@ -201,7 +201,7 @@ int main(int argc, char**argv)
     std::cout << *thrust::max_element( gyy.begin(), gyy.end()) << "\t";
     std::cout << hxX << "\t";
     std::cout << hyX << "\t";
-    std::cout<<t.diff()/(double)number_sw<<"s"<<std::endl;
+    std::cout<<t.diff()/(double)number<<"s"<<std::endl;
 
     dg::blas1::transfer( error_direct, X);
     ncerr = nc_put_var_double( ncid, psiID, X.data());
diff --git a/inc/geometries/separatrix_orthogonal_t.cu b/inc/geometries/separatrix_orthogonal_t.cu
index 0d104f0ea..899d8c478 100644
--- a/inc/geometries/separatrix_orthogonal_t.cu
+++ b/inc/geometries/separatrix_orthogonal_t.cu
@@ -122,18 +122,12 @@ int main( int argc, char* argv[])
     double Z_X = -1.1*gp.elongation*gp.a;
     dg::geo::findXpoint( c.get_psip(), R_X, Z_X);
 
-    //solovev::Psip psip( gp); 
-    //std::cout << "Psi min "<<psip(gp.R_0, 0)<<"\n";
-    //solovev::PsipR psipR(gp); solovev::PsipZ psipZ(gp);
-    //solovev::LaplacePsip laplacePsip(gp); 
     double R0 = gp.R_0, Z0 = 0;
     dg::geo::SeparatrixOrthogonal generator(c.get_psip(), psi_0, R_X,Z_X, R0, Z0,0);
     //dg::geo::SimpleOrthogonalX generator(c.get_psip(), psi_0, R_X,Z_X, R0, Z0,0);
-    //dg::OrthogonalGridX3d g3d(generator, psi_0, fx_0, fy_0, n, Nx, Ny,Nz, dg::DIR, dg::NEU);
-    //dg::CurvilinearGridX2d g2d = g3d.perp_grid();
     dg::EquidistXRefinement equi(add_x, add_y, 1,1);
     dg::CurvilinearRefinedProductGridX3d g3d(equi, generator, fx_0, fy_0, n, Nx, Ny,Nz, dg::DIR, dg::NEU);
-    dg::CurvilinearRefinedGridX2d g2d = g3d.perp_grid();
+    dg::CurvilinearRefinedGridX2d g2d(equi, generator, fx_0, fy_0, n, Nx, Ny,dg::DIR, dg::NEU);
     t.toc();
     dg::GridX3d g3d_periodic(g3d.x0(), g3d.x1(), g3d.y0(), g3d.y1(), g3d.z0(), g3d.z1(), g3d.fx(), g3d.fy(), g3d.n(), g3d.Nx(), g3d.Ny(), 2); 
     std::cout << "Construction took "<<t.diff()<<"s"<<std::endl;
diff --git a/inc/geometries/simple_orthogonal_t.cu b/inc/geometries/simple_orthogonal_t.cu
index 819b0e623..d1db34668 100644
--- a/inc/geometries/simple_orthogonal_t.cu
+++ b/inc/geometries/simple_orthogonal_t.cu
@@ -67,9 +67,6 @@ int main( int argc, char* argv[])
     std::cout << "Type psi_0 and psi_1\n";
     double psi_0, psi_1;
     std::cin >> psi_0>> psi_1;
-    std::cout << "Type new_n, multiple_x and multiple_y \n";
-    double n_ref, multiple_x, multiple_y;
-    std::cin >> n_ref>>multiple_x >> multiple_y;
     gp.display( std::cout);
     dg::Timer t;
     //solovev::detail::Fpsi fpsi( gp, -10);
@@ -116,7 +113,7 @@ int main( int argc, char* argv[])
 
     //compute and write deformation into netcdf
     dg::SparseTensor<dg::HVec> metric = g2d.metric();
-    dg::HVec g_xx = metric.value(0,0), g_xy = metric.value(0,1), g_yy=metric.value(1,1);
+    dg::HVec g_xx = metric.value(0,0), g_yy=metric.value(1,1);
     dg::SparseElement<dg::HVec> vol_ = dg::tensor::volume(metric);
     dg::HVec vol = vol_.value();
     dg::blas1::pointwiseDivide( g_yy, g_xx, temp0);
@@ -130,20 +127,14 @@ int main( int argc, char* argv[])
 
     std::cout << "Construction successful!\n";
 
-    //compute error in volume element
-    dg::blas1::pointwiseDot( g_xy, g_xy, temp1); double error = sqrt( dg::blas2::dot( temp1, w2d, temp1));
-    std::cout<< "    Error in Off-diagonal is "<<error<<"\n";
-
     //compare determinant vs volume form
     dg::blas1::pointwiseDot( g_xx, g_yy, temp0);
-    dg::blas1::pointwiseDot( g_xy, g_xy, temp1);
-    dg::blas1::axpby( 1., temp0, -1., temp1, temp0);
     dg::blas1::transform( temp0, temp0, dg::SQRT<double>());
     dg::blas1::pointwiseDivide( ones, temp0, temp0);
     dg::blas1::transfer( temp0, X);
     err = nc_put_var_double( ncid, volID, periodify(X, g2d_periodic).data());
     dg::blas1::axpby( 1., temp0, -1., vol, temp0);
-    error = sqrt(dg::blas2::dot( temp0, w2d, temp0)/dg::blas2::dot( vol, w2d, vol));
+    double error = sqrt(dg::blas2::dot( temp0, w2d, temp0)/dg::blas2::dot( vol, w2d, vol));
     std::cout << "Rel Consistency  of volume is "<<error<<"\n";
 
     /*
-- 
GitLab


From 971e1aa02ab0222b9f7be2ab3678b8278745dba2 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 21 Aug 2017 02:42:57 -0700
Subject: [PATCH 194/453] removed CylindricalGrid template parameter in src
 files

---
 src/asela/asela.cu         |  8 ++++----
 src/asela/asela_hpc.cu     | 10 +++++-----
 src/asela/asela_mpi.cu     | 10 +++++-----
 src/asela2D/asela.cu       |  8 ++++----
 src/asela2D/asela_hpc.cu   | 10 +++++-----
 src/asela2D/asela_mpi.cu   | 10 +++++-----
 src/feltor/feltor.cu       |  8 ++++----
 src/feltor/feltor_hpc.cu   | 10 +++++-----
 src/feltor/feltor_mpi.cu   | 10 +++++-----
 src/feltor2D/feltor.cu     |  8 ++++----
 src/feltor2D/feltor_hpc.cu | 10 +++++-----
 src/feltor2D/feltor_mpi.cu | 10 +++++-----
 src/heat/heat.cu           |  6 +++---
 src/heat/heat_hpc.cu       | 10 +++++-----
 14 files changed, 64 insertions(+), 64 deletions(-)

diff --git a/src/asela/asela.cu b/src/asela/asela.cu
index 13f7e60cf..a3c81ef5e 100644
--- a/src/asela/asela.cu
+++ b/src/asela/asela.cu
@@ -23,7 +23,7 @@
    - integrates the ToeflR - functor and 
    - directly visualizes results on the screen using parameters in window_params.txt
 */
-typedef dg::FieldAligned< dg::CylindricalGrid3d<dg::DVec>, dg::IDMatrix, dg::DVec> DFA;
+typedef dg::FieldAligned< dg::CylindricalGrid3d, dg::IDMatrix, dg::DVec> DFA;
 
 using namespace dg::geo::solovev;
 
@@ -70,11 +70,11 @@ int main( int argc, char* argv[])
     double Zmax=p.boxscaleZp*gp.a*gp.elongation;
     //Make grid
 
-    dg::CylindricalGrid3d<dg::DVec> grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n, p.Nx, p.Ny, p.Nz, p.bc, p.bc, dg::PER);      //create RHS 
+    dg::CylindricalGrid3d grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n, p.Nx, p.Ny, p.Nz, p.bc, p.bc, dg::PER);      //create RHS 
     std::cout << "Constructing Asela...\n";
-    eule::Asela<dg::CylindricalGrid3d<dg::DVec>, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec> asela( grid, p,gp); //initialize before rolkar!
+    eule::Asela<dg::CylindricalGrid3d, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec> asela( grid, p,gp); //initialize before rolkar!
     std::cout << "Constructing Rolkar...\n";
-    eule::Rolkar<dg::CylindricalGrid3d<dg::DVec>, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec> rolkar(  grid, p, gp, asela.ds(), asela.dsDIR());
+    eule::Rolkar<dg::CylindricalGrid3d, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec> rolkar(  grid, p, gp, asela.ds(), asela.dsDIR());
     std::cout << "Done!\n";
 
    /////////////////////The initial field///////////////////////////////////////////
diff --git a/src/asela/asela_hpc.cu b/src/asela/asela_hpc.cu
index 9ab9136af..253de85eb 100644
--- a/src/asela/asela_hpc.cu
+++ b/src/asela/asela_hpc.cu
@@ -21,7 +21,7 @@
 
 */
 
-typedef dg::FieldAligned< dg::CylindricalGrid3d<dg::DVec>, dg::IDMatrix, dg::DVec> DFA;
+typedef dg::FieldAligned< dg::CylindricalGrid3d, dg::IDMatrix, dg::DVec> DFA;
 using namespace dg::geo::solovev;
 int main( int argc, char* argv[])
 {
@@ -52,15 +52,15 @@ int main( int argc, char* argv[])
     double Rmax=gp.R_0+p.boxscaleRp*gp.a; 
     double Zmax=p.boxscaleZp*gp.a*gp.elongation;
     //Make grids
-    dg::CylindricalGrid3d<dg::DVec> grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n, p.Nx, p.Ny, p.Nz, p.bc, p.bc, dg::PER);  
-    dg::CylindricalGrid3d<dg::DVec > grid_out( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n_out, p.Nx_out, p.Ny_out, p.Nz_out, p.bc, p.bc, dg::PER);  
+    dg::CylindricalGrid3d grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n, p.Nx, p.Ny, p.Nz, p.bc, p.bc, dg::PER);  
+    dg::CylindricalGrid3d grid_out( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n_out, p.Nx_out, p.Ny_out, p.Nz_out, p.bc, p.bc, dg::PER);  
 
      
     //create RHS 
     std::cout << "Constructing Asela...\n";
-    eule::Asela<dg::CylindricalGrid3d<dg::DVec>, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec> asela( grid, p, gp); //initialize before rolkar!
+    eule::Asela<dg::CylindricalGrid3d, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec> asela( grid, p, gp); //initialize before rolkar!
     std::cout << "Constructing Rolkar...\n";
-    eule::Rolkar< dg::CylindricalGrid3d<dg::DVec>, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec > rolkar( grid, p, gp, asela.ds(), asela.dsDIR());
+    eule::Rolkar< dg::CylindricalGrid3d, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec > rolkar( grid, p, gp, asela.ds(), asela.dsDIR());
     std::cout << "Done!\n";
 
     /////////////////////The initial field//////////////////////////////////////////
diff --git a/src/asela/asela_mpi.cu b/src/asela/asela_mpi.cu
index 04901aa23..5d669187b 100644
--- a/src/asela/asela_mpi.cu
+++ b/src/asela/asela_mpi.cu
@@ -24,7 +24,7 @@
         output dimensions must be divisible by the mpi process numbers
 */
 
-typedef dg::MPI_FieldAligned< dg::CylindricalMPIGrid3d<dg::MDVec>, dg::IDMatrix,dg::BijectiveComm< dg::iDVec, dg::DVec >, dg::DVec> DFA;
+typedef dg::MPI_FieldAligned< dg::CylindricalMPIGrid3d, dg::IDMatrix,dg::BijectiveComm< dg::iDVec, dg::DVec >, dg::DVec> DFA;
 using namespace dg::geo::solovev;
 int main( int argc, char* argv[])
 {
@@ -84,15 +84,15 @@ int main( int argc, char* argv[])
     double Rmax=gp.R_0+p.boxscaleRp*gp.a; 
     double Zmax=p.boxscaleZp*gp.a*gp.elongation;
     //Make grids
-     dg::CylindricalMPIGrid3d<dg::MDVec> grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n, p.Nx, p.Ny, p.Nz, p.bc, p.bc, dg::PER, comm);  
-     dg::CylindricalMPIGrid3d<dg::MDVec> grid_out( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n_out, p.Nx_out, p.Ny_out, p.Nz_out, p.bc, p.bc, dg::PER, comm);  
+     dg::CylindricalMPIGrid3d grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n, p.Nx, p.Ny, p.Nz, p.bc, p.bc, dg::PER, comm);  
+     dg::CylindricalMPIGrid3d grid_out( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n_out, p.Nx_out, p.Ny_out, p.Nz_out, p.bc, p.bc, dg::PER, comm);  
 
      
     //create RHS 
     if(rank==0)std::cout << "Constructing Asela...\n";
-    eule::Asela<dg::CylindricalMPIGrid3d<dg::MDVec>, dg::DS<DFA, dg::MDMatrix, dg::MDVec>, dg::MDMatrix, dg::MDVec> asela( grid, p, gp); //initialize before rolkar!
+    eule::Asela<dg::CylindricalMPIGrid3d, dg::DS<DFA, dg::MDMatrix, dg::MDVec>, dg::MDMatrix, dg::MDVec> asela( grid, p, gp); //initialize before rolkar!
     if(rank==0)std::cout << "Constructing Rolkar...\n";
-    eule::Rolkar< dg::CylindricalMPIGrid3d<dg::MDVec>, dg::DS<DFA, dg::MDMatrix, dg::MDVec>, dg::MDMatrix, dg::MDVec > rolkar( grid, p, gp, asela.ds(), asela.dsDIR());
+    eule::Rolkar< dg::CylindricalMPIGrid3d, dg::DS<DFA, dg::MDMatrix, dg::MDVec>, dg::MDMatrix, dg::MDVec > rolkar( grid, p, gp, asela.ds(), asela.dsDIR());
     if(rank==0)std::cout << "Done!\n";
 
     /////////////////////The initial field/////////////////////////////////////////
diff --git a/src/asela2D/asela.cu b/src/asela2D/asela.cu
index 8e48867b6..50b1a9e73 100644
--- a/src/asela2D/asela.cu
+++ b/src/asela2D/asela.cu
@@ -20,7 +20,7 @@
    - directly visualizes results on the screen using parameters in window_params.txt
 */
 
-typedef dg::FieldAligned< dg::CylindricalGrid3d<dg::DVec>, dg::IDMatrix, dg::DVec> DFA;
+typedef dg::FieldAligned< dg::CylindricalGrid3d, dg::IDMatrix, dg::DVec> DFA;
 using namespace dg::geo::solovev;
 int main( int argc, char* argv[])
 {
@@ -63,12 +63,12 @@ int main( int argc, char* argv[])
     double Rmax=gp.R_0+p.boxscaleRp*gp.a; 
     double Zmax=p.boxscaleZp*gp.a*gp.elongation;
     //Make grid
-    dg::CylindricalGrid3d<dg::DVec > grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n, p.Nx, p.Ny, 1, p.bc, p.bc, dg::PER);      //create RHS 
+    dg::CylindricalGrid3d grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n, p.Nx, p.Ny, 1, p.bc, p.bc, dg::PER);      //create RHS 
     std::cout << "Constructing Asela...\n";
-    eule::Asela<dg::CylindricalGrid3d<dg::DVec>, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec > asela( grid, p, gp); //initialize before rolkar!
+    eule::Asela<dg::CylindricalGrid3d, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec > asela( grid, p, gp); //initialize before rolkar!
 
     std::cout << "Constructing Rolkar...\n";
-    eule::Rolkar<dg::CylindricalGrid3d<dg::DVec>, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec> rolkar( grid, p, gp, asela.ds(), asela.dsDIR());
+    eule::Rolkar<dg::CylindricalGrid3d, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec> rolkar( grid, p, gp, asela.ds(), asela.dsDIR());
     std::cout << "Done!\n";
 
     /////////////////////The initial field///////////////////////////////////////////
diff --git a/src/asela2D/asela_hpc.cu b/src/asela2D/asela_hpc.cu
index ed0ce242b..3f560cd14 100644
--- a/src/asela2D/asela_hpc.cu
+++ b/src/asela2D/asela_hpc.cu
@@ -23,7 +23,7 @@
         density fields are the real densities in XSPACE ( not logarithmic values)
 */
 
-typedef dg::FieldAligned< dg::CylindricalGrid3d<dg::DVec>, dg::IDMatrix, dg::DVec> DFA;
+typedef dg::FieldAligned< dg::CylindricalGrid3d, dg::IDMatrix, dg::DVec> DFA;
 using namespace dg::geo::solovev;
 int main( int argc, char* argv[])
 {
@@ -54,13 +54,13 @@ int main( int argc, char* argv[])
     double Rmax=gp.R_0+p.boxscaleRp*gp.a; 
     double Zmax=p.boxscaleZp*gp.a*gp.elongation;
     //Make grids
-    dg::CylindricalGrid3d<dg::DVec> grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n, p.Nx, p.Ny, 1, p.bc, p.bc, dg::PER);  
-    dg::CylindricalGrid3d<dg::DVec> grid_out( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n_out, p.Nx_out, p.Ny_out,1,p.bc, p.bc, dg::PER);  
+    dg::CylindricalGrid3d grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n, p.Nx, p.Ny, 1, p.bc, p.bc, dg::PER);  
+    dg::CylindricalGrid3d grid_out( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n_out, p.Nx_out, p.Ny_out,1,p.bc, p.bc, dg::PER);  
     //create RHS 
     std::cout << "Constructing Asela...\n";
-    eule::Asela<dg::CylindricalGrid3d<dg::DVec>, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec > asela( grid, p, gp); //initialize before rolkar!
+    eule::Asela<dg::CylindricalGrid3d, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec > asela( grid, p, gp); //initialize before rolkar!
     std::cout << "Constructing Rolkar...\n";
-    eule::Rolkar<dg::CylindricalGrid3d<dg::DVec>, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec> rolkar( grid, p, gp, asela.ds(), asela.dsDIR());
+    eule::Rolkar<dg::CylindricalGrid3d, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec> rolkar( grid, p, gp, asela.ds(), asela.dsDIR());
     std::cout << "Done!\n";
 
     /////////////////////The initial field///////////////////////////////////////////
diff --git a/src/asela2D/asela_mpi.cu b/src/asela2D/asela_mpi.cu
index 7650c27f9..2d4c52f54 100644
--- a/src/asela2D/asela_mpi.cu
+++ b/src/asela2D/asela_mpi.cu
@@ -23,7 +23,7 @@
         density fields are the real densities in XSPACE ( not logarithmic values)
 */
 
-typedef dg::MPI_FieldAligned< dg::CylindricalMPIGrid3d<dg::MDVec>, dg::IDMatrix,dg::BijectiveComm< dg::iDVec, dg::DVec >, dg::DVec> DFA;
+typedef dg::MPI_FieldAligned< dg::CylindricalMPIGrid3d, dg::IDMatrix,dg::BijectiveComm< dg::iDVec, dg::DVec >, dg::DVec> DFA;
 using namespace dg::geo::solovev;
 int main( int argc, char* argv[])
 {
@@ -72,13 +72,13 @@ int main( int argc, char* argv[])
    
     //Make grids: both the dimensions of grid and grid_out must be dividable by the mpi process numbers in that direction
 
-    dg::CylindricalMPIGrid3d<dg::MDVec> grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n, p.Nx, p.Ny, 1, p.bc, p.bc, dg::PER, comm);  
-    dg::CylindricalMPIGrid3d<dg::MDVec> grid_out( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n_out, p.Nx_out, p.Ny_out, 1, p.bc, p.bc, dg::PER, comm);  
+    dg::CylindricalMPIGrid3d grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n, p.Nx, p.Ny, 1, p.bc, p.bc, dg::PER, comm);  
+    dg::CylindricalMPIGrid3d grid_out( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n_out, p.Nx_out, p.Ny_out, 1, p.bc, p.bc, dg::PER, comm);  
      
     if(rank==0)std::cout << "Constructing Asela...\n";
-    eule::Asela<dg::CylindricalMPIGrid3d<dg::MDVec>, dg::DS<DFA, dg::MDMatrix, dg::MDVec>, dg::MDMatrix, dg::MDVec> asela( grid, p, gp); //initialize before rolkar!
+    eule::Asela<dg::CylindricalMPIGrid3d, dg::DS<DFA, dg::MDMatrix, dg::MDVec>, dg::MDMatrix, dg::MDVec> asela( grid, p, gp); //initialize before rolkar!
     if(rank==0)std::cout << "Constructing Rolkar...\n";
-    eule::Rolkar< dg::CylindricalMPIGrid3d<dg::MDVec>, dg::DS<DFA, dg::MDMatrix, dg::MDVec>, dg::MDMatrix, dg::MDVec > rolkar( grid, p, gp, asela.ds(), asela.dsDIR());
+    eule::Rolkar< dg::CylindricalMPIGrid3d, dg::DS<DFA, dg::MDMatrix, dg::MDVec>, dg::MDMatrix, dg::MDVec > rolkar( grid, p, gp, asela.ds(), asela.dsDIR());
     if(rank==0)std::cout << "Done!\n";
 
     /////////////////////The initial field///////////////////////////////////////////
diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index bfe4fbc7a..1ee7ecda3 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -20,7 +20,7 @@
    - integrates the Feltor - functor and 
    - directly visualizes results on the screen using parameters in window_params.txt
 */
-typedef dg::FieldAligned< dg::CylindricalGrid3d<dg::DVec>, dg::IDMatrix, dg::DVec> DFA;
+typedef dg::FieldAligned< dg::CylindricalGrid3d, dg::IDMatrix, dg::DVec> DFA;
 
 using namespace dg::geo::solovev;
 
@@ -66,13 +66,13 @@ int main( int argc, char* argv[])
     double Rmax=gp.R_0+p.boxscaleRp*gp.a; 
     double Zmax=p.boxscaleZp*gp.a*gp.elongation;
     //Make grid
-    dg::CylindricalGrid3d<dg::DVec> grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n, p.Nx, p.Ny, p.Nz, p.bc, p.bc, dg::PER);  
+    dg::CylindricalGrid3d grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n, p.Nx, p.Ny, p.Nz, p.bc, p.bc, dg::PER);  
 
     //create RHS 
     std::cout << "Constructing Feltor...\n";
-    eule::Feltor<dg::CylindricalGrid3d<dg::DVec>, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec> feltor( grid, p, gp); //initialize before rolkar!
+    eule::Feltor<dg::CylindricalGrid3d, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec> feltor( grid, p, gp); //initialize before rolkar!
     std::cout << "Constructing Rolkar...\n";
-    eule::Rolkar<dg::CylindricalGrid3d<dg::DVec>, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec> rolkar( grid, p, gp, feltor.ds(), feltor.dsDIR());
+    eule::Rolkar<dg::CylindricalGrid3d, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec> rolkar( grid, p, gp, feltor.ds(), feltor.dsDIR());
     std::cout << "Done!\n";
 
     /////////////////////The initial field///////////////////////////////////////////
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 4887baba3..b82c117fc 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -24,7 +24,7 @@
 
 */
 
-typedef dg::FieldAligned< dg::CylindricalGrid3d<dg::DVec>, dg::IDMatrix, dg::DVec> DFA;
+typedef dg::FieldAligned< dg::CylindricalGrid3d, dg::IDMatrix, dg::DVec> DFA;
 using namespace dg::geo::solovev;
 int main( int argc, char* argv[])
 {
@@ -55,14 +55,14 @@ int main( int argc, char* argv[])
     double Rmax=gp.R_0+p.boxscaleRp*gp.a; 
     double Zmax=p.boxscaleZp*gp.a*gp.elongation;
     //Make grids
-    dg::CylindricalGrid3d<dg::DVec> grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n, p.Nx, p.Ny, p.Nz, p.bc, p.bc, dg::PER);  
-    dg::CylindricalGrid3d<dg::DVec > grid_out( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n_out, p.Nx_out, p.Ny_out, p.Nz_out, p.bc, p.bc, dg::PER);  
+    dg::CylindricalGrid3d grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n, p.Nx, p.Ny, p.Nz, p.bc, p.bc, dg::PER);  
+    dg::CylindricalGrid3d grid_out( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n_out, p.Nx_out, p.Ny_out, p.Nz_out, p.bc, p.bc, dg::PER);  
      
     //create RHS 
     std::cout << "Constructing Feltor...\n";
-    eule::Feltor<dg::CylindricalGrid3d<dg::DVec>, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec> feltor( grid, p, gp); //initialize before rolkar!
+    eule::Feltor<dg::CylindricalGrid3d, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec> feltor( grid, p, gp); //initialize before rolkar!
     std::cout << "Constructing Rolkar...\n";
-    eule::Rolkar< dg::CylindricalGrid3d<dg::DVec>, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec > rolkar( grid, p, gp, feltor.ds(), feltor.dsDIR());
+    eule::Rolkar< dg::CylindricalGrid3d, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec > rolkar( grid, p, gp, feltor.ds(), feltor.dsDIR());
     std::cout << "Done!\n";
 
     /////////////////////The initial field//////////////////////////////////////////
diff --git a/src/feltor/feltor_mpi.cu b/src/feltor/feltor_mpi.cu
index efec9eaa4..d118ee66e 100644
--- a/src/feltor/feltor_mpi.cu
+++ b/src/feltor/feltor_mpi.cu
@@ -24,7 +24,7 @@
         output dimensions must be divisible by the mpi process numbers
 */
 
-typedef dg::MPI_FieldAligned< dg::CylindricalMPIGrid3d<dg::MDVec>, dg::IDMatrix,dg::BijectiveComm< dg::iDVec, dg::DVec >, dg::DVec> DFA;
+typedef dg::MPI_FieldAligned< dg::CylindricalMPIGrid3d, dg::IDMatrix,dg::BijectiveComm< dg::iDVec, dg::DVec >, dg::DVec> DFA;
 using namespace dg::geo::solovev;
 int main( int argc, char* argv[])
 {
@@ -84,14 +84,14 @@ int main( int argc, char* argv[])
     double Rmax=gp.R_0+p.boxscaleRp*gp.a; 
     double Zmax=p.boxscaleZp*gp.a*gp.elongation;
     //Make grids
-     dg::CylindricalMPIGrid3d<dg::MDVec> grid(     Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n,     p.Nx,     p.Ny,     p.Nz,     p.bc, p.bc, dg::PER, comm);  
-     dg::CylindricalMPIGrid3d<dg::MDVec> grid_out( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n_out, p.Nx_out, p.Ny_out, p.Nz_out, p.bc, p.bc, dg::PER, comm);  
+     dg::CylindricalMPIGrid3d grid(     Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n,     p.Nx,     p.Ny,     p.Nz,     p.bc, p.bc, dg::PER, comm);  
+     dg::CylindricalMPIGrid3d grid_out( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n_out, p.Nx_out, p.Ny_out, p.Nz_out, p.bc, p.bc, dg::PER, comm);  
      
     //create RHS 
     if(rank==0)std::cout << "Constructing Feltor...\n";
-    eule::Feltor<dg::CylindricalMPIGrid3d<dg::MDVec>, dg::DS<DFA, dg::MDMatrix, dg::MDVec>, dg::MDMatrix, dg::MDVec> feltor( grid, p, gp); //initialize before rolkar!
+    eule::Feltor<dg::CylindricalMPIGrid3d, dg::DS<DFA, dg::MDMatrix, dg::MDVec>, dg::MDMatrix, dg::MDVec> feltor( grid, p, gp); //initialize before rolkar!
     if(rank==0)std::cout << "Constructing Rolkar...\n";
-    eule::Rolkar< dg::CylindricalMPIGrid3d<dg::MDVec>, dg::DS<DFA, dg::MDMatrix, dg::MDVec>, dg::MDMatrix, dg::MDVec > rolkar( grid, p, gp, feltor.ds(), feltor.dsDIR());
+    eule::Rolkar< dg::CylindricalMPIGrid3d, dg::DS<DFA, dg::MDMatrix, dg::MDVec>, dg::MDMatrix, dg::MDVec > rolkar( grid, p, gp, feltor.ds(), feltor.dsDIR());
     if(rank==0)std::cout << "Done!\n";
 
     /////////////////////The initial field/////////////////////////////////////////
diff --git a/src/feltor2D/feltor.cu b/src/feltor2D/feltor.cu
index 3abfbb910..42d2dcdad 100644
--- a/src/feltor2D/feltor.cu
+++ b/src/feltor2D/feltor.cu
@@ -21,7 +21,7 @@
 */
 
 
-typedef dg::FieldAligned< dg::CylindricalGrid3d<dg::DVec>, dg::IDMatrix, dg::DVec> DFA;
+typedef dg::FieldAligned< dg::CylindricalGrid3d, dg::IDMatrix, dg::DVec> DFA;
 using namespace dg::geo::solovev;
 int main( int argc, char* argv[])
 {
@@ -64,12 +64,12 @@ int main( int argc, char* argv[])
     double Rmax=gp.R_0+p.boxscaleRp*gp.a; 
     double Zmax=p.boxscaleZp*gp.a*gp.elongation;
     //Make grid
-     dg::CylindricalGrid3d<dg::DVec > grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n, p.Nx, p.Ny, 1, p.bc, p.bc, dg::PER);  
+     dg::CylindricalGrid3d grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n, p.Nx, p.Ny, 1, p.bc, p.bc, dg::PER);  
     //create RHS 
     std::cout << "Constructing Feltor...\n";
-    eule::Feltor<dg::CylindricalGrid3d<dg::DVec>, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec > feltor( grid, p, gp); //initialize before rolkar!
+    eule::Feltor<dg::CylindricalGrid3d, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec > feltor( grid, p, gp); //initialize before rolkar!
     std::cout << "Constructing Rolkar...\n";
-    eule::Rolkar<dg::CylindricalGrid3d<dg::DVec>, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec> rolkar( grid, p, gp, feltor.ds(), feltor.dsDIR());
+    eule::Rolkar<dg::CylindricalGrid3d, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec> rolkar( grid, p, gp, feltor.ds(), feltor.dsDIR());
     std::cout << "Done!\n";
 
     /////////////////////The initial field///////////////////////////////////////////
diff --git a/src/feltor2D/feltor_hpc.cu b/src/feltor2D/feltor_hpc.cu
index cf986a88c..7e29224a8 100644
--- a/src/feltor2D/feltor_hpc.cu
+++ b/src/feltor2D/feltor_hpc.cu
@@ -23,7 +23,7 @@
         density fields are the real densities in XSPACE ( not logarithmic values)
 */
 
-typedef dg::FieldAligned< dg::CylindricalGrid3d<dg::DVec>, dg::IDMatrix, dg::DVec> DFA;
+typedef dg::FieldAligned< dg::CylindricalGrid3d, dg::IDMatrix, dg::DVec> DFA;
 using namespace dg::geo::solovev;
 int main( int argc, char* argv[])
 {
@@ -54,13 +54,13 @@ int main( int argc, char* argv[])
     double Rmax=gp.R_0+p.boxscaleRp*gp.a; 
     double Zmax=p.boxscaleZp*gp.a*gp.elongation;
     //Make grids
-    dg::CylindricalGrid3d<dg::DVec> grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n, p.Nx, p.Ny, 1, p.bc, p.bc, dg::PER);  
-    dg::CylindricalGrid3d<dg::DVec> grid_out( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n_out, p.Nx_out, p.Ny_out,1,p.bc, p.bc, dg::PER);  
+    dg::CylindricalGrid3d grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n, p.Nx, p.Ny, 1, p.bc, p.bc, dg::PER);  
+    dg::CylindricalGrid3d grid_out( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n_out, p.Nx_out, p.Ny_out,1,p.bc, p.bc, dg::PER);  
     //create RHS 
     std::cout << "Constructing Feltor...\n";
-    eule::Feltor<dg::CylindricalGrid3d<dg::DVec>, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec > feltor( grid, p, gp); //initialize before rolkar!
+    eule::Feltor<dg::CylindricalGrid3d, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec > feltor( grid, p, gp); //initialize before rolkar!
     std::cout << "Constructing Rolkar...\n";
-    eule::Rolkar<dg::CylindricalGrid3d<dg::DVec>, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec> rolkar( grid, p, gp, feltor.ds(), feltor.dsDIR());
+    eule::Rolkar<dg::CylindricalGrid3d, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec> rolkar( grid, p, gp, feltor.ds(), feltor.dsDIR());
     std::cout << "Done!\n";
 
     /////////////////////The initial field///////////////////////////////////////////
diff --git a/src/feltor2D/feltor_mpi.cu b/src/feltor2D/feltor_mpi.cu
index 099cefbba..4f504d989 100644
--- a/src/feltor2D/feltor_mpi.cu
+++ b/src/feltor2D/feltor_mpi.cu
@@ -23,7 +23,7 @@
         density fields are the real densities in XSPACE ( not logarithmic values)
 */
 
-typedef dg::MPI_FieldAligned< dg::CylindricalMPIGrid3d<dg::MDVec>, dg::IDMatrix,dg::BijectiveComm< dg::iDVec, dg::DVec >, dg::DVec> DFA;
+typedef dg::MPI_FieldAligned< dg::CylindricalMPIGrid3d, dg::IDMatrix,dg::BijectiveComm< dg::iDVec, dg::DVec >, dg::DVec> DFA;
 using namespace dg::geo::solovev;
 int main( int argc, char* argv[])
 {
@@ -72,13 +72,13 @@ int main( int argc, char* argv[])
    
     //Make grids: both the dimensions of grid and grid_out must be dividable by the mpi process numbers in that direction
 
-    dg::CylindricalMPIGrid3d<dg::MDVec> grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n, p.Nx, p.Ny, 1, p.bc, p.bc, dg::PER, comm);  
-    dg::CylindricalMPIGrid3d<dg::MDVec> grid_out( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n_out, p.Nx_out, p.Ny_out, 1, p.bc, p.bc, dg::PER, comm);  
+    dg::CylindricalMPIGrid3d grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n, p.Nx, p.Ny, 1, p.bc, p.bc, dg::PER, comm);  
+    dg::CylindricalMPIGrid3d grid_out( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n_out, p.Nx_out, p.Ny_out, 1, p.bc, p.bc, dg::PER, comm);  
      
     if(rank==0)std::cout << "Constructing Feltor...\n";
-    eule::Feltor<dg::CylindricalMPIGrid3d<dg::MDVec>, dg::DS<DFA, dg::MDMatrix, dg::MDVec>, dg::MDMatrix, dg::MDVec> feltor( grid, p, gp); //initialize before rolkar!
+    eule::Feltor<dg::CylindricalMPIGrid3d, dg::DS<DFA, dg::MDMatrix, dg::MDVec>, dg::MDMatrix, dg::MDVec> feltor( grid, p, gp); //initialize before rolkar!
     if(rank==0)std::cout << "Constructing Rolkar...\n";
-    eule::Rolkar< dg::CylindricalMPIGrid3d<dg::MDVec>, dg::DS<DFA, dg::MDMatrix, dg::MDVec>, dg::MDMatrix, dg::MDVec > rolkar( grid, p, gp, feltor.ds(), feltor.dsDIR());
+    eule::Rolkar< dg::CylindricalMPIGrid3d, dg::DS<DFA, dg::MDMatrix, dg::MDVec>, dg::MDMatrix, dg::MDVec > rolkar( grid, p, gp, feltor.ds(), feltor.dsDIR());
     if(rank==0)std::cout << "Done!\n";
 
     /////////////////////The initial field///////////////////////////////////////////
diff --git a/src/heat/heat.cu b/src/heat/heat.cu
index c64136a00..1af95be9d 100644
--- a/src/heat/heat.cu
+++ b/src/heat/heat.cu
@@ -20,7 +20,7 @@
 
 #include "heat.cuh"
 
-typedef dg::FieldAligned< dg::CylindricalGrid3d<dg::DVec>, dg::IDMatrix, dg::DVec> DFA;
+typedef dg::FieldAligned< dg::CylindricalGrid3d, dg::IDMatrix, dg::DVec> DFA;
 using namespace dg::geo::solovev;
 
 int main( int argc, char* argv[])
@@ -63,7 +63,7 @@ int main( int argc, char* argv[])
     double Rmax=gp.R_0+p.boxscaleRp*gp.a; 
     double Zmax=p.boxscaleZp*gp.a*gp.elongation;
 
-     dg::CylindricalGrid3d<dg::DVec> grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n, p.Nx, p.Ny, p.Nz, p.bc, p.bc, dg::PER);  
+     dg::CylindricalGrid3d grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n, p.Nx, p.Ny, p.Nz, p.bc, p.bc, dg::PER);  
 
 //     dg::DVec w3d_ = dg::create::volume( grid);
 //     dg::DVec v3d_ = dg::create::inv_volume( grid);
@@ -115,7 +115,7 @@ int main( int argc, char* argv[])
     std::cout << "initialize feltor" << std::endl;
     eule::Feltor<dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec > feltor( grid, p,gp); //initialize before rolkar!
     std::cout << "initialize rolkar" << std::endl;
-    eule::Rolkar<dg::CylindricalGrid3d<dg::DVec> , dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec > rolkar( grid, p,gp);
+    eule::Rolkar<dg::CylindricalGrid3d , dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec > rolkar( grid, p,gp);
 
     ////////////////////////////////The initial field////////////////////////////////
  //initial perturbation
diff --git a/src/heat/heat_hpc.cu b/src/heat/heat_hpc.cu
index c742291dc..985c3c225 100644
--- a/src/heat/heat_hpc.cu
+++ b/src/heat/heat_hpc.cu
@@ -22,7 +22,7 @@
 
 #include "heat.cuh"
 
-typedef dg::FieldAligned< dg::CylindricalGrid3d<dg::DVec>, dg::IDMatrix, dg::DVec> DFA;
+typedef dg::FieldAligned< dg::CylindricalGrid3d, dg::IDMatrix, dg::DVec> DFA;
 using namespace dg::geo::solovev;
 
 int main( int argc, char* argv[])
@@ -54,8 +54,8 @@ int main( int argc, char* argv[])
     double Zmax=p.boxscaleZp*gp.a*gp.elongation;
 
     //Make grids
-    dg::CylindricalGrid3d<dg::DVec> grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n, p.Nx, p.Ny, p.Nz, p.bc, p.bc, dg::PER);  
-    dg::CylindricalGrid3d<dg::DVec> grid_out( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n_out, p.Nx_out, p.Ny_out,p.Nz_out,p.bc, p.bc, dg::PER); 
+    dg::CylindricalGrid3d grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n, p.Nx, p.Ny, p.Nz, p.bc, p.bc, dg::PER);  
+    dg::CylindricalGrid3d grid_out( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n_out, p.Nx_out, p.Ny_out,p.Nz_out,p.bc, p.bc, dg::PER); 
     dg::DVec w3d =  dg::create::volume(grid);
     dg::DVec w3dout =  dg::create::volume(grid_out);
 
@@ -92,7 +92,7 @@ int main( int argc, char* argv[])
         double Zminin =-pin.boxscaleZm*gpin.a*gpin.elongation;
         double Rmaxin = gpin.R_0 + pin.boxscaleRp*gpin.a; 
         double Zmaxin = pin.boxscaleZp*gpin.a*gpin.elongation;
-        dg::CylindricalGrid3d<dg::DVec > grid_in( Rminin,Rmaxin, Zminin,Zmaxin, 0, 2.*M_PI, pin.n, pin.Nx, pin.Ny, pin.Nz, pin.bc, pin.bc, dg::PER);
+        dg::CylindricalGrid3d grid_in( Rminin,Rmaxin, Zminin,Zmaxin, 0, 2.*M_PI, pin.n, pin.Nx, pin.Ny, pin.Nz, pin.bc, pin.bc, dg::PER);
         size_t start3din[4]  = {pin.maxout, 0, 0, 0};
         size_t count3din[4]  = {1, grid_in.Nz(), grid_in.n()*grid_in.Ny(), grid_in.n()*grid_in.Nx()};
         std::string namesin[1] = {"T"}; 
@@ -106,7 +106,7 @@ int main( int argc, char* argv[])
     std::cout << "Constructing Feltor...\n";
     eule::Feltor<dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec > feltor( grid, p,gp); 
     std::cout << "Constructing Rolkar...\n";
-    eule::Rolkar<dg::CylindricalGrid3d<dg::DVec>, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec > rolkar( grid, p,gp);
+    eule::Rolkar<dg::CylindricalGrid3d, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec > rolkar( grid, p,gp);
     std::cout << "Done!\n";
 
     /////////////////////The initial field///////////////////////////////////////////
-- 
GitLab


From bc7de90fde37bd6bd7da480e231e9aaf3c213d4a Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 21 Aug 2017 04:37:30 -0700
Subject: [PATCH 195/453] documented Geometry and made GeneralElliptic use
 Handle class

---
 inc/dg/arakawa.h     |  8 ++++----
 inc/dg/dg_doc.h      |  8 ++++----
 inc/dg/elliptic.h    | 30 +++++++++++++++---------------
 inc/dg/elliptic_b.cu |  6 +++---
 inc/dg/helmholtz.h   |  8 ++++----
 inc/dg/poisson.h     | 12 ++++++------
 6 files changed, 36 insertions(+), 36 deletions(-)

diff --git a/inc/dg/arakawa.h b/inc/dg/arakawa.h
index 040325afe..120319120 100644
--- a/inc/dg/arakawa.h
+++ b/inc/dg/arakawa.h
@@ -32,14 +32,14 @@ struct ArakawaX
      * @brief Create Arakawa on a grid
      * @param g The grid
      */
-    ArakawaX( Geometry g);
+    ArakawaX( const Geometry& g);
     /**
      * @brief Create Arakawa on a grid using different boundary conditions
      * @param g The grid
      * @param bcx The boundary condition in x
      * @param bcy The boundary condition in y
      */
-    ArakawaX( Geometry g, bc bcx, bc bcy);
+    ArakawaX( const Geometry& g, bc bcx, bc bcy);
 
     /**
      * @brief Compute poisson's bracket
@@ -92,7 +92,7 @@ struct ArakawaX
 };
 
 template<class Geometry, class Matrix, class container>
-ArakawaX<Geometry, Matrix, container>::ArakawaX( Geometry g ): 
+ArakawaX<Geometry, Matrix, container>::ArakawaX( const Geometry& g ): 
     dxlhs( dg::evaluate( one, g) ), dxrhs(dxlhs), dylhs(dxlhs), dyrhs( dxlhs), helper_( dxlhs), 
     bdxf( dg::create::dx( g, g.bcx())),
     bdyf( dg::create::dy( g, g.bcy()))
@@ -102,7 +102,7 @@ ArakawaX<Geometry, Matrix, container>::ArakawaX( Geometry g ):
     dg::tensor::sqrt(perp_vol_inv_);
 }
 template<class Geometry, class Matrix, class container>
-ArakawaX<Geometry, Matrix, container>::ArakawaX( Geometry g, bc bcx, bc bcy): 
+ArakawaX<Geometry, Matrix, container>::ArakawaX( const Geometry& g, bc bcx, bc bcy): 
     dxlhs( dg::evaluate( one, g) ), dxrhs(dxlhs), dylhs(dxlhs), dyrhs( dxlhs), helper_( dxlhs),
     bdxf(dg::create::dx( g, bcx)),
     bdyf(dg::create::dy( g, bcy))
diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index 61ea5ab27..22b87ad8f 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -110,10 +110,10 @@
   *  - double operator()(double, double, double) const
   */
  /** @class hide_geometry
-  * @tparam Geometry One of the geometry classes. The functions dg::create::dx( g, bcx) and
-  * dg::create::dy( g, bcy) must be callable and return an instance of the Matrix class. 
-  * Furthermore dg::evaluate( one, g) must return an instance of the container class.
-     as do calls to dg::create::weights(g) and dg::create::inv_weights(g)
+  * @tparam Geometry A type that is or derives from one of the abstract geometry base classes (e.g. aGeometry2d, aGeometry3d, aMPIGeometry2d, ...)
+  * The functions dg::create::dx() and dg::create::dy() must be callable and return an instance convertible to the Matrix class. 
+  * Furthermore dg::evaluate() must return an instance of the container class.
+     as do calls to dg::create::weights() and dg::create::inv_weights()
   */
 
  /** @class hide_container_lvl1
diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index c292612d7..e1673a5d7 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -1,14 +1,15 @@
 #pragma once
 
 #include "blas.h"
-#include "geometry/geometry.h"
 #include "enums.h"
+#include "backend/memory.h"
 #include "backend/evaluation.cuh"
 #include "backend/derivatives.h"
 #ifdef MPI_VERSION
 #include "backend/mpi_derivatives.h"
 #include "backend/mpi_evaluation.h"
 #endif
+#include "geometry/geometry.h"
 
 /*! @file 
 
@@ -63,7 +64,7 @@ class Elliptic
      * @param jfactor (\f$ = \alpha \f$ ) scale jump terms (1 is a good value but in some cases 0.1 or 0.01 might be better)
      * @note chi is assumed 1 per default
      */
-    Elliptic( Geometry g, norm no = not_normed, direction dir = forward, double jfactor=1.): 
+    Elliptic( const Geometry& g, norm no = not_normed, direction dir = forward, double jfactor=1.): 
         no_(no), jfactor_(jfactor)
     { 
         construct( g, g.bcx(), g.bcy(), dir);
@@ -79,7 +80,7 @@ class Elliptic
 
      * @param jfactor scale jump terms (1 is a good value but in some cases 0.1 or 0.01 might be better)
      */
-    Elliptic( Geometry g, bc bcx, bc bcy, norm no = not_normed, direction dir = forward, double jfactor=1.): 
+    Elliptic( const Geometry& g, bc bcx, bc bcy, norm no = not_normed, direction dir = forward, double jfactor=1.): 
         no_(no), jfactor_(jfactor)
     { 
         construct( g, bcx, bcy, dir);
@@ -238,7 +239,7 @@ struct GeneralElliptic
      * @param no Not normed for elliptic equations, normed else
      * @param dir Direction of the right first derivative
      */
-    GeneralElliptic( Geometry g, norm no = not_normed, direction dir = forward): 
+    GeneralElliptic( const Geometry& g, norm no = not_normed, direction dir = forward): 
         leftx ( dg::create::dx( g, inverse( g.bcx()), inverse(dir))),
         lefty ( dg::create::dy( g, inverse( g.bcy()), inverse(dir))),
         leftz ( dg::create::dz( g, inverse( g.bcz()), inverse(dir))),
@@ -250,7 +251,7 @@ struct GeneralElliptic
         weights_(dg::create::volume(g)), precond_(dg::create::inv_weights(g)), 
         xchi( dg::evaluate( one, g) ), ychi( xchi), zchi( xchi), 
         xx(xchi), yy(xx), zz(xx), temp0( xx), temp1(temp0),
-        no_(no), g_(g)
+        no_(no)
     { 
         vol_=dg::tensor::determinant(g.metric());
         dg::tensor::invert(vol_);
@@ -266,7 +267,7 @@ struct GeneralElliptic
      * @param no Not normed for elliptic equations, normed else
      * @param dir Direction of the right first derivative
      */
-    GeneralElliptic( Geometry g, bc bcx, bc bcy, bc bcz, norm no = not_normed, direction dir = forward): 
+    GeneralElliptic( const Geometry& g, bc bcx, bc bcy, bc bcz, norm no = not_normed, direction dir = forward): 
         leftx ( dg::create::dx( g, inverse( bcx), inverse(dir))),
         lefty ( dg::create::dy( g, inverse( bcy), inverse(dir))),
         leftz ( dg::create::dz( g, inverse( bcz), inverse(dir))),
@@ -278,7 +279,7 @@ struct GeneralElliptic
         weights_(dg::create::volume(g)), precond_(dg::create::inv_weights(g)), 
         xchi( dg::evaluate( one, g) ), ychi( xchi), zchi( xchi), 
         xx(xchi), yy(xx), zz(xx), temp0( xx), temp1(temp0),
-        no_(no), g_(g)
+        no_(no)
     { 
         vol_=dg::tensor::determinant(g.metric());
         dg::tensor::invert(vol_);
@@ -405,7 +406,6 @@ struct GeneralElliptic
     container xchi, ychi, zchi, xx, yy, zz, temp0, temp1;
     norm no_;
     SparseElement<container> vol_;
-    Geometry g_;
 };
 
 /**
@@ -441,7 +441,7 @@ struct GeneralEllipticSym
      * @param no Not normed for elliptic equations, normed else
      * @param dir Direction of the right first derivative
      */
-    GeneralEllipticSym( Geometry g, norm no = not_normed, direction dir = forward): 
+    GeneralEllipticSym( const Geometry& g, norm no = not_normed, direction dir = forward): 
         ellipticForward_( g, no, dir), ellipticBackward_(g,no,inverse(dir)),
         weights_(dg::create::volume(g)), precond_(dg::create::inv_weights(g)), 
         temp_( dg::evaluate( one, g) )
@@ -457,7 +457,7 @@ struct GeneralEllipticSym
      * @param no Not normed for elliptic equations, normed else
      * @param dir Direction of the right first derivative
      */
-    GeneralEllipticSym( Geometry g, bc bcx, bc bcy,bc bcz, norm no = not_normed, direction dir = forward): 
+    GeneralEllipticSym( const Geometry& g, bc bcx, bc bcy,bc bcz, norm no = not_normed, direction dir = forward): 
         ellipticForward_( g, bcx, bcy, no, dir), ellipticBackward_(g,bcx,bcy,no,inverse(dir)),
         weights_(dg::create::volume(g)), precond_(dg::create::inv_weights(g)), 
         temp_( dg::evaluate( one, g) )
@@ -575,7 +575,7 @@ struct TensorElliptic
      * @param no Not normed for elliptic equations, normed else
      * @param dir Direction of the right first derivative
      */
-    TensorElliptic( Geometry g, norm no = not_normed, direction dir = forward): 
+    TensorElliptic( const Geometry& g, norm no = not_normed, direction dir = forward): 
         no_(no), g_(g)
     { 
         construct( g, g.bcx(), g.bcy(), dir);
@@ -588,7 +588,7 @@ struct TensorElliptic
      * @param no Not normed for elliptic equations, normed else
      * @param dir Direction of the right first derivative
      */
-    TensorElliptic( Geometry g, bc bcx, bc bcy, norm no = not_normed, direction dir = forward): 
+    TensorElliptic( const Geometry& g, bc bcx, bc bcy, norm no = not_normed, direction dir = forward): 
         no_(no), g_(g)
     { 
         construct( g, bcx, bcy, dir);
@@ -616,7 +616,7 @@ struct TensorElliptic
     void transform_and_set( const ChiRR& chiRR, const ChiRZ& chiRZ, const ChiZZ& chiZZ)
     {
         typename GeometryTraits<Geometry>::host_vector chiXX, chiXY, chiYY;
-        dg::pushForwardPerp( chiRR, chiRZ, chiZZ, chiXX, chiXY, chiYY, g_);
+        dg::pushForwardPerp( chiRR, chiRZ, chiZZ, chiXX, chiXY, chiYY, g_.get());
         dg::blas1::transfer( chiXX, chixx_);
         dg::blas1::transfer( chiXY, chixy_);
         dg::blas1::transfer( chiYY, chiyy_);
@@ -671,7 +671,7 @@ struct TensorElliptic
             dg::blas2::symv( weights_wo_vol, y, y);
     }
     private:
-    void construct( Geometry g, bc bcx, bc bcy, direction dir)
+    void construct( const Geometry& g, bc bcx, bc bcy, direction dir)
     {
         dg::blas2::transfer( dg::create::dx( g, inverse( bcx), inverse(dir)), leftx);
         dg::blas2::transfer( dg::create::dy( g, inverse( bcy), inverse(dir)), lefty);
@@ -711,7 +711,7 @@ struct TensorElliptic
     container chixx_, chixy_, chiyy_, tempx_, tempy_, gradx_;
     SparseElement<container> vol_;
     norm no_;
-    Geometry g_;
+    Handle<Geometry> g_;
 };
 
 ///@cond
diff --git a/inc/dg/elliptic_b.cu b/inc/dg/elliptic_b.cu
index 46abf0b7d..e60fe22f5 100644
--- a/inc/dg/elliptic_b.cu
+++ b/inc/dg/elliptic_b.cu
@@ -45,8 +45,8 @@ int main()
     std::cout << "TEST CYLINDRICAL LAPLACIAN\n";
     std::cout << "Create Laplacian\n";
     t.tic();
-    dg::Elliptic<dg::CylindricalGrid3d, dg::DMatrix, dg::DVec> laplace(grid, dg::not_normed, dg::centered);
-    dg::Elliptic<dg::CylindricalGrid3d, dg::fDMatrix, dg::fDVec> flaplace(grid, dg::not_normed, dg::centered);
+    dg::Elliptic<dg::aGeometry3d, dg::DMatrix, dg::DVec> laplace(grid, dg::not_normed, dg::centered);
+    dg::Elliptic<dg::aGeometry3d, dg::fDMatrix, dg::fDVec> flaplace(grid, dg::not_normed, dg::centered);
     dg::DMatrix DX = dg::create::dx( grid);
     t.toc();
     std::cout<< "Creation took "<<t.diff()<<"s\n";
@@ -61,7 +61,7 @@ int main()
     dg::blas2::symv( w3d, b, b);
     dg::fDVec fx;
     dg::blas1::transfer(x,fx);
-    dg::Inverse<dg::Elliptic<dg::CylindricalGrid3d, dg::fDMatrix, dg::fDVec>, dg::fDVec> inverse( flaplace, fx, 10, 1e-15, 0);
+    dg::Inverse<dg::Elliptic<dg::aGeometry3d, dg::fDMatrix, dg::fDVec>, dg::fDVec> inverse( flaplace, fx, 10, 1e-15, 0);
     
     std::cout << "For a precision of "<< eps<<" ..."<<std::endl;
     t.tic();
diff --git a/inc/dg/helmholtz.h b/inc/dg/helmholtz.h
index 4cfb5d777..faa6d41d9 100644
--- a/inc/dg/helmholtz.h
+++ b/inc/dg/helmholtz.h
@@ -35,7 +35,7 @@ struct Helmholtz
      * @param jfactor The jfactor used in the Laplace operator (probably 1 is always the best factor but one never knows...)
      * @note The default value of \f$\chi\f$ is one
      */
-    Helmholtz( Geometry g, double alpha = 1., direction dir = dg::forward, double jfactor=1.):
+    Helmholtz( const Geometry& g, double alpha = 1., direction dir = dg::forward, double jfactor=1.):
         laplaceM_(g, normed, dir, jfactor), 
         temp_(dg::evaluate(dg::one, g)),
         alpha_(alpha)
@@ -52,7 +52,7 @@ struct Helmholtz
      * @param jfactor The jfactor used in the Laplace operator (probably 1 is always the best factor but one never knows...)
      * @note The default value of \f$\chi\f$ is one
      */
-    Helmholtz( Geometry g, bc bcx, bc bcy, double alpha = 1., direction dir = dg::forward, double jfactor=1.):
+    Helmholtz( const Geometry& g, bc bcx, bc bcy, double alpha = 1., direction dir = dg::forward, double jfactor=1.):
         laplaceM_(g, bcx,bcy,normed, dir, jfactor), 
         temp_(dg::evaluate(dg::one, g)), 
         alpha_(alpha)
@@ -151,7 +151,7 @@ struct Helmholtz2
      * @param jfactor The jfactor used in the Laplace operator (probably 1 is always the best factor but one never knows...)
      * @note The default value of \f$\chi\f$ is one
      */
-    Helmholtz2( Geometry g, double alpha = 1., direction dir = dg::forward, double jfactor=1.):
+    Helmholtz2( const Geometry& g, double alpha = 1., direction dir = dg::forward, double jfactor=1.):
         laplaceM_(g, normed, dir, jfactor), 
         temp1_(dg::evaluate(dg::one, g)),temp2_(temp1_), chi_(temp1_),
         alpha_(alpha)
@@ -168,7 +168,7 @@ struct Helmholtz2
      * @param jfactor The jfactor used in the Laplace operator (probably 1 is always the best factor but one never knows...)
      * @note The default value of \f$\chi\f$ is one
      */
-    Helmholtz2( Geometry g, bc bcx, bc bcy, double alpha = 1., direction dir = dg::forward, double jfactor=1.):
+    Helmholtz2( const Geometry& g, bc bcx, bc bcy, double alpha = 1., direction dir = dg::forward, double jfactor=1.):
         laplaceM_(g, bcx,bcy,normed, dir, jfactor), 
         temp1_(dg::evaluate(dg::one, g)), temp2_(temp1_),chi_(temp1_),
         alpha_(alpha)
diff --git a/inc/dg/poisson.h b/inc/dg/poisson.h
index 8dfb7decf..2e25a890f 100644
--- a/inc/dg/poisson.h
+++ b/inc/dg/poisson.h
@@ -33,14 +33,14 @@ struct Poisson
      * @brief Create Poisson on a grid
      * @param g The grid
      */
-    Poisson( Geometry g);
+    Poisson( const Geometry& g);
     /**
      * @brief Create Poisson on a grid using different boundary conditions
      * @param g The grid
      * @param bcx The boundary condition in x
      * @param bcy The boundary condition in y
      */
-    Poisson( Geometry g, bc bcx, bc bcy);
+    Poisson( const Geometry& g, bc bcx, bc bcy);
     /**
      * @brief Create Poisson on a grid using different boundary conditions
      * @param g The grid
@@ -49,7 +49,7 @@ struct Poisson
      * @param bcylhs The lhs boundary condition in y
      * @param bcyrhs The rhs boundary condition in y
      */
-    Poisson( Geometry g, bc bcxlhs, bc bcylhs, bc bcxrhs, bc bcyrhs );
+    Poisson( const Geometry& g, bc bcxlhs, bc bcylhs, bc bcxrhs, bc bcyrhs );
     /**
      * @brief Compute poisson's bracket
      *
@@ -116,7 +116,7 @@ struct Poisson
 //idea: backward transform lhs and rhs and then use bdxf and bdyf , then forward transform
 //needs less memory!! and is faster
 template< class Geometry, class Matrix, class container>
-Poisson<Geometry, Matrix, container>::Poisson( Geometry g ): 
+Poisson<Geometry, Matrix, container>::Poisson( const Geometry& g ): 
     dxlhslhs_( dg::evaluate( one, g) ), dxrhsrhs_(dxlhslhs_), dylhslhs_(dxlhslhs_), dyrhsrhs_( dxlhslhs_), helper_( dxlhslhs_),
     dxlhs_(dg::create::dx( g, g.bcx(),dg::centered)),
     dylhs_(dg::create::dy( g, g.bcy(),dg::centered)),
@@ -129,7 +129,7 @@ Poisson<Geometry, Matrix, container>::Poisson( Geometry g ):
 }
 
 template< class Geometry, class Matrix, class container>
-Poisson<Geometry, Matrix, container>::Poisson( Geometry g, bc bcx, bc bcy): 
+Poisson<Geometry, Matrix, container>::Poisson( const Geometry& g, bc bcx, bc bcy): 
     dxlhslhs_( dg::evaluate( one, g) ), dxrhsrhs_(dxlhslhs_), dylhslhs_(dxlhslhs_), dyrhsrhs_( dxlhslhs_), helper_( dxlhslhs_),
     dxlhs_(dg::create::dx( g, bcx,dg::centered)),
     dylhs_(dg::create::dy( g, bcy,dg::centered)),
@@ -142,7 +142,7 @@ Poisson<Geometry, Matrix, container>::Poisson( Geometry g, bc bcx, bc bcy):
 }
 
 template< class Geometry, class Matrix, class container>
-Poisson<Geometry, Matrix, container>::Poisson(  Geometry g, bc bcxlhs, bc bcylhs, bc bcxrhs, bc bcyrhs): 
+Poisson<Geometry, Matrix, container>::Poisson(  const Geometry& g, bc bcxlhs, bc bcylhs, bc bcxrhs, bc bcyrhs): 
     dxlhslhs_( dg::evaluate( one, g) ), dxrhsrhs_(dxlhslhs_), dylhslhs_(dxlhslhs_), dyrhsrhs_( dxlhslhs_), helper_( dxlhslhs_),
     dxlhs_(dg::create::dx( g, bcxlhs,dg::centered)),
     dylhs_(dg::create::dy( g, bcylhs,dg::centered)),
-- 
GitLab


From 5b573cbda151ab54edac25429217481b376b4db7 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 21 Aug 2017 04:57:45 -0700
Subject: [PATCH 196/453] moved curvilinear grids and generators back to
 geometries folder

---
 inc/dg/geometry/geometry.h                    |   2 -
 inc/geometries/average.h                      |   4 +-
 inc/{dg/geometry => geometries}/curvilinear.h |   0
 .../geometry => geometries}/curvilinearX.h    |   0
 inc/{dg/geometry => geometries}/generator.h   |   0
 inc/{dg/geometry => geometries}/generatorX.h  |   0
 inc/geometries/hector.h                       |   4 +-
 .../geometry => geometries}/mpi_curvilinear.h |   0
 .../refined_curvilinearX.h                    |   0
 inc/geometries/ribeiro.h                      |   2 +-
 inc/geometries/ribeiroX.h                     |   2 +-
 inc/geometries/separatrix_orthogonal.h        |   4 +-
 inc/geometries/solovev_doc.h                  | 104 ++----------------
 13 files changed, 19 insertions(+), 103 deletions(-)
 rename inc/{dg/geometry => geometries}/curvilinear.h (100%)
 rename inc/{dg/geometry => geometries}/curvilinearX.h (100%)
 rename inc/{dg/geometry => geometries}/generator.h (100%)
 rename inc/{dg/geometry => geometries}/generatorX.h (100%)
 rename inc/{dg/geometry => geometries}/mpi_curvilinear.h (100%)
 rename inc/{dg/geometry => geometries}/refined_curvilinearX.h (100%)

diff --git a/inc/dg/geometry/geometry.h b/inc/dg/geometry/geometry.h
index 1d6ced142..4d2100a7f 100644
--- a/inc/dg/geometry/geometry.h
+++ b/inc/dg/geometry/geometry.h
@@ -10,11 +10,9 @@
 #include "../backend/mpi_precon.h"
 #endif//MPI_VERSION
 #include "base_geometry.h"
-#include "curvilinear.h"
 //#include "cartesianX.h"
 #ifdef MPI_VERSION
 #include "mpi_base.h"
-#include "mpi_curvilinear.h"
 #endif//MPI_VERSION
 #include "tensor.h"
 #include "transform.h"
diff --git a/inc/geometries/average.h b/inc/geometries/average.h
index 47d2ddc14..04f449302 100644
--- a/inc/geometries/average.h
+++ b/inc/geometries/average.h
@@ -91,7 +91,7 @@ struct Alpha
  \f[ \langle f\rangle(\psi_0) = \frac{1}{A} \int dV \delta(\psi_p(R,Z)-\psi_0) |\nabla\psi_p|f(R,Z) \f]
 
  with \f$ A = \int dV \delta(\psi_p(R,Z)-\psi_0)|\nabla\psi_p|\f$
- * @tparam container  The container class of the vector to average
+ * @copydoc hide_container
  * @ingroup misc
  */
 template <class container = thrust::host_vector<double> >
@@ -146,7 +146,7 @@ struct FluxSurfaceAverage
  * \f[ q(\psi_0) = \frac{1}{2\pi} \int dV |\nabla\psi_p| \delta(\psi_p-\psi_0) \alpha( R,Z) \f]
 
 where \f$ \alpha\f$ is the dg::geo::Alpha functor.
- * @tparam container The container class to use aContainer
+ * @copydoc hide_container
  * @ingroup misc
  *
  */
diff --git a/inc/dg/geometry/curvilinear.h b/inc/geometries/curvilinear.h
similarity index 100%
rename from inc/dg/geometry/curvilinear.h
rename to inc/geometries/curvilinear.h
diff --git a/inc/dg/geometry/curvilinearX.h b/inc/geometries/curvilinearX.h
similarity index 100%
rename from inc/dg/geometry/curvilinearX.h
rename to inc/geometries/curvilinearX.h
diff --git a/inc/dg/geometry/generator.h b/inc/geometries/generator.h
similarity index 100%
rename from inc/dg/geometry/generator.h
rename to inc/geometries/generator.h
diff --git a/inc/dg/geometry/generatorX.h b/inc/geometries/generatorX.h
similarity index 100%
rename from inc/dg/geometry/generatorX.h
rename to inc/geometries/generatorX.h
diff --git a/inc/geometries/hector.h b/inc/geometries/hector.h
index ef0840c6c..3de7bd6cd 100644
--- a/inc/geometries/hector.h
+++ b/inc/geometries/hector.h
@@ -198,12 +198,12 @@ void transform(
 ///@endcond
 
 /**
- * @brief The High PrEcision Conformal grid generaTOR (models aGenerator)
+ * @brief The High PrEcision Conformal grid generaTOR 
  *
  * @ingroup generators
  * @tparam IMatrix The interpolation matrix type
  * @tparam Matrix  The matrix type in the elliptic equation
- * @tparam container models aContainer 
+ * @copydoc hide_container
  */
 template <class IMatrix = dg::IHMatrix, class Matrix = dg::HMatrix, class container = dg::HVec>
 struct Hector : public aGenerator2d
diff --git a/inc/dg/geometry/mpi_curvilinear.h b/inc/geometries/mpi_curvilinear.h
similarity index 100%
rename from inc/dg/geometry/mpi_curvilinear.h
rename to inc/geometries/mpi_curvilinear.h
diff --git a/inc/dg/geometry/refined_curvilinearX.h b/inc/geometries/refined_curvilinearX.h
similarity index 100%
rename from inc/dg/geometry/refined_curvilinearX.h
rename to inc/geometries/refined_curvilinearX.h
diff --git a/inc/geometries/ribeiro.h b/inc/geometries/ribeiro.h
index c81622e40..1a7743cc2 100644
--- a/inc/geometries/ribeiro.h
+++ b/inc/geometries/ribeiro.h
@@ -181,7 +181,7 @@ struct FieldFinv
 ///@endcond
 
 /**
- * @brief A two-dimensional grid based on "almost-conformal" coordinates by %Ribeiro and Scott 2010 (models aGenerator)
+ * @brief A two-dimensional grid based on "almost-conformal" coordinates by %Ribeiro and Scott 2010 
  * @ingroup generators
  */
 struct Ribeiro : public aGenerator2d
diff --git a/inc/geometries/ribeiroX.h b/inc/geometries/ribeiroX.h
index 526c48697..e54faeccd 100644
--- a/inc/geometries/ribeiroX.h
+++ b/inc/geometries/ribeiroX.h
@@ -236,7 +236,7 @@ struct XFieldFinv
 ///@endcond
 
 /**
- * @brief A two-dimensional grid based on "almost-conformal" coordinates by %Ribeiro and Scott 2010 (models aGeneratorX)
+ * @brief A two-dimensional grid based on "almost-conformal" coordinates by %Ribeiro and Scott 2010 
  * @ingroup generators
  * @tparam Psi All the template parameters must model aBinaryOperator i.e. the bracket operator() must be callable with two arguments and return a double. 
  */
diff --git a/inc/geometries/separatrix_orthogonal.h b/inc/geometries/separatrix_orthogonal.h
index 867d3ebab..3117be32b 100644
--- a/inc/geometries/separatrix_orthogonal.h
+++ b/inc/geometries/separatrix_orthogonal.h
@@ -114,7 +114,7 @@ void computeX_rzy( const BinaryFunctorsLvl1& psi,
 ///@endcond
 
 /**
- * @brief Choose points on inside or outside line (models aGeneratorX)
+ * @brief Choose points on inside or outside line 
  *
  * @ingroup generators
  */
@@ -179,7 +179,7 @@ struct SimpleOrthogonalX : public aGeneratorX2d
 };
 
 /**
- * @brief Choose points on separatrix (models aGeneratorX)
+ * @brief Choose points on separatrix 
  *
  * @ingroup generators
  */
diff --git a/inc/geometries/solovev_doc.h b/inc/geometries/solovev_doc.h
index f8f6a30e8..c8fd9e27d 100644
--- a/inc/geometries/solovev_doc.h
+++ b/inc/geometries/solovev_doc.h
@@ -1,32 +1,27 @@
 #error Documentation only
 /*! 
  * 
- * @defgroup grids 1. Grids 
- * @defgroup generators 2. Grid generators
+ * @defgroup generators 1. Grid generators
  *
       All the grids introduced by this extension can be constructed with 
-      generator classes. A generator class is a class that models aGenerator. 
- * @defgroup fluxfunctions 3. New functors based on the magnetic field geometry
+      generator classes. 
+ * @defgroup fluxfunctions 2. New functors based on the magnetic field geometry
 
  All functors in this section model two or three-dimensional functions, i.e. they all overload the operator() like aBinaryOperator
  * @{
-      @defgroup geom 3.1 new flux functions and derivatives
-      @defgroup magnetic 3.2 magnetic field and associated functors
-      @defgroup profiles 3.3 miscellaneous functors based on flux functions
+      @defgroup geom 2.1 new flux functions and derivatives
+      @defgroup magnetic 2.2 magnetic field and associated functors
+      @defgroup profiles 2.3 miscellaneous functors based on flux functions
  * @}
- * @defgroup misc 4. Miscellaneous additions
+ * @defgroup misc 3. Miscellaneous additions
  *
  * Objects that are used to define and integrate the magnetic field lines. 
  * All objects can be used in the evaluation() function.
-   @defgroup temp 99. Template models
-   Documentation for template models
  * 
  */
 /*! @mainpage
  * This extension adds new features to the FELTOR core dg library. 
  *
- * - the conformal, orthogonal and curvilinear grid classes are added to the
- * dg namespace. 
  * - several grid generator classes are added, among them our new Hector class
  *   are added to the dg::geo namespace
  * - a bunch of new functors implementing various magnetic field geometries
@@ -35,84 +30,7 @@
  * and one used to integrate the field lines for parallel derivatives all in the dg::geo namespace.
  */
 
-
-/**
-* @brief The generatorX  template model
-
-A generator is there to construct coordinates from some coordinates
-\f$ x,y\f$ to the computational domain \f$\zeta, \eta\f$, which
-is a product space and has an X-point topology. 
- @attention this is not a real class it's there for documentation only
- @attention parameter names can be different
- @ingroup temp
-*/
-struct aGeneratorX
-{
-    bool isOrthogonal() const; //!< true if coordinate system is orthogonal
-    bool isConformal() const; //!< true if coordinate system is conformal
-    double f0() const; //!< the normalization constant  of the \f$\zeta\f$ coordinate i.e. \f$ \zeta_0 = f_0\psi_0 \f$ and \f$ \zeta_1 = -f_\zeta\zeta_0/(1-f_\zeta) \f$ 
-    /**
-    * @brief Generate grid points and elements of the Jacobian 
-    *
-    * @param zeta1d (input) a list of \f$ N_\zeta\f$ points \f$ f_0\psi_0<\zeta_i< -f_\zeta\zeta_0/(1-f_\zeta)\f$
-    * @param eta1d (input) a list of \f$ N_\eta\f$ points \f$ 0<\eta_j<\f$height() 
-    * @param x (output) the list of \f$ N_\eta N_\zeta\f$ coordinates \f$ x(\zeta_i, \eta_j)\f$ 
-    * @param y (output) the list of \f$ N_\eta N_\zeta\f$ coordinates \f$ y(\zeta_i, \eta_j)\f$ 
-    * @param nodeX0 is the index of the first point in eta1d  after the first jump in topology in \f$ \eta\f$
-    * @param nodeX1 is the index of the first point in eta1d  after the second jump in topology in \f$ \eta\f$
-    * @param zetaX (output) the list of \f$ N_\eta N_\zeta\f$ elements \f$ \partial\zeta/\partial x (\zeta_i, \eta_j)\f$ 
-    * @param zetaY (output) the list of \f$ N_\eta N_\zeta\f$ elements \f$ \partial\zeta/\partial y (\zeta_i, \eta_j)\f$ 
-    * @param etaX (output) the list of \f$ N_\eta N_\zeta\f$ elements \f$ \partial\eta/\partial x (\zeta_i, \eta_j)\f$ 
-    * @param etaY (output) the list of \f$ N_\eta N_\zeta\f$ elements \f$ \partial\eta/\partial y (\zeta_i, \eta_j)\f$ 
-    @note the \f$ \zeta\f$ coordinate is contiguous in memory
-    */
-    void operator()( 
-         const thrust::host_vector<double>& zeta1d, 
-         const thrust::host_vector<double>& eta1d, 
-         const unsigned nodeX0, const unsigned nodeX1, 
-         thrust::host_vector<double>& x, 
-         thrust::host_vector<double>& y, 
-         thrust::host_vector<double>& zetaX, 
-         thrust::host_vector<double>& zetaY, 
-         thrust::host_vector<double>& etaX, 
-         thrust::host_vector<double>& etaY) ;
-};
-
-
-/**
-* @brief The functor template model
-
-A functor in the dg framework can be used in the evaluate or pullback
-functions. The only function to overload is the operator() member.
- @attention this is not a real class it's there for documentation only
- @attention parameter names can be different
- @ingroup temp
-*/
-struct aBinaryOperator
-{
-    double operator()(double x, double y); //!< 2d version
-};
-
-/**
-* @brief The container template model
-
-A container in the dg framework must be usable in the blas functions
-of the main dg library. 
- Also it must be compatible to a thrust::host_vector<double> in the blas1::transfer function.
- Can be one of
- - thrust::host_vector<double>
- - thrust::device_vector<double> 
- - cusp::array1d<double, cusp::device_memory>
- - MPI_Vector<thrust::host_vector<double> >
- - MPI_Vector<thrust::device_vector<double> >
- - MPI_Vector<cusp::array1d<double, cusp::device_memory> >
-
-
- @attention this is not a real class it's there for documentation only
- @attention parameter names can be different
- @ingroup temp
-*/
-struct aContainer
-{
-};
-
+ /** @class hide_container
+  * @tparam container A data container class for which the blas1 functionality is overloaded. Also we assume that the type is copyable/assignable. Currently this is one of 
+  *   dg::HVec, dg::DVec, dg::MHVec or dg::MDVec
+  */
-- 
GitLab


From c2839c0cd2b3c0d041809a59559f9fe81f4c00e3 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 21 Aug 2017 05:06:09 -0700
Subject: [PATCH 197/453] geometries documentation

---
 inc/geometries/geometries.h  | 6 ------
 inc/geometries/solovev_doc.h | 1 +
 2 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/inc/geometries/geometries.h b/inc/geometries/geometries.h
index 8db69497f..621e0047b 100644
--- a/inc/geometries/geometries.h
+++ b/inc/geometries/geometries.h
@@ -1,15 +1,9 @@
 #pragma once
 
 //include grids
-#include "conformal.h"
-#include "orthogonal.h"
 #include "curvilinear.h"
-#include "refined_conformal.h"
-#include "refined_orthogonal.h"
 #include "refined_curvilinear.h"
 #ifdef MPI_VERSION
-#include "mpi_conformal.h"
-#include "mpi_orthogonal.h"
 #include "mpi_curvilinear.h"
 #endif
 
diff --git a/inc/geometries/solovev_doc.h b/inc/geometries/solovev_doc.h
index c8fd9e27d..28c39a15e 100644
--- a/inc/geometries/solovev_doc.h
+++ b/inc/geometries/solovev_doc.h
@@ -1,6 +1,7 @@
 #error Documentation only
 /*! 
  * 
+ * @defgroup geometry 0. New Geometry classes
  * @defgroup generators 1. Grid generators
  *
       All the grids introduced by this extension can be constructed with 
-- 
GitLab


From cc0ae468f7ea6f46695079519f42ebeb5302e387 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 21 Aug 2017 06:31:24 -0700
Subject: [PATCH 198/453] more on documentation

---
 inc/dg/Doxyfile                       |   2 +-
 inc/file/Doxyfile                     |   2 +-
 inc/geometries/Doxyfile               | 140 +++++++++++++-------------
 inc/geometries/curvilinear.h          |   2 +-
 inc/geometries/curvilinearX.h         |   2 +-
 inc/geometries/mpi_curvilinear.h      |   2 +-
 inc/geometries/refined_curvilinearX.h |   2 +-
 inc/geometries/solovev_doc.h          |   2 +-
 8 files changed, 77 insertions(+), 77 deletions(-)

diff --git a/inc/dg/Doxyfile b/inc/dg/Doxyfile
index cb293e7c1..c585ef5dd 100644
--- a/inc/dg/Doxyfile
+++ b/inc/dg/Doxyfile
@@ -2094,7 +2094,7 @@ TAGFILES               =
 # tag file that is based on the input files it reads. See section "Linking to
 # external documentation" for more information about the usage of tag files.
 
-GENERATE_TAGFILE       =
+GENERATE_TAGFILE       = doc/dg.tag
 
 # If the ALLEXTERNALS tag is set to YES, all external class will be listed in
 # the class index. If set to NO, only the inherited external classes will be
diff --git a/inc/file/Doxyfile b/inc/file/Doxyfile
index 744d136ad..ce14304ec 100644
--- a/inc/file/Doxyfile
+++ b/inc/file/Doxyfile
@@ -2091,7 +2091,7 @@ TAGFILES               =
 # tag file that is based on the input files it reads. See section "Linking to
 # external documentation" for more information about the usage of tag files.
 
-GENERATE_TAGFILE       =
+GENERATE_TAGFILE       = doc/file.tag
 
 # If the ALLEXTERNALS tag is set to YES, all external class will be listed in
 # the class index. If set to NO, only the inherited external classes will be
diff --git a/inc/geometries/Doxyfile b/inc/geometries/Doxyfile
index 3b5a4295f..af680f771 100644
--- a/inc/geometries/Doxyfile
+++ b/inc/geometries/Doxyfile
@@ -38,7 +38,7 @@ PROJECT_NAME           = Geometry
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = 
+PROJECT_NUMBER         =
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
@@ -51,7 +51,7 @@ PROJECT_BRIEF          = "Library for magnetic flux functions, grids and grid ge
 # pixels and the maximum width should not exceed 200 pixels. Doxygen will copy
 # the logo to the output directory.
 
-PROJECT_LOGO           = 
+PROJECT_LOGO           =
 
 # The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path
 # into which the generated documentation will be written. If a relative path is
@@ -162,7 +162,7 @@ FULL_PATH_NAMES        = YES
 # will be relative from the directory where doxygen is started.
 # This tag requires that the tag FULL_PATH_NAMES is set to YES.
 
-STRIP_FROM_PATH        = 
+STRIP_FROM_PATH        =
 
 # The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the
 # path mentioned in the documentation of a class, which tells the reader which
@@ -171,7 +171,7 @@ STRIP_FROM_PATH        =
 # specify the list of include paths that are normally passed to the compiler
 # using the -I flag.
 
-STRIP_FROM_INC_PATH    = 
+STRIP_FROM_INC_PATH    =
 
 # If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but
 # less readable) file names. This can be useful is your file systems doesn't
@@ -238,13 +238,13 @@ TAB_SIZE               = 8
 # "Side Effects:". You can put \n's in the value part of an alias to insert
 # newlines.
 
-ALIASES                = 
+ALIASES                =
 
 # This tag can be used to specify a number of word-keyword mappings (TCL only).
 # A mapping has the form "name=value". For example adding "class=itcl::class"
 # will allow you to use the command class in the itcl::class meaning.
 
-TCL_SUBST              = 
+TCL_SUBST              =
 
 # Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources
 # only. Doxygen will then generate output that is more tailored for C. For
@@ -291,7 +291,7 @@ OPTIMIZE_OUTPUT_VHDL   = NO
 # Note that for custom extensions you also need to set FILE_PATTERNS otherwise
 # the files are not read by doxygen.
 
-EXTENSION_MAPPING      = 
+EXTENSION_MAPPING      =
 
 # If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments
 # according to the Markdown format, which allows for more readable
@@ -639,7 +639,7 @@ GENERATE_DEPRECATEDLIST= YES
 # sections, marked by \if <section_label> ... \endif and \cond <section_label>
 # ... \endcond blocks.
 
-ENABLED_SECTIONS       = 
+ENABLED_SECTIONS       =
 
 # The MAX_INITIALIZER_LINES tag determines the maximum number of lines that the
 # initial value of a variable or macro / define can have for it to appear in the
@@ -681,7 +681,7 @@ SHOW_NAMESPACES        = YES
 # by doxygen. Whatever the program writes to standard output is used as the file
 # version. For an example see the documentation.
 
-FILE_VERSION_FILTER    = 
+FILE_VERSION_FILTER    =
 
 # The LAYOUT_FILE tag can be used to specify a layout file which will be parsed
 # by doxygen. The layout file controls the global structure of the generated
@@ -694,7 +694,7 @@ FILE_VERSION_FILTER    =
 # DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE
 # tag is left empty.
 
-LAYOUT_FILE            = 
+LAYOUT_FILE            =
 
 # The CITE_BIB_FILES tag can be used to specify one or more bib files containing
 # the reference definitions. This must be a list of .bib files. The .bib
@@ -704,7 +704,7 @@ LAYOUT_FILE            =
 # LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the
 # search path. See also \cite for info how to create references.
 
-CITE_BIB_FILES         = 
+CITE_BIB_FILES         =
 
 #---------------------------------------------------------------------------
 # Configuration options related to warning and progress messages
@@ -769,7 +769,7 @@ WARN_FORMAT            = "$file:$line: $text"
 # messages should be written. If left blank the output is written to standard
 # error (stderr).
 
-WARN_LOGFILE           = 
+WARN_LOGFILE           =
 
 #---------------------------------------------------------------------------
 # Configuration options related to the input files
@@ -822,7 +822,7 @@ RECURSIVE              = NO
 # Note that relative paths are relative to the directory from which doxygen is
 # run.
 
-EXCLUDE                = 
+EXCLUDE                =
 
 # The EXCLUDE_SYMLINKS tag can be used to select whether or not files or
 # directories that are symbolic links (a Unix file system feature) are excluded
@@ -838,7 +838,7 @@ EXCLUDE_SYMLINKS       = NO
 # Note that the wildcards are matched against the file with absolute path, so to
 # exclude all test directories for example use the pattern */test/*
 
-EXCLUDE_PATTERNS       = 
+EXCLUDE_PATTERNS       =
 
 # The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names
 # (namespaces, classes, functions, etc.) that should be excluded from the
@@ -849,13 +849,13 @@ EXCLUDE_PATTERNS       =
 # Note that the wildcards are matched against the file with absolute path, so to
 # exclude all test directories use the pattern */test/*
 
-EXCLUDE_SYMBOLS        = 
+EXCLUDE_SYMBOLS        =
 
 # The EXAMPLE_PATH tag can be used to specify one or more files or directories
 # that contain example code fragments that are included (see the \include
 # command).
 
-EXAMPLE_PATH           = 
+EXAMPLE_PATH           =
 
 # If the value of the EXAMPLE_PATH tag contains directories, you can use the
 # EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and
@@ -875,7 +875,7 @@ EXAMPLE_RECURSIVE      = NO
 # that contain images that are to be included in the documentation (see the
 # \image command).
 
-IMAGE_PATH             = 
+IMAGE_PATH             =
 
 # The INPUT_FILTER tag can be used to specify a program that doxygen should
 # invoke to filter for each input file. Doxygen will invoke the filter program
@@ -896,7 +896,7 @@ IMAGE_PATH             =
 # need to set EXTENSION_MAPPING for the extension otherwise the files are not
 # properly processed by doxygen.
 
-INPUT_FILTER           = 
+INPUT_FILTER           =
 
 # The FILTER_PATTERNS tag can be used to specify filters on a per file pattern
 # basis. Doxygen will compare the file name with each pattern and apply the
@@ -909,7 +909,7 @@ INPUT_FILTER           =
 # need to set EXTENSION_MAPPING for the extension otherwise the files are not
 # properly processed by doxygen.
 
-FILTER_PATTERNS        = 
+FILTER_PATTERNS        =
 
 # If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using
 # INPUT_FILTER) will also be used to filter the input files that are used for
@@ -924,14 +924,14 @@ FILTER_SOURCE_FILES    = NO
 # *.ext= (so without naming a filter).
 # This tag requires that the tag FILTER_SOURCE_FILES is set to YES.
 
-FILTER_SOURCE_PATTERNS = 
+FILTER_SOURCE_PATTERNS =
 
 # If the USE_MDFILE_AS_MAINPAGE tag refers to the name of a markdown file that
 # is part of the input, its contents will be placed on the main page
 # (index.html). This can be useful if you have a project on for instance GitHub
 # and want to reuse the introduction page also for the doxygen output.
 
-USE_MDFILE_AS_MAINPAGE = 
+USE_MDFILE_AS_MAINPAGE =
 
 #---------------------------------------------------------------------------
 # Configuration options related to source browsing
@@ -1036,7 +1036,7 @@ CLANG_ASSISTED_PARSING = NO
 # specified with INPUT and INCLUDE_PATH.
 # This tag requires that the tag CLANG_ASSISTED_PARSING is set to YES.
 
-CLANG_OPTIONS          = 
+CLANG_OPTIONS          =
 
 #---------------------------------------------------------------------------
 # Configuration options related to the alphabetical class index
@@ -1062,7 +1062,7 @@ COLS_IN_ALPHA_INDEX    = 5
 # while generating the index headers.
 # This tag requires that the tag ALPHABETICAL_INDEX is set to YES.
 
-IGNORE_PREFIX          = 
+IGNORE_PREFIX          =
 
 #---------------------------------------------------------------------------
 # Configuration options related to the HTML output
@@ -1106,7 +1106,7 @@ HTML_FILE_EXTENSION    = .html
 # of the possible markers and block names see the documentation.
 # This tag requires that the tag GENERATE_HTML is set to YES.
 
-HTML_HEADER            = 
+HTML_HEADER            =
 
 # The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each
 # generated HTML page. If the tag is left blank doxygen will generate a standard
@@ -1116,7 +1116,7 @@ HTML_HEADER            =
 # that doxygen normally uses.
 # This tag requires that the tag GENERATE_HTML is set to YES.
 
-HTML_FOOTER            = 
+HTML_FOOTER            =
 
 # The HTML_STYLESHEET tag can be used to specify a user-defined cascading style
 # sheet that is used by each HTML page. It can be used to fine-tune the look of
@@ -1128,7 +1128,7 @@ HTML_FOOTER            =
 # obsolete.
 # This tag requires that the tag GENERATE_HTML is set to YES.
 
-HTML_STYLESHEET        = 
+HTML_STYLESHEET        =
 
 # The HTML_EXTRA_STYLESHEET tag can be used to specify additional user-defined
 # cascading style sheets that are included after the standard style sheets
@@ -1141,7 +1141,7 @@ HTML_STYLESHEET        =
 # list). For an example see the documentation.
 # This tag requires that the tag GENERATE_HTML is set to YES.
 
-HTML_EXTRA_STYLESHEET  = 
+HTML_EXTRA_STYLESHEET  =
 
 # The HTML_EXTRA_FILES tag can be used to specify one or more extra images or
 # other source files which should be copied to the HTML output directory. Note
@@ -1151,7 +1151,7 @@ HTML_EXTRA_STYLESHEET  =
 # files will be copied as-is; there are no commands or markers available.
 # This tag requires that the tag GENERATE_HTML is set to YES.
 
-HTML_EXTRA_FILES       = 
+HTML_EXTRA_FILES       =
 
 # The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen
 # will adjust the colors in the style sheet and background images according to
@@ -1280,7 +1280,7 @@ GENERATE_HTMLHELP      = NO
 # written to the html output directory.
 # This tag requires that the tag GENERATE_HTMLHELP is set to YES.
 
-CHM_FILE               = 
+CHM_FILE               =
 
 # The HHC_LOCATION tag can be used to specify the location (absolute path
 # including file name) of the HTML help compiler (hhc.exe). If non-empty,
@@ -1288,7 +1288,7 @@ CHM_FILE               =
 # The file has to be specified with full path.
 # This tag requires that the tag GENERATE_HTMLHELP is set to YES.
 
-HHC_LOCATION           = 
+HHC_LOCATION           =
 
 # The GENERATE_CHI flag controls if a separate .chi index file is generated
 # (YES) or that it should be included in the master .chm file (NO).
@@ -1301,7 +1301,7 @@ GENERATE_CHI           = NO
 # and project file content.
 # This tag requires that the tag GENERATE_HTMLHELP is set to YES.
 
-CHM_INDEX_ENCODING     = 
+CHM_INDEX_ENCODING     =
 
 # The BINARY_TOC flag controls whether a binary table of contents is generated
 # (YES) or a normal table of contents (NO) in the .chm file. Furthermore it
@@ -1332,7 +1332,7 @@ GENERATE_QHP           = NO
 # the HTML output folder.
 # This tag requires that the tag GENERATE_QHP is set to YES.
 
-QCH_FILE               = 
+QCH_FILE               =
 
 # The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help
 # Project output. For more information please see Qt Help Project / Namespace
@@ -1357,7 +1357,7 @@ QHP_VIRTUAL_FOLDER     = doc
 # filters).
 # This tag requires that the tag GENERATE_QHP is set to YES.
 
-QHP_CUST_FILTER_NAME   = 
+QHP_CUST_FILTER_NAME   =
 
 # The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the
 # custom filter to add. For more information please see Qt Help Project / Custom
@@ -1365,21 +1365,21 @@ QHP_CUST_FILTER_NAME   =
 # filters).
 # This tag requires that the tag GENERATE_QHP is set to YES.
 
-QHP_CUST_FILTER_ATTRS  = 
+QHP_CUST_FILTER_ATTRS  =
 
 # The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this
 # project's filter section matches. Qt Help Project / Filter Attributes (see:
 # http://qt-project.org/doc/qt-4.8/qthelpproject.html#filter-attributes).
 # This tag requires that the tag GENERATE_QHP is set to YES.
 
-QHP_SECT_FILTER_ATTRS  = 
+QHP_SECT_FILTER_ATTRS  =
 
 # The QHG_LOCATION tag can be used to specify the location of Qt's
 # qhelpgenerator. If non-empty doxygen will try to run qhelpgenerator on the
 # generated .qhp file.
 # This tag requires that the tag GENERATE_QHP is set to YES.
 
-QHG_LOCATION           = 
+QHG_LOCATION           =
 
 # If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files will be
 # generated, together with the HTML files, they form an Eclipse help plugin. To
@@ -1512,7 +1512,7 @@ MATHJAX_RELPATH        = /usr/share/javascript/mathjax/
 # MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols
 # This tag requires that the tag USE_MATHJAX is set to YES.
 
-MATHJAX_EXTENSIONS     = 
+MATHJAX_EXTENSIONS     =
 
 # The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces
 # of code that will be used on startup of the MathJax code. See the MathJax site
@@ -1520,7 +1520,7 @@ MATHJAX_EXTENSIONS     =
 # example see the documentation.
 # This tag requires that the tag USE_MATHJAX is set to YES.
 
-MATHJAX_CODEFILE       = 
+MATHJAX_CODEFILE       =
 
 # When the SEARCHENGINE tag is enabled doxygen will generate a search box for
 # the HTML output. The underlying search engine uses javascript and DHTML and
@@ -1580,7 +1580,7 @@ EXTERNAL_SEARCH        = NO
 # Searching" for details.
 # This tag requires that the tag SEARCHENGINE is set to YES.
 
-SEARCHENGINE_URL       = 
+SEARCHENGINE_URL       =
 
 # When SERVER_BASED_SEARCH and EXTERNAL_SEARCH are both enabled the unindexed
 # search data is written to a file for indexing by an external tool. With the
@@ -1596,7 +1596,7 @@ SEARCHDATA_FILE        = searchdata.xml
 # projects and redirect the results back to the right project.
 # This tag requires that the tag SEARCHENGINE is set to YES.
 
-EXTERNAL_SEARCH_ID     = 
+EXTERNAL_SEARCH_ID     =
 
 # The EXTRA_SEARCH_MAPPINGS tag can be used to enable searching through doxygen
 # projects other than the one defined by this configuration file, but that are
@@ -1606,7 +1606,7 @@ EXTERNAL_SEARCH_ID     =
 # EXTRA_SEARCH_MAPPINGS = tagname1=loc1 tagname2=loc2 ...
 # This tag requires that the tag SEARCHENGINE is set to YES.
 
-EXTRA_SEARCH_MAPPINGS  = 
+EXTRA_SEARCH_MAPPINGS  =
 
 #---------------------------------------------------------------------------
 # Configuration options related to the LaTeX output
@@ -1670,7 +1670,7 @@ PAPER_TYPE             = a4
 # If left blank no extra packages will be included.
 # This tag requires that the tag GENERATE_LATEX is set to YES.
 
-EXTRA_PACKAGES         = 
+EXTRA_PACKAGES         =
 
 # The LATEX_HEADER tag can be used to specify a personal LaTeX header for the
 # generated LaTeX document. The header should contain everything until the first
@@ -1686,7 +1686,7 @@ EXTRA_PACKAGES         =
 # to HTML_HEADER.
 # This tag requires that the tag GENERATE_LATEX is set to YES.
 
-LATEX_HEADER           = 
+LATEX_HEADER           =
 
 # The LATEX_FOOTER tag can be used to specify a personal LaTeX footer for the
 # generated LaTeX document. The footer should contain everything after the last
@@ -1697,7 +1697,7 @@ LATEX_HEADER           =
 # Note: Only use a user-defined footer if you know what you are doing!
 # This tag requires that the tag GENERATE_LATEX is set to YES.
 
-LATEX_FOOTER           = 
+LATEX_FOOTER           =
 
 # The LATEX_EXTRA_STYLESHEET tag can be used to specify additional user-defined
 # LaTeX style sheets that are included after the standard style sheets created
@@ -1708,7 +1708,7 @@ LATEX_FOOTER           =
 # list).
 # This tag requires that the tag GENERATE_LATEX is set to YES.
 
-LATEX_EXTRA_STYLESHEET = 
+LATEX_EXTRA_STYLESHEET =
 
 # The LATEX_EXTRA_FILES tag can be used to specify one or more extra images or
 # other source files which should be copied to the LATEX_OUTPUT output
@@ -1716,7 +1716,7 @@ LATEX_EXTRA_STYLESHEET =
 # markers available.
 # This tag requires that the tag GENERATE_LATEX is set to YES.
 
-LATEX_EXTRA_FILES      = 
+LATEX_EXTRA_FILES      =
 
 # If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated is
 # prepared for conversion to PDF (using ps2pdf or pdflatex). The PDF file will
@@ -1824,14 +1824,14 @@ RTF_HYPERLINKS         = NO
 # default style sheet that doxygen normally uses.
 # This tag requires that the tag GENERATE_RTF is set to YES.
 
-RTF_STYLESHEET_FILE    = 
+RTF_STYLESHEET_FILE    =
 
 # Set optional variables used in the generation of an RTF document. Syntax is
 # similar to doxygen's config file. A template extensions file can be generated
 # using doxygen -e rtf extensionFile.
 # This tag requires that the tag GENERATE_RTF is set to YES.
 
-RTF_EXTENSIONS_FILE    = 
+RTF_EXTENSIONS_FILE    =
 
 # If the RTF_SOURCE_CODE tag is set to YES then doxygen will include source code
 # with syntax highlighting in the RTF output.
@@ -1876,7 +1876,7 @@ MAN_EXTENSION          = .3
 # MAN_EXTENSION with the initial . removed.
 # This tag requires that the tag GENERATE_MAN is set to YES.
 
-MAN_SUBDIR             = 
+MAN_SUBDIR             =
 
 # If the MAN_LINKS tag is set to YES and doxygen generates man output, then it
 # will generate one additional man file for each entity documented in the real
@@ -1989,7 +1989,7 @@ PERLMOD_PRETTY         = YES
 # overwrite each other's variables.
 # This tag requires that the tag GENERATE_PERLMOD is set to YES.
 
-PERLMOD_MAKEVAR_PREFIX = 
+PERLMOD_MAKEVAR_PREFIX =
 
 #---------------------------------------------------------------------------
 # Configuration options related to the preprocessor
@@ -2030,7 +2030,7 @@ SEARCH_INCLUDES        = YES
 # preprocessor.
 # This tag requires that the tag SEARCH_INCLUDES is set to YES.
 
-INCLUDE_PATH           = 
+INCLUDE_PATH           =
 
 # You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard
 # patterns (like *.h and *.hpp) to filter out the header-files in the
@@ -2038,7 +2038,7 @@ INCLUDE_PATH           =
 # used.
 # This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
 
-INCLUDE_FILE_PATTERNS  = 
+INCLUDE_FILE_PATTERNS  =
 
 # The PREDEFINED tag can be used to specify one or more macro names that are
 # defined before the preprocessor is started (similar to the -D option of e.g.
@@ -2048,7 +2048,7 @@ INCLUDE_FILE_PATTERNS  =
 # recursively expanded use the := operator instead of the = operator.
 # This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
 
-PREDEFINED             = 
+PREDEFINED             =
 
 # If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this
 # tag can be used to specify a list of macro names that should be expanded. The
@@ -2057,7 +2057,7 @@ PREDEFINED             =
 # definition found in the source code.
 # This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
 
-EXPAND_AS_DEFINED      = 
+EXPAND_AS_DEFINED      =
 
 # If the SKIP_FUNCTION_MACROS tag is set to YES then doxygen's preprocessor will
 # remove all references to function-like macros that are alone on a line, have
@@ -2086,13 +2086,13 @@ SKIP_FUNCTION_MACROS   = YES
 # the path). If a tag file is not located in the directory in which doxygen is
 # run, you must also specify the path to the tagfile here.
 
-TAGFILES               = 
+TAGFILES               = ../dg/doc/dg.tag=../../../dg/doc/html
 
 # When a file name is specified after GENERATE_TAGFILE, doxygen will create a
 # tag file that is based on the input files it reads. See section "Linking to
 # external documentation" for more information about the usage of tag files.
 
-GENERATE_TAGFILE       = 
+GENERATE_TAGFILE       = doc/geometries.tag
 
 # If the ALLEXTERNALS tag is set to YES, all external class will be listed in
 # the class index. If set to NO, only the inherited external classes will be
@@ -2106,14 +2106,14 @@ ALLEXTERNALS           = NO
 # listed.
 # The default value is: YES.
 
-EXTERNAL_GROUPS        = YES
+EXTERNAL_GROUPS        = NO
 
 # If the EXTERNAL_PAGES tag is set to YES, all external pages will be listed in
 # the related pages index. If set to NO, only the current project's pages will
 # be listed.
 # The default value is: YES.
 
-EXTERNAL_PAGES         = YES
+EXTERNAL_PAGES         = NO
 
 # The PERL_PATH should be the absolute path and name of the perl script
 # interpreter (i.e. the result of 'which perl').
@@ -2132,7 +2132,7 @@ PERL_PATH              = /usr/bin/perl
 # powerful graphs.
 # The default value is: YES.
 
-CLASS_DIAGRAMS         = YES
+CLASS_DIAGRAMS         = NO
 
 # You can define message sequence charts within doxygen comments using the \msc
 # command. Doxygen will then run the mscgen tool (see:
@@ -2141,14 +2141,14 @@ CLASS_DIAGRAMS         = YES
 # the mscgen tool resides. If left empty the tool is assumed to be found in the
 # default search path.
 
-MSCGEN_PATH            = 
+MSCGEN_PATH            =
 
 # You can include diagrams made with dia in doxygen documentation. Doxygen will
 # then run dia to produce the diagram and insert it in the documentation. The
 # DIA_PATH tag allows you to specify the directory where the dia binary resides.
 # If left empty dia is assumed to be found in the default search path.
 
-DIA_PATH               = 
+DIA_PATH               =
 
 # If set to YES the inheritance and collaboration graphs will hide inheritance
 # and usage relations if the target is undocumented or is not a class.
@@ -2163,7 +2163,7 @@ HIDE_UNDOC_RELATIONS   = YES
 # set to NO
 # The default value is: YES.
 
-HAVE_DOT               = NO
+HAVE_DOT               = YES
 
 # The DOT_NUM_THREADS specifies the number of dot invocations doxygen is allowed
 # to run in parallel. When set to 0 doxygen will base this on the number of
@@ -2197,7 +2197,7 @@ DOT_FONTSIZE           = 10
 # the path where dot can find it using this tag.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
-DOT_FONTPATH           = 
+DOT_FONTPATH           =
 
 # If the CLASS_GRAPH tag is set to YES then doxygen will generate a graph for
 # each documented class showing the direct and indirect inheritance relations.
@@ -2214,7 +2214,7 @@ CLASS_GRAPH            = YES
 # The default value is: YES.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
-COLLABORATION_GRAPH    = YES
+COLLABORATION_GRAPH    = NO
 
 # If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for
 # groups, showing the direct groups dependencies.
@@ -2343,26 +2343,26 @@ INTERACTIVE_SVG        = NO
 # found. If left blank, it is assumed the dot tool can be found in the path.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
-DOT_PATH               = 
+DOT_PATH               =
 
 # The DOTFILE_DIRS tag can be used to specify one or more directories that
 # contain dot files that are included in the documentation (see the \dotfile
 # command).
 # This tag requires that the tag HAVE_DOT is set to YES.
 
-DOTFILE_DIRS           = 
+DOTFILE_DIRS           =
 
 # The MSCFILE_DIRS tag can be used to specify one or more directories that
 # contain msc files that are included in the documentation (see the \mscfile
 # command).
 
-MSCFILE_DIRS           = 
+MSCFILE_DIRS           =
 
 # The DIAFILE_DIRS tag can be used to specify one or more directories that
 # contain dia files that are included in the documentation (see the \diafile
 # command).
 
-DIAFILE_DIRS           = 
+DIAFILE_DIRS           =
 
 # When using plantuml, the PLANTUML_JAR_PATH tag should be used to specify the
 # path where java can find the plantuml.jar file. If left blank, it is assumed
@@ -2370,12 +2370,12 @@ DIAFILE_DIRS           =
 # generate a warning when it encounters a \startuml command in this case and
 # will not generate output for the diagram.
 
-PLANTUML_JAR_PATH      = 
+PLANTUML_JAR_PATH      =
 
 # When using plantuml, the specified paths are searched for files specified by
 # the !include statement in a plantuml block.
 
-PLANTUML_INCLUDE_PATH  = 
+PLANTUML_INCLUDE_PATH  =
 
 # The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of nodes
 # that will be shown in the graph. If the number of nodes in a graph becomes
diff --git a/inc/geometries/curvilinear.h b/inc/geometries/curvilinear.h
index be2f222ac..b2a1dff5e 100644
--- a/inc/geometries/curvilinear.h
+++ b/inc/geometries/curvilinear.h
@@ -7,7 +7,7 @@
 
 namespace dg
 {
-///@addtogroup geometry
+///@addtogroup grids
 ///@{
 
 ///@cond
diff --git a/inc/geometries/curvilinearX.h b/inc/geometries/curvilinearX.h
index 73fc3af22..50b9a6645 100644
--- a/inc/geometries/curvilinearX.h
+++ b/inc/geometries/curvilinearX.h
@@ -10,7 +10,7 @@
 
 namespace dg
 {
-///@addtogroup geometry
+///@addtogroup grids
 ///@{
 
 /**
diff --git a/inc/geometries/mpi_curvilinear.h b/inc/geometries/mpi_curvilinear.h
index 777da1957..a40afeef7 100644
--- a/inc/geometries/mpi_curvilinear.h
+++ b/inc/geometries/mpi_curvilinear.h
@@ -17,7 +17,7 @@ namespace dg
 struct CurvilinearProductMPIGrid3d; 
 ///@endcond
 //
-///@addtogroup geometry
+///@addtogroup grids
 ///@{
 /**
  * @brief A two-dimensional MPI grid based on curvilinear coordinates
diff --git a/inc/geometries/refined_curvilinearX.h b/inc/geometries/refined_curvilinearX.h
index 308854798..d68aa10ed 100644
--- a/inc/geometries/refined_curvilinearX.h
+++ b/inc/geometries/refined_curvilinearX.h
@@ -6,7 +6,7 @@
 
 namespace dg
 {
-///@addtogroup geometry
+///@addtogroup grids
 ///@{
 
 /**
diff --git a/inc/geometries/solovev_doc.h b/inc/geometries/solovev_doc.h
index 28c39a15e..7d7638c46 100644
--- a/inc/geometries/solovev_doc.h
+++ b/inc/geometries/solovev_doc.h
@@ -1,7 +1,7 @@
 #error Documentation only
 /*! 
  * 
- * @defgroup geometry 0. New Geometry classes
+ * @defgroup grids 0. New Geometry classes
  * @defgroup generators 1. Grid generators
  *
       All the grids introduced by this extension can be constructed with 
-- 
GitLab


From 21175a8d6a2808836f0a75da7c353fa00bb1e58a Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 21 Aug 2017 06:46:58 -0700
Subject: [PATCH 199/453] documentation script for global documentation

---
 inc/doc/Makefile        |  8 ++++++
 inc/doc/generate_doc.sh | 37 ++++++++++++++++++++++++
 inc/doc/header.html     | 63 +++++++++++++++++++++++++++++++++++++++++
 3 files changed, 108 insertions(+)
 create mode 100644 inc/doc/Makefile
 create mode 100644 inc/doc/generate_doc.sh
 create mode 100644 inc/doc/header.html

diff --git a/inc/doc/Makefile b/inc/doc/Makefile
new file mode 100644
index 000000000..5cad6c68a
--- /dev/null
+++ b/inc/doc/Makefile
@@ -0,0 +1,8 @@
+
+doc:
+	sh generate_doc.sh
+	ln -s dg/html/modules.html index.html
+
+
+clean:
+	rm -rf dg file geometries dg.tag file.tag index.html
diff --git a/inc/doc/generate_doc.sh b/inc/doc/generate_doc.sh
new file mode 100644
index 000000000..2aaccbc06
--- /dev/null
+++ b/inc/doc/generate_doc.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+#this file automatically loops over inc directories and runs doxygen
+# on the Doxyfiles and outputs documentation in the current directory
+# It generates tag files so that there can be links between inc documentations and uses the html_header feature of doxygen to create the top menu bar
+
+#pay attention on name collisions in documentations through tag files
+independent="dg file"
+dependent="geometries"
+tagfiles=""
+
+#first generate tag files for independen documentations
+for i in $independent;
+do 
+    (cat ../$i/Doxyfile; \
+    mkdir -p $i
+	echo "INPUT = ../$i/"; \
+    #echo "GENERATE_HTML=NO";\
+	echo "OUTPUT_DIRECTORY = ./$i"; \
+	echo "HTML_HEADER = header.html"; \
+	#echo "MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"; \ ## uncomment if run for official feltor page
+    echo "GENERATE_TAGFILE = ./${i}.tag" ) | doxygen - ;
+    tagfiles="$tagfiles ./${i}.tag=../../${i}/html"
+done;
+
+#generate dependent documentations
+for i in $dependent;
+do 
+    (cat ../$i/Doxyfile; \
+	echo "INPUT = ../$i/"; \
+    echo "OUTPUT_DIRECTORY = ./$i/";  \
+	echo "HTML_HEADER = header.html"; \
+    echo "EXTERNAL_GROUPS=NO" ;\
+    echo "EXTERNAL_PAGES=NO" ;\
+	#echo "MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"; \ ## uncomment if run for official feltor page
+    echo "TAGFILES = $tagfiles") | doxygen - ;
+done;
+
diff --git a/inc/doc/header.html b/inc/doc/header.html
new file mode 100644
index 000000000..9a911750f
--- /dev/null
+++ b/inc/doc/header.html
@@ -0,0 +1,63 @@
+<!-- HTML header for doxygen 1.8.6-->
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+<meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/>
+<meta http-equiv="X-UA-Compatible" content="IE=9"/>
+<meta name="generator" content="Doxygen $doxygenversion"/>
+<!--BEGIN PROJECT_NAME--><title>$projectname: $title</title><!--END PROJECT_NAME-->
+<!--BEGIN !PROJECT_NAME--><title>$title</title><!--END !PROJECT_NAME-->
+<link href="$relpath^tabs.css" rel="stylesheet" type="text/css"/>
+<script type="text/javascript" src="$relpath^jquery.js"></script>
+<script type="text/javascript" src="$relpath^dynsections.js"></script>
+$treeview
+$search
+$mathjax
+<link href="$relpath^$stylesheet" rel="stylesheet" type="text/css" />
+$extrastylesheet
+</head>
+<body>
+<div id="top"><!-- do not remove this div, it is closed by doxygen! -->
+
+<!--BEGIN TITLEAREA-->
+<div id="titlearea">
+    <!--My own title area-->
+  <ul class="tablist"> 
+      <!-- <li><a href="../../../index.html">/</a><li>-->
+    <li><a href="../../dg/html/modules.html">dG</a></li>
+    <li><a href="../../geometries/html/modules.html">geometry</a></li>
+    <li><a href="../../file/html/namespacefile.html">file</a></li>
+  </ul>
+  <!--End My own title area-->
+<table cellspacing="0" cellpadding="0">
+ <tbody>
+ <tr style="height: 56px;">
+  <!--BEGIN PROJECT_LOGO-->
+  <td id="projectlogo"><img alt="Logo" src="$relpath^$projectlogo"/></td>
+  <!--END PROJECT_LOGO-->
+  <!--BEGIN PROJECT_NAME-->
+  <td style="padding-left: 0.5em;">
+   <div id="projectname">$projectname
+   <!--BEGIN PROJECT_NUMBER-->&#160;<span id="projectnumber">$projectnumber</span><!--END PROJECT_NUMBER-->
+   </div>
+   <!--BEGIN PROJECT_BRIEF--><div id="projectbrief">$projectbrief</div><!--END PROJECT_BRIEF-->
+  </td>
+  <!--END PROJECT_NAME-->
+  <!--BEGIN !PROJECT_NAME-->
+   <!--BEGIN PROJECT_BRIEF-->
+    <td style="padding-left: 0.5em;">
+    <div id="projectbrief">$projectbrief</div>
+    </td>
+   <!--END PROJECT_BRIEF-->
+  <!--END !PROJECT_NAME-->
+  <!--BEGIN DISABLE_INDEX-->
+   <!--BEGIN SEARCHENGINE-->
+   <td>$searchbox</td>
+   <!--END SEARCHENGINE-->
+  <!--END DISABLE_INDEX-->
+ </tr>
+ </tbody>
+</table>
+</div>
+<!--END TITLEAREA-->
+<!-- end header part -->
-- 
GitLab


From a76977ac56ec65ccb43eab8c6fc28d0dc343484c Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 21 Aug 2017 08:49:21 -0700
Subject: [PATCH 200/453] wrote Makefile for documentation

---
 inc/doc/Makefile             | 41 +++++++++++++++++++++++++++++++++---
 inc/doc/generate_doc.sh      | 37 --------------------------------
 inc/geometries/Doxyfile      |  2 +-
 inc/geometries/solovev_doc.h | 14 ++++++------
 4 files changed, 46 insertions(+), 48 deletions(-)
 delete mode 100644 inc/doc/generate_doc.sh

diff --git a/inc/doc/Makefile b/inc/doc/Makefile
index 5cad6c68a..4b5ebcb2b 100644
--- a/inc/doc/Makefile
+++ b/inc/doc/Makefile
@@ -1,8 +1,43 @@
+tagfiles="./dg.tag=../../dg/html ./file.tag=../../file/html"
+.PHONY: dg file geometries
 
-doc:
-	sh generate_doc.sh
-	ln -s dg/html/modules.html index.html
+all: doc
+
+dg:
+	(cat ../dg/Doxyfile; \
+    mkdir -p dg; \
+	echo "INPUT = ../dg/"; \
+	echo "OUTPUT_DIRECTORY = ./dg"; \
+	echo "HTML_HEADER = header.html"; \
+	echo "GENERATE_TAGFILE = ./dg.tag" ) | doxygen - ;
+	#echo "MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"; \ ## uncomment if run for official feltor page
+
+
+file:
+	(cat ../file/Doxyfile; \
+    mkdir -p file; \
+	echo "INPUT = ../file/"; \
+	echo "OUTPUT_DIRECTORY = ./file"; \
+	echo "HTML_HEADER = header.html"; \
+	echo "GENERATE_TAGFILE = ./file.tag" ) | doxygen - ;
+	#echo "MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"; \ ## uncomment if run for official feltor page
+
+geometries:
+	(cat ../geometries/Doxyfile; \
+	echo "INPUT = ../geometries/"; \
+    echo "OUTPUT_DIRECTORY = ./geometries/";  \
+	echo "HTML_HEADER = header.html"; \
+    echo "EXTERNAL_GROUPS=NO" ;\
+    echo "EXTERNAL_PAGES=NO" ;\
+    echo "TAGFILES = $(tagfiles)") | doxygen - ;
+	#echo "MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"; \ ## uncomment if run for official feltor page
+
+doc: dg file geometries
+	ln -fs dg/html/modules.html index.html
+	echo "Open with: firefox index.html"
 
 
 clean:
 	rm -rf dg file geometries dg.tag file.tag index.html
+
+
diff --git a/inc/doc/generate_doc.sh b/inc/doc/generate_doc.sh
deleted file mode 100644
index 2aaccbc06..000000000
--- a/inc/doc/generate_doc.sh
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/bin/bash
-#this file automatically loops over inc directories and runs doxygen
-# on the Doxyfiles and outputs documentation in the current directory
-# It generates tag files so that there can be links between inc documentations and uses the html_header feature of doxygen to create the top menu bar
-
-#pay attention on name collisions in documentations through tag files
-independent="dg file"
-dependent="geometries"
-tagfiles=""
-
-#first generate tag files for independen documentations
-for i in $independent;
-do 
-    (cat ../$i/Doxyfile; \
-    mkdir -p $i
-	echo "INPUT = ../$i/"; \
-    #echo "GENERATE_HTML=NO";\
-	echo "OUTPUT_DIRECTORY = ./$i"; \
-	echo "HTML_HEADER = header.html"; \
-	#echo "MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"; \ ## uncomment if run for official feltor page
-    echo "GENERATE_TAGFILE = ./${i}.tag" ) | doxygen - ;
-    tagfiles="$tagfiles ./${i}.tag=../../${i}/html"
-done;
-
-#generate dependent documentations
-for i in $dependent;
-do 
-    (cat ../$i/Doxyfile; \
-	echo "INPUT = ../$i/"; \
-    echo "OUTPUT_DIRECTORY = ./$i/";  \
-	echo "HTML_HEADER = header.html"; \
-    echo "EXTERNAL_GROUPS=NO" ;\
-    echo "EXTERNAL_PAGES=NO" ;\
-	#echo "MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"; \ ## uncomment if run for official feltor page
-    echo "TAGFILES = $tagfiles") | doxygen - ;
-done;
-
diff --git a/inc/geometries/Doxyfile b/inc/geometries/Doxyfile
index af680f771..97effe2a2 100644
--- a/inc/geometries/Doxyfile
+++ b/inc/geometries/Doxyfile
@@ -2092,7 +2092,7 @@ TAGFILES               = ../dg/doc/dg.tag=../../../dg/doc/html
 # tag file that is based on the input files it reads. See section "Linking to
 # external documentation" for more information about the usage of tag files.
 
-GENERATE_TAGFILE       = doc/geometries.tag
+GENERATE_TAGFILE       = 
 
 # If the ALLEXTERNALS tag is set to YES, all external class will be listed in
 # the class index. If set to NO, only the inherited external classes will be
diff --git a/inc/geometries/solovev_doc.h b/inc/geometries/solovev_doc.h
index 7d7638c46..b4836572f 100644
--- a/inc/geometries/solovev_doc.h
+++ b/inc/geometries/solovev_doc.h
@@ -1,20 +1,20 @@
 #error Documentation only
 /*! 
  * 
- * @defgroup grids 0. New Geometry classes
- * @defgroup generators 1. Grid generators
+ * @defgroup grids 1. New Geometry classes
+ * @defgroup generators 2. Grid generators
  *
       All the grids introduced by this extension can be constructed with 
       generator classes. 
- * @defgroup fluxfunctions 2. New functors based on the magnetic field geometry
+ * @defgroup fluxfunctions 3. New functors based on the magnetic field geometry
 
  All functors in this section model two or three-dimensional functions, i.e. they all overload the operator() like aBinaryOperator
  * @{
-      @defgroup geom 2.1 new flux functions and derivatives
-      @defgroup magnetic 2.2 magnetic field and associated functors
-      @defgroup profiles 2.3 miscellaneous functors based on flux functions
+      @defgroup geom 3.1 new flux functions and derivatives
+      @defgroup magnetic 3.2 magnetic field and associated functors
+      @defgroup profiles 3.3 miscellaneous functors based on flux functions
  * @}
- * @defgroup misc 3. Miscellaneous additions
+ * @defgroup misc 4. Miscellaneous additions
  *
  * Objects that are used to define and integrate the magnetic field lines. 
  * All objects can be used in the evaluation() function.
-- 
GitLab


From bf9f5e0fba487b1b0ba6220cbb038f713b04d381 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Mon, 21 Aug 2017 21:32:16 +0200
Subject: [PATCH 201/453] added alpha,x,beta,y symv routine

---
 inc/dg/Makefile                               |  2 +-
 inc/dg/arakawa.h                              | 10 +-
 inc/dg/backend/mpi_matrix.h                   | 96 ++++++++++++++++++-
 inc/dg/backend/mpi_matrix_blas.h              |  6 ++
 inc/dg/backend/selfmade_blas.cuh              |  4 +-
 inc/dg/backend/sparseblockmat.cuh             | 17 +++-
 inc/dg/backend/sparseblockmat.h               | 18 +++-
 inc/dg/backend/sparseblockmat_gpu_kernels.cuh | 41 ++++----
 inc/dg/backend/sparseblockmat_omp_kernels.h   | 63 ++++++------
 inc/dg/blas2.h                                |  5 +
 10 files changed, 190 insertions(+), 72 deletions(-)

diff --git a/inc/dg/Makefile b/inc/dg/Makefile
index b93bd4b54..70988f2b1 100644
--- a/inc/dg/Makefile
+++ b/inc/dg/Makefile
@@ -18,7 +18,7 @@ all: $(CPPFILES:%.cpp=%) $(CUFILES:%.cu=%)
 	$(MPICC) $(INCLUDE) -DDG_DEBUG $(MPICFLAGS) $< -o $@ -g
 
 %_mpib: %_mpib.cu
-	$(MPICC) $(OPT) $(MPICFLAGS) $< -o $@ $(INCLUDE) 
+	$(MPICC) $(OPT) $(MPICFLAGS) $< -o $@ $(INCLUDE) -g
 
 %_t: %_t.cu
 	$(CC) $(OPT) $(INCLUDE) -DDG_DEBUG $(CFLAGS) $< -o $@ 
diff --git a/inc/dg/arakawa.h b/inc/dg/arakawa.h
index 281fbf3b8..65bcd781c 100644
--- a/inc/dg/arakawa.h
+++ b/inc/dg/arakawa.h
@@ -144,12 +144,10 @@ void ArakawaX< Geometry, Matrix, container>::operator()( const container& lhs, c
     //blas1::axpby( 1., dxlhs,  -0., helper); //x+ - +x
     //blas1::axpby( 0., result, -1., dylhs);  //+x - x+
 
-    blas2::symv( bdyf, helper_, result);      //dy*(dxl*r - l*dxr) -> result
-    blas2::symv( bdxf, dylhs, dxlhs);      //dx*(l*dyr - dyl*r) -> dxlhs
-    //now sum everything up
-    blas1::axpby( 1., dxlhs, 1., result); //result + dxlhs -> result
-    blas1::axpby( 1., dxrhs, 1., result); //result + dyrhs -> result
-    geo::dividePerpVolume( result, grid);
+    blas2::symv( 1., bdyf, helper_, 1., dxrhs);
+    blas2::symv( 1., bdxf, dylhs, 1., dxrhs);
+    geo::dividePerpVolume( dxrhs, grid);
+    result.swap(dxrhs);
 }
 
 }//namespace dg
diff --git a/inc/dg/backend/mpi_matrix.h b/inc/dg/backend/mpi_matrix.h
index f5bcfad6a..b9808ca35 100644
--- a/inc/dg/backend/mpi_matrix.h
+++ b/inc/dg/backend/mpi_matrix.h
@@ -82,6 +82,53 @@ struct RowColDistMat
     */
     const Collective& collective() const{return c_;}
     
+    /**
+    * @brief Matrix Vector product
+    *
+    * First the inner elements are computed with a call to doSymv then 
+    * the collect function of the communication object is called. 
+    * Finally the outer elements are added with a call to doSymv for the outer matrix
+    * @tparam container container class of the vector elements
+    * @param x input
+    * @param y output
+    */
+    template<class container> 
+    void symv( double alpha, const MPI_Vector<container>& x, double beta, MPI_Vector<container>& y) const
+    {
+        assert( x.communicator() == y.communicator());
+        assert( x.communicator() == c_.communicator());
+        //int rank;
+        //MPI_Comm_rank( MPI_COMM_WORLD, &rank);
+        //dg::Timer t;
+        if( c_.size() == 0) //no communication needed
+        {
+            dg::blas2::detail::doSymv( alpha, m_i, x.data(), beta, y.data(), 
+                       typename dg::MatrixTraits<LocalMatrixInner>::matrix_category(), 
+                       typename dg::VectorTraits<container>::vector_category() );
+            return;
+
+        }
+        //t.tic();
+        //1. compute inner points
+        dg::blas2::detail::doSymv( alpha, m_i, x.data(), beta, y.data(), 
+                       typename dg::MatrixTraits<LocalMatrixInner>::matrix_category(), 
+                       typename dg::VectorTraits<container>::vector_category() );
+        //t.toc();
+        //if(rank==0)std::cout << "Inner points took "<<t.diff()<<"s\n";
+        //2. communicate outer points
+        //t.tic();
+        const container& temp = c_.collect( x.data());
+        //t.toc();
+        //if(rank==0)std::cout << "Collect      took "<<t.diff()<<"s\n";
+        //3. compute and add outer points
+        //t.tic();
+        dg::blas2::detail::doSymv(alpha, m_o, temp, 1., y.data(), 
+                       typename dg::MatrixTraits<LocalMatrixOuter>::matrix_category(), 
+                       typename dg::VectorTraits<container>::vector_category() );
+        //t.toc();
+        //if(rank==0)std::cout << "Outer points took "<<t.diff()<<"s\n";
+    }
+
     /**
     * @brief Matrix Vector product
     *
@@ -130,7 +177,6 @@ struct RowColDistMat
         //t.toc();
         //if(rank==0)std::cout << "Outer points took "<<t.diff()<<"s\n";
     }
-
         
     private:
     LocalMatrixInner m_i;
@@ -197,6 +243,34 @@ struct RowDistMat
     * @param y
     */
     template<class container> 
+    void symv( double alpha, const MPI_Vector<container>& x, double beta, MPI_Vector<container>& y)
+    {
+        assert( x.communicator() == y.communicator());
+        assert( x.communicator() == c_.communicator());
+        //int rank;
+        //MPI_Comm_rank( MPI_COMM_WORLD, &rank);
+        //dg::Timer t;
+
+        //t.tic();
+        if( c_.size() == 0) //no communication needed
+        {
+            dg::blas2::detail::doSymv( alpha, m_, x.data(), beta, y.data(), 
+                       typename dg::MatrixTraits<LocalMatrix>::matrix_category(), 
+                       typename dg::VectorTraits<container>::vector_category() );
+            return;
+
+        }
+        container temp = c_.collect( x.data());
+        //t.toc();
+        //if(rank==0)std::cout << "collect took "<<t.diff()<<"s\n";
+        //t.tic();
+        dg::blas2::detail::doSymv( alpha, m_, temp, beta, y.data(), 
+                       typename dg::MatrixTraits<LocalMatrix>::matrix_category(), 
+                       typename dg::VectorTraits<container>::vector_category() );
+        //t.toc();
+        //if(rank==0)std::cout << "symv    took "<<t.diff()<<"s\n";
+    }
+    template<class container> 
     void symv( const MPI_Vector<container>& x, MPI_Vector<container>& y)
     {
         assert( x.communicator() == y.communicator());
@@ -282,6 +356,26 @@ struct ColDistMat
     * @param y
     */
     template<class container> 
+    void symv( double alpha, const MPI_Vector<container>& x, double beta, MPI_Vector<container>& y)
+    {
+        assert( x.communicator() == y.communicator());
+        assert( x.communicator() == c_.communicator());
+        if( c_.size() == 0) //no communication needed
+        {
+            dg::blas2::detail::doSymv( alpha, m_, x.data(), beta, y.data(), 
+                       typename dg::MatrixTraits<LocalMatrix>::matrix_category(), 
+                       typename dg::VectorTraits<container>::vector_category(),
+                       typename dg::VectorTraits<container>::vector_category() );
+            return;
+
+        }
+        container temp( c_.size());
+        dg::blas2::detail::doSymv( alpha, m_, x.data(), beta, temp, 
+                       typename dg::MatrixTraits<LocalMatrix>::matrix_category(), 
+                       typename dg::VectorTraits<container>::vector_category() );
+        c_.send_and_reduce( temp, y.data());
+    }
+    template<class container> 
     void symv( const MPI_Vector<container>& x, MPI_Vector<container>& y)
     {
         assert( x.communicator() == y.communicator());
diff --git a/inc/dg/backend/mpi_matrix_blas.h b/inc/dg/backend/mpi_matrix_blas.h
index 32c2091fb..c37ccdd2b 100644
--- a/inc/dg/backend/mpi_matrix_blas.h
+++ b/inc/dg/backend/mpi_matrix_blas.h
@@ -22,6 +22,12 @@ inline void doSymv( Matrix& m, Vector1& x, Vector2& y, MPIMatrixTag, MPIVectorTa
     m.symv( x, y);
 }
 
+template< class Matrix, class Vector>
+inline void doSymv( typename MatrixTraits<Matrix>::value_type alpha, const Matrix& m, const Vector& x, typename MatrixTraits<Matrix>::value_type beta, Vector& y, MPIMatrixTag, MPIVectorTag )
+{
+    m.symv( alpha, x, beta, y);
+}
+
 template< class Matrix, class Vector1, class Vector2>
 inline void doGemv( Matrix& m, Vector1&x, Vector2& y, MPIMatrixTag, MPIVectorTag, MPIVectorTag  )
 {
diff --git a/inc/dg/backend/selfmade_blas.cuh b/inc/dg/backend/selfmade_blas.cuh
index 45d973951..0e10a8203 100644
--- a/inc/dg/backend/selfmade_blas.cuh
+++ b/inc/dg/backend/selfmade_blas.cuh
@@ -38,10 +38,10 @@ inline void doGemv(
 
 template< class Matrix, class Vector>
 inline void doSymv(  
-              typename Vector::value_type alpha, 
+              typename VectorTraits<Vector>::value_type alpha, 
               const Matrix& m,
               const Vector& x, 
-              typename Vector::value_type beta, 
+              typename VectorTraits<Vector>::value_type beta, 
               Vector& y, 
               SelfMadeMatrixTag,
               AnyVectorTag)
diff --git a/inc/dg/backend/sparseblockmat.cuh b/inc/dg/backend/sparseblockmat.cuh
index ad95b463b..0bba470c2 100644
--- a/inc/dg/backend/sparseblockmat.cuh
+++ b/inc/dg/backend/sparseblockmat.cuh
@@ -43,7 +43,16 @@ struct EllSparseBlockMatDevice
     * @param y output may not equal input
     */
     template <class deviceContainer>
-    void symv(const deviceContainer& x, deviceContainer& y) const;
+    void symv(value_type alpha, const deviceContainer& x, value_type beta, deviceContainer& y) const;
+    /**
+    * @brief Apply the matrix to a vector
+    *
+    * @param x input
+    * @param y output may not equal input
+    */
+    template <class deviceContainer>
+    void symv( const deviceContainer& x, deviceContainer& y) const{symv(1., x, 0., y);}
+
     /**
     * @brief Display internal data to a stream
     *
@@ -53,7 +62,7 @@ struct EllSparseBlockMatDevice
     private:
     typedef thrust::device_vector<int> IVec;
     template <class deviceContainer>
-    void launch_multiply_kernel(const deviceContainer& x, deviceContainer& y) const;
+    void launch_multiply_kernel(value_type alpha, const deviceContainer& x, value_type beta, deviceContainer& y) const;
     
     thrust::device_vector<value_type> data;
     IVec cols_idx, data_idx; 
@@ -172,9 +181,9 @@ void CooSparseBlockMatDevice<value_type>::display( std::ostream& os) const
 }
 template<class value_type>
 template<class DeviceContainer>
-inline void EllSparseBlockMatDevice<value_type>::symv( const DeviceContainer& x, DeviceContainer& y) const
+inline void EllSparseBlockMatDevice<value_type>::symv( value_type alpha, const DeviceContainer& x, value_type beta, DeviceContainer& y) const
 {
-    launch_multiply_kernel( x,y);
+    launch_multiply_kernel( alpha, x, beta, y);
 }
 template<class value_type>
 template<class DeviceContainer>
diff --git a/inc/dg/backend/sparseblockmat.h b/inc/dg/backend/sparseblockmat.h
index 83d57c7a8..bafe2a762 100644
--- a/inc/dg/backend/sparseblockmat.h
+++ b/inc/dg/backend/sparseblockmat.h
@@ -65,7 +65,15 @@ struct EllSparseBlockMat
     * @param x input
     * @param y output may not equal input
     */
-    void symv(const thrust::host_vector<value_type>& x, thrust::host_vector<value_type>& y) const;
+    void symv(value_type alpha, const thrust::host_vector<value_type>& x, value_type beta, thrust::host_vector<value_type>& y) const;
+    /**
+    * @brief Apply the matrix to a vector
+    *
+    * @param x input
+    * @param y output may not equal input
+    */
+    void symv(const thrust::host_vector<value_type>& x, thrust::host_vector<value_type>& y) const {symv( 1., x, 0., y);}
+
     /**
      * @brief Sets ranges from 0 to left_size and 0 to right_size
      */
@@ -182,7 +190,7 @@ struct CooSparseBlockMat
 ///@cond
 
 template<class value_type>
-void EllSparseBlockMat<value_type>::symv(const thrust::host_vector<value_type>& x, thrust::host_vector<value_type>& y) const
+void EllSparseBlockMat<value_type>::symv(value_type alpha, const thrust::host_vector<value_type>& x, value_type beta, thrust::host_vector<value_type>& y) const
 {
     assert( y.size() == (unsigned)num_rows*n*left_size*right_size);
     assert( x.size() == (unsigned)num_cols*n*left_size*right_size);
@@ -194,10 +202,10 @@ void EllSparseBlockMat<value_type>::symv(const thrust::host_vector<value_type>&
     for( int j=right_range[0]; j<right_range[1]; j++)
     {
         int I = ((s*num_rows + i)*n+k)*right_size+j;
-        y[I] =0;
+        y[I] *= beta;
         for( int d=0; d<blocks_per_line; d++)
         for( int q=0; q<n; q++) //multiplication-loop
-            y[I] += data[ (data_idx[i*blocks_per_line+d]*n + k)*n+q]*
+            y[I] += alpha*data[ (data_idx[i*blocks_per_line+d]*n + k)*n+q]*
                 x[((s*num_cols + cols_idx[i*blocks_per_line+d])*n+q)*right_size+j];
     }
 }
@@ -259,7 +267,6 @@ void CooSparseBlockMat<value_type>::symv( value_type alpha, const thrust::host_v
 {
     assert( y.size() == (unsigned)num_rows*n*left_size*right_size);
     assert( x.size() == (unsigned)num_cols*n*left_size*right_size);
-    assert( beta == 1);
 
     //simplest implementation
     for( int s=0; s<left_size; s++)
@@ -268,6 +275,7 @@ void CooSparseBlockMat<value_type>::symv( value_type alpha, const thrust::host_v
     for( int j=0; j<right_size; j++)
     {
         int I = ((s*num_rows + rows_idx[i])*n+k)*right_size+j;
+        y[I] *= beta;
         for( int q=0; q<n; q++) //multiplication-loop
             y[I] += alpha*data[ (data_idx[i]*n + k)*n+q]*
                 x[((s*num_cols + cols_idx[i])*n+q)*right_size+j];
diff --git a/inc/dg/backend/sparseblockmat_gpu_kernels.cuh b/inc/dg/backend/sparseblockmat_gpu_kernels.cuh
index 9d5510249..7fc4fa482 100644
--- a/inc/dg/backend/sparseblockmat_gpu_kernels.cuh
+++ b/inc/dg/backend/sparseblockmat_gpu_kernels.cuh
@@ -5,7 +5,7 @@ namespace dg
 
 // general multiply kernel
 template<class value_type>
- __global__ void ell_multiply_kernel(
+ __global__ void ell_multiply_kernel( value_type alpha, value_type beta,
          const value_type* data, const int* cols_idx, const int* data_idx, 
          const int num_rows, const int num_cols, const int blocks_per_line,
          const int n, const int size,
@@ -34,14 +34,14 @@ template<class value_type>
                 temp +=data[ B+q]* x[(J+q)*right+j];
         }
         int idx = ((s*num_rows+i)*n+k)*right+j;
-        y[idx]=temp;
+        y[idx]=alpha*temp+beta*y[idx];
     }
 
 }
 
 // multiply kernel, n=3, 3 blocks per line
 template<class value_type>
- __global__ void ell_multiply_kernel33(
+ __global__ void ell_multiply_kernel33(value_type alpha, value_type beta,
          const value_type* data, const int* cols_idx, const int* data_idx, 
          const int num_rows, const int num_cols,
          const int size,
@@ -80,13 +80,13 @@ template<class value_type>
             temp +=data[ B+2]* x[(J+2)*right+j];
         }
         int idx = ((s*num_rows+i)*3+k)*right+j;
-        y[idx]=temp;
+        y[idx]=alpha*temp+beta*y[idx];
     }
 }
 
 // multiply kernel, n=3, 2 blocks per line
 template<class value_type>
- __global__ void ell_multiply_kernel32(
+ __global__ void ell_multiply_kernel32(value_type alpha, value_type beta,
          const value_type* data, const int* cols_idx, const int* data_idx, 
          const int num_rows, const int num_cols, 
          const int size,
@@ -120,14 +120,14 @@ template<class value_type>
             temp +=data[ B+1]* x[(J+1)*right+j];
             temp +=data[ B+2]* x[(J+2)*right+j];
         }
-        y[idx]=temp;
+        y[idx]=alpha*temp+beta*y[idx];
     }
 
 }
 
 // multiply kernel, n=3, 3 blocks per line, right = 1
 template<class value_type>
- __global__ void ell_multiply_kernel33x(
+ __global__ void ell_multiply_kernel33x(value_type alpha, value_type beta,
          const value_type* data, const int* cols_idx, const int* data_idx, 
          const int num_rows, const int num_cols,
          const int size,
@@ -160,13 +160,13 @@ template<class value_type>
             temp +=data[ B+2]* x[(J+2)];
         }
         int idx = (s*num_rows+i)*3+k;
-        y[idx] = temp;
+        y[idx] = alpha*temp+beta*y[idx];
     }
 }
 
 // multiply kernel, n=3, 2 blocks per line, right = 1
 template<class value_type>
- __global__ void ell_multiply_kernel32x(
+ __global__ void ell_multiply_kernel32x(value_type alpha, value_type beta,
          const value_type* data, const int* cols_idx, const int* data_idx, 
          const int num_rows, const int num_cols,
          const int size,
@@ -195,7 +195,7 @@ template<class value_type>
             temp +=data[ B1+1]* x[(J1+1)];
             temp +=data[ B1+2]* x[(J1+2)];
         }
-        y[idx]=temp;
+        y[idx] = alpha*temp+beta*y[idx];
     }
 
 }
@@ -207,7 +207,7 @@ template<class value_type>
          const int num_rows, const int num_cols, const int entry,
          const int n, 
          const int left, const int right, 
-         value_type alpha, const value_type* x, value_type *y
+         value_type alpha, const value_type* x, value_type beta, value_type *y
          )
 {
     int size = left*n*right;
@@ -225,14 +225,14 @@ template<class value_type>
         int J = cols_idx[entry];
         for( int q=0; q<n; q++) //multiplication-loop
             temp += data[ (B*n + k)*n+q]* x[((s*num_cols + J)*n+q)*right+j];
-        y[I] += alpha*temp;
+        y[I] = alpha*temp + beta*y[I];
     }
 
 }
 
 template<class value_type>
 template<class DeviceContainer>
-void EllSparseBlockMatDevice<value_type>::launch_multiply_kernel( const DeviceContainer& x, DeviceContainer& y) const
+void EllSparseBlockMatDevice<value_type>::launch_multiply_kernel(value_type alpha, const DeviceContainer& x, value_type beta, DeviceContainer& y) const
 {
     assert( y.size() == (unsigned)num_rows*n*left_size*right_size);
     assert( x.size() == (unsigned)num_cols*n*left_size*right_size);
@@ -252,23 +252,23 @@ void EllSparseBlockMatDevice<value_type>::launch_multiply_kernel( const DeviceCo
         if( blocks_per_line == 3)
         {
             if( right_size == 1)
-                ell_multiply_kernel33x<value_type> <<<NUM_BLOCKS, BLOCK_SIZE>>> ( data_ptr, cols_ptr, block_ptr, num_rows, num_cols, size, x_ptr,y_ptr);
+                ell_multiply_kernel33x<value_type> <<<NUM_BLOCKS, BLOCK_SIZE>>> (alpha,beta, data_ptr, cols_ptr, block_ptr, num_rows, num_cols, size, x_ptr,y_ptr);
             else
-                ell_multiply_kernel33<value_type> <<<NUM_BLOCKS, BLOCK_SIZE>>> ( data_ptr, cols_ptr, block_ptr, num_rows, num_cols, size, right_size, right_range_ptr, x_ptr,y_ptr);
+                ell_multiply_kernel33<value_type> <<<NUM_BLOCKS, BLOCK_SIZE>>> (alpha,beta, data_ptr, cols_ptr, block_ptr, num_rows, num_cols, size, right_size, right_range_ptr, x_ptr,y_ptr);
         }
         else if( blocks_per_line == 2)
         {
             if( right_size == 1)
-                ell_multiply_kernel32x<value_type> <<<NUM_BLOCKS, BLOCK_SIZE>>> ( data_ptr, cols_ptr, block_ptr, num_rows, num_cols, size, x_ptr,y_ptr);
+                ell_multiply_kernel32x<value_type> <<<NUM_BLOCKS, BLOCK_SIZE>>> (alpha,beta, data_ptr, cols_ptr, block_ptr, num_rows, num_cols, size, x_ptr,y_ptr);
             else
-                ell_multiply_kernel32<value_type> <<<NUM_BLOCKS, BLOCK_SIZE>>> ( data_ptr, cols_ptr, block_ptr, num_rows, num_cols, size, right_size, right_range_ptr, x_ptr,y_ptr);
+                ell_multiply_kernel32<value_type> <<<NUM_BLOCKS, BLOCK_SIZE>>> (alpha,beta, data_ptr, cols_ptr, block_ptr, num_rows, num_cols, size, right_size, right_range_ptr, x_ptr,y_ptr);
         }
         else
-            ell_multiply_kernel<value_type> <<<NUM_BLOCKS, BLOCK_SIZE>>> ( 
+            ell_multiply_kernel<value_type> <<<NUM_BLOCKS, BLOCK_SIZE>>> (alpha,beta, 
                 data_ptr, cols_ptr, block_ptr, num_rows, num_cols, blocks_per_line, 3, size, right_size, right_range_ptr, x_ptr,y_ptr);
     }
     else
-        ell_multiply_kernel<value_type> <<<NUM_BLOCKS, BLOCK_SIZE>>> ( 
+        ell_multiply_kernel<value_type> <<<NUM_BLOCKS, BLOCK_SIZE>>> (alpha,beta, 
             data_ptr, cols_ptr, block_ptr, num_rows, num_cols, blocks_per_line, n, size, right_size, right_range_ptr, x_ptr,y_ptr);
 }
 
@@ -278,7 +278,6 @@ void CooSparseBlockMatDevice<value_type>::launch_multiply_kernel( value_type alp
 {
     assert( y.size() == (unsigned)num_rows*n*left_size*right_size);
     assert( x.size() == (unsigned)num_cols*n*left_size*right_size);
-    assert( beta == 1);
     //set up kernel parameters
     const size_t BLOCK_SIZE = 256; 
     const size_t size = left_size*right_size*n;
@@ -293,7 +292,7 @@ void CooSparseBlockMatDevice<value_type>::launch_multiply_kernel( value_type alp
     for( int i=0; i<num_entries; i++)
     {
         coo_multiply_kernel<value_type> <<<NUM_BLOCKS, BLOCK_SIZE>>> ( 
-            data_ptr, rows_ptr, cols_ptr, block_ptr, num_rows, num_cols, i, n, left_size, right_size, alpha, x_ptr,y_ptr);
+            data_ptr, rows_ptr, cols_ptr, block_ptr, num_rows, num_cols, i, n, left_size, right_size, alpha, x_ptr, beta, y_ptr);
     }
 }
 
diff --git a/inc/dg/backend/sparseblockmat_omp_kernels.h b/inc/dg/backend/sparseblockmat_omp_kernels.h
index 2cb637f29..240d93e51 100644
--- a/inc/dg/backend/sparseblockmat_omp_kernels.h
+++ b/inc/dg/backend/sparseblockmat_omp_kernels.h
@@ -3,7 +3,7 @@ namespace dg{
 
 // multiply kernel
 template<class value_type>
-void ell_multiply_kernel(
+void ell_multiply_kernel( value_type alpha, value_type beta,
          const value_type* data, const int* cols_idx, const int* data_idx, 
          const int num_rows, const int num_cols, const int blocks_per_line,
          const int n, 
@@ -20,13 +20,13 @@ void ell_multiply_kernel(
     for( int j=right_range[0]; j<right_range[1]; j++)
     {
         int I = ((s*num_rows + i)*n+k)*right_size+j;
-        y[I] =0;
+        y[I] *=beta;
         for( int d=0; d<blocks_per_line; d++)
         {
             int B = (data_idx[i*blocks_per_line+d]*n+k)*n;
             int J = (s*num_cols+cols_idx[i*blocks_per_line+d])*n;
             for( int q=0; q<n; q++) //multiplication-loop
-                y[I] += data[ B+q]*
+                y[I] += alpha*data[ B+q]*
                     x[(J+q)*right_size+j];
         }
     }
@@ -34,7 +34,7 @@ void ell_multiply_kernel(
 
 // multiply kernel n=3, 3 blocks per line
 template<class value_type>
-void ell_multiply_kernel33(
+void ell_multiply_kernel33( value_type alpha, value_type beta,
          const value_type* data, const int* cols_idx, const int* data_idx, 
          const int num_rows, const int num_cols, 
          const int left_size, const int right_size, 
@@ -76,7 +76,7 @@ void ell_multiply_kernel33(
         temp +=data[ B2+1]* x[(J2+1)*right_size+j];
         temp +=data[ B2+2]* x[(J2+2)*right_size+j];
         int I = ((s*num_rows + i)*3+k)*right_size+j;
-        y[I]=temp;
+        y[I]=alpha*temp+beta*y[I];
     }
 #pragma omp parallel for collapse(2)
     for( int s=0; s<left_size; s++)
@@ -103,7 +103,7 @@ void ell_multiply_kernel33(
         temp +=data[ B2+1]* x[(J2+1)*right_size+j];
         temp +=data[ B2+2]* x[(J2+2)*right_size+j];
         int I = ((s*num_rows + i)*3+k)*right_size+j;
-        y[I]=temp;
+        y[I]=alpha*temp+beta*y[I];
     }
 #pragma omp parallel for 
     for( int s=0; s<left_size; s++)
@@ -130,16 +130,16 @@ void ell_multiply_kernel33(
         temp +=data[ B2+1]* x[(J2+1)*right_size+j];
         temp +=data[ B2+2]* x[(J2+2)*right_size+j];
         int I = ((s*num_rows + i)*3+k)*right_size+j;
-        y[I]=temp;
+        y[I]=alpha*temp+beta*y[I];
     }
     }
     else 
-        ell_multiply_kernel( data, cols_idx, data_idx, num_rows, num_cols, 3, 3, left_size, right_size, right_range,  x, y);
+        ell_multiply_kernel(alpha,beta, data, cols_idx, data_idx, num_rows, num_cols, 3, 3, left_size, right_size, right_range,  x, y);
 }
 
 // multiply kernel, n=3, 2 blocks per line
 template<class value_type>
-void ell_multiply_kernel32(
+void ell_multiply_kernel32( value_type alpha, value_type beta,
          const value_type* data, const int* cols_idx, const int* data_idx, 
          const int num_rows, const int num_cols,
          const int left_size, const int right_size, 
@@ -177,7 +177,7 @@ void ell_multiply_kernel32(
         temp +=data[ B1+1]* x[(J1+1)*right_size+j];
         temp +=data[ B1+2]* x[(J1+2)*right_size+j];
         int I = ((s*num_rows + i)*3+k)*right_size+j;
-        y[I]=temp;
+        y[I]=alpha*temp+beta*y[I];
     }
 #pragma omp parallel for collapse(2)
     for( int s=0; s<left_size; s++)
@@ -197,7 +197,7 @@ void ell_multiply_kernel32(
         temp +=data[ B1+1]* x[(J1+1)*right_size+j];
         temp +=data[ B1+2]* x[(J1+2)*right_size+j];
         int I = ((s*num_rows + i)*3+k)*right_size+j;
-        y[I]=temp;
+        y[I]=alpha*temp+beta*y[I];
     }
 #pragma omp parallel for 
     for( int s=0; s<left_size; s++)
@@ -217,16 +217,16 @@ void ell_multiply_kernel32(
         temp +=data[ B1+1]* x[(J1+1)*right_size+j];
         temp +=data[ B1+2]* x[(J1+2)*right_size+j];
         int I = ((s*num_rows + i)*3+k)*right_size+j;
-        y[I]=temp;
+        y[I]=alpha*temp+beta*y[I];
     }
     }
     else
-        ell_multiply_kernel( data, cols_idx, data_idx, num_rows, num_cols, 2, 3, left_size, right_size, right_range,  x, y);
+        ell_multiply_kernel(alpha,beta, data, cols_idx, data_idx, num_rows, num_cols, 2, 3, left_size, right_size, right_range,  x, y);
 
 }
 // multiply kernel, n=3, 3 blocks per line, right_size = 1
 template<class value_type>
-void ell_multiply_kernel33x(
+void ell_multiply_kernel33x( value_type alpha, value_type beta,
          const value_type* data, const int* cols_idx, const int* data_idx, 
          const int num_rows, const int num_cols,
          const int left_size,
@@ -266,7 +266,7 @@ void ell_multiply_kernel33x(
         temp +=data[ B2+1]* x[(J2+1)];
         temp +=data[ B2+2]* x[(J2+2)];
         int I = ((s*num_rows + i)*3+k);
-        y[I]=temp;
+        y[I]=alpha*temp+beta*y[I];
     }
 
 #pragma omp parallel for// collapse(2)
@@ -292,7 +292,7 @@ void ell_multiply_kernel33x(
         temp +=data[ B2+2]* x[(J2+2)];
 
         int I = ((s*num_rows + i)*3+k);
-        y[I]=temp;
+        y[I]=alpha*temp+beta*y[I];
     }
 #pragma omp parallel for
     for( int s=0; s<left_size; s++)
@@ -319,19 +319,19 @@ void ell_multiply_kernel33x(
         temp +=data[ B2+2]* x[(J2+2)];
 
         int I = ((s*num_rows + i)*3+k);
-        y[I]=temp;
+        y[I]=alpha*temp+beta*y[I];
     }
     }
     else 
     {
         int right_range[2] = {0,1};
-        ell_multiply_kernel( data, cols_idx, data_idx, num_rows, num_cols, 3, 3, left_size, 1, right_range,  x, y);
+        ell_multiply_kernel(alpha,beta, data, cols_idx, data_idx, num_rows, num_cols, 3, 3, left_size, 1, right_range,  x, y);
     }
 }
 
 // multiply kernel, n=3, 2 blocks per line, right_size = 1
 template<class value_type>
-void ell_multiply_kernel32x(
+void ell_multiply_kernel32x( value_type alpha, value_type beta,
          const value_type* data, const int* cols_idx, const int* data_idx, 
          const int num_rows, const int num_cols,
          const int left_size, 
@@ -367,7 +367,7 @@ void ell_multiply_kernel32x(
         temp +=data[ B1+1]* x[(J1+1)];
         temp +=data[ B1+2]* x[(J1+2)];
         int I = ((s*num_rows + i)*3+k);
-        y[I]=temp;
+        y[I]=alpha*temp+beta*y[I];
     }
 #pragma omp parallel for
     for( int s=0; s<left_size; s++)
@@ -386,7 +386,7 @@ void ell_multiply_kernel32x(
         temp +=data[ B1+1]* x[(J1+1)];
         temp +=data[ B1+2]* x[(J1+2)];
         int I = ((s*num_rows + i)*3+k);
-        y[I]=temp;
+        y[I]=alpha*temp+beta*y[I];
     }
 #pragma omp parallel for
     for( int s=0; s<left_size; s++)
@@ -405,13 +405,13 @@ void ell_multiply_kernel32x(
         temp +=data[ B1+1]* x[(J1+1)];
         temp +=data[ B1+2]* x[(J1+2)];
         int I = ((s*num_rows + i)*3+k);
-        y[I]=temp;
+        y[I]=alpha*temp+beta*y[I];
     }
     }
     else
     {
         int right_range[2] = {0,1};
-        ell_multiply_kernel( data, cols_idx, data_idx, num_rows, num_cols, 2, 3, left_size, 1, right_range, x, y);
+        ell_multiply_kernel(alpha,beta, data, cols_idx, data_idx, num_rows, num_cols, 2, 3, left_size, 1, right_range, x, y);
     }
 
 
@@ -419,7 +419,7 @@ void ell_multiply_kernel32x(
 
 template<class value_type>
 template<class DeviceContainer>
-void EllSparseBlockMatDevice<value_type>::launch_multiply_kernel( const DeviceContainer& x, DeviceContainer& y) const
+void EllSparseBlockMatDevice<value_type>::launch_multiply_kernel( value_type alpha, const DeviceContainer& x, value_type beta, DeviceContainer& y) const
 {
     assert( y.size() == (unsigned)num_rows*n*left_size*right_size);
     assert( x.size() == (unsigned)num_cols*n*left_size*right_size);
@@ -435,23 +435,23 @@ void EllSparseBlockMatDevice<value_type>::launch_multiply_kernel( const DeviceCo
         if( blocks_per_line == 3)
         {
             if( right_size == 1)
-                ell_multiply_kernel33x<value_type> ( data_ptr, cols_ptr, block_ptr, num_rows, num_cols, left_size, x_ptr,y_ptr);
+                ell_multiply_kernel33x<value_type> ( alpha, beta, data_ptr, cols_ptr, block_ptr, num_rows, num_cols, left_size, x_ptr,y_ptr);
             else
-                ell_multiply_kernel33<value_type> ( data_ptr, cols_ptr, block_ptr, num_rows, num_cols, left_size, right_size, right_range_ptr,  x_ptr,y_ptr);
+                ell_multiply_kernel33<value_type> ( alpha, beta, data_ptr, cols_ptr, block_ptr, num_rows, num_cols, left_size, right_size, right_range_ptr,  x_ptr,y_ptr);
         }
         else if( blocks_per_line == 2)
         {
             if( right_size == 1)
-                ell_multiply_kernel32x<value_type> ( data_ptr, cols_ptr, block_ptr, num_rows, num_cols, left_size,  x_ptr,y_ptr);
+                ell_multiply_kernel32x<value_type> ( alpha, beta, data_ptr, cols_ptr, block_ptr, num_rows, num_cols, left_size,  x_ptr,y_ptr);
             else
-                ell_multiply_kernel32<value_type> ( data_ptr, cols_ptr, block_ptr, num_rows, num_cols, left_size, right_size, right_range_ptr,  x_ptr,y_ptr);
+                ell_multiply_kernel32<value_type> ( alpha, beta, data_ptr, cols_ptr, block_ptr, num_rows, num_cols, left_size, right_size, right_range_ptr,  x_ptr,y_ptr);
         }
         else
-            ell_multiply_kernel<value_type>( 
+            ell_multiply_kernel<value_type>(alpha, beta,  
                 data_ptr, cols_ptr, block_ptr, num_rows, num_cols, blocks_per_line, 3, left_size, right_size, right_range_ptr,  x_ptr,y_ptr);
     }
     else
-        ell_multiply_kernel<value_type>  ( 
+        ell_multiply_kernel<value_type>  (alpha, beta,  
             data_ptr, cols_ptr, block_ptr, num_rows, num_cols, blocks_per_line, n, left_size, right_size, right_range_ptr,  x_ptr,y_ptr);
 }
 
@@ -461,7 +461,6 @@ void CooSparseBlockMatDevice<value_type>::launch_multiply_kernel( value_type alp
 {
     assert( y.size() == (unsigned)num_rows*n*left_size*right_size);
     assert( x.size() == (unsigned)num_cols*n*left_size*right_size);
-    assert( beta == 1);
 
     for( int i=0; i<num_entries; i++)
 #pragma omp parallel for collapse(3)
@@ -474,7 +473,7 @@ void CooSparseBlockMatDevice<value_type>::launch_multiply_kernel( value_type alp
         for( int q=0; q<n; q++) //multiplication-loop
             temp+= data[ (data_idx[i]*n + k)*n+q]*
                 x[((s*num_cols + cols_idx[i])*n+q)*right_size+j];
-        y[I] += alpha*temp;
+        y[I] = alpha*temp + beta*y[I];
     }
 }
 
diff --git a/inc/dg/blas2.h b/inc/dg/blas2.h
index 3303617bb..fffec1692 100644
--- a/inc/dg/blas2.h
+++ b/inc/dg/blas2.h
@@ -103,6 +103,11 @@ inline void symv( typename MatrixTraits<Precon>::value_type alpha,
                   typename MatrixTraits<Precon>::value_type beta, 
                   Vector& y)
 {
+    if(alpha == (typename MatrixTraits<Precon>::value_type)0) 
+    {
+        dg::blas1::scal( y, alpha);
+        return;
+    }
     dg::blas2::detail::doSymv( alpha, P, x, beta, y, 
                        typename dg::MatrixTraits<Precon>::matrix_category(), 
                        typename dg::VectorTraits<Vector>::vector_category() );
-- 
GitLab


From f7bafcadd0271ad4538ba39f0a7f026da1844e71 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Mon, 21 Aug 2017 21:43:07 +0200
Subject: [PATCH 202/453] modified elliptic.h with symv a,x,b,y function

---
 inc/dg/elliptic.h | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index 331228bfa..ec210587a 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -148,26 +148,23 @@ class Elliptic
     {
         //compute gradient
         dg::blas2::gemv( rightx, x, tempx); //R_x*f 
-        dg::blas2::gemv( righty, x, tempy); //R_y*f
+        dg::blas2::gemv( righty, x, y); //R_y*f
 
-        dg::geo::volRaisePerpIndex( tempx, tempy, gradx, y, g_);
+        dg::geo::volRaisePerpIndex( tempx, y, gradx, tempy, g_);
 
         //multiply with chi 
         dg::blas1::pointwiseDot( xchi, gradx, gradx); //Chi*R_x*x 
-        dg::blas1::pointwiseDot( xchi, y, y); //Chi*R_x*x 
+        dg::blas1::pointwiseDot( xchi, tempy, tempy); //Chi*R_x*x 
 
         //now take divergence
-        dg::blas2::gemv( leftx, gradx, tempx);  
-        dg::blas2::gemv( lefty, y, tempy);  
-        dg::blas1::axpby( -1., tempx, -1., tempy, y); //-D_xx - D_yy 
+        dg::blas2::symv( lefty, tempy, y);  
+        dg::blas2::symv( -1., leftx, gradx, -1., y);  
         if( no_ == normed)
             dg::geo::divideVolume( y, g_);
 
         //add jump terms
-        dg::blas2::symv( jumpX, x, tempx);
-        dg::blas1::axpby( jfactor_, tempx, 1., y, y); 
-        dg::blas2::symv( jumpY, x, tempy);
-        dg::blas1::axpby( jfactor_, tempy, 1., y, y); 
+        dg::blas2::symv( jfactor_, jumpX, x, 1., y);
+        dg::blas2::symv( jfactor_, jumpY, x, 1., y);
         if( no_ == not_normed)//multiply weights without volume
             dg::blas2::symv( weights_wo_vol, y, y);
 
-- 
GitLab


From b21aec51d6b3ebc01b78f80284dcee1c0a2f95ed Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Tue, 22 Aug 2017 04:38:08 -0700
Subject: [PATCH 203/453] update conformal_elliptic_b.cu

---
 inc/geometries/conformal_elliptic_b.cu    | 70 +++++++++++------------
 inc/geometries/curvilinear.h              |  2 +-
 inc/geometries/curvilinearX.h             |  4 +-
 inc/geometries/flux_t.cu                  |  2 +-
 inc/geometries/geometries.h               |  1 -
 inc/geometries/geometryX_elliptic_b.cu    |  2 +-
 inc/geometries/hector_t.cu                |  2 +-
 inc/geometries/mpi_curvilinear.h          |  1 +
 inc/geometries/refined_curvilinearX.h     |  2 +-
 inc/geometries/ribeiro_t.cu               |  2 +-
 inc/geometries/separatrix_orthogonal_t.cu |  4 +-
 inc/geometries/simple_orthogonal.h        |  1 +
 inc/geometries/simple_orthogonal_t.cu     |  1 +
 13 files changed, 47 insertions(+), 47 deletions(-)

diff --git a/inc/geometries/conformal_elliptic_b.cu b/inc/geometries/conformal_elliptic_b.cu
index f26cf7568..678e701e2 100644
--- a/inc/geometries/conformal_elliptic_b.cu
+++ b/inc/geometries/conformal_elliptic_b.cu
@@ -1,13 +1,10 @@
 #include <iostream>
 
-#include "dg/backend/timer.cuh"
 #include "dg/backend/grid.h"
 #include "dg/elliptic.h"
 #include "dg/cg.h"
 
 //Grids
-#include "conformal.h"
-#include "orthogonal.h"
 #include "curvilinear.h"
 //generators
 #include "simple_orthogonal.h"
@@ -16,29 +13,30 @@
 #include "solovev.h"
 //#include "guenther.h"
 #include "testfunctors.h"
+#include "dg/backend/timer.cuh"
 
 using namespace dg::geo::solovev;
 
 const unsigned nIter=6;
 template<class Geometry>
-void compute_error_elliptic( const MagneticField& c, double R_0, const Geometry& g2d, double psi_0, double psi_1, double eps)
+void compute_error_elliptic( const dg::geo::TokamakMagneticField& c, const Geometry& g2d, double psi_0, double psi_1, double eps)
 {
     dg::DVec x =    dg::evaluate( dg::zero, g2d);
     /////////////////////////////DirNeu/////FLUXALIGNED//////////////////////
     //dg::Elliptic<Geometry, dg::DMatrix, dg::DVec> pol( g2d, dg::DIR_NEU, dg::PER, dg::not_normed, dg::forward);
-    //const dg::DVec b =    dg::pullback( dg::geo::EllipticDirPerM<MagneticField>(c, R_0, psi_0, 2.*psi_1-psi_0, 0), g2d);
-    //const dg::DVec chi =  dg::pullback( dg::geo::Bmodule<MagneticField>(c, R_0), g2d);
-    //const dg::DVec solution = dg::pullback( dg::geo::FuncDirPer<MagneticField>(c, psi_0, 2.*psi_1-psi_0, 0 ), g2d);
+    //const dg::DVec b =    dg::pullback( dg::geo::EllipticDirPerM(c, psi_0, 2.*psi_1-psi_0, 0), g2d);
+    //const dg::DVec chi =  dg::pullback( dg::geo::Bmodule(c), g2d);
+    //const dg::DVec solution = dg::pullback( dg::geo::FuncDirPer(c, psi_0, 2.*psi_1-psi_0, 0 ), g2d);
     /////////////////////////////Dir/////FIELALIGNED SIN///////////////////
     //dg::Elliptic<Geometry, dg::DMatrix, dg::DVec> pol( g2d, dg::DIR, dg::PER, dg::not_normed, dg::forward);
-    //const dg::DVec b =    dg::pullback( dg::geo::EllipticDirPerM<MagneticField>(c, R_0, psi_0, psi_1, 4), g2d);
-    //const dg::DVec chi =  dg::pullback( dg::geo::Bmodule<MagneticField>(c, R_0), g2d);
-    //const dg::DVec solution = dg::pullback( dg::geo::FuncDirPer<MagneticField>(c, psi_0, psi_1, 4 ), g2d);
+    //const dg::DVec b =    dg::pullback( dg::geo::EllipticDirPerM(c, psi_0, psi_1, 4), g2d);
+    //const dg::DVec chi =  dg::pullback( dg::geo::Bmodule(c), g2d);
+    //const dg::DVec solution = dg::pullback( dg::geo::FuncDirPer(c, psi_0, psi_1, 4 ), g2d);
     /////////////////////////////Dir//////BLOB/////////////////////////////
     dg::Elliptic<Geometry, dg::DMatrix, dg::DVec> pol( g2d, dg::DIR, dg::PER, dg::not_normed, dg::forward);
-    const dg::DVec b =    dg::pullback( dg::geo::EllipticDirNeuM<MagneticField>(c, R_0, psi_0, psi_1, 440, -220, 40.,1.), g2d);
-    const dg::DVec chi =  dg::pullback( dg::geo::BmodTheta<MagneticField>(c, R_0), g2d);
-    const dg::DVec solution = dg::pullback(dg::geo::FuncDirNeu<MagneticField>(c, psi_0, psi_1, 440, -220, 40.,1.), g2d);
+    const dg::DVec b =    dg::pullback( dg::geo::EllipticDirNeuM(c, psi_0, psi_1, 440, -220, 40.,1.), g2d);
+    const dg::DVec chi =  dg::pullback( dg::geo::BmodTheta(c), g2d);
+    const dg::DVec solution = dg::pullback(dg::geo::FuncDirNeu(c, psi_0, psi_1, 440, -220, 40.,1.), g2d);
     ///////////////////////////////////////////////////////////////////////
     const dg::DVec vol2d = dg::create::volume( g2d);
     pol.set_chi( chi);
@@ -61,7 +59,8 @@ void compute_error_elliptic( const MagneticField& c, double R_0, const Geometry&
 template<class Geometry>
 void compute_cellsize( const Geometry& g2d)
 {
-    dg::DVec gyy = g2d.g_xx(), gxx=g2d.g_yy(), vol = g2d.vol();
+    dg::SparseTensor<dg::DVec> metric = g2d.metric();
+    dg::DVec gyy = metric.value(1,1), gxx=metric.value(0,0), vol = dg::tensor::volume(metric).value();
     dg::blas1::transform( gxx, gxx, dg::SQRT<double>());
     dg::blas1::transform( gyy, gyy, dg::SQRT<double>());
     dg::blas1::pointwiseDot( gxx, vol, gxx);
@@ -101,16 +100,16 @@ int main(int argc, char**argv)
     //write parameters from file into variables
     GeomParameters gp(js);
     gp.display( std::cout);
-    MagneticField c( gp); 
+    dg::geo::TokamakMagneticField c = dg::geo::solovev::createMagField( gp); 
     const double eps = 1e-10;
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     std::cout << "eps\tn\t Nx\t Ny \t # iterations \t error  \t time/iteration (s)\t hx_max\t hy_max\t hx_min\t hy_min \n";
     std::cout << "Orthogonal:\n";
-    dg::geo::SimpleOrthogonal<Psip, PsipR, PsipZ, LaplacePsip> generator0(c.psip, c.psipR, c.psipZ, c.laplacePsip, psi_0, psi_1, gp.R_0, 0., 0);
+    dg::geo::SimpleOrthogonal generator0(c.get_psip(), psi_0, psi_1, gp.R_0, 0., 0);
     for( unsigned i=0; i<nIter; i++)
     {
-        dg::OrthogonalGrid2d<dg::DVec> g2d(generator0, n, Nx, Ny);
-        compute_error_elliptic(c, gp.R_0, g2d, psi_0, psi_1, eps);
+        dg::CurvilinearGrid2d g2d(generator0, n, Nx, Ny);
+        compute_error_elliptic(c, g2d, psi_0, psi_1, eps);
         compute_cellsize(g2d);
         std::cout <<std::endl;
         Nx*=2; Ny*=2;
@@ -118,11 +117,11 @@ int main(int argc, char**argv)
     Nx=NxIni, Ny=NyIni;
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     std::cout << "Orthogonal Adapted:\n";
-    dg::geo::SimpleOrthogonal<Psip, PsipR, PsipZ, LaplacePsip> generator1(c.psip, c.psipR, c.psipZ, c.laplacePsip, psi_0, psi_1, gp.R_0, 0., 1);
+    dg::geo::SimpleOrthogonal generator1(c.get_psip(), psi_0, psi_1, gp.R_0, 0., 1);
     for( unsigned i=0; i<nIter; i++)
     {
-        dg::OrthogonalGrid2d<dg::DVec> g2d(generator1, n, Nx, Ny);
-        compute_error_elliptic(c, gp.R_0, g2d, psi_0, psi_1, eps);
+        dg::CurvilinearGrid2d g2d(generator1, n, Nx, Ny);
+        compute_error_elliptic(c, g2d, psi_0, psi_1, eps);
         compute_cellsize(g2d);
         std::cout <<std::endl;
         Nx*=2; Ny*=2;
@@ -130,11 +129,11 @@ int main(int argc, char**argv)
     Nx=NxIni, Ny=NyIni;
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     std::cout << "Conformal:\n";
-    dg::geo::Hector<dg::IHMatrix, dg::HMatrix, dg::HVec> hectorConf( c.psip, c.psipR, c.psipZ, c.psipRR, c.psipRZ, c.psipZZ, psi_0, psi_1, gp.R_0, 0., nGrid,NxGrid,NyGrid, 1e-10, true);
+    dg::geo::Hector<dg::IHMatrix, dg::HMatrix, dg::HVec> hectorConf( c.get_psip(), psi_0, psi_1, gp.R_0, 0., nGrid,NxGrid,NyGrid, 1e-10, true);
     for( unsigned i=0; i<nIter; i++)
     {
-        dg::ConformalGrid2d<dg::DVec> g2d(hectorConf, n, Nx, Ny);
-        compute_error_elliptic(c, gp.R_0, g2d, psi_0, psi_1,eps);
+        dg::CurvilinearGrid2d g2d(hectorConf, n, Nx, Ny);
+        compute_error_elliptic(c, g2d, psi_0, psi_1,eps);
         compute_cellsize(g2d);
         std::cout <<std::endl;
         Nx*=2; Ny*=2;
@@ -142,12 +141,12 @@ int main(int argc, char**argv)
     Nx=NxIni, Ny=NyIni;
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     std::cout << "ConformalMonitor:\n";
-    dg::geo::LiseikinCollective<PsipR, PsipZ, PsipRR, PsipRZ, PsipZZ> lc( c.psipR, c.psipZ, c.psipRR, c.psipRZ, c.psipZZ, 0.1, 0.001);
-    dg::geo::Hector<dg::IHMatrix, dg::HMatrix, dg::HVec> hectorMonitor( c.psip, c.psipR, c.psipZ, c.psipRR, c.psipRZ, c.psipZZ, lc.chi_XX, lc.chi_XY, lc.chi_YY, lc.divChiX, lc.divChiY, psi_0, psi_1, gp.R_0, 0., nGrid,NxGrid,NyGrid, 1e-10, true);
+    dg::geo::BinarySymmTensorLvl1 lc = dg::geo::make_LiseikinCollective( c.get_psip(), 0.1, 0.001);
+    dg::geo::Hector<dg::IHMatrix, dg::HMatrix, dg::HVec> hectorMonitor( c.get_psip(), lc, psi_0, psi_1, gp.R_0, 0., nGrid,NxGrid,NyGrid, 1e-10, true);
     for( unsigned i=0; i<nIter; i++)
     {
-        dg::CurvilinearGrid2d<dg::DVec> g2d(hectorMonitor, n, Nx, Ny);
-        compute_error_elliptic(c, gp.R_0, g2d, psi_0, psi_1,eps);
+        dg::CurvilinearGrid2d g2d(hectorMonitor, n, Nx, Ny);
+        compute_error_elliptic(c, g2d, psi_0, psi_1,eps);
         compute_cellsize(g2d);
         std::cout <<std::endl;
         Nx*=2; Ny*=2;
@@ -155,12 +154,12 @@ int main(int argc, char**argv)
     Nx=NxIni, Ny=NyIni;
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     std::cout << "ConformalAdaption:\n";
-    dg::geo::NablaPsiInvCollective<PsipR, PsipZ, PsipRR, PsipRZ, PsipZZ> nc( c.psipR, c.psipZ, c.psipRR, c.psipRZ, c.psipZZ);
-    dg::geo::Hector<dg::IHMatrix, dg::HMatrix, dg::HVec> hectorAdapt( c.psip, c.psipR, c.psipZ, c.psipRR, c.psipRZ, c.psipZZ, nc.nablaPsiInv, nc.nablaPsiInvX, nc.nablaPsiInvY, psi_0, psi_1, gp.R_0, 0., nGrid,NxGrid,NyGrid, 1e-10, true);
+    dg::geo::BinaryFunctorsLvl1 nc = dg::geo::make_NablaPsiInvCollective( c.get_psip());
+    dg::geo::Hector<dg::IHMatrix, dg::HMatrix, dg::HVec> hectorAdapt( c.get_psip(), nc, psi_0, psi_1, gp.R_0, 0., nGrid,NxGrid,NyGrid, 1e-10, true);
     for( unsigned i=0; i<nIter; i++)
     {
-        dg::OrthogonalGrid2d<dg::DVec> g2d(hectorAdapt, n, Nx, Ny);
-        compute_error_elliptic(c, gp.R_0, g2d, psi_0, psi_1,eps);
+        dg::CurvilinearGrid2d g2d(hectorAdapt, n, Nx, Ny);
+        compute_error_elliptic(c, g2d, psi_0, psi_1,eps);
         compute_cellsize(g2d);
         std::cout <<std::endl;
         Nx*=2; Ny*=2;
@@ -168,12 +167,11 @@ int main(int argc, char**argv)
     Nx=NxIni, Ny=NyIni;
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     std::cout << "Ribeiro:\n";
-    dg::geo::Ribeiro<Psip, PsipR, PsipZ, PsipRR, PsipRZ, PsipZZ>
-      ribeiro( c.psip, c.psipR, c.psipZ, c.psipRR, c.psipRZ, c.psipZZ, psi_0, psi_1, gp.R_0, 0.);
+    dg::geo::Ribeiro ribeiro( c.get_psip(), psi_0, psi_1, gp.R_0, 0.);
     for( unsigned i=0; i<nIter; i++)
     {
-        dg::CurvilinearGrid2d<dg::DVec> g2d(ribeiro, n, Nx, Ny);
-        compute_error_elliptic(c, gp.R_0, g2d, psi_0, psi_1,eps);
+        dg::CurvilinearGrid2d g2d(ribeiro, n, Nx, Ny);
+        compute_error_elliptic(c, g2d, psi_0, psi_1,eps);
         compute_cellsize(g2d);
         std::cout <<std::endl;
         Nx*=2; Ny*=2;
diff --git a/inc/geometries/curvilinear.h b/inc/geometries/curvilinear.h
index b2a1dff5e..f4fc14ccb 100644
--- a/inc/geometries/curvilinear.h
+++ b/inc/geometries/curvilinear.h
@@ -2,7 +2,7 @@
 
 #include "dg/backend/memory.h"
 #include "dg/blas1.h"
-#include "base_geometry.h"
+#include "dg/geometry/base_geometry.h"
 #include "generator.h"
 
 namespace dg
diff --git a/inc/geometries/curvilinearX.h b/inc/geometries/curvilinearX.h
index 50b9a6645..17126d43c 100644
--- a/inc/geometries/curvilinearX.h
+++ b/inc/geometries/curvilinearX.h
@@ -4,8 +4,8 @@
 #include "dg/backend/evaluationX.cuh"
 #include "dg/backend/functions.h"
 #include "dg/blas1.h"
-#include "base_geometryX.h"
-#include "generatorX.h"
+#include "dg/geometry/base_geometryX.h"
+#include "dg/geometry/generatorX.h"
 #include "curvilinear.h"
 
 namespace dg
diff --git a/inc/geometries/flux_t.cu b/inc/geometries/flux_t.cu
index a41ac461b..0e091c928 100644
--- a/inc/geometries/flux_t.cu
+++ b/inc/geometries/flux_t.cu
@@ -9,7 +9,7 @@
 #include "dg/functors.h"
 
 #include "dg/backend/timer.cuh"
-#include "dg/geometry/curvilinear.h"
+#include "curvilinear.h"
 //#include "guenther.h"
 #include "solovev.h"
 #include "flux.h"
diff --git a/inc/geometries/geometries.h b/inc/geometries/geometries.h
index 621e0047b..5185d6bb4 100644
--- a/inc/geometries/geometries.h
+++ b/inc/geometries/geometries.h
@@ -2,7 +2,6 @@
 
 //include grids
 #include "curvilinear.h"
-#include "refined_curvilinear.h"
 #ifdef MPI_VERSION
 #include "mpi_curvilinear.h"
 #endif
diff --git a/inc/geometries/geometryX_elliptic_b.cu b/inc/geometries/geometryX_elliptic_b.cu
index bf5e05333..d305c381b 100644
--- a/inc/geometries/geometryX_elliptic_b.cu
+++ b/inc/geometries/geometryX_elliptic_b.cu
@@ -14,7 +14,7 @@
 #include "solovev.h"
 #include "taylor.h"
 //#include "ribeiroX.h"
-#include "dg/geometry/curvilinearX.h"
+#include "curvilinearX.h"
 #include "separatrix_orthogonal.h"
 #include "testfunctors.h"
 
diff --git a/inc/geometries/hector_t.cu b/inc/geometries/hector_t.cu
index 33ccb6542..75831e16e 100644
--- a/inc/geometries/hector_t.cu
+++ b/inc/geometries/hector_t.cu
@@ -7,7 +7,7 @@
 
 #include "dg/backend/xspacelib.cuh"
 #include "dg/functors.h"
-#include "dg/geometry/curvilinear.h"
+#include "curvilinear.h"
 
 #include "dg/backend/timer.cuh"
 //#include "guenther.h"
diff --git a/inc/geometries/mpi_curvilinear.h b/inc/geometries/mpi_curvilinear.h
index a40afeef7..fd5d40176 100644
--- a/inc/geometries/mpi_curvilinear.h
+++ b/inc/geometries/mpi_curvilinear.h
@@ -5,6 +5,7 @@
 #include "dg/backend/mpi_evaluation.h"
 #include "dg/backend/mpi_grid.h"
 #include "dg/backend/mpi_vector.h"
+#include "dg/geometry/mpi_base.h"
 #include "curvilinear.h"
 #include "generator.h"
 
diff --git a/inc/geometries/refined_curvilinearX.h b/inc/geometries/refined_curvilinearX.h
index d68aa10ed..9cf5034a0 100644
--- a/inc/geometries/refined_curvilinearX.h
+++ b/inc/geometries/refined_curvilinearX.h
@@ -1,7 +1,7 @@
 #pragma once
 
+#include "dg/geometry/refined_gridX.h"
 #include "generatorX.h"
-#include "refined_gridX.h"
 #include "curvilinear.h"
 
 namespace dg
diff --git a/inc/geometries/ribeiro_t.cu b/inc/geometries/ribeiro_t.cu
index 6931bcb57..d6b4bfe55 100644
--- a/inc/geometries/ribeiro_t.cu
+++ b/inc/geometries/ribeiro_t.cu
@@ -9,7 +9,7 @@
 #include "dg/functors.h"
 
 #include "dg/backend/timer.cuh"
-#include "dg/geometry/curvilinear.h"
+#include "curvilinear.h"
 //#include "guenther.h"
 #include "solovev.h"
 #include "ribeiro.h"
diff --git a/inc/geometries/separatrix_orthogonal_t.cu b/inc/geometries/separatrix_orthogonal_t.cu
index 899d8c478..b3e07fafa 100644
--- a/inc/geometries/separatrix_orthogonal_t.cu
+++ b/inc/geometries/separatrix_orthogonal_t.cu
@@ -13,8 +13,8 @@
 #include "taylor.h"
 //#include "guenther.h"
 #include "dg/geometry/transform.h"
-#include "dg/geometry/curvilinearX.h"
-#include "dg/geometry/refined_curvilinearX.h"
+#include "refined_curvilinearX.h"
+#include "curvilinearX.h"
 #include "separatrix_orthogonal.h"
 #include "init.h"
 
diff --git a/inc/geometries/simple_orthogonal.h b/inc/geometries/simple_orthogonal.h
index c1a5cc72e..dd6ab3839 100644
--- a/inc/geometries/simple_orthogonal.h
+++ b/inc/geometries/simple_orthogonal.h
@@ -9,6 +9,7 @@
 #include "dg/runge_kutta.h"
 #include "dg/nullstelle.h"
 #include "dg/geometry/geometry.h"
+#include "generator.h"
 #include "utilities.h"
 
 
diff --git a/inc/geometries/simple_orthogonal_t.cu b/inc/geometries/simple_orthogonal_t.cu
index d1db34668..07ecec698 100644
--- a/inc/geometries/simple_orthogonal_t.cu
+++ b/inc/geometries/simple_orthogonal_t.cu
@@ -16,6 +16,7 @@
 //#include "ds.h"
 #include "init.h"
 #include "testfunctors.h"
+#include "curvilinear.h"
 
 #include "file/nc_utilities.h"
 using namespace dg::geo::solovev;
-- 
GitLab


From 529859800e36f0990d61b856e6648c9b8053e216 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Tue, 22 Aug 2017 06:40:40 -0700
Subject: [PATCH 204/453] update README.md with newest doc folder

---
 README.md           | 6 ++++--
 inc/geometries/ds.h | 4 ++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 4c80834a3..3b0b2a00c 100644
--- a/README.md
+++ b/README.md
@@ -111,9 +111,11 @@ input/output parameters, etc. can be generated as a pdf with
 ## 2. Further reading
 Please check out our [wiki pages](https://github.com/feltor-dev/feltor/wiki) for some general information and user oriented documentation. 
 Moreover, we maintain tex files in every src folder for technical documentation, 
- which can be compiled with 
+ which can be compiled using pdflatex with 
 `make doc ` in the respective src folder.
-The [developer oriented documentation](http://feltor-dev.github.io/feltor/inc/dg/html/modules.html) of the dG library was generated with [Doxygen](http://www.doxygen.org) from source code.
+The [developer oriented documentation](http://feltor-dev.github.io/feltor/inc/dg/html/modules.html) of the dG library was generated with [Doxygen](http://www.doxygen.org). 
+You can generate a local version from source code
+by typing `make doc` in the folder `path/to/feltor/inc/doc` and then open `index.html` with your favorite browser.
 
 ## 3. Contributions and Acknowledgements
 For instructions on how to contribute read the [wiki page](https://github.com/feltor-dev/feltor/wiki/Contributions).
diff --git a/inc/geometries/ds.h b/inc/geometries/ds.h
index 7687d99e1..d4cfeddc1 100644
--- a/inc/geometries/ds.h
+++ b/inc/geometries/ds.h
@@ -42,8 +42,8 @@ struct DS
     * @param dir the direction affects both the operator() and the symv function
     @param jumpX determines if a jump matrix is added in X-direction
     */
-    template<class MagneticField, class Geometry>
-    DS(const MagneticField& field, Geometry, dg::norm no=dg::normed, dg::direction dir = dg::centered, bool dependsOnX = true, bool dependsOnY=true, unsigned mx=1, unsigned my=1);
+    template<class Geometry>
+    DS(const dg::geo::TokamakMagneticField& field, const Geometry& g, dg::norm no=dg::normed, dg::direction dir = dg::centered, bool dependsOnX = true, bool dependsOnY=true, bool dependsOnZ=false, unsigned mx=1, unsigned my=1);
 
     /**
     * @brief Apply the forward derivative on a 3d vector
-- 
GitLab


From 80782236e07321c6e54e3261f87083454c671f65 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <Matthias.Wiesenberger@uibk.ac.at>
Date: Tue, 22 Aug 2017 16:09:50 +0200
Subject: [PATCH 205/453] update Marconi configuration

---
 config/devices/devices.mk | 4 ++--
 config/marconi.mk         | 6 +++++-
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/config/devices/devices.mk b/config/devices/devices.mk
index d6ed50bb2..c5d0b1c2f 100644
--- a/config/devices/devices.mk
+++ b/config/devices/devices.mk
@@ -26,7 +26,7 @@ endif #device=omp
 ifeq ($(strip $(device)),mic)
 CFLAGS+=-Wall -x c++
 CFLAGS+= -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_OMP
-CFLAGS+= $(OMPFLAG) -mmic 
+CFLAGS+= $(OMPFLAG) #-mmic 
 MPICFLAGS+=$(CFLAGS) #includes values in CFLAGS defined later
-OPT+=-xMIC-AVX512 -fma -finline-functions -align
+OPT=-O3 -xMIC-AVX512 -fma -finline-functions -align -restrict
 endif #device=mic
diff --git a/config/marconi.mk b/config/marconi.mk
index bf18c9cff..58eacc638 100644
--- a/config/marconi.mk
+++ b/config/marconi.mk
@@ -5,7 +5,7 @@ INCLUDE += -I$(NETCDF_INC) -I$(HDF5_INC)
 GLFLAGS  = -lm
 CC=icc
 MPICC=mpiicc
-OPT=-O3 -xHost -restrict
+OPT=-O3 -xHost -restrict # overwritten for mic in devices.mk
 #MPICFLAGS+= -DMPICH_IGNORE_CXX_SEEK
 OMPFLAG=-qopenmp
 JSONLIB=-L$(HOME)/include/json/../../src/lib_json -ljsoncpp # json library for input parameters
@@ -22,5 +22,9 @@ endif
 #module load netcdf/4.4.1--intelmpi--2017--binary 
 
 
+###########configure mic jobs with#########################
+#mcdram=cache:numa=quadrant
+#export KMP_AFFINITY=scatter
+#export OM_NUM_THREADS=68
 
 
-- 
GitLab


From 51c5339dd51c48d6a20164936088c32f5c2d6dc9 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Tue, 22 Aug 2017 23:32:01 +0200
Subject: [PATCH 206/453] applied RESTRICT Macro from Harald

---
 inc/dg/backend/sparseblockmat_omp_kernels.h | 34 +++++++++++++--------
 1 file changed, 22 insertions(+), 12 deletions(-)

diff --git a/inc/dg/backend/sparseblockmat_omp_kernels.h b/inc/dg/backend/sparseblockmat_omp_kernels.h
index e1cdaca1f..e939408d4 100644
--- a/inc/dg/backend/sparseblockmat_omp_kernels.h
+++ b/inc/dg/backend/sparseblockmat_omp_kernels.h
@@ -1,15 +1,25 @@
 
+#if defined(__INTEL_COMPILER)
+  // On Intel compiler, you need to pass the -restrict compiler flag in addition to your own compiler flags.
+# define RESTRICT restrict
+#elif defined(__GNUG__)
+# define RESTRICT __restrict__
+#else
+# warning Missing restrict keyword for this compiler
+# define RESTRICT
+#endif
+
 namespace dg{
 
 // multiply kernel
 template<class value_type>
 void ell_multiply_kernel(
-         const value_type * restrict data, const int * restrict cols_idx, const int * restrict data_idx, 
+         const value_type * RESTRICT data, const int * RESTRICT cols_idx, const int * RESTRICT data_idx, 
          const int num_rows, const int num_cols, const int blocks_per_line,
          const int n, 
          const int left_size, const int right_size, 
-         const int * restrict right_range,
-         const value_type * restrict x, value_type * restrict y
+         const int * RESTRICT right_range,
+         const value_type * RESTRICT x, value_type * RESTRICT y
          )
 {
     //simplest implementation
@@ -35,11 +45,11 @@ void ell_multiply_kernel(
 // multiply kernel n=3, 3 blocks per line
 template<class value_type>
 void ell_multiply_kernel33(
-         const value_type * restrict data, const int * restrict cols_idx, const int * restrict data_idx, 
+         const value_type * RESTRICT data, const int * RESTRICT cols_idx, const int * RESTRICT data_idx, 
          const int num_rows, const int num_cols, 
          const int left_size, const int right_size, 
          const int* right_range,
-         const value_type * restrict x, value_type * restrict y
+         const value_type * RESTRICT x, value_type * RESTRICT y
          )
 {
  // left_size = 1
@@ -97,11 +107,11 @@ void ell_multiply_kernel33(
 // multiply kernel, n=3, 2 blocks per line
 template<class value_type>
 void ell_multiply_kernel32(
-         const value_type * restrict data, const int * restrict cols_idx, const int * restrict data_idx, 
+         const value_type * RESTRICT data, const int * RESTRICT cols_idx, const int * RESTRICT data_idx, 
          const int num_rows, const int num_cols,
          const int left_size, const int right_size, 
-         const int * restrict right_range,
-         const value_type * restrict x, value_type * restrict y
+         const int * RESTRICT right_range,
+         const value_type * RESTRICT x, value_type * RESTRICT y
          )
 {
     bool forward = true, backward = true;
@@ -184,10 +194,10 @@ void ell_multiply_kernel32(
 // multiply kernel, n=3, 3 blocks per line, right_size = 1
 template<class value_type>
 void ell_multiply_kernel33x(
-         const value_type * restrict data, const int * restrict cols_idx, const int * restrict data_idx, 
+         const value_type * RESTRICT data, const int * RESTRICT cols_idx, const int * RESTRICT data_idx, 
          const int num_rows, const int num_cols,
          const int left_size,
-         const value_type * restrict x, value_type * restrict y
+         const value_type * RESTRICT x, value_type * RESTRICT y
          )
 {
     bool trivial = true;
@@ -286,10 +296,10 @@ void ell_multiply_kernel33x(
 // multiply kernel, n=3, 2 blocks per line, right_size = 1
 template<class value_type>
 void ell_multiply_kernel32x(
-         const value_type * restrict data, const int * restrict cols_idx, const int * restrict data_idx, 
+         const value_type * RESTRICT data, const int * RESTRICT cols_idx, const int * RESTRICT data_idx, 
          const int num_rows, const int num_cols,
          const int left_size, 
-         const value_type * restrict x, value_type * restrict y
+         const value_type * RESTRICT x, value_type * RESTRICT y
          )
 {
     bool forward = true, backward = true;
-- 
GitLab


From 9135762d955092a96344d507d14e3803c1cc355f Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Wed, 23 Aug 2017 00:31:26 +0200
Subject: [PATCH 207/453] found bug in sparseblockmat_omp and implemented new
 arakawa function

---
 inc/dg/arakawa.h                            | 106 ++++++++++++++------
 inc/dg/backend/sparseblockmat_omp_kernels.h |  86 +---------------
 2 files changed, 79 insertions(+), 113 deletions(-)

diff --git a/inc/dg/arakawa.h b/inc/dg/arakawa.h
index 65bcd781c..e1352bd0b 100644
--- a/inc/dg/arakawa.h
+++ b/inc/dg/arakawa.h
@@ -11,6 +11,49 @@
 #include "backend/mpi_evaluation.h"
 #endif
 
+template<class container>
+void pointwiseDot( double alpha, const std::vector<const container* >& x1, const std::vector<const container* >& y1, 
+                   double beta,  const std::vector<const container* >& x2, const std::vector<const container* >& y2, 
+                   double gamma, std::vector<container* > & z)
+{
+    unsigned K=x1.size();
+    //const double * RESTRICT x1_ptr[K]; 
+    //const double * RESTRICT y1_ptr[K]; 
+    //const double * RESTRICT x2_ptr[K]; 
+    //const double * RESTRICT y2_ptr[K]; 
+    //double * RESTRICT z_ptr[K];
+    const double *  x1_ptr[K]; 
+    const double *  y1_ptr[K]; 
+    const double *  x2_ptr[K]; 
+    const double *  y2_ptr[K]; 
+    double *  z_ptr[K];
+    for(unsigned i=0; i<K; i++)
+    {
+        x1_ptr[i] = thrust::raw_pointer_cast( &(x1[i]->data()[0]));
+        x2_ptr[i] = thrust::raw_pointer_cast( &(x2[i]->data()[0]));
+        y1_ptr[i] = thrust::raw_pointer_cast( &(y1[i]->data()[0]));
+        y2_ptr[i] = thrust::raw_pointer_cast( &(y2[i]->data()[0]));
+         z_ptr[i] = thrust::raw_pointer_cast( &(z[i]->data()[0]));
+    }
+    unsigned size = x1[0]->size();
+#pragma omp parallel
+{
+    double temp[K];
+#pragma omp for simd
+    for( unsigned i=0; i<size; i++)
+    {
+        for( unsigned l=0; l<K; l++)
+        {
+            temp[l] = alpha*x1_ptr[l][i]*y1_ptr[l][i] 
+                      +beta*x2_ptr[l][i]*y2_ptr[l][i]
+                      +gamma*z_ptr[l][i];
+        }
+        for( unsigned l=0; l<K; l++)
+            z_ptr[l][i] = temp[l];
+    }
+}
+}
+
 /*! @file 
   
   object for computation of Poisson bracket
@@ -120,34 +163,41 @@ void ArakawaX< Geometry, Matrix, container>::operator()( const container& lhs, c
     blas2::symv( bdxf, rhs, dxrhs);
     blas2::symv( bdyf, rhs, dyrhs);
 
-    // order is important now
-    // +x (1) -> result und (2) -> blhs
-    blas1::pointwiseDot( lhs, dyrhs, result);
-    blas1::pointwiseDot( lhs, dxrhs, helper_);
-
-    // ++ (1) -> dyrhs and (2) -> dxrhs
-    blas1::pointwiseDot( dxlhs, dyrhs, dyrhs);
-    blas1::pointwiseDot( dylhs, dxrhs, dxrhs);
-
-    // x+ (1) -> dxlhs and (2) -> dylhs
-    blas1::pointwiseDot( dxlhs, rhs, dxlhs);
-    blas1::pointwiseDot( dylhs, rhs, dylhs);
-
-    blas1::axpby( 1./3., dyrhs, -1./3., dxrhs);  //dxl*dyr - dyl*dxr -> dxrhs
-    //everything which needs a dx 
-    blas1::axpby( 1./3., dxlhs, -1./3., helper_);   //dxl*r - l*dxr     -> helper 
-    //everything which needs a dy
-    blas1::axpby( 1./3., result, -1./3., dylhs); //l*dyr - dyl*r     -> dylhs
-
-    //blas1::axpby( 0., dyrhs,  -0., dxrhs); //++
-    ////for testing purposes (note that you need to set criss-cross)
-    //blas1::axpby( 1., dxlhs,  -0., helper); //x+ - +x
-    //blas1::axpby( 0., result, -1., dylhs);  //+x - x+
-
-    blas2::symv( 1., bdyf, helper_, 1., dxrhs);
-    blas2::symv( 1., bdxf, dylhs, 1., dxrhs);
-    geo::dividePerpVolume( dxrhs, grid);
-    result.swap(dxrhs);
+    std::vector<const container* > s0(3), t0(3), s1(3), t1(3); 
+    std::vector<container* > s2(3);
+    s0[0] = &dxlhs, t0[0] = &dyrhs, s1[0] = &dylhs, t1[0] = &dxrhs;
+    s0[1] =   &lhs, t0[1] = &dyrhs, s1[1] = &dylhs, t1[1] =   &rhs;
+    s0[2] = &dxlhs, t0[2] =   &rhs, s1[2] =   &lhs, t1[2] = &dxrhs;
+    s2[0] = &result, s2[1] = &dxlhs, s2[2] = &dxrhs;
+    pointwiseDot( 1./3., s0, t0, -1./3., s1, t1, 0., s2);
+    
+    //// order is important now
+    //// +x (1) -> result und (2) -> blhs
+    //blas1::pointwiseDot( lhs, dyrhs, result);
+    //blas1::pointwiseDot( lhs, dxrhs, helper_);
+
+    //// ++ (1) -> dyrhs and (2) -> dxrhs
+    //blas1::pointwiseDot( dxlhs, dyrhs, dyrhs);
+    //blas1::pointwiseDot( dylhs, dxrhs, dxrhs);
+
+    //// x+ (1) -> dxlhs and (2) -> dylhs
+    //blas1::pointwiseDot( dxlhs, rhs, dxlhs);
+    //blas1::pointwiseDot( dylhs, rhs, dylhs);
+
+    //blas1::axpby( 1./3., dyrhs, -1./3., dxrhs);  //dxl*dyr - dyl*dxr -> dxrhs
+    ////everything which needs a dx 
+    //blas1::axpby( 1./3., dxlhs, -1./3., helper_);   //dxl*r - l*dxr     -> helper 
+    ////everything which needs a dy
+    //blas1::axpby( 1./3., result, -1./3., dylhs); //l*dyr - dyl*r     -> dylhs
+
+    ////blas1::axpby( 0., dyrhs,  -0., dxrhs); //++
+    //////for testing purposes (note that you need to set criss-cross)
+    ////blas1::axpby( 1., dxlhs,  -0., helper); //x+ - +x
+    ////blas1::axpby( 0., result, -1., dylhs);  //+x - x+
+
+    blas2::symv( 1., bdxf, dxlhs, 1., result);
+    blas2::symv( 1., bdyf, dxrhs, 1., result);
+    geo::dividePerpVolume( result, grid);
 }
 
 }//namespace dg
diff --git a/inc/dg/backend/sparseblockmat_omp_kernels.h b/inc/dg/backend/sparseblockmat_omp_kernels.h
index 883aa3fdb..838e588a8 100644
--- a/inc/dg/backend/sparseblockmat_omp_kernels.h
+++ b/inc/dg/backend/sparseblockmat_omp_kernels.h
@@ -65,89 +65,6 @@ void ell_multiply_kernel33( value_type alpha, value_type beta,
         }
     if( trivial)
     {
-<<<<<<< HEAD
-#pragma omp parallel for 
-    for( int s=0; s<left_size; s++)
-    for( int i=0; i<1; i++)
-    for( int k=0; k<3; k++)
-    for( int j=right_range[0]; j<right_range[1]; j++)
-    {
-        value_type temp = 0;
-        int B0 = (data_idx[i*3+0]*3+k)*3;
-        int B1 = (data_idx[i*3+1]*3+k)*3;
-        int B2 = (data_idx[i*3+2]*3+k)*3;
-        int J0 = (s*num_cols+cols_idx[i*3+0])*3;
-        int J1 = (s*num_cols+cols_idx[i*3+1])*3;
-        int J2 = (s*num_cols+cols_idx[i*3+2])*3;
-        temp +=data[ B0+0]* x[(J0+0)*right_size+j];
-        temp +=data[ B0+1]* x[(J0+1)*right_size+j];
-        temp +=data[ B0+2]* x[(J0+2)*right_size+j];
-
-        temp +=data[ B1+0]* x[(J1+0)*right_size+j];
-        temp +=data[ B1+1]* x[(J1+1)*right_size+j];
-        temp +=data[ B1+2]* x[(J1+2)*right_size+j];
-
-        temp +=data[ B2+0]* x[(J2+0)*right_size+j];
-        temp +=data[ B2+1]* x[(J2+1)*right_size+j];
-        temp +=data[ B2+2]* x[(J2+2)*right_size+j];
-        int I = ((s*num_rows + i)*3+k)*right_size+j;
-        y[I]=alpha*temp+beta*y[I];
-    }
-#pragma omp parallel for collapse(2)
-    for( int s=0; s<left_size; s++)
-    for( int i=1; i<num_rows-1; i++)
-    for( int k=0; k<3; k++)
-    for( int j=right_range[0]; j<right_range[1]; j++)
-    {
-        value_type temp = 0;
-        int B0 = (0*3+k)*3;
-        int B1 = (1*3+k)*3;
-        int B2 = (2*3+k)*3;
-        int J0 = (s*num_cols+i+0-1)*3;
-        int J1 = (s*num_cols+i+1-1)*3;
-        int J2 = (s*num_cols+i+2-1)*3;
-        temp +=data[ B0+0]* x[(J0+0)*right_size+j];
-        temp +=data[ B0+1]* x[(J0+1)*right_size+j];
-        temp +=data[ B0+2]* x[(J0+2)*right_size+j];
-
-        temp +=data[ B1+0]* x[(J1+0)*right_size+j];
-        temp +=data[ B1+1]* x[(J1+1)*right_size+j];
-        temp +=data[ B1+2]* x[(J1+2)*right_size+j];
-
-        temp +=data[ B2+0]* x[(J2+0)*right_size+j];
-        temp +=data[ B2+1]* x[(J2+1)*right_size+j];
-        temp +=data[ B2+2]* x[(J2+2)*right_size+j];
-        int I = ((s*num_rows + i)*3+k)*right_size+j;
-        y[I]=alpha*temp+beta*y[I];
-    }
-#pragma omp parallel for 
-    for( int s=0; s<left_size; s++)
-    for( int i=num_rows-1; i<num_rows; i++)
-    for( int k=0; k<3; k++)
-    for( int j=right_range[0]; j<right_range[1]; j++)
-    {
-        value_type temp = 0;
-        int B0 = (data_idx[i*3+0]*3+k)*3;
-        int B1 = (data_idx[i*3+1]*3+k)*3;
-        int B2 = (data_idx[i*3+2]*3+k)*3;
-        int J0 = (s*num_cols+cols_idx[i*3+0])*3;
-        int J1 = (s*num_cols+cols_idx[i*3+1])*3;
-        int J2 = (s*num_cols+cols_idx[i*3+2])*3;
-        temp +=data[ B0+0]* x[(J0+0)*right_size+j];
-        temp +=data[ B0+1]* x[(J0+1)*right_size+j];
-        temp +=data[ B0+2]* x[(J0+2)*right_size+j];
-
-        temp +=data[ B1+0]* x[(J1+0)*right_size+j];
-        temp +=data[ B1+1]* x[(J1+1)*right_size+j];
-        temp +=data[ B1+2]* x[(J1+2)*right_size+j];
-
-        temp +=data[ B2+0]* x[(J2+0)*right_size+j];
-        temp +=data[ B2+1]* x[(J2+1)*right_size+j];
-        temp +=data[ B2+2]* x[(J2+2)*right_size+j];
-        int I = ((s*num_rows + i)*3+k)*right_size+j;
-        y[I]=alpha*temp+beta*y[I];
-    }
-=======
 	#pragma omp parallel for collapse(2)
 	for( int s=0; s<left_size; s++)
 	{
@@ -177,12 +94,11 @@ void ell_multiply_kernel33( value_type alpha, value_type beta,
 				    temp +=data[ B2+1]* x[(J2+1)*right_size+j];
 				    temp +=data[ B2+2]* x[(J2+2)*right_size+j];
 				    int I = ((s*num_rows + i)*3+k)*right_size+j;
-				    y[I]=temp;
+                    y[I]=alpha*temp+beta*y[I];
 				}
 			}
 		}
 	}
->>>>>>> servat
     }
     else 
         ell_multiply_kernel(alpha,beta, data, cols_idx, data_idx, num_rows, num_cols, 3, 3, left_size, right_size, right_range,  x, y);
-- 
GitLab


From c5cd982dae744549de083dc8027b038b598d5958 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Wed, 23 Aug 2017 00:41:40 +0200
Subject: [PATCH 208/453] made use of RESTRICT keyword

---
 inc/dg/arakawa.h | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/inc/dg/arakawa.h b/inc/dg/arakawa.h
index e1352bd0b..986451512 100644
--- a/inc/dg/arakawa.h
+++ b/inc/dg/arakawa.h
@@ -17,16 +17,16 @@ void pointwiseDot( double alpha, const std::vector<const container* >& x1, const
                    double gamma, std::vector<container* > & z)
 {
     unsigned K=x1.size();
-    //const double * RESTRICT x1_ptr[K]; 
-    //const double * RESTRICT y1_ptr[K]; 
-    //const double * RESTRICT x2_ptr[K]; 
-    //const double * RESTRICT y2_ptr[K]; 
-    //double * RESTRICT z_ptr[K];
-    const double *  x1_ptr[K]; 
-    const double *  y1_ptr[K]; 
-    const double *  x2_ptr[K]; 
-    const double *  y2_ptr[K]; 
-    double *  z_ptr[K];
+    const double * RESTRICT x1_ptr[K]; 
+    const double * RESTRICT y1_ptr[K]; 
+    const double * RESTRICT x2_ptr[K]; 
+    const double * RESTRICT y2_ptr[K]; 
+    double * RESTRICT z_ptr[K];
+    //const double *  x1_ptr[K]; 
+    //const double *  y1_ptr[K]; 
+    //const double *  x2_ptr[K]; 
+    //const double *  y2_ptr[K]; 
+    //double *  z_ptr[K];
     for(unsigned i=0; i<K; i++)
     {
         x1_ptr[i] = thrust::raw_pointer_cast( &(x1[i]->data()[0]));
-- 
GitLab


From 758d73c871ae8d122d423ad307666011f34939e2 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <Matthias.Wiesenberger@uibk.ac.at>
Date: Wed, 23 Aug 2017 11:57:54 +0200
Subject: [PATCH 209/453] added interactive job submission line in marconi
 config script

---
 config/marconi.mk | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/config/marconi.mk b/config/marconi.mk
index 58eacc638..3101f515b 100644
--- a/config/marconi.mk
+++ b/config/marconi.mk
@@ -24,7 +24,8 @@ endif
 
 ###########configure mic jobs with#########################
 #mcdram=cache:numa=quadrant
-#export KMP_AFFINITY=scatter
+#export KMP_AFFINITY=scatter #important
 #export OM_NUM_THREADS=68
+#qsub -I -qxfuaknldebug -A FUA21_FELTOR -l select=1:ncpus=68:mcdram=cach:numa=quadrant -l walltime=0:20:00
 
 
-- 
GitLab


From 92eb32612a4269110e97910065271f6cbd8a7432 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <Matthias.Wiesenberger@uibk.ac.at>
Date: Thu, 24 Aug 2017 15:55:47 +0200
Subject: [PATCH 210/453] update marconi.mk also in newds branch

---
 config/devices/devices.mk | 3 ++-
 config/marconi.mk         | 7 ++++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/config/devices/devices.mk b/config/devices/devices.mk
index 0bbd99955..c5d0b1c2f 100644
--- a/config/devices/devices.mk
+++ b/config/devices/devices.mk
@@ -26,6 +26,7 @@ endif #device=omp
 ifeq ($(strip $(device)),mic)
 CFLAGS+=-Wall -x c++
 CFLAGS+= -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_OMP
-CFLAGS+= $(OMPFLAG) -mmic 
+CFLAGS+= $(OMPFLAG) #-mmic 
 MPICFLAGS+=$(CFLAGS) #includes values in CFLAGS defined later
+OPT=-O3 -xMIC-AVX512 -fma -finline-functions -align -restrict
 endif #device=mic
diff --git a/config/marconi.mk b/config/marconi.mk
index 6e77e8a57..3101f515b 100644
--- a/config/marconi.mk
+++ b/config/marconi.mk
@@ -5,7 +5,7 @@ INCLUDE += -I$(NETCDF_INC) -I$(HDF5_INC)
 GLFLAGS  = -lm
 CC=icc
 MPICC=mpiicc
-OPT=-O3 -xHost
+OPT=-O3 -xHost -restrict # overwritten for mic in devices.mk
 #MPICFLAGS+= -DMPICH_IGNORE_CXX_SEEK
 OMPFLAG=-qopenmp
 JSONLIB=-L$(HOME)/include/json/../../src/lib_json -ljsoncpp # json library for input parameters
@@ -22,5 +22,10 @@ endif
 #module load netcdf/4.4.1--intelmpi--2017--binary 
 
 
+###########configure mic jobs with#########################
+#mcdram=cache:numa=quadrant
+#export KMP_AFFINITY=scatter #important
+#export OM_NUM_THREADS=68
+#qsub -I -qxfuaknldebug -A FUA21_FELTOR -l select=1:ncpus=68:mcdram=cach:numa=quadrant -l walltime=0:20:00
 
 
-- 
GitLab


From 6f4d2e10096a1b46c1ef8fbe1dabe2db995a189c Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <Matthias.Wiesenberger@uibk.ac.at>
Date: Thu, 24 Aug 2017 16:07:14 +0200
Subject: [PATCH 211/453] debugged some mpi programs in geometries folder

---
 inc/geometries/geometry_elliptic_b.cu    |  1 +
 inc/geometries/geometry_elliptic_mpib.cu |  7 +++----
 inc/geometries/mpi_curvilinear.h         |  1 +
 inc/geometries/ribeiro_mpit.cu           | 10 +++++-----
 4 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/inc/geometries/geometry_elliptic_b.cu b/inc/geometries/geometry_elliptic_b.cu
index 7b857940d..e8430e9da 100644
--- a/inc/geometries/geometry_elliptic_b.cu
+++ b/inc/geometries/geometry_elliptic_b.cu
@@ -11,6 +11,7 @@
 #include "solovev.h"
 #include "guenther.h"
 #include "simple_orthogonal.h"
+#include "curvilinear.h"
 #include "testfunctors.h"
 
 
diff --git a/inc/geometries/geometry_elliptic_mpib.cu b/inc/geometries/geometry_elliptic_mpib.cu
index 9d521f947..de1a5adb4 100644
--- a/inc/geometries/geometry_elliptic_mpib.cu
+++ b/inc/geometries/geometry_elliptic_mpib.cu
@@ -13,8 +13,7 @@
 
 #include "solovev.h"
 //#include "guenther.h"
-#include "mpi_conformal.h"
-#include "mpi_orthogonal.h"
+#include "mpi_curvilinear.h"
 #include "simple_orthogonal.h"
 #include "testfunctors.h"
 
@@ -85,8 +84,8 @@ int main(int argc, char**argv)
     dg::HVec X( g2d.size()), Y(X); //P = dg::pullback( dg::coo3, g);
     for( unsigned i=0; i<g2d.size(); i++)
     {
-        X[i] = g2d.map()[0].data().[i];
-        Y[i] = g2d.map()[1].data().[i];
+        X[i] = g2d.map()[0].data()[i];
+        Y[i] = g2d.map()[1].data()[i];
     }
     ncerr = nc_put_vara_double( ncid, coordsID[0], start, count, X.data());
     ncerr = nc_put_vara_double( ncid, coordsID[1], start, count, Y.data());
diff --git a/inc/geometries/mpi_curvilinear.h b/inc/geometries/mpi_curvilinear.h
index fd5d40176..b6aa3580b 100644
--- a/inc/geometries/mpi_curvilinear.h
+++ b/inc/geometries/mpi_curvilinear.h
@@ -100,6 +100,7 @@ struct CurvilinearProductMPIGrid3d : public dg::aMPIGeometry3d
     perpendicular_grid perp_grid() const { return perpendicular_grid(*this);}
 
     const aGenerator2d& generator() const{return handle_.get();}
+    virtual CurvilinearProductMPIGrid3d* clone()const{return new CurvilinearProductMPIGrid3d(*this);}
     private:
     MPI_Comm get_reduced_comm( MPI_Comm src)
     {
diff --git a/inc/geometries/ribeiro_mpit.cu b/inc/geometries/ribeiro_mpit.cu
index c36e5005e..b3bdeb25e 100644
--- a/inc/geometries/ribeiro_mpit.cu
+++ b/inc/geometries/ribeiro_mpit.cu
@@ -12,7 +12,7 @@
 
 #include "dg/backend/timer.cuh"
 #include "dg/backend/mpi_init.h"
-#include "dg/geometry/mpi_curvilinear.h"
+#include "mpi_curvilinear.h"
 //#include "guenther.h"
 #include "solovev.h"
 #include "ribeiro.h"
@@ -64,7 +64,7 @@ int main( int argc, char* argv[])
     if(rank==0)std::cout << "Constructing grid ... \n";
     t.tic();
     dg::geo::Ribeiro ribeiro( psip, psi_0, psi_1, gp.R_0, 0., 1);
-    dg::CurvilinearMPIGrid3d g3d(&ribeiro, n, Nx, Ny,Nz, dg::DIR,dg::PER, dg::PER,comm);
+    dg::CurvilinearProductMPIGrid3d g3d(ribeiro, n, Nx, Ny,Nz, dg::DIR,dg::PER, dg::PER,comm);
     dg::CurvilinearMPIGrid2d g2d = g3d.perp_grid();
     t.toc();
     if(rank==0)std::cout << "Construction took "<<t.diff()<<"s"<<std::endl;
@@ -94,7 +94,7 @@ int main( int argc, char* argv[])
     err = nc_var_par_access( ncid, defID, NC_COLLECTIVE);
     err = nc_var_par_access( ncid, divBID, NC_COLLECTIVE);
 
-    dg::MHVec psi_p = dg::pullback( psip, g2d);
+    dg::MHVec psi_p = dg::pullback( psip.f(), g2d);
     //g.display();
     err = nc_put_vara_double( ncid, onesID, start, count, psi_p.data().data());
     dg::HVec X( g2d.size()), Y(X); //P = dg::pullback( dg::coo3, g);
@@ -110,7 +110,7 @@ int main( int argc, char* argv[])
     err = nc_put_vara_double( ncid, coordsID[0], start,count, X.data());
     err = nc_put_vara_double( ncid, coordsID[1], start,count, Y.data());
 
-    dg::SparseTensor<dg::HVec> metric = g2d.metric();
+    dg::SparseTensor<dg::MHVec> metric = g2d.metric();
     dg::MHVec g_xx = metric.value(0,0), g_xy = metric.value(0,1), g_yy=metric.value(1,1);
     dg::SparseElement<dg::MHVec> vol_ = dg::tensor::volume(metric);
     dg::MHVec vol = vol_.value();
@@ -160,7 +160,7 @@ int main( int argc, char* argv[])
     if(rank==0)std::cout << "TEST VOLUME IS:\n";
     if( psi_0 < psi_1) gp.psipmax = psi_1, gp.psipmin = psi_0;
     else               gp.psipmax = psi_0, gp.psipmin = psi_1;
-    dg::geo::Iris<Psip> iris(c.psip, gp.psipmin, gp.psipmax);
+    dg::geo::Iris iris(psip.f(), gp.psipmin, gp.psipmax);
     //dg::CylindricalGrid3d<dg::HVec> g3d( gp.R_0 -2.*gp.a, gp.R_0 + 2*gp.a, -2*gp.a, 2*gp.a, 0, 2*M_PI, 3, 2200, 2200, 1, dg::PER, dg::PER, dg::PER);
     dg::CartesianMPIGrid2d g2dC( gp.R_0 -2.*gp.a, gp.R_0 + 2.*gp.a, -2.*gp.a, 2.*gp.a, 1, 2e3, 2e3, dg::DIR, dg::PER, g2d.communicator());
     dg::MHVec vec  = dg::evaluate( iris, g2dC);
-- 
GitLab


From 405ef70977c2d79268b84ea42ec8ebab627cbcf6 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Thu, 24 Aug 2017 07:49:21 -0700
Subject: [PATCH 212/453] debugging advection programs in geometries

---
 inc/geometries/conformal_elliptic_b.cu    |  2 +-
 inc/geometries/curvilinearX.h             |  2 +-
 inc/geometries/flux_t.cu                  |  2 +-
 inc/geometries/geometryX_elliptic_b.cu    |  2 +-
 inc/geometries/geometry_advection_b.cu    | 74 ++++++++++-------------
 inc/geometries/geometry_advection_mpib.cu | 71 +++++++++-------------
 inc/geometries/geometry_diag.cu           |  2 +-
 inc/geometries/geometry_elliptic_b.cu     |  2 +-
 inc/geometries/guenther.h                 |  4 ++
 inc/geometries/magnetic_field.h           |  4 +-
 inc/geometries/ribeiro.h                  |  1 +
 inc/geometries/separatrix_orthogonal.h    |  2 +-
 inc/geometries/separatrix_orthogonal_t.cu |  7 +--
 inc/geometries/simple_orthogonal_t.cu     |  7 +--
 inc/geometries/solovev.h                  |  7 +++
 inc/geometries/taylor.h                   |  4 ++
 inc/geometries/toroidal.h                 |  5 ++
 17 files changed, 96 insertions(+), 102 deletions(-)

diff --git a/inc/geometries/conformal_elliptic_b.cu b/inc/geometries/conformal_elliptic_b.cu
index 678e701e2..88c5c4afd 100644
--- a/inc/geometries/conformal_elliptic_b.cu
+++ b/inc/geometries/conformal_elliptic_b.cu
@@ -100,7 +100,7 @@ int main(int argc, char**argv)
     //write parameters from file into variables
     GeomParameters gp(js);
     gp.display( std::cout);
-    dg::geo::TokamakMagneticField c = dg::geo::solovev::createMagField( gp); 
+    dg::geo::TokamakMagneticField c = dg::geo::createSolovevField( gp); 
     const double eps = 1e-10;
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     std::cout << "eps\tn\t Nx\t Ny \t # iterations \t error  \t time/iteration (s)\t hx_max\t hy_max\t hx_min\t hy_min \n";
diff --git a/inc/geometries/curvilinearX.h b/inc/geometries/curvilinearX.h
index 17126d43c..19a194869 100644
--- a/inc/geometries/curvilinearX.h
+++ b/inc/geometries/curvilinearX.h
@@ -5,7 +5,7 @@
 #include "dg/backend/functions.h"
 #include "dg/blas1.h"
 #include "dg/geometry/base_geometryX.h"
-#include "dg/geometry/generatorX.h"
+#include "generatorX.h"
 #include "curvilinear.h"
 
 namespace dg
diff --git a/inc/geometries/flux_t.cu b/inc/geometries/flux_t.cu
index 0e091c928..e72428b03 100644
--- a/inc/geometries/flux_t.cu
+++ b/inc/geometries/flux_t.cu
@@ -72,7 +72,7 @@ int main( int argc, char* argv[])
     //solovev::detail::Fpsi fpsi( gp, -10);
     std::cout << "Constructing flux grid ... \n";
     t.tic();
-    dg::geo::TokamakMagneticField c = createMagField( gp);
+    dg::geo::TokamakMagneticField c = dg::geo::createSolovevField( gp);
     dg::geo::FluxGenerator flux( c.get_psip(), c.get_ipol(), psi_0, psi_1, gp.R_0, 0., 1);
     dg::CurvilinearProductGrid3d g3d(flux, n, Nx, Ny,Nz, dg::DIR);
     dg::CurvilinearGrid2d g2d = g3d.perp_grid();
diff --git a/inc/geometries/geometryX_elliptic_b.cu b/inc/geometries/geometryX_elliptic_b.cu
index d305c381b..69b3990b0 100644
--- a/inc/geometries/geometryX_elliptic_b.cu
+++ b/inc/geometries/geometryX_elliptic_b.cu
@@ -51,7 +51,7 @@ int main(int argc, char**argv)
     t.tic();
 
     ////////////////construct Generator////////////////////////////////////
-    dg::geo::TokamakMagneticField c = dg::geo::taylor::createMagField(gp);
+    dg::geo::TokamakMagneticField c = dg::geo::createTaylorField(gp);
     std::cout << "Psi min "<<c.psip()(gp.R_0, 0)<<"\n";
     double R0 = gp.R_0, Z0 = 0;
     //double R_X = gp.R_0-1.4*gp.triangularity*gp.a;
diff --git a/inc/geometries/geometry_advection_b.cu b/inc/geometries/geometry_advection_b.cu
index 53406b146..9d3555172 100644
--- a/inc/geometries/geometry_advection_b.cu
+++ b/inc/geometries/geometry_advection_b.cu
@@ -1,13 +1,10 @@
 #include <iostream>
 #include <iomanip>
 
-#include "dg/backend/timer.cuh"
 #include "dg/arakawa.h"
 #include "dg/poisson.h"
-#include "dg/geometry.h"
+#include "dg/geometry/geometry.h"
 
-#include "conformal.h"
-#include "orthogonal.h"
 #include "curvilinear.h"
 
 #include "flux.h"
@@ -16,30 +13,28 @@
 #include "solovev.h"
 #include "magnetic_field.h"
 #include "testfunctors.h"
+#include "dg/backend/timer.cuh"
 
-using namespace dg::geo::solovev;
-
-template< class MagneticField>
 struct FuncDirPer2
 {
-    FuncDirPer2( MagneticField c, double R0, double psi_0, double psi_1):
-        R_0_(R0), psi0_(psi_0), psi1_(psi_1), psip_(c.psip), psipR_(c.psipR), psipZ_(c.psipZ){}
+    FuncDirPer2( dg::geo::TokamakMagneticField c, double psi_0, double psi_1):
+        R_0_(c.R0()), psi0_(psi_0), psi1_(psi_1), psip_(c.psip()), psipR_(c.psipR()), psipZ_(c.psipZ()){}
     double operator()(double R, double Z, double phi) const {
         return this->operator()(R,Z);
     }
     double operator()(double R, double Z) const {
-        double psip = psip_(R,Z);
+        double psip = psip_.get()(R,Z);
         return (psip-psi0_)*(psip-psi1_)*cos(theta(R,Z));
     }
     double dR( double R, double Z)const
     {
-        double psip = psip_(R,Z), psipR = psipR_(R,Z), theta_ = theta(R,Z);
+        double psip = psip_.get()(R,Z), psipR = psipR_.get()(R,Z), theta_ = theta(R,Z);
         return (2.*psip*psipR - (psi0_+psi1_)*psipR)*cos(theta_) 
             - (psip-psi0_)*(psip-psi1_)*sin(theta_)*thetaR(R,Z);
     }
     double dZ( double R, double Z)const
     {
-        double psip = psip_(R,Z), psipZ = psipZ_(R,Z), theta_=theta(R,Z);
+        double psip = psip_.get()(R,Z), psipZ = psipZ_.get()(R,Z), theta_=theta(R,Z);
         return (2*psip*psipZ - (psi0_+psi1_)*psipZ)*cos(theta_) 
             - (psip-psi0_)*(psip-psi1_)*sin(theta_)*thetaZ(R,Z);
     }
@@ -61,16 +56,13 @@ struct FuncDirPer2
     }
     double R_0_;
     double psi0_, psi1_;
-    Psip psip_;
-    PsipR psipR_;
-    PsipZ psipZ_;
+    dg::Handle<dg::geo::aBinaryFunctor> psip_, psipR_,  psipZ_;
 };
 
-template<class MagneticField>
 struct ArakawaDirPer
 {
-    ArakawaDirPer( MagneticField c, double R0, double psi_0, double psi_1): 
-        f_(c, R0, psi_0, psi_1, 4), g_(c, R0, psi_0, psi_1){ }
+    ArakawaDirPer( dg::geo::TokamakMagneticField c, double psi_0, double psi_1): 
+        f_(c, psi_0, psi_1, 4), g_(c, psi_0, psi_1){ }
     double operator()(double R, double Z, double phi) const {
         return this->operator()(R,Z);
     }
@@ -78,14 +70,13 @@ struct ArakawaDirPer
         return f_.dR( R,Z)*g_.dZ(R,Z) - f_.dZ(R,Z)*g_.dR(R,Z);
     }
     private:
-    dg::geo::FuncDirPer<MagneticField> f_;
-    FuncDirPer2<MagneticField> g_;
+    dg::geo::FuncDirPer f_;
+    FuncDirPer2 g_;
 };
 
-template<class MagneticField>
 struct VariationDirPer
 {
-    VariationDirPer( MagneticField c, double R0, double psi_0, double psi_1): f_(c, R0, psi_0, psi_1,4. ){}
+    VariationDirPer( dg::geo::TokamakMagneticField c, double psi_0, double psi_1): f_(c, psi_0, psi_1,4. ){}
     double operator()(double R, double Z, double phi) const {
         return this->operator()(R,Z);}
 
@@ -93,26 +84,25 @@ struct VariationDirPer
         return f_.dR( R,Z)*f_.dR(R,Z) + f_.dZ(R,Z)*f_.dZ(R,Z);
     }
     private:
-    dg::geo::FuncDirPer<MagneticField> f_;
+    dg::geo::FuncDirPer f_;
 };
 
-template< class MagneticField>
 struct CurvatureDirPer
 {
-    CurvatureDirPer( MagneticField c, double R0, double psi_0, double psi_1): f_(c, R0, psi_0, psi_1,4.), curvR(c, R0), curvZ(c, R0){}
+    CurvatureDirPer( dg::geo::TokamakMagneticField c, double psi_0, double psi_1): f_(c, psi_0, psi_1,4.), curvR(c), curvZ(c){}
     double operator()(double R, double Z, double phi) const {
         return this->operator()(R,Z);}
     double operator()(double R, double Z) const {
         return curvR( R,Z)*f_.dR(R,Z) + curvZ(R,Z)*f_.dZ(R,Z);
     }
     private:
-    dg::geo::FuncDirPer<MagneticField> f_;
-    dg::geo::CurvatureNablaBR<MagneticField> curvR;
-    dg::geo::CurvatureNablaBZ<MagneticField> curvZ;
+    dg::geo::FuncDirPer f_;
+    dg::geo::CurvatureNablaBR curvR;
+    dg::geo::CurvatureNablaBZ curvZ;
 };
 
 
-typedef dg::CurvilinearGrid2d<dg::DVec> Geometry;
+typedef dg::CurvilinearGrid2d Geometry;
 
 int main(int argc, char** argv)
 {
@@ -131,9 +121,10 @@ int main(int argc, char** argv)
         std::ifstream is(argv[1]);
         reader.parse(is,js,false);
     }
-    GeomParameters gp(js);
-    Psip psip( gp); 
-    std::cout << "Psi min "<<psip(gp.R_0, 0)<<"\n";
+    dg::geo::solovev::GeomParameters gp(js);
+    dg::geo::TokamakMagneticField c = dg::geo::createSolovevField( gp);
+
+    std::cout << "Psi min "<<c.psip()(gp.R_0, 0)<<"\n";
     std::cout << "Type psi_0 (-20) and psi_1 (-4)\n";
     double psi_0, psi_1;
     std::cin >> psi_0>> psi_1;
@@ -143,11 +134,8 @@ int main(int argc, char** argv)
     //solovev::detail::Fpsi fpsi( gp, -10);
     std::cout << "Constructing grid ... \n";
     t.tic();
-    MagneticField c( gp);
-    dg::geo::RibeiroFluxGenerator<Psip, PsipR, PsipZ, PsipRR, PsipRZ, PsipZZ>
-        ribeiro( c.psip, c.psipR, c.psipZ, c.psipRR, c.psipRZ, c.psipZZ, psi_0, psi_1, gp.R_0, 0., 1);
-    //dg::geo::SimpleOrthogonal<Psip, PsipR, PsipZ, LaplacePsip>
-    //    ribeiro( c.psip, c.psipR, c.psipZ, c.laplacePsip, psi_0, psi_1, gp.R_0, 0., 1);
+    dg::geo::RibeiroFluxGenerator ribeiro( c.get_psip(), psi_0, psi_1, gp.R_0, 0., 1);
+    //dg::geo::SimpleOrthogonal ribeiro( c.get_psip(), psi_0, psi_1, gp.R_0, 0., 1);
     Geometry grid(ribeiro, n, Nx, Ny, dg::DIR); //2d
     t.toc();
     std::cout << "Construction took "<<t.diff()<<"s"<<std::endl;
@@ -157,10 +145,10 @@ int main(int argc, char** argv)
     std::cout <<std::fixed<< std::setprecision(6)<<std::endl;
 
 
-    dg::geo::FuncDirPer<MagneticField> left(c, gp.R_0, psi_0, psi_1, 4);
-    FuncDirPer2<MagneticField> right( c, gp.R_0, psi_0, psi_1);
-    ArakawaDirPer<MagneticField> jacobian( c, gp.R_0, psi_0, psi_1);
-    VariationDirPer<MagneticField> variationLHS(c, gp.R_0, psi_0, psi_1);
+    dg::geo::FuncDirPer left(c, psi_0, psi_1, 4);
+    FuncDirPer2 right( c, psi_0, psi_1);
+    ArakawaDirPer jacobian( c, psi_0, psi_1);
+    VariationDirPer variationLHS(c, psi_0, psi_1);
 
     const dg::DVec lhs = dg::pullback( left, grid);
     dg::DVec jac(lhs);
@@ -219,7 +207,7 @@ int main(int argc, char** argv)
     std::cout << "TESTING CURVATURE 3D\n";
     dg::DVec curvX, curvY;
     dg::HVec tempX, tempY;
-    dg::geo::pushForwardPerp(dg::geo::CurvatureNablaBR<MagneticField>(c, gp.R_0), dg::geo::CurvatureNablaBZ<MagneticField>(c, gp.R_0), tempX, tempY, grid);
+    dg::pushForwardPerp(dg::geo::CurvatureNablaBR(c), dg::geo::CurvatureNablaBZ(c), tempX, tempY, grid);
     dg::blas1::transfer(  tempX, curvX);
     dg::blas1::transfer(  tempY, curvY);
     dg::DMatrix dx, dy;
@@ -232,7 +220,7 @@ int main(int argc, char** argv)
     dg::blas1::pointwiseDot( 1., tempy, curvY, 1.,  tempx);
     const double normCurv = dg::blas2::dot( tempx, vol, tempx);
 
-    CurvatureDirPer<MagneticField> curv(c, gp.R_0, psi_0, psi_1);
+    CurvatureDirPer curv(c, psi_0, psi_1);
     dg::DVec curvature;
     dg::blas1::transfer( dg::pullback(curv, grid), curvature);
 
diff --git a/inc/geometries/geometry_advection_mpib.cu b/inc/geometries/geometry_advection_mpib.cu
index fef0b7fac..0d4d38897 100644
--- a/inc/geometries/geometry_advection_mpib.cu
+++ b/inc/geometries/geometry_advection_mpib.cu
@@ -5,7 +5,7 @@
 
 #include "dg/arakawa.h"
 #include "dg/poisson.h"
-#include "dg/geometry.h"
+#include "dg/geometry/geometry.h"
 #include "dg/backend/mpi_init.h"
 #include "dg/backend/timer.cuh"
 
@@ -14,30 +14,27 @@
 #include "flux.h"
 #include "simple_orthogonal.h"
 #include "mpi_curvilinear.h"
-#include "mpi_orthogonal.h"
 
-using namespace dg::geo::solovev;
-template< class MagneticField>
 struct FuncDirPer2
 {
-    FuncDirPer2( MagneticField c, double R0, double psi_0, double psi_1):
-        R_0_(R0), psi0_(psi_0), psi1_(psi_1), psip_(c.psip), psipR_(c.psipR), psipZ_(c.psipZ){}
+    FuncDirPer2( dg::geo::TokamakMagneticField c, double psi_0, double psi_1):
+        R_0_(c.R0()), psi0_(psi_0), psi1_(psi_1), psip_(c.psip()), psipR_(c.psipR()), psipZ_(c.psipZ()){}
     double operator()(double R, double Z, double phi) const {
         return this->operator()(R,Z);
     }
     double operator()(double R, double Z) const {
-        double psip = psip_(R,Z);
+        double psip = psip_.get()(R,Z);
         return (psip-psi0_)*(psip-psi1_)*cos(theta(R,Z));
     }
     double dR( double R, double Z)const
     {
-        double psip = psip_(R,Z), psipR = psipR_(R,Z), theta_ = theta(R,Z);
+        double psip = psip_.get()(R,Z), psipR = psipR_.get()(R,Z), theta_ = theta(R,Z);
         return (2.*psip*psipR - (psi0_+psi1_)*psipR)*cos(theta_) 
             - (psip-psi0_)*(psip-psi1_)*sin(theta_)*thetaR(R,Z);
     }
     double dZ( double R, double Z)const
     {
-        double psip = psip_(R,Z), psipZ = psipZ_(R,Z), theta_=theta(R,Z);
+        double psip = psip_.get()(R,Z), psipZ = psipZ_.get()(R,Z), theta_=theta(R,Z);
         return (2*psip*psipZ - (psi0_+psi1_)*psipZ)*cos(theta_) 
             - (psip-psi0_)*(psip-psi1_)*sin(theta_)*thetaZ(R,Z);
     }
@@ -59,16 +56,13 @@ struct FuncDirPer2
     }
     double R_0_;
     double psi0_, psi1_;
-    Psip psip_;
-    PsipR psipR_;
-    PsipZ psipZ_;
+    dg::Handle<dg::geo::aBinaryFunctor> psip_, psipR_,  psipZ_;
 };
 
-template<class MagneticField>
 struct ArakawaDirPer
 {
-    ArakawaDirPer( MagneticField c, double R0, double psi_0, double psi_1): 
-        f_(c, R0, psi_0, psi_1, 4), g_(c, R0, psi_0, psi_1){}
+    ArakawaDirPer( dg::geo::TokamakMagneticField c, double psi_0, double psi_1): 
+        f_(c, psi_0, psi_1, 4), g_(c, psi_0, psi_1){ }
     double operator()(double R, double Z, double phi) const {
         return this->operator()(R,Z);
     }
@@ -76,14 +70,13 @@ struct ArakawaDirPer
         return f_.dR( R,Z)*g_.dZ(R,Z) - f_.dZ(R,Z)*g_.dR(R,Z);
     }
     private:
-    dg::geo::FuncDirPer<MagneticField> f_;
-    FuncDirPer2<MagneticField> g_;
+    dg::geo::FuncDirPer f_;
+    FuncDirPer2 g_;
 };
 
-template<class MagneticField>
 struct VariationDirPer
 {
-    VariationDirPer( MagneticField c, double R0, double psi_0, double psi_1): f_(c, R0, psi_0, psi_1,4. ){}
+    VariationDirPer( dg::geo::TokamakMagneticField c, double psi_0, double psi_1): f_(c, psi_0, psi_1,4. ){}
     double operator()(double R, double Z, double phi) const {
         return this->operator()(R,Z);}
 
@@ -91,26 +84,25 @@ struct VariationDirPer
         return f_.dR( R,Z)*f_.dR(R,Z) + f_.dZ(R,Z)*f_.dZ(R,Z);
     }
     private:
-    dg::geo::FuncDirPer<MagneticField> f_;
+    dg::geo::FuncDirPer f_;
 };
 
-template< class MagneticField>
 struct CurvatureDirPer
 {
-    CurvatureDirPer( MagneticField c, double R0, double psi_0, double psi_1): f_(c, R0, psi_0, psi_1,4.), curvR(c, R0), curvZ(c, R0){}
+    CurvatureDirPer( dg::geo::TokamakMagneticField c, double psi_0, double psi_1): f_(c, psi_0, psi_1,4.), curvR(c), curvZ(c){}
     double operator()(double R, double Z, double phi) const {
         return this->operator()(R,Z);}
     double operator()(double R, double Z) const {
         return curvR( R,Z)*f_.dR(R,Z) + curvZ(R,Z)*f_.dZ(R,Z);
     }
     private:
-    dg::geo::FuncDirPer<MagneticField> f_;
-    dg::geo::CurvatureNablaBR<MagneticField> curvR;
-    dg::geo::CurvatureNablaBZ<MagneticField> curvZ;
+    dg::geo::FuncDirPer f_;
+    dg::geo::CurvatureNablaBR curvR;
+    dg::geo::CurvatureNablaBZ curvZ;
 };
 
 
-typedef  dg::CurvilinearMPIGrid2d<dg::DVec> Geometry;
+typedef  dg::CurvilinearMPIGrid2d Geometry;
 
 int main(int argc, char** argv)
 {
@@ -132,9 +124,9 @@ int main(int argc, char** argv)
         std::ifstream is(argv[1]);
         reader.parse(is,js,false);
     }
-    GeomParameters gp(js);
-    Psip psip( gp); 
-    if(rank==0)std::cout << "Psi min "<<psip(gp.R_0, 0)<<"\n";
+    dg::geo::solovev::GeomParameters gp(js);
+    dg::geo::TokamakMagneticField c = dg::geo::createSolovevField( gp);
+    if(rank==0)std::cout << "Psi min "<<c.psip()(gp.R_0, 0)<<"\n";
     if(rank==0)std::cout << "Type psi_0 and psi_1\n";
     double psi_0, psi_1;
     if(rank==0)std::cin >> psi_0>> psi_1;
@@ -148,22 +140,19 @@ int main(int argc, char** argv)
         MPI_Comm planeComm;
         int remain_dims[] = {true,true,false}; //true true false
         MPI_Cart_sub( comm, remain_dims, &planeComm);
-    MagneticField c( gp);
-    dg::geo::RibeiroFluxGenerator<Psip, PsipR, PsipZ, PsipRR, PsipRZ, PsipZZ>
-        generator( c.psip, c.psipR, c.psipZ, c.psipRR, c.psipRZ, c.psipZZ, psi_0, psi_1, gp.R_0, 0., 1);
-    //dg::geo::SimpleOrthogonal<Psip, PsipR, PsipZ, LaplacePsip> 
-        //generator( c.psip, c.psipR, c.psipZ, c.laplacePsip, psi_0, psi_1, gp.R_0, 0., 1);
-    Geometry grid(generator, n, Nx, Ny,dg::DIR, planeComm); //2d
+    dg::geo::RibeiroFluxGenerator generator( c.get_psip(), psi_0, psi_1, gp.R_0, 0., 1);
+    //dg::geo::SimpleOrthogonal generator( c.get_psip(), psi_0, psi_1, gp.R_0, 0., 1);
+    Geometry grid(generator, n, Nx, Ny, dg::DIR, dg::PER, planeComm); //2d
     t.toc();
     if(rank==0)std::cout << "Construction took "<<t.diff()<<"s\n";
 
     dg::MDVec vol = dg::create::volume( grid);
     if(rank==0)std::cout <<std::fixed<< std::setprecision(2)<<std::endl;
 
-    dg::geo::FuncDirPer<MagneticField> left(c, gp.R_0, psi_0, psi_1, 4);
-    FuncDirPer2<MagneticField> right( c, gp.R_0, psi_0, psi_1);
-    ArakawaDirPer<MagneticField> jacobian( c, gp.R_0, psi_0, psi_1);
-    VariationDirPer<MagneticField> variationLHS(c, gp.R_0, psi_0, psi_1);
+    dg::geo::FuncDirPer left(c, psi_0, psi_1, 4);
+    FuncDirPer2 right( c, psi_0, psi_1);
+    ArakawaDirPer jacobian( c, psi_0, psi_1);
+    VariationDirPer variationLHS(c, psi_0, psi_1);
 
     const dg::MDVec lhs = dg::pullback( left, grid);
     dg::MDVec jac(lhs);
@@ -216,7 +205,7 @@ int main(int argc, char** argv)
     if(rank==0)std::cout << "TESTING CURVATURE 3D\n";
     dg::MDVec curvX, curvY;
     dg::MHVec tempX, tempY;
-    dg::geo::pushForwardPerp(dg::geo::CurvatureNablaBR<MagneticField>(c, gp.R_0), dg::geo::CurvatureNablaBZ<MagneticField>(c, gp.R_0), tempX, tempY, grid);
+    dg::pushForwardPerp(dg::geo::CurvatureNablaBR(c), dg::geo::CurvatureNablaBZ(c), tempX, tempY, grid);
     dg::blas1::transfer(  tempX, curvX);
     dg::blas1::transfer(  tempY, curvY);
     dg::MDMatrix dx, dy;
@@ -229,7 +218,7 @@ int main(int argc, char** argv)
     dg::blas1::pointwiseDot( 1., tempy, curvY, 1.,  tempx);
     norm = dg::blas2::dot( tempx, vol, tempx);
 
-    CurvatureDirPer<MagneticField> curv(c, gp.R_0, psi_0, psi_1);
+    CurvatureDirPer curv(c, psi_0, psi_1);
     dg::MDVec curvature;
     dg::blas1::transfer( dg::pullback(curv, grid), curvature);
 
diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index 9486b0371..de4918a35 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -121,7 +121,7 @@ int main( int argc, char* argv[])
     double Zmax=p.boxscaleZp*gp.a*gp.elongation;
  
     //construct all geometry quantities
-    dg::geo::TokamakMagneticField c = dg::geo::taylor::createMagField(gp);
+    dg::geo::TokamakMagneticField c = dg::geo::createTaylorField(gp);
     const double R_X = gp.R_0-1.1*gp.triangularity*gp.a;
     const double Z_X = -1.1*gp.elongation*gp.a;
     const double R_H = gp.R_0-gp.triangularity*gp.a;
diff --git a/inc/geometries/geometry_elliptic_b.cu b/inc/geometries/geometry_elliptic_b.cu
index e8430e9da..758b1e189 100644
--- a/inc/geometries/geometry_elliptic_b.cu
+++ b/inc/geometries/geometry_elliptic_b.cu
@@ -37,7 +37,7 @@ int main(int argc, char**argv)
     }
     //write parameters from file into variables
     dg::geo::solovev::GeomParameters gp(js);
-    dg::geo::TokamakMagneticField c = dg::geo::solovev::createMagField(gp);
+    dg::geo::TokamakMagneticField c = dg::geo::createSolovevField(gp);
     gp.display( std::cout);
     dg::Timer t;
     std::cout << "Psi min "<<c.psip()(gp.R_0, 0)<<"\n";
diff --git a/inc/geometries/guenther.h b/inc/geometries/guenther.h
index 54a7ee7ca..8b97c62a8 100644
--- a/inc/geometries/guenther.h
+++ b/inc/geometries/guenther.h
@@ -506,5 +506,9 @@ struct Divb
 };
 ///@endcond
 } //namespace guenther
+TokamakMagneticField createGuentherField( double R_0, double I_0)
+{
+    return TokamakMagneticField( R_0, guenther::createPsip(R_0), guenther::createIpol(I_0));
+}
 } //namespace geo
 }//namespace dg
diff --git a/inc/geometries/magnetic_field.h b/inc/geometries/magnetic_field.h
index ac494a999..b43d2d011 100644
--- a/inc/geometries/magnetic_field.h
+++ b/inc/geometries/magnetic_field.h
@@ -15,9 +15,9 @@ namespace geo
 ///@addtogroup magnetic
 ///@{
 /**
-* @brief container class of R, psi and ipol 
+* @brief container class of R, psi and Ipol 
 
- This is the representation of magnetic fields that can be modeled in the form
+ This is the representation of toroidally axisymmetric magnetic fields that can be modeled in the form
  \f[
  \vec B = \frac{R_0}{R} \left( I \hat e_\varphi + \nabla \psi_p \times \hat e_\varphi\right)
  \f]
diff --git a/inc/geometries/ribeiro.h b/inc/geometries/ribeiro.h
index 1a7743cc2..c43a428cc 100644
--- a/inc/geometries/ribeiro.h
+++ b/inc/geometries/ribeiro.h
@@ -9,6 +9,7 @@
 #include "dg/functors.h"
 #include "dg/runge_kutta.h"
 #include "dg/nullstelle.h"
+#include "generator.h"
 #include "utilities.h"
 
 
diff --git a/inc/geometries/separatrix_orthogonal.h b/inc/geometries/separatrix_orthogonal.h
index 3117be32b..1e9f5a28f 100644
--- a/inc/geometries/separatrix_orthogonal.h
+++ b/inc/geometries/separatrix_orthogonal.h
@@ -4,8 +4,8 @@
 #include "dg/backend/interpolationX.cuh"
 #include "dg/backend/evaluationX.cuh"
 #include "dg/backend/weightsX.cuh"
-#include "dg/geometry/generatorX.h"
 #include "dg/runge_kutta.h"
+#include "generatorX.h"
 #include "utilitiesX.h"
 
 #include "simple_orthogonal.h"
diff --git a/inc/geometries/separatrix_orthogonal_t.cu b/inc/geometries/separatrix_orthogonal_t.cu
index b3e07fafa..8ee070c75 100644
--- a/inc/geometries/separatrix_orthogonal_t.cu
+++ b/inc/geometries/separatrix_orthogonal_t.cu
@@ -36,9 +36,6 @@ double sine( double x) {return sin(x);}
 double cosine( double x) {return cos(x);}
 //typedef dg::FieldAligned< dg::CurvilinearGridX3d<dg::HVec> , dg::IHMatrix, dg::HVec> HFA;
 
-//using namespace dg::geo::solovev;
-using namespace dg::geo::taylor;
-
 thrust::host_vector<double> periodify( const thrust::host_vector<double>& in, const dg::GridX3d& g)
 {
     assert( g.Nz() == 2);
@@ -102,7 +99,7 @@ int main( int argc, char* argv[])
         std::ifstream is(argv[1]);
         reader.parse(is,js,false);
     }
-    GeomParameters gp(js);
+    dg::geo::taylor::GeomParameters gp(js);
     dg::Timer t;
     std::cout << "Type psi_0 \n";
     double psi_0 = -16;
@@ -116,7 +113,7 @@ int main( int argc, char* argv[])
     gp.display( std::cout);
     std::cout << "Constructing orthogonal grid ... \n";
     t.tic();
-    dg::geo::TokamakMagneticField c = dg::geo::taylor::createMagField(gp);
+    dg::geo::TokamakMagneticField c = dg::geo::createTaylorField(gp);
     std::cout << "Psi min "<<c.psip()(gp.R_0, 0)<<"\n";
     double R_X = gp.R_0-1.1*gp.triangularity*gp.a;
     double Z_X = -1.1*gp.elongation*gp.a;
diff --git a/inc/geometries/simple_orthogonal_t.cu b/inc/geometries/simple_orthogonal_t.cu
index 07ecec698..35aaedc51 100644
--- a/inc/geometries/simple_orthogonal_t.cu
+++ b/inc/geometries/simple_orthogonal_t.cu
@@ -19,7 +19,6 @@
 #include "curvilinear.h"
 
 #include "file/nc_utilities.h"
-using namespace dg::geo::solovev;
 
 thrust::host_vector<double> periodify( const thrust::host_vector<double>& in, const dg::Grid2d& g)
 {
@@ -62,8 +61,8 @@ int main( int argc, char* argv[])
         std::ifstream is(argv[1]);
         reader.parse(is,js,false);
     }
-    GeomParameters gp(js);
-    dg::geo::BinaryFunctorsLvl2 psip=createPsip(gp);
+    dg::geo::solovev::GeomParameters gp(js);
+    dg::geo::BinaryFunctorsLvl2 psip=dg::geo::solovev::createPsip(gp);
     std::cout << "Psi min "<<psip.f()(gp.R_0, 0)<<"\n";
     std::cout << "Type psi_0 and psi_1\n";
     double psi_0, psi_1;
@@ -210,7 +209,7 @@ int main( int argc, char* argv[])
 //     std::cout << "ana. norm of gradLnB is "<<norm<<"\n";
 //     dg::blas1::axpby( 1., gradB, -1., gradLnB, gradLnB);
      //X = g2d.lapx();
-    dg::geo::TokamakMagneticField c=createMagField(gp);
+    dg::geo::TokamakMagneticField c=dg::geo::createSolovevField(gp);
      X = dg::pullback(dg::geo::FuncDirNeu(c, psi_0, psi_1, 550, -150, 30., 1), g2d);
      err = nc_put_var_double( ncid, divBID, periodify(X, g2d_periodic).data());
 //     double norm2 = sqrt(dg::blas2::dot(gradLnB, vol3d,gradLnB));
diff --git a/inc/geometries/solovev.h b/inc/geometries/solovev.h
index 0a8c1b7c4..6016370a8 100644
--- a/inc/geometries/solovev.h
+++ b/inc/geometries/solovev.h
@@ -362,6 +362,7 @@ BinaryFunctorsLvl1 createIpol( GeomParameters gp)
     BinaryFunctorsLvl1 ipol( new Ipol(gp), new IpolR(gp), new IpolZ(gp));
     return ipol;
 }
+
 TokamakMagneticField createMagField( GeomParameters gp)
 {
     return TokamakMagneticField( gp.R_0, createPsip(gp), createIpol(gp));
@@ -552,6 +553,12 @@ struct PsipRZ: public aCloneableBinaryFunctor<PsipRZ>
 
 
 } //namespace solovev
+
+TokamakMagneticField createSolovevField( solovev::GeomParameters gp)
+{
+    return TokamakMagneticField( gp.R_0, solovev::createPsip(gp), solovev::createIpol(gp));
+}
+
 } //namespace geo
 } //namespace dg
 
diff --git a/inc/geometries/taylor.h b/inc/geometries/taylor.h
index 34db3e8b9..f0361a618 100644
--- a/inc/geometries/taylor.h
+++ b/inc/geometries/taylor.h
@@ -311,6 +311,10 @@ dg::geo::TokamakMagneticField createMagField( solovev::GeomParameters gp)
 ///@}
 
 } //namespace taylor
+dg::geo::TokamakMagneticField createTaylorField( solovev::GeomParameters gp)
+{
+    return TokamakMagneticField( gp.R_0, dg::geo::taylor::createPsip(gp), dg::geo::taylor::createIpol(gp));
+}
 } //namespace geo
 
 }//namespace dg
diff --git a/inc/geometries/toroidal.h b/inc/geometries/toroidal.h
index 2b31f5213..d82765d38 100644
--- a/inc/geometries/toroidal.h
+++ b/inc/geometries/toroidal.h
@@ -29,5 +29,10 @@ TokamakMagneticField createMagField( double R0)
 }
 
 }//namespace toroidal
+
+TokamakMagneticField createToroidalField( double R0)
+{
+    return TokamakMagneticField( R0, toiroidal::createPsip(), toiroidal::createIpol());
+}
 }//namespace geo
 }//namespace dg
-- 
GitLab


From 092395a2a40ccfe38ef895ba7a23bbc89339c2d8 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Thu, 24 Aug 2017 08:28:37 -0700
Subject: [PATCH 213/453] debugging curvilinear_advection

---
 inc/dg/geometry/multiply_t.cu          |  3 ++-
 inc/dg/geometry/tensor.h               |  4 ++--
 inc/dg/geometry/transform.h            |  2 +-
 inc/geometries/flux_t.cu               |  4 +++-
 inc/geometries/geometry_advection_b.cu | 16 ++++++++--------
 5 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/inc/dg/geometry/multiply_t.cu b/inc/dg/geometry/multiply_t.cu
index d557e8205..e5ca51019 100644
--- a/inc/dg/geometry/multiply_t.cu
+++ b/inc/dg/geometry/multiply_t.cu
@@ -68,7 +68,7 @@ int main()
     dg::tensor::multiply2d(t, inout0, inout1, work0, work1);
     std::cout << "Result         is ["<<work0[0]<<" "<<work1[0]<<"]\n";
     dg::tensor::multiply2d_inplace(t, inout0, inout1, work1);
-    std::cout << "Result inplace is ["<<inout0[0]<<" "<<inout1[0]<<"]\n";
+    std::cout << "Result inplace is ["<<inout0[0]<<" "<<inout1[0]<<"]\n T is \n";
     t.idx(0,2) = 2; std::swap( t.idx(1,1), t.idx(2,1));  print(t);
     std::cout << "Multiply T with [8,9,2]\n";
     dg::tensor::multiply3d(t, eight, nine,two, work0, work1, work2);
@@ -77,6 +77,7 @@ int main()
     dg::tensor::multiply3d_inplace(t, inout0, inout1,inout2, work1,work2);
     std::cout << "Result inplace is ["<<inout0[0]<<" "<<inout1[0]<<" "<<inout2[0]<<"]\n";
     std::cout << "Determinant of T: "<<dg::tensor::determinant(t).value()[0]<<" (320)\n";
+    std::cout << "Perp Determinant of T: "<<dg::tensor::determinant(t.perp()).value()[0]<<" (-32)\n";
     std::swap(t.idx(2,1), t.idx(2,0)); 
     t.value(0) = five;
     t.idx(1,1) = 0;
diff --git a/inc/dg/geometry/tensor.h b/inc/dg/geometry/tensor.h
index 1d76f113b..4acfa4d72 100644
--- a/inc/dg/geometry/tensor.h
+++ b/inc/dg/geometry/tensor.h
@@ -470,8 +470,8 @@ SparseTensor<container> SparseTensor<container>::perp() const
     if( isEmpty()) return t;
     for(unsigned i=0; i<3; i++)
     {
-        if( t.isSet(2,i)) t.mat_idx_(2,i)=-1;
-        if( t.isSet(i,2)) t.mat_idx_(i,2)=-1;
+        t.mat_idx_(2,i)=-1;
+        t.mat_idx_(i,2)=-1;
     }
     t.clear_unused_values();
     return t;
diff --git a/inc/dg/geometry/transform.h b/inc/dg/geometry/transform.h
index d030ad551..840fab52e 100644
--- a/inc/dg/geometry/transform.h
+++ b/inc/dg/geometry/transform.h
@@ -230,7 +230,7 @@ template< class Geometry>
 typename GeometryTraits<Geometry>::host_vector inv_volume( const Geometry& g)
 {
     typedef typename GeometryTraits< Geometry>::host_vector host_vector;
-    SparseElement<host_vector> inv_vol = dg::tensor::determinant(g.metric());
+    SparseElement<host_vector> inv_vol = dg::tensor::determinant(g.metric().perp());
     dg::tensor::sqrt(inv_vol);
     host_vector temp = dg::create::inv_weights( g);
     dg::tensor::pointwiseDot( inv_vol,temp, temp);
diff --git a/inc/geometries/flux_t.cu b/inc/geometries/flux_t.cu
index e72428b03..ebbf3be39 100644
--- a/inc/geometries/flux_t.cu
+++ b/inc/geometries/flux_t.cu
@@ -143,8 +143,10 @@ int main( int argc, char* argv[])
     dg::CartesianGrid2d g2dC( gp.R_0 -2.0*gp.a, gp.R_0 + 2.0*gp.a, -2.0*gp.a,2.0*gp.a,1, 2e3, 2e3, dg::PER, dg::PER);
     dg::HVec vec  = dg::evaluate( iris, g2dC);
     dg::HVec R  = dg::evaluate( dg::cooX2d, g2dC);
+    dg::HVec onesC = dg::evaluate( dg::one, g2dC);
     dg::HVec g2d_weights = dg::create::volume( g2dC);
-    double volumeRZP = 2.*M_PI*dg::blas2::dot( vec, g2d_weights, R);
+    //double volumeRZP = 2.*M_PI*dg::blas2::dot( vec, g2d_weights, R);
+    double volumeRZP = dg::blas2::dot( vec, g2d_weights, onesC);
     std::cout << "volumeXYP is "<< volume   <<std::endl;
     std::cout << "volumeRZP is "<< volumeRZP<<std::endl;
     std::cout << "relative difference in volume is "<<fabs(volumeRZP - volume)/volume<<std::endl;
diff --git a/inc/geometries/geometry_advection_b.cu b/inc/geometries/geometry_advection_b.cu
index 9d3555172..50a61e354 100644
--- a/inc/geometries/geometry_advection_b.cu
+++ b/inc/geometries/geometry_advection_b.cu
@@ -102,13 +102,12 @@ struct CurvatureDirPer
 };
 
 
-typedef dg::CurvilinearGrid2d Geometry;
 
 int main(int argc, char** argv)
 {
-    std::cout << "Type n, Nx, Ny, Nz\n";
-    unsigned n, Nx, Ny, Nz;
-    std::cin >> n>> Nx>>Ny>>Nz;   
+    std::cout << "Type n, Nx, Ny\n";
+    unsigned n, Nx, Ny;
+    std::cin >> n>> Nx>>Ny;   
     Json::Reader reader;
     Json::Value js;
     if( argc==1)
@@ -134,9 +133,10 @@ int main(int argc, char** argv)
     //solovev::detail::Fpsi fpsi( gp, -10);
     std::cout << "Constructing grid ... \n";
     t.tic();
-    dg::geo::RibeiroFluxGenerator ribeiro( c.get_psip(), psi_0, psi_1, gp.R_0, 0., 1);
+    //dg::geo::RibeiroFluxGenerator ribeiro( c.get_psip(), psi_0, psi_1, gp.R_0, 0., 1);
+    dg::geo::FluxGenerator ribeiro( c.get_psip(), c.get_ipol(), psi_0, psi_1, gp.R_0, 0., 1);
     //dg::geo::SimpleOrthogonal ribeiro( c.get_psip(), psi_0, psi_1, gp.R_0, 0., 1);
-    Geometry grid(ribeiro, n, Nx, Ny, dg::DIR); //2d
+    dg::CurvilinearGrid2d grid(ribeiro, n, Nx, Ny, dg::DIR); //2d
     t.toc();
     std::cout << "Construction took "<<t.diff()<<"s"<<std::endl;
     grid.display();
@@ -159,7 +159,7 @@ int main(int argc, char** argv)
 
     ///////////////////////////////////////////////////////////////////////
     std::cout << "TESTING ARAKAWA\n";
-    dg::ArakawaX<Geometry, dg::DMatrix, dg::DVec> arakawa( grid);
+    dg::ArakawaX<dg::aGeometry2d, dg::DMatrix, dg::DVec> arakawa( grid);
     arakawa( lhs, rhs, jac);
     const double norm = dg::blas2::dot( sol, vol, sol);
     std::cout << std::scientific;
@@ -184,7 +184,7 @@ int main(int argc, char** argv)
     std::cout << "Variation rel. distance to solution "<<sqrt( result/normVar)<<std::endl; //don't forget sqrt when comuting errors
     ///////////////////////////////////////////////////////////////////////
     std::cout << "TESTING POISSON\n";
-    dg::Poisson<Geometry, dg::DMatrix, dg::DVec> poisson( grid);
+    dg::Poisson<dg::aGeometry2d, dg::DMatrix, dg::DVec> poisson( grid);
     poisson( lhs, rhs, jac);
     result = dg::blas2::dot( eins, vol, jac);
     std::cout << "Mean     Jacobian is "<<result<<"\n";
-- 
GitLab


From d2173ee27a3da96091eeebac8e3d3414c845bf67 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 24 Aug 2017 20:41:50 +0200
Subject: [PATCH 214/453] corrected but in Curvilinear2d Grid construction

---
 inc/dg/geometry/transform.h           |  2 +-
 inc/geometries/curvilinear.h          |  4 +---
 inc/geometries/curvilinearX.h         |  5 +++--
 inc/geometries/flux_t.cu              | 20 +++++++++++++-------
 inc/geometries/refined_curvilinearX.h |  5 +++--
 5 files changed, 21 insertions(+), 15 deletions(-)

diff --git a/inc/dg/geometry/transform.h b/inc/dg/geometry/transform.h
index 840fab52e..d030ad551 100644
--- a/inc/dg/geometry/transform.h
+++ b/inc/dg/geometry/transform.h
@@ -230,7 +230,7 @@ template< class Geometry>
 typename GeometryTraits<Geometry>::host_vector inv_volume( const Geometry& g)
 {
     typedef typename GeometryTraits< Geometry>::host_vector host_vector;
-    SparseElement<host_vector> inv_vol = dg::tensor::determinant(g.metric().perp());
+    SparseElement<host_vector> inv_vol = dg::tensor::determinant(g.metric());
     dg::tensor::sqrt(inv_vol);
     host_vector temp = dg::create::inv_weights( g);
     dg::tensor::pointwiseDot( inv_vol,temp, temp);
diff --git a/inc/geometries/curvilinear.h b/inc/geometries/curvilinear.h
index f4fc14ccb..ca22ca3fa 100644
--- a/inc/geometries/curvilinear.h
+++ b/inc/geometries/curvilinear.h
@@ -167,9 +167,7 @@ struct CurvilinearGrid2d : public dg::aGeometry2d
     void construct( unsigned n, unsigned Nx, unsigned Ny)
     {
         CurvilinearProductGrid3d g( handle_.get(), n,Nx,Ny,1,bcx());
-        jac_=g.jacobian();
-        map_=g.map();
-        metric_=g.metric();
+        *this = CurvilinearGrid2d(g);
     }
     virtual SparseTensor<thrust::host_vector<double> > do_compute_jacobian( ) const {
         return jac_;
diff --git a/inc/geometries/curvilinearX.h b/inc/geometries/curvilinearX.h
index 19a194869..f0a928a75 100644
--- a/inc/geometries/curvilinearX.h
+++ b/inc/geometries/curvilinearX.h
@@ -121,9 +121,10 @@ struct CurvilinearGridX2d : public dg::aGeometryX2d
     void construct( double fx, double fy, unsigned n, unsigned Nx, unsigned Ny)
     {
         CurvilinearProductGridX3d g( handle_.get(),fx,fy,n,Nx,Ny,1,bcx());
-        jac_=g.jacobian();
         map_=g.map();
-        metric_=g.metric();
+        jac_=g.jacobian().perp();
+        metric_=g.metric().perp();
+        map_.pop_back();
     }
     virtual SparseTensor<thrust::host_vector<double> > do_compute_jacobian( ) const {
         return jac_;
diff --git a/inc/geometries/flux_t.cu b/inc/geometries/flux_t.cu
index ebbf3be39..074654a25 100644
--- a/inc/geometries/flux_t.cu
+++ b/inc/geometries/flux_t.cu
@@ -75,7 +75,7 @@ int main( int argc, char* argv[])
     dg::geo::TokamakMagneticField c = dg::geo::createSolovevField( gp);
     dg::geo::FluxGenerator flux( c.get_psip(), c.get_ipol(), psi_0, psi_1, gp.R_0, 0., 1);
     dg::CurvilinearProductGrid3d g3d(flux, n, Nx, Ny,Nz, dg::DIR);
-    dg::CurvilinearGrid2d g2d = g3d.perp_grid();
+    dg::CurvilinearGrid2d g2d(flux, n, Nx,Ny, dg::DIR);
     dg::Grid2d g2d_periodic(g2d.x0(), g2d.x1(), g2d.y0(), g2d.y1(), g2d.n(), g2d.Nx(), g2d.Ny()+1); 
     t.toc();
     std::cout << "Construction took "<<t.diff()<<"s"<<std::endl;
@@ -132,9 +132,12 @@ int main( int argc, char* argv[])
     double error = sqrt(dg::blas2::dot( temp0, w3d, temp0)/dg::blas2::dot(vol_.value(), w3d, vol_.value()));
     std::cout << "Rel Consistency  of volume is "<<error<<"\n";
 
-    const dg::HVec vol = dg::create::volume( g3d);
+    const dg::HVec vol3d = dg::create::volume( g3d);
     dg::HVec ones3d = dg::evaluate( dg::one, g3d);
-    double volume = dg::blas1::dot( vol, ones3d);
+    double volume3d = dg::blas1::dot( vol3d, ones3d);
+    const dg::HVec vol2d = dg::create::volume( g2d);
+    dg::HVec ones2d = dg::evaluate( dg::one, g2d);
+    double volume2d = dg::blas1::dot( vol2d, ones2d);
 
     std::cout << "TEST VOLUME IS:\n";
     if( psi_0 < psi_1) gp.psipmax = psi_1, gp.psipmin = psi_0;
@@ -145,11 +148,14 @@ int main( int argc, char* argv[])
     dg::HVec R  = dg::evaluate( dg::cooX2d, g2dC);
     dg::HVec onesC = dg::evaluate( dg::one, g2dC);
     dg::HVec g2d_weights = dg::create::volume( g2dC);
-    //double volumeRZP = 2.*M_PI*dg::blas2::dot( vec, g2d_weights, R);
-    double volumeRZP = dg::blas2::dot( vec, g2d_weights, onesC);
-    std::cout << "volumeXYP is "<< volume   <<std::endl;
+    double volumeRZP = 2.*M_PI*dg::blas2::dot( vec, g2d_weights, R);
+    double volumeRZ = dg::blas2::dot( vec, g2d_weights, onesC);
+    std::cout << "volumeXYP is "<< volume3d  <<std::endl;
     std::cout << "volumeRZP is "<< volumeRZP<<std::endl;
-    std::cout << "relative difference in volume is "<<fabs(volumeRZP - volume)/volume<<std::endl;
+    std::cout << "volumeXY  is "<< volume2d  <<std::endl;
+    std::cout << "volumeRZ  is "<< volumeRZ<<std::endl;
+    std::cout << "relative difference in volume3d is "<<fabs(volumeRZP - volume3d)/volume3d<<std::endl;
+    std::cout << "relative difference in volume2d is "<<fabs(volumeRZ - volume2d)/volume2d<<std::endl;
     std::cout << "Note that the error might also come from the volume in RZP!\n"; //since integration of jacobian is fairly good probably
 
     ///////////////////////////TEST 3d grid//////////////////////////////////////
diff --git a/inc/geometries/refined_curvilinearX.h b/inc/geometries/refined_curvilinearX.h
index 9cf5034a0..df73db911 100644
--- a/inc/geometries/refined_curvilinearX.h
+++ b/inc/geometries/refined_curvilinearX.h
@@ -131,9 +131,10 @@ struct CurvilinearRefinedGridX2d : public dg::aGeometryX2d
     void construct(double fx, double fy, unsigned n, unsigned Nx, unsigned Ny)
     {
         CurvilinearRefinedProductGridX3d g( ref_.get(), handle_.get(),fx,fy,n,Nx,Ny,1,bcx());
-        jac_=g.jacobian();
         map_=g.map();
-        metric_=g.metric();
+        jac_=g.jacobian().perp();
+        metric_=g.metric().perp();
+        map_.pop_back();
     }
     virtual SparseTensor<thrust::host_vector<double> > do_compute_jacobian( ) const {
         return jac_;
-- 
GitLab


From ebc8563f2cea6829904d79a3641e2d636db7bf6d Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 24 Aug 2017 20:56:22 +0200
Subject: [PATCH 215/453] corrected size bug in mpi curvilinear grid

---
 inc/geometries/flux_t.cu         | 4 ++--
 inc/geometries/mpi_curvilinear.h | 1 +
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/inc/geometries/flux_t.cu b/inc/geometries/flux_t.cu
index 074654a25..518711f10 100644
--- a/inc/geometries/flux_t.cu
+++ b/inc/geometries/flux_t.cu
@@ -151,9 +151,9 @@ int main( int argc, char* argv[])
     double volumeRZP = 2.*M_PI*dg::blas2::dot( vec, g2d_weights, R);
     double volumeRZ = dg::blas2::dot( vec, g2d_weights, onesC);
     std::cout << "volumeXYP is "<< volume3d  <<std::endl;
-    std::cout << "volumeRZP is "<< volumeRZP<<std::endl;
+    std::cout << "volumeRZP is "<< volumeRZP <<std::endl;
     std::cout << "volumeXY  is "<< volume2d  <<std::endl;
-    std::cout << "volumeRZ  is "<< volumeRZ<<std::endl;
+    std::cout << "volumeRZ  is "<< volumeRZ  <<std::endl;
     std::cout << "relative difference in volume3d is "<<fabs(volumeRZP - volume3d)/volume3d<<std::endl;
     std::cout << "relative difference in volume2d is "<<fabs(volumeRZ - volume2d)/volume2d<<std::endl;
     std::cout << "Note that the error might also come from the volume in RZP!\n"; //since integration of jacobian is fairly good probably
diff --git a/inc/geometries/mpi_curvilinear.h b/inc/geometries/mpi_curvilinear.h
index b6aa3580b..da79600dc 100644
--- a/inc/geometries/mpi_curvilinear.h
+++ b/inc/geometries/mpi_curvilinear.h
@@ -65,6 +65,7 @@ struct CurvilinearMPIGrid2d : public dg::aMPIGeometry2d
             jac_.value(i) = global2local( jacobian.value(i), *this);
         for( unsigned i=0; i<metric.values().size(); i++)
             metric_.value(i) = global2local( metric.value(i), *this);
+        map_.resize(map.size());
         for( unsigned i=0; i<map.size(); i++)
             map_[i] = global2local( map[i], *this);
     }
-- 
GitLab


From da8395f61304844d04de3e8b9b494d0ced876627 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Fri, 25 Aug 2017 02:45:35 -0700
Subject: [PATCH 216/453] work on fieldaligned integration and updated
 documentation in runge_kutta

---
 inc/dg/dg_doc.h               |  4 +-
 inc/dg/runge_kutta.h          | 85 ++++++++++++++---------------------
 inc/geometries/fieldaligned.h | 23 ++++------
 inc/geometries/solovev_doc.h  |  2 +-
 4 files changed, 45 insertions(+), 69 deletions(-)

diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index 22b87ad8f..8db2f518c 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -117,7 +117,7 @@
   */
 
  /** @class hide_container_lvl1
-  * @tparam container A data container class for which the blas1 functionality is overloaded. Also we assume that the type is copyable/assignable. Currently this is one of 
+  * @tparam container A data container class for which the blas1 functionality is overloaded. Also we assume that the type is copyable/assignable and has a swap member function. Currently this is one of 
   *   dg::HVec, dg::DVec, dg::MHVec or dg::MDVec
   */
  /** @class hide_matrix_container
@@ -127,7 +127,7 @@
   *  - dg::MHMatrix with dg::MHVec
   *  - dg::MDMatrix with dg::MDVec
   *
-  * @tparam container A data container class for which the blas1 functionality is overloaded. Currently this is one of 
+  * @tparam container A data container class for which the blas1 functionality is overloaded. Also we assume that the type is copyable/assignable and has a swap member function. Currently this is one of 
   *   dg::HVec, dg::DVec, dg::MHVec or dg::MDVec
   */
 
diff --git a/inc/dg/runge_kutta.h b/inc/dg/runge_kutta.h
index 4584878d1..e9aec77fd 100644
--- a/inc/dg/runge_kutta.h
+++ b/inc/dg/runge_kutta.h
@@ -481,19 +481,13 @@ struct NotANumber : public std::exception
     char const* what() const throw(){ return "NaN returned!";}
 };
 
+///@addtogroup time
+///@{
 /**
- * @brief Integrate differential equation with a s-stage RK scheme and a fixed number of steps
+ * @brief Integrate differential equation with a stage s Runge-Kutta scheme and a fixed number of steps
  *
- * @ingroup time 
- * @tparam RHS The right-hand side class
- * @copydoc hide_container_lvl1
  * @tparam s # of stages (1, 2, 3, 4, 6, 17)
- * @param rhs The right-hand-side
- * @param begin initial condition (size 3)
- * @param end (write-only) contains solution on output
- * @param T_min initial time
- * @param T_max final time
- * @param N # of steps to use
+ * @copydetails stepperRK1()
  */
 template< class RHS, class container, unsigned s>
 void stepperRK(RHS& rhs, const container& begin, container& end, double T_min, double T_max, unsigned N )
@@ -510,72 +504,55 @@ void stepperRK(RHS& rhs, const container& begin, container& end, double T_min, d
     }
 }
 
+/**
+ * @brief Integrate differential equation with a stage 1 Runge-Kutta scheme and a fixed number of steps
+ *
+ * @tparam RHS The right-hand side class
+ * @copydoc hide_container_lvl1
+ * @param rhs The right-hand-side
+ * @param begin initial condition 
+ * @param end (write-only) contains solution on output
+ * @param T_min initial time
+ * @param T_max final time
+ * @param N number of steps 
+ */
 template< class RHS, class container>
 void stepperRK1(RHS& rhs, const container& begin, container& end, double T_min, double T_max, unsigned N )
 {
     stepperRK<RHS, container, 1>( rhs, begin, end, T_min, T_max, N);
 }
+///@brief Integrate differential equation with a stage 2 Runge-Kutta scheme and a fixed number of steps
+///@copydetails stepperRK1()
 template< class RHS, class container>
 void stepperRK2(RHS& rhs, const container& begin, container& end, double T_min, double T_max, unsigned N )
 {
     stepperRK<RHS, container, 2>( rhs, begin, end, T_min, T_max, N);
 }
 
+///@brief Integrate differential equation with a stage 3 Runge-Kutta scheme and a fixed number of steps
+///@copydetails stepperRK1()
 template< class RHS, class container>
 void stepperRK3(RHS& rhs, const container& begin, container& end, double T_min, double T_max, unsigned N )
 {
     stepperRK<RHS, container, 3>( rhs, begin, end, T_min, T_max, N);
 }
 
-/**
- * @brief Integrates the differential equation using RK 4 and a fixed number of steps
- *
- * @ingroup time 
- * @tparam RHS The right-hand side class
- * @copydoc hide_container_lvl1
- * @param rhs The right-hand-side
- * @param begin initial condition 
- * @param end (write-only) contains solution on output
- * @param T_min initial time
- * @param T_max final time
- * @param N number of steps 
- */
+///@brief Integrate differential equation with a stage 4 Runge-Kutta scheme and a fixed number of steps
+///@copydetails stepperRK1()
 template< class RHS, class container>
 void stepperRK4(RHS& rhs, const container& begin, container& end, double T_min, double T_max, unsigned N )
 {
     stepperRK<RHS, container, 4>( rhs, begin, end, T_min, T_max, N);
 }
-/**
- * @brief Integrates the differential equation using RK 6 and a fixed number of steps
- *
- * @ingroup time 
- * @tparam RHS The right-hand side class
- * @copydoc hide_container_lvl1
- * @param rhs The right-hand-side
- * @param begin initial condition 
- * @param end (write-only) contains solution on output
- * @param T_min initial time
- * @param T_max final time
- * @param N number of steps 
- */
+///@brief Integrate differential equation with a stage 6 Runge-Kutta scheme and a fixed number of steps
+///@copydetails stepperRK1()
 template< class RHS, class container>
 void stepperRK6(RHS& rhs, const container& begin, container& end, double T_min, double T_max, unsigned N )
 {
     stepperRK<RHS, container, 6>( rhs, begin, end, T_min, T_max, N);
 }
-/**
- * @brief Integrates the differential equation using RK 17 and a fixed number of steps
- *
- * @ingroup time 
- * @tparam RHS The right-hand side class
- * @copydoc hide_container_lvl1
- * @param rhs The right-hand-side
- * @param begin initial condition 
- * @param end (write-only) contains solution on output
- * @param T_min initial time
- * @param T_max final time
- * @param N number of steps 
- */
+///@brief Integrate differential equation with a stage 17 Runge-Kutta scheme and a fixed number of steps
+///@copydetails stepperRK1()
 template< class RHS, class container>
 void stepperRK17(RHS& rhs, const container& begin, container& end, double T_min, double T_max, unsigned N )
 {
@@ -583,10 +560,8 @@ void stepperRK17(RHS& rhs, const container& begin, container& end, double T_min,
 }
 
 
-///@addtogroup time
-///@{
 /**
- * @brief Integrates the differential equation using a s stage RK scheme and a rudimentary stepsize-control
+ * @brief Integrates the differential equation using a stage s Runge-Kutta scheme, a rudimentary stepsize-control and monitoring the sanity of integration
  *
  * Doubles the number of timesteps until the desired accuracy is reached
  *
@@ -657,17 +632,23 @@ int integrateRK(RHS& rhs, const container& begin, container& end, double T_max,
 
 }
 
+/// @brief Integrates the differential equation using a stage 4 Runge-Kutta scheme, a rudimentary stepsize-control and monitoring the sanity of integration
+///@copydetails integrateRK()
 template< class RHS, class container>
 int integrateRK4(RHS& rhs, const container& begin, container& end, double T_max, double eps_abs )
 {
     return integrateRK<RHS, container, 4>( rhs, begin, end, T_max, eps_abs);
 }
 
+/// @brief Integrates the differential equation using a stage 6 Runge-Kutta scheme, a rudimentary stepsize-control and monitoring the sanity of integration
+/// @copydetails integrateRK()
 template< class RHS, class container>
 int integrateRK6(RHS& rhs, const container& begin, container& end, double T_max, double eps_abs )
 {
     return integrateRK<RHS, container, 6>( rhs, begin, end, T_max, eps_abs);
 }
+/// @brief Integrates the differential equation using a stage 17 Runge-Kutta scheme, a rudimentary stepsize-control and monitoring the sanity of integration
+///@copydetails integrateRK()
 template< class RHS, class container>
 int integrateRK17(RHS& rhs, const container& begin, container& end, double T_max, double eps_abs )
 {
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index 9448ffa10..c7576e13c 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -99,21 +99,16 @@ struct DSField
 {
     DSField( const BinaryVectorLvl0& v, const GeometryPerp& g)
     {
-        thrust::host_vector<double> vx, vy, vz;
         thrust::host_vector<double> b_zeta, b_eta;
-        dg::geo::pushForwardPerp( fieldR, fieldZ, b_zeta, b_eta, g);
+        dg::pushForwardPerp( v.x(), v.y(), b_zeta, b_eta, g);
         FieldP<MagneticField> fieldP(c);
-        thrust::host_vector<double> b_phi = dg::pullback( fieldP, g);
-        thrust::host_vector<double> b_mod = dg::pullback( bmod, g);
-        dg::blas1::pointwiseDivide( b_zeta, b_phi, b_zeta);
-        dg::blas1::pointwiseDivide( b_eta,  b_phi, b_eta);
-        dg::blas1::pointwiseDivide( b_mod,  b_phi, b_mod);
+        thrust::host_vector<double> b_phi = dg::pullback( v.z(), g);
         dxdz_ = dg::forward_transform( b_zeta, g );
         dypz_ = dg::forward_transform( b_eta, g );
-        dsdz_ = dg::forward_transform( b_mod, g );
+        dsdz_ = dg::forward_transform( b_phi, g );
     }
 
-    void operator()(thrust::host_vector<double> y, thrust::host_vector<double>& yp)
+    void operator()(const thrust::host_vector<double>& y, thrust::host_vector<double>& yp)
     {
         g_.shift_topologic( y[0], y[1], y[0], y[1]); //shift points onto domain
         if( !g_.contains( y[0], y[1])) yp[0] = yp[1]= yp[2] = 0;
@@ -328,7 +323,7 @@ struct FieldAligned
         If there is no limiter the boundary condition is periodic.
     */
     template <class Geometry, class Limiter>
-    FieldAligned(const BinaryVectorLvl0& vec, Geometry grid, unsigned multiplyX, unsigned multiplyY, double eps = 1e-4, Limiter limit = DefaultLimiter(), dg::bc globalbcz = dg::DIR, bool dependsOnX=true, bool dependsOnY=true, double deltaPhi = -1);
+    FieldAligned(const BinaryVectorLvl0& vec, const Geometry& grid, unsigned multiplyX, unsigned multiplyY, double eps = 1e-4, Limiter limit = DefaultLimiter(), dg::bc globalbcz = dg::DIR, bool dependsOnX=true, bool dependsOnY=true, double deltaPhi = -1);
 
     /**
     * @brief Set boundary conditions in the limiter region
@@ -458,7 +453,7 @@ struct FieldAligned
 
 template<class I, class container>
 template <class MagneticField, class Geometry, class Limiter>
-FieldAligned<I, container>::FieldAligned(MagneticField mag, Geometry grid, unsigned mx, unsigned my, double eps, Limiter limit, dg::bc globalbcz, double deltaPhi):
+FieldAligned<I, container>::FieldAligned(const BinaryVectorLvl0& mag, const Geometry& grid, unsigned mx, unsigned my, double eps, Limiter limit, dg::bc globalbcz, bool dependsOnX, bool dependsOnY, double deltaPhi):
         hz_( dg::evaluate( dg::zero, grid)), hp_( hz_), hm_( hz_), 
         Nz_(grid.Nz()), bcz_(grid.bcz())
 {
@@ -470,11 +465,11 @@ FieldAligned<I, container>::FieldAligned(MagneticField mag, Geometry grid, unsig
     right_ = left_ = dg::evaluate( zero, g2dCoarse);
     ghostM.resize( perp_size_); ghostP.resize( perp_size_);
     //Set starting points
-    typename Geometry::perpendicular_grid g2dFine = g2dCoarse.resize( (double)mX, (double)mY);
+    typename Geometry::perpendicular_grid g2dFine = g2dCoarse.muliplyCellNumbers( (double)mX, (double)mY);
     
-    std::vector<thrust::host_vector<double> > y( 3, dg::evaluate( dg::cooX2d, g2dFine)); // x
+    std::vector<thrust::host_vector<double> > y( 3, dg::evaluate( dg::cooX2d, g2dFine)); //x
     y[1] = dg::evaluate( dg::cooY2d, g2dFine); //y
-    y[2] = dg::evaluate( dg::zero, g2dFine);
+    y[2] = dg::evaluate( dg::zero, g2dFine); //s
     std::vector<thrust::host_vector<double> > yp( 3, dg::evaluate(dg::zero, g2dFine)), ym(yp); 
     if( deltaPhi <=0) deltaPhi = grid.hz();
     else assert( grid.Nz() == 1 || grid.hz()==deltaPhi);
diff --git a/inc/geometries/solovev_doc.h b/inc/geometries/solovev_doc.h
index b4836572f..71dc5f19b 100644
--- a/inc/geometries/solovev_doc.h
+++ b/inc/geometries/solovev_doc.h
@@ -32,6 +32,6 @@
  */
 
  /** @class hide_container
-  * @tparam container A data container class for which the blas1 functionality is overloaded. Also we assume that the type is copyable/assignable. Currently this is one of 
+  * @tparam container A data container class for which the blas1 functionality is overloaded. Also we assume that the type is copyable/assignable and has a swap member function. Currently this is one of 
   *   dg::HVec, dg::DVec, dg::MHVec or dg::MDVec
   */
-- 
GitLab


From 6dedf750c5b025c236677cdce1148db35d4e5798 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Fri, 25 Aug 2017 04:29:04 -0700
Subject: [PATCH 217/453] update fieldaligned and contains documentation

---
 inc/dg/backend/grid.h         |  6 +--
 inc/dg/backend/gridX.h        |  6 +--
 inc/dg/runge_kutta.h          | 19 +---------
 inc/geometries/fieldaligned.h | 71 ++++++++++-------------------------
 4 files changed, 26 insertions(+), 76 deletions(-)

diff --git a/inc/dg/backend/grid.h b/inc/dg/backend/grid.h
index c2b72a17f..a9b8d01d2 100644
--- a/inc/dg/backend/grid.h
+++ b/inc/dg/backend/grid.h
@@ -174,7 +174,7 @@ struct Grid1d
      * @note Doesn't check periodicity!!
      * @param x point to check
      *
-     * @return true if x0()<x<x1(), false else
+     * @return true if x0()<=x<=x1(), false else
      */
     bool contains( double x)const
     {
@@ -360,7 +360,7 @@ struct aTopology2d
      * @param x x-coordinate to check
      * @param y y-coordinate to check
      *
-     * @return true if x0()<x<x1() and y0()<y<y1(), false else
+     * @return true if x0()<=x<=x1() and y0()<=y<=y1(), false else
      */
     bool contains( double x, double y)const
     {
@@ -625,7 +625,7 @@ struct aTopology3d
      * @param y y-coordinate to check
      * @param z z-coordinate to check
      *
-     * @return true if x0()<x<x1() and y0()<y<y1() and z0()<z<z1() , false else
+     * @return true if x0()<=x<=x1() and y0()<=y<=y1() and z0()<=z<=z1() , false else
      */
     bool contains( double x, double y, double z)const
     {
diff --git a/inc/dg/backend/gridX.h b/inc/dg/backend/gridX.h
index 0b8e5e9d4..49eba04fd 100644
--- a/inc/dg/backend/gridX.h
+++ b/inc/dg/backend/gridX.h
@@ -179,7 +179,7 @@ struct GridX1d
      * @note doesn't check periodicity!!
      * @param x point to check
      *
-     * @return true if x is between x0 and x1, false else
+     * @return true if x0()<=x<=x1(), false else
      */
     bool contains( double x) const
     {
@@ -425,7 +425,7 @@ struct aTopologyX2d
      * @param x x-point to check
      * @param y y-point to check
      *
-     * @return true if (x,y) is inside the grid, false else
+     * @return true if x0()<=x<=x1() and y0()<=y<=y1(), false else
      */
     bool contains( double x, double y)const
     {
@@ -733,7 +733,7 @@ struct aTopologyX3d
      * @param y y-point to check
      * @param z z-point to check
      *
-     * @return true if x is between x0 and x1, false else
+     * @return true if x0()<=x<=x1() and y0()<=y<=y1() and z0()<=z<=z1() , false else
      */
     bool contains( double x, double y, double z)const
     {
diff --git a/inc/dg/runge_kutta.h b/inc/dg/runge_kutta.h
index e9aec77fd..e84b20fd0 100644
--- a/inc/dg/runge_kutta.h
+++ b/inc/dg/runge_kutta.h
@@ -463,24 +463,6 @@ void RK_classic<s, container>::operator()( Functor& f, const container& u0, cont
         blas1::axpby( dt*rk_classic<s>::b[i], k_[i],1., u1);
 }
 
-/**
- * @brief Thrown by the integrateRK4 function if the rhs is badly conditioned
- */
-struct NotANumber : public std::exception
-{
-    /**
-     * @brief Construct 
-     *
-     */
-    NotANumber( ) {}
-    /**
-     * @brief What string
-     *
-     * @return string "NaN returned!"
-     */
-    char const* what() const throw(){ return "NaN returned!";}
-};
-
 ///@addtogroup time
 ///@{
 /**
@@ -573,6 +555,7 @@ void stepperRK17(RHS& rhs, const container& begin, container& end, double T_min,
  * @param end (write-only) contains solution on output
  * @param T_max time difference
  * @param eps_abs desired absolute accuracy
+ * @return 0 if converged, -1 and a warning to std::cerr when isnan appears, -2 if failed to reach eps_abs
  */
 template< class RHS, class container, unsigned s>
 int integrateRK(RHS& rhs, const container& begin, container& end, double T_max, double eps_abs )
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index c7576e13c..6a8bc3ede 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -54,10 +54,9 @@ struct DefaultField
  * @tparam MagneticField models aTokamakMagneticField
  * @ingroup misc
  */ 
-template<class MagneticField>
-struct Field
+struct DSFieldCylindrical
 {
-    Field( const MagneticField& c):c_(c), invB_(c), R_0_(c.R_0) { }
+    DSFieldCylindrical( const dg::geo::TokamakMagneticField& c):c_(c), invB_(c), R_0_(c.R0()) { }
     /**
      * @brief \f[ \frac{d \hat{R} }{ d \varphi}  = \frac{\hat{R}}{\hat{I}} \frac{\partial\hat{\psi}_p}{\partial \hat{Z}}, \hspace {3 mm}
      \frac{d \hat{Z} }{ d \varphi}  =- \frac{\hat{R}}{\hat{I}} \frac{\partial \hat{\psi}_p}{\partial \hat{R}} , \hspace {3 mm}
@@ -65,10 +64,10 @@ struct Field
      */ 
     void operator()( const dg::HVec& y, dg::HVec& yp) const
     {
-        double ipol = c_.ipol(y[0],y[1]);
+        double ipol = c_.ipol()(y[0],y[1]);
         yp[2] =  y[0]*y[0]/invB_(y[0],y[1])/ipol/R_0_;       //ds/dphi =  R^2 B/I/R_0_hat
-        yp[0] =  y[0]*c_.psipZ(y[0],y[1])/ipol;              //dR/dphi =  R/I Psip_Z
-        yp[1] = -y[0]*c_.psipR(y[0],y[1])/ipol ;             //dZ/dphi = -R/I Psip_R
+        yp[0] =  y[0]*c_.psipZ()(y[0],y[1])/ipol;              //dR/dphi =  R/I Psip_Z
+        yp[1] = -y[0]*c_.psipR()(y[0],y[1])/ipol ;             //dZ/dphi = -R/I Psip_R
 
     }
     double error( const dg::HVec& x0, const dg::HVec& x1)
@@ -89,7 +88,7 @@ struct Field
     }
     
     private:
-    MagneticField c_;
+    dg::geo::TokamakMagneticField c_;
     InvB invB_;
     double R_0_;
 };
@@ -97,7 +96,8 @@ struct Field
 template< class GeometryPerp>
 struct DSField
 {
-    DSField( const BinaryVectorLvl0& v, const GeometryPerp& g)
+    
+    DSField( const dg::geo::BinaryVectorLvl0& v, const GeometryPerp& g)
     {
         thrust::host_vector<double> b_zeta, b_eta;
         dg::pushForwardPerp( v.x(), v.y(), b_zeta, b_eta, g);
@@ -107,7 +107,8 @@ struct DSField
         dypz_ = dg::forward_transform( b_eta, g );
         dsdz_ = dg::forward_transform( b_phi, g );
     }
-
+    //interpolate the vectors given in the constructor on the given point
+    //if point lies outside of grid boundaries zero is returned
     void operator()(const thrust::host_vector<double>& y, thrust::host_vector<double>& yp)
     {
         g_.shift_topologic( y[0], y[1], y[0], y[1]); //shift points onto domain
@@ -131,7 +132,7 @@ struct DSField
             return false;
         }
         //if new integrated point outside domain
-        if ((1e-5 > end[0]  ) || (1e10 < end[0])  ||(-1e10  > end[1]  ) || (1e10 < end[1])||(-1e10 > end[2]  ) || (1e10 < end[2])  )
+        if ((10*g_.x0() > end[0]  ) || (10*g_.x1() < end[0])  ||(10*g_.y0()  > end[1]  ) || (10*g_.y1() < end[1])||(-1e10 > end[2]  ) || (1e10 < end[2])  )
         {
             return false;
         }
@@ -143,35 +144,11 @@ struct DSField
 
 };
 
-/**
- * @brief Default Limiter means there is a limiter everywhere
- */
-struct DefaultLimiter
-{
-    /**
-     * @brief return 1
-     *
-     * @param x x value
-     * @param y y value
-     * @return 1
-     */
-    double operator()(double x, double y) { return 1; }
-};
+///@brief Default Limiter means there is a limiter everywhere
+typedef ONE DefaultLimiter;
 
-/**
- * @brief No Limiter 
- */
-struct NoLimiter
-{
-    /**
-     * @brief return 0
-     *
-     * @param x x value
-     * @param y y value
-     * @return 0
-     */
-    double operator()(double x, double y) { return 0.; }
-};
+///@brief No Limiter 
+typedef ZERO NoLimiter;
 
 /**
  * @brief Integrate a field line to find whether the result lies inside or outside of the box
@@ -192,29 +169,19 @@ struct BoxIntegrator
     BoxIntegrator( Field field, const Grid& g, double eps): field_(field), g_(g), coords_(3), coordsp_(3), eps_(eps) {}
     /**
      * @brief Set the starting coordinates for next field line integration
-     *
      * @param coords the new coords (must have size = 3)
      */
     void set_coords( const thrust::host_vector<double>& coords){ coords_ = coords;}
+
     /**
      * @brief Integrate from 0 to deltaPhi
-     *
      * @param deltaPhi upper integration boundary
-     *
-     * @return 1 if point is inside the box, -1 else
+     * @return 1 if result is inside the box, -1 else
      */
     double operator()( double deltaPhi)
     {
-        try{
-            dg::integrateRK4( field_, coords_, coordsp_, deltaPhi, eps_);
-        }
-        catch( dg::NotANumber& exception) { return -1;}
-        if (!(coordsp_[0] >= g_.x0() && coordsp_[0] <= g_.x1())) {
-            return -1;
-        }
-        if (!(coordsp_[1] >= g_.y0() && coordsp_[1] <= g_.y1())) {
-            return -1;
-        }
+        dg::integrateRK4( field_, coords_, coordsp_, deltaPhi, eps_);
+        if( !g_.contains( coordsp_[0], coordsp_[1]) ) return -1;
         return +1;
     }
     private:
-- 
GitLab


From 9e1f2f9085dabae501b79b806921466a8e5c59d9 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Fri, 25 Aug 2017 07:39:16 -0700
Subject: [PATCH 218/453] debugging fieldaligned, changed behaviour of DIR and
 NEU global boundary conditions in fieldaligned

---
 inc/geometries/ds.h           |   2 +-
 inc/geometries/fieldaligned.h | 152 +++++++++++++++++-----------------
 inc/geometries/flux_t.cu      |  41 ++++-----
 3 files changed, 98 insertions(+), 97 deletions(-)

diff --git a/inc/geometries/ds.h b/inc/geometries/ds.h
index d4cfeddc1..83196195e 100644
--- a/inc/geometries/ds.h
+++ b/inc/geometries/ds.h
@@ -222,7 +222,7 @@ DS< I, M,container>::DS(const BinaryVectorLvl1& vector, Geometry grid, dg::norm
         tempP( dg::evaluate( dg::zero, grid())), temp0( tempP), tempM( tempP), 
         f(tempP), dsf(tempP),
         vol3d( dg::create::volume( grid)), inv3d( dg::create::inv_volume( grid)),
-        invB(dg::pullback(dg::geo::InvB<MagneticField>(mag),grid)), 
+        invB(dg::pullback(dg::geo::InvB(mag),grid)), 
         no_(no), dir_(dir), apply_jumpX_(jumpX)
 {
     volume_ = dg::evaluate( dg::one, grid);
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index 6a8bc3ede..16209686e 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -6,12 +6,14 @@
 #include "dg/backend/grid.h"
 #include "dg/blas.h"
 #include "dg/backend/interpolation.cuh"
+#include "dg/backend/projection.cuh"
 #include "dg/backend/functions.h"
 
 #include "dg/functors.h"
 #include "dg/nullstelle.h"
 #include "dg/runge_kutta.h"
 #include "magnetic_field.h"
+#include "fluxfunctions.h"
 
 namespace dg{
 
@@ -49,11 +51,6 @@ struct DefaultField
 
 };
 
-/**
- * @brief Integrates the equations for a field line 
- * @tparam MagneticField models aTokamakMagneticField
- * @ingroup misc
- */ 
 struct DSFieldCylindrical
 {
     DSFieldCylindrical( const dg::geo::TokamakMagneticField& c):c_(c), invB_(c), R_0_(c.R0()) { }
@@ -66,10 +63,10 @@ struct DSFieldCylindrical
     {
         double ipol = c_.ipol()(y[0],y[1]);
         yp[2] =  y[0]*y[0]/invB_(y[0],y[1])/ipol/R_0_;       //ds/dphi =  R^2 B/I/R_0_hat
-        yp[0] =  y[0]*c_.psipZ()(y[0],y[1])/ipol;              //dR/dphi =  R/I Psip_Z
-        yp[1] = -y[0]*c_.psipR()(y[0],y[1])/ipol ;             //dZ/dphi = -R/I Psip_R
-
+        yp[0] =  y[0]*c_.psipZ()(y[0],y[1])/ipol;            //dR/dphi =  R/I Psip_Z
+        yp[1] = -y[0]*c_.psipR()(y[0],y[1])/ipol ;           //dZ/dphi = -R/I Psip_R
     }
+
     double error( const dg::HVec& x0, const dg::HVec& x1)
     {
         return sqrt( (x0[0]-x1[0])*(x0[0]-x1[0]) +(x0[1]-x1[1])*(x0[1]-x1[1])+(x0[2]-x1[2])*(x0[2]-x1[2]));
@@ -89,36 +86,36 @@ struct DSFieldCylindrical
     
     private:
     dg::geo::TokamakMagneticField c_;
-    InvB invB_;
+    dg::geo::InvB invB_;
     double R_0_;
 };
 
-template< class GeometryPerp>
 struct DSField
 {
     
-    DSField( const dg::geo::BinaryVectorLvl0& v, const GeometryPerp& g)
+    //z component of v may not vanish
+    DSField( const dg::geo::BinaryVectorLvl0& v, const aGeometry2d& g)
     {
         thrust::host_vector<double> b_zeta, b_eta;
         dg::pushForwardPerp( v.x(), v.y(), b_zeta, b_eta, g);
-        FieldP<MagneticField> fieldP(c);
         thrust::host_vector<double> b_phi = dg::pullback( v.z(), g);
-        dxdz_ = dg::forward_transform( b_zeta, g );
-        dypz_ = dg::forward_transform( b_eta, g );
-        dsdz_ = dg::forward_transform( b_phi, g );
+        dzetadphi_ = dg::create::forward_transform( b_zeta, g );
+        detadphi_ = dg::create::forward_transform( b_eta, g );
+        dsdphi_ = dg::create::forward_transform( b_phi, g );
     }
     //interpolate the vectors given in the constructor on the given point
     //if point lies outside of grid boundaries zero is returned
     void operator()(const thrust::host_vector<double>& y, thrust::host_vector<double>& yp)
     {
-        g_.shift_topologic( y[0], y[1], y[0], y[1]); //shift points onto domain
-        if( !g_.contains( y[0], y[1])) yp[0] = yp[1]= yp[2] = 0;
+        yp[0] = y[0], yp[1] = y[1];
+        g_.get().shift_topologic( y[0], y[1], yp[0], yp[1]); //shift points onto domain
+        if( !g_.get().contains( y[0], y[1])) yp[0] = yp[1]= yp[2] = 0;
         else
         {
             //else interpolate
-            yp[0] = interpolate( y[0], y[1], dzetadphi_, g_);
-            yp[1] = interpolate( y[0], y[1], detadphi_, g_);
-            yp[2] = interpolate( y[0], y[1], dsphi_, g_);
+            yp[0] = interpolate( y[0], y[1], dzetadphi_, g_.get());
+            yp[1] = interpolate( y[0], y[1], detadphi_, g_.get());
+            yp[2] = interpolate( y[0], y[1], dsdphi_, g_.get());
         }
     }
 
@@ -132,7 +129,7 @@ struct DSField
             return false;
         }
         //if new integrated point outside domain
-        if ((10*g_.x0() > end[0]  ) || (10*g_.x1() < end[0])  ||(10*g_.y0()  > end[1]  ) || (10*g_.y1() < end[1])||(-1e10 > end[2]  ) || (1e10 < end[2])  )
+        if ((10*g_.get().x0() > end[0]  ) || (10*g_.get().x1() < end[0])  ||(10*g_.get().y0()  > end[1]  ) || (10*g_.get().y1() < end[1])||(-1e10 > end[2]  ) || (1e10 < end[2])  )
         {
             return false;
         }
@@ -140,7 +137,7 @@ struct DSField
     }
     private:
     thrust::host_vector<double> dzetadphi_, detadphi_, dsdphi_;
-    GeometryPerp g_;
+    dg::Handle<aGeometry2d> g_;
 
 };
 
@@ -152,9 +149,8 @@ typedef ZERO NoLimiter;
 
 /**
  * @brief Integrate a field line to find whether the result lies inside or outside of the box
- *
  * @tparam Field Must be usable in the integrateRK4 function
- * @tparam Grid must provide 2d boundaries x0(), x1(), y0(), and y1()
+ * @tparam Grid must provide 2d contains function
  */
 template < class Field, class Grid>
 struct BoxIntegrator
@@ -181,12 +177,12 @@ struct BoxIntegrator
     double operator()( double deltaPhi)
     {
         dg::integrateRK4( field_, coords_, coordsp_, deltaPhi, eps_);
-        if( !g_.contains( coordsp_[0], coordsp_[1]) ) return -1;
+        if( !g_.get().contains( coordsp_[0], coordsp_[1]) ) return -1;
         return +1;
     }
     private:
     Field field_;
-    Grid g_;
+    dg::Handle<Grid> g_;
     thrust::host_vector<double> coords_, coordsp_;
     double eps_;
 };
@@ -195,7 +191,7 @@ struct BoxIntegrator
  * @brief Integrate one field line in a given box, Result is guaranteed to lie inside the box
  *
  * @tparam Field Must be usable in the integrateRK4 function
- * @tparam Grid must provide 2d boundaries x0(), x1(), y0(), and y1()
+ * @tparam Grid must provide 2d contains function
  * @param field The field to use
  * @param grid instance of the Grid class 
  * @param coords0 The initial condition
@@ -210,17 +206,17 @@ void boxintegrator( Field& field, const Grid& grid,
         thrust::host_vector<double>& coords1, 
         double& phi1, double eps, dg::bc globalbcz)
 {
-    dg::integrateRK4( field, coords0, coords1, phi1, eps); //+ integration
+    dg::integrateRK4( field, coords0, coords1, phi1, eps); //integration
     //First catch periodic domain
     grid.shift_topologic( coords0[0], coords0[1], coords1[0], coords1[1]);
     if ( !grid.contains( coords1[0], coords1[1]))   //Punkt liegt immer noch außerhalb 
     {
 #ifdef DG_DEBUG
-        std::cerr << "point "<<coords1[0]<<" "<<coords1[1]<<" "<<coords1[3]<<" "<<coords1[4]<<" is somewhere else!\n";
+        std::cerr << "point "<<coords1[0]<<" "<<coords1[1]<<" is somewhere else!\n";
 #endif //DG_DEBUG
-        if( globalbcz == dg::DIR)
+        double deltaS = coords1[2];
+        if( globalbcz == dg::DIR || globalbcz == dg::NEU)
         {
-            //idea: maybe we should take the "wrong" long Delta s instead of the short one to avoid deteriorated CFL condition
             BoxIntegrator<Field, Grid> boxy( field, grid, eps);
             boxy.set_coords( coords0); //nimm alte koordinaten
             if( phi1 > 0)
@@ -241,17 +237,15 @@ void boxintegrator( Field& field, const Grid& grid,
             if (!(coords1[0] < grid.x1())) { coords1[0]=grid.x1();}
             if (!(coords1[1] > grid.y0())) { coords1[1]=grid.y0();}
             if (!(coords1[1] < grid.y1())) { coords1[1]=grid.y1();}
-        }
-        else if (globalbcz == dg::NEU )
-        {
-             coords1[0] = coords0[0]; coords1[1] = coords0[1];  //this is clearly wrong -> makes ds a d\varphi
+            //idea: maybe we should take the "wrong" long Delta s instead of the short one to avoid deteriorated CFL condition:
+            coords1[2] = deltaS;
         }
         else if (globalbcz == DIR_NEU )std::cerr << "DIR_NEU NOT IMPLEMENTED "<<std::endl;
         else if (globalbcz == NEU_DIR )std::cerr << "NEU_DIR NOT IMPLEMENTED "<<std::endl;
         else if (globalbcz == dg::PER )std::cerr << "PER NOT IMPLEMENTED "<<std::endl;
     }
 }
-////////////////////////////////////FieldAlignedCLASS////////////////////////////////////////////
+//////////////////////////////FieldAlignedCLASS////////////////////////////////////////////
 /**
 * @brief Class for the evaluation of a parallel derivative
 *
@@ -259,10 +253,11 @@ void boxintegrator( Field& field, const Grid& grid,
 \mathbf{b}\cdot \nabla = b_R\partial_R + b_Z\partial_Z + b_\phi\partial_\phi \f$, \f$\nabla_\parallel^\dagger\f$ and \f$\Delta_\parallel=\nabla_\parallel^\dagger\cdot\nabla_\parallel\f$ in
 cylindrical coordinates
 * @ingroup utilities
-* @tparam Matrix The matrix class of the interpolation matrix
+* @tparam Geometry The Geometry class 
+* @tparam IMatrix The matrix class of the interpolation matrix
 * @tparam container The container-class on which the interpolation matrix operates on (does not need to be dg::HVec)
 */
-template<class IMatrix, class container >
+template<class Geometry, class IMatrix, class container >
 struct FieldAligned
 {
 
@@ -289,8 +284,8 @@ struct FieldAligned
         by the bcz variable from the grid and can be changed by the set_boundaries function. 
         If there is no limiter the boundary condition is periodic.
     */
-    template <class Geometry, class Limiter>
-    FieldAligned(const BinaryVectorLvl0& vec, const Geometry& grid, unsigned multiplyX, unsigned multiplyY, double eps = 1e-4, Limiter limit = DefaultLimiter(), dg::bc globalbcz = dg::DIR, bool dependsOnX=true, bool dependsOnY=true, double deltaPhi = -1);
+    template <class Limiter>
+    FieldAligned(const dg::geo::BinaryVectorLvl0& vec, const Geometry& grid, unsigned multiplyX, unsigned multiplyY, double eps = 1e-4, Limiter limit = DefaultLimiter(), dg::bc globalbcz = dg::DIR, bool dependsOnX=true, bool dependsOnY=true, double deltaPhi = -1);
 
     /**
     * @brief Set boundary conditions in the limiter region
@@ -374,17 +369,19 @@ struct FieldAligned
     /**
     * @brief Applies the interpolation to the next planes 
     *
+    * @param which specify what interpolation should be applied
     * @param in input 
     * @param out output may not equal intpu
     */
-    void einsPlus( const container& in, container& out);
+    void einsPlus( enum whichMatrix which, const container& in, container& out);
     /**
     * @brief Applies the interpolation to the previous planes
     *
+    * @param which specify what interpolation should be applied
     * @param in input 
     * @param out output may not equal intpu
     */
-    void einsMinus( const container& in, container& out);
+    void einsMinus(enum whichMatrix which, const container& in, container& out);
 
     /**
     * @brief hz is the distance between the plus and minus planes
@@ -413,17 +410,20 @@ struct FieldAligned
     dg::bc bcz_;
     container left_, right_;
     container limiter_;
+    dg::Handle<Geometry> g_;
 };
 
 ///@cond 
 ////////////////////////////////////DEFINITIONS////////////////////////////////////////
 
-template<class I, class container>
-template <class MagneticField, class Geometry, class Limiter>
-FieldAligned<I, container>::FieldAligned(const BinaryVectorLvl0& mag, const Geometry& grid, unsigned mx, unsigned my, double eps, Limiter limit, dg::bc globalbcz, bool dependsOnX, bool dependsOnY, double deltaPhi):
+template<class Geometry, class IMatrix, class container>
+template <class Limiter>
+FieldAligned<Geometry, IMatrix, container>::FieldAligned(const dg::geo::BinaryVectorLvl0& mag, const Geometry& grid, unsigned mx, unsigned my, double eps, Limiter limit, dg::bc globalbcz, bool dependsOnX, bool dependsOnY, double deltaPhi):
         hz_( dg::evaluate( dg::zero, grid)), hp_( hz_), hm_( hz_), 
-        Nz_(grid.Nz()), bcz_(grid.bcz())
+        Nz_(grid.Nz()), bcz_(grid.bcz()), g_(grid)
 {
+    if( deltaPhi <=0) deltaPhi = grid.hz();
+    else assert( grid.Nz() == 1 || grid.hz()==deltaPhi);
     //Resize vector to 2D grid size
 
     typename Geometry::perpendicular_grid g2dCoarse = grid.perp_grid( );
@@ -432,31 +432,29 @@ FieldAligned<I, container>::FieldAligned(const BinaryVectorLvl0& mag, const Geom
     right_ = left_ = dg::evaluate( zero, g2dCoarse);
     ghostM.resize( perp_size_); ghostP.resize( perp_size_);
     //Set starting points
-    typename Geometry::perpendicular_grid g2dFine = g2dCoarse.muliplyCellNumbers( (double)mX, (double)mY);
+    typename Geometry::perpendicular_grid g2dFine = g2dCoarse.muliplyCellNumbers( (double)mx, (double)my);
     
     std::vector<thrust::host_vector<double> > y( 3, dg::evaluate( dg::cooX2d, g2dFine)); //x
     y[1] = dg::evaluate( dg::cooY2d, g2dFine); //y
     y[2] = dg::evaluate( dg::zero, g2dFine); //s
     std::vector<thrust::host_vector<double> > yp( 3, dg::evaluate(dg::zero, g2dFine)), ym(yp); 
-    if( deltaPhi <=0) deltaPhi = grid.hz();
-    else assert( grid.Nz() == 1 || grid.hz()==deltaPhi);
     dg::Timer t;
     t.tic();
     //construct field on high polynomial grid, then integrate it
     typename Geometry::perpendicular_grid g2dField = g2dCoarse;
     g2dField.set( 11, g2dField.Nx(), g2dField.Ny());
-    dg::DSField<typename Geometry::perpendicular_grid> field( mag, g2dField);
+    dg::DSField field( mag, g2dField);
 #ifdef _OPENMP
 #pragma omp parallel for shared(field)
 #endif //_OPENMP
-    for( unsigned i=0; i<g2dF.size(); i++)
+    for( unsigned i=0; i<g2dFine.size(); i++)
     {
         thrust::host_vector<double> coords(3), coordsP(3), coordsM(3);
         coords[0] = y[0][i], coords[1] = y[1][i], coords[2] = y[2][i]; //x,y,s
         double phi1 = deltaPhi;
-        boxintegrator( field, g2d, coords, coordsP, phi1, eps, globalbcz);
+        boxintegrator( field, g2dFine, coords, coordsP, phi1, eps, globalbcz);
         phi1 =  - deltaPhi;
-        boxintegrator( field, g2d, coords, coordsM, phi1, eps, globalbcz);
+        boxintegrator( field, g2dFine, coords, coordsM, phi1, eps, globalbcz);
         yp[0][i] = coordsP[0], yp[1][i] = coordsP[1], yp[2][i] = coordsP[2];
         ym[0][i] = coordsM[0], ym[1][i] = coordsM[1], ym[2][i] = coordsM[2];
     }
@@ -473,7 +471,7 @@ FieldAligned<I, container>::FieldAligned(const BinaryVectorLvl0& mag, const Geom
     cusp::multiply( projection, plusFine, plus);
     cusp::multiply( projection, minusFine, minus);
     t.toc();
-    std::cout<< "Multiplication of PI took: "<<t.diff()<<"s\n";
+    std::cout<< "Multiplication of P*I took: "<<t.diff()<<"s\n";
     //Transposed matrices work only for csr_matrix due to bad matrix form for ell_matrix!!!
     cusp::transpose( plus, plusT);
     cusp::transpose( minus, minusT);     
@@ -492,8 +490,8 @@ FieldAligned<I, container>::FieldAligned(const BinaryVectorLvl0& mag, const Geom
  
 }
 
-template<class I, class container>
-void FieldAligned<I,container>::set_boundaries( dg::bc bcz, const container& global, double scal_left, double scal_right)
+template<class G, class I, class container>
+void FieldAligned<G, I,container>::set_boundaries( dg::bc bcz, const container& global, double scal_left, double scal_right)
 {
     cView left( global.cbegin(), global.cbegin() + perp_size_);
     cView right( global.cbegin()+(Nz_-1)*perp_size_, global.cbegin() + Nz_*perp_size_);
@@ -506,26 +504,26 @@ void FieldAligned<I,container>::set_boundaries( dg::bc bcz, const container& glo
     bcz_ = bcz;
 }
 
-template< class I, class container>
-template< class BinaryOp, class Grid>
-container FieldAligned<I,container>::evaluate( BinaryOp binary, unsigned p0, Grid grid) const
+template< class G, class I, class container>
+template< class BinaryOp>
+container FieldAligned<G, I,container>::evaluate( BinaryOp binary, unsigned p0) const
 {
-    return evaluate( binary, dg::CONSTANT(1), p0, 0, grid);
+    return evaluate( binary, dg::CONSTANT(1), p0, 0);
 }
 
-template<class I, class container>
-template< class BinaryOp, class UnaryOp, class Grid>
-container FieldAligned<I,container>::evaluate( BinaryOp binary, UnaryOp unary, unsigned p0, unsigned rounds, Grid g_) const
+template<class G, class I, class container>
+template< class BinaryOp, class UnaryOp>
+container FieldAligned<G, I,container>::evaluate( BinaryOp binary, UnaryOp unary, unsigned p0, unsigned rounds) const
 {
     //idea: simply apply I+/I- enough times on the init2d vector to get the result in each plane
     //unary function is always such that the p0 plane is at x=0
-    assert( g_.Nz() > 1);
-    assert( p0 < g_.Nz());
-    const typename G::perpendicular_grid g2d = g_.perp_grid();
-    assert( g2d.size() == perp_size_ && Nz_== g_.Nz());
+    assert( g_.get().Nz() > 1);
+    assert( p0 < g_.get().Nz());
+    const typename G::perpendicular_grid g2d = g_.get().perp_grid();
+    assert( g2d.size() == perp_size_ && Nz_== g_.get().Nz());
     container init2d = dg::pullback( binary, g2d), temp(init2d), tempP(init2d), tempM(init2d);
-    container vec3d = dg::evaluate( dg::zero, g_);
-    std::vector<container>  plus2d( g_.Nz(), (container)dg::evaluate(dg::zero, g2d) ), minus2d( plus2d), result( plus2d);
+    container vec3d = dg::evaluate( dg::zero, g_.get());
+    std::vector<container>  plus2d( g_.get().Nz(), (container)dg::evaluate(dg::zero, g2d) ), minus2d( plus2d), result( plus2d);
     unsigned turns = rounds; 
     if( turns ==0) turns++;
     //first apply Interpolation many times, scale and store results
@@ -542,8 +540,8 @@ container FieldAligned<I,container>::evaluate( BinaryOp binary, UnaryOp unary, u
                 dg::blas2::symv( minus, tempM, temp);
                 temp.swap( tempM);
             }
-            dg::blas1::scal( tempP, unary(  (double)rep*g_.hz() ) );
-            dg::blas1::scal( tempM, unary( -(double)rep*g_.hz() ) );
+            dg::blas1::scal( tempP, unary(  (double)rep*g_.get().hz() ) );
+            dg::blas1::scal( tempM, unary( -(double)rep*g_.get().hz() ) );
             dg::blas1::axpby( 1., tempP, 1., plus2d[i0]);
             dg::blas1::axpby( 1., tempM, 1., minus2d[i0]);
         }
@@ -579,15 +577,15 @@ container FieldAligned<I,container>::evaluate( BinaryOp binary, UnaryOp unary, u
 }
 
 
-template< class I, class container>
-void FieldAligned<I, container >::operator()(enum whichMatrix which, const container& f, container& fe)
+template<class G, class I, class container>
+void FieldAligned<G, I, container >::operator()(enum whichMatrix which, const container& f, container& fe)
 {
     if(which == einsPlus || which == einsMinusT) einsPlus( which, f, fe);
     if(which == einsMinus || which == einsPlusT) einsMinus( which, f, fe);
 }
 
-template< class I, class container>
-void FieldAligned<I, container>::einsPlus( enum whichMatrix which, const container& f, container& fpe)
+template< class G, class I, class container>
+void FieldAligned<G, I, container>::einsPlus( enum whichMatrix which, const container& f, container& fpe)
 {
     View ghostPV( ghostP.begin(), ghostP.end());
     View ghostMV( ghostM.begin(), ghostM.end());
@@ -621,8 +619,8 @@ void FieldAligned<I, container>::einsPlus( enum whichMatrix which, const contain
     }
 }
 
-template< class I, class container>
-void FieldAligned<I, container>::einsMinus( enum whichMatrix which, const container& f, container& fme)
+template< class G, class I, class container>
+void FieldAligned<G, I, container>::einsMinus( enum whichMatrix which, const container& f, container& fme)
 {
     //note that thrust functions don't work on views
     View ghostPV( ghostP.begin(), ghostP.end());
diff --git a/inc/geometries/flux_t.cu b/inc/geometries/flux_t.cu
index 518711f10..3750050c9 100644
--- a/inc/geometries/flux_t.cu
+++ b/inc/geometries/flux_t.cu
@@ -11,9 +11,10 @@
 #include "dg/backend/timer.cuh"
 #include "curvilinear.h"
 //#include "guenther.h"
+#include "testfunctors.h"
 #include "solovev.h"
 #include "flux.h"
-//#include "ds.h"
+#include "fieldaligned.h"
 #include "init.h"
 
 #include "file/nc_utilities.h"
@@ -158,21 +159,25 @@ int main( int argc, char* argv[])
     std::cout << "relative difference in volume2d is "<<fabs(volumeRZ - volume2d)/volume2d<<std::endl;
     std::cout << "Note that the error might also come from the volume in RZP!\n"; //since integration of jacobian is fairly good probably
 
-    ///////////////////////////TEST 3d grid//////////////////////////////////////
-    //std::cout << "Start DS test!"<<std::endl;
-    //const dg::HVec vol3d = dg::create::volume( g3d);
-    //t.tic();
-    //DFA fieldaligned( flux::Field( gp, g3d.x(), g3d.f_x()), g3d, gp.rk4eps, dg::NoLimiter()); 
-    //dg::DS<DFA, dg::DMatrix, dg::HVec> ds( fieldaligned, flux::Field(gp, g3d.x(), g3d.f_x()), dg::normed, dg::centered);
-
-    //
-    //t.toc();
-    //std::cout << "Construction took "<<t.diff()<<"s\n";
-    //dg::HVec B = dg::pullback( solovev::InvB(gp), g3d), divB(B);
-    //dg::HVec lnB = dg::pullback( solovev::LnB(gp), g3d), gradB(B);
-    //dg::HVec gradLnB = dg::pullback( solovev::GradLnB(gp), g3d);
-    //dg::blas1::pointwiseDivide( ones3d, B, B);
-    //dg::HVec function = dg::pullback( solovev::FuncNeu(gp), g3d), derivative(function);
+    /////////////////////////TEST 3d grid//////////////////////////////////////
+    std::cout << "Start DS test!"<<std::endl;
+    t.tic();
+    dg::geo::BHatR bhatR(c);
+    dg::geo::BHatZ bhatZ(c);
+    dg::geo::BHatP bhatP(c);
+    //dg::geo::BinaryVectorLvl0 bhat( dg::geo::BHatR(c), dg::geo::BHatZ(c), dg::geo::BHatP(c));
+    dg::geo::BinaryVectorLvl0 bhat( bhatR, bhatZ, bhatP);
+    dg::FieldAligned<dg::aGeometry3d, dg::IHMatrix, dg::HVec> fieldaligned( bhat, g3d, 1,1,gp.rk4eps, dg::NoLimiter() ); 
+    //dg::DS<dg::IHMatrix, dg::HMatrix, dg::HVec> ds( fieldaligned, flux::Field(gp, g3d.x(), g3d.f_x()), dg::normed, dg::centered);
+
+    
+    t.toc();
+    std::cout << "Construction took "<<t.diff()<<"s\n";
+    dg::HVec B = dg::pullback( dg::geo::InvB(c), g3d), divB(B);
+    dg::HVec lnB = dg::pullback( dg::geo::LnB(c), g3d), gradB(B);
+    dg::HVec gradLnB = dg::pullback( dg::geo::GradLnB(c), g3d);
+    dg::blas1::pointwiseDivide( ones3d, B, B);
+    dg::HVec function = dg::pullback( dg::geo::FuncNeu(c), g3d), derivative(function);
     //ds( function, derivative);
 
     //ds.centeredT( B, divB);
@@ -188,9 +193,7 @@ int main( int argc, char* argv[])
     //err = nc_put_var_double( ncid, varID[4], periodify(X, g2d_periodic).data());
     //double norm2 = sqrt(dg::blas2::dot(gradLnB, vol3d,gradLnB));
     //std::cout << "rel. error of lnB is    "<<norm2/norm<<"\n";
-    err = nc_close( ncid);
-
-
 
+    err = nc_close( ncid);
     return 0;
 }
-- 
GitLab


From dd75ea4e6a296c1a3edf887838264f0a42417578 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sun, 27 Aug 2017 20:55:02 +0200
Subject: [PATCH 219/453] added ep.tex file to src/ep

---
 src/ep/Makefile |   9 +-
 src/ep/ep.tex   | 286 ++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 293 insertions(+), 2 deletions(-)
 create mode 100644 src/ep/ep.tex

diff --git a/src/ep/Makefile b/src/ep/Makefile
index dbda78f6d..d0f674055 100644
--- a/src/ep/Makefile
+++ b/src/ep/Makefile
@@ -14,7 +14,12 @@ toefl_mpi: toefl_mpi.cu toeflR.cuh
 toeflR: toeflR.cu toeflR.cuh 
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(GLFLAGS) $(JSONLIB) -DDG_BENCHMARK  -g
 
-.PHONY: clean
+doc: 
+	mkdir -p doc; \
+	cd doc; \
+	pdflatex ../ep.tex;
+
+.PHONY: clean doc
 
 clean:
-	rm -f toefl_mpi
+	rm -rf toeflR toefl_mpi doc
diff --git a/src/ep/ep.tex b/src/ep/ep.tex
new file mode 100644
index 000000000..241ed57b4
--- /dev/null
+++ b/src/ep/ep.tex
@@ -0,0 +1,286 @@
+%\documentclass[12pt]{article}
+%\documentclass[12pt]{scrartcl}
+\documentclass{hitec}
+\settextfraction{0.9} % indent text
+\usepackage{csquotes}
+\usepackage[hidelinks]{hyperref} % doi links are short and usefull?
+\hypersetup{%
+    colorlinks=true,
+    linkcolor=blue,
+    urlcolor=magenta
+}
+\urlstyle{rm}
+\usepackage[english]{babel}
+\usepackage{mathtools} % loads and extends amsmath
+\usepackage{amssymb}
+% packages not used
+%\usepackage{graphicx}
+%\usepackage{amsthm}
+%\usepackage{subfig}
+\usepackage{bm}
+\usepackage{longtable}
+\usepackage{booktabs}
+\usepackage{ragged2e} % maybe use \RaggedRight for tables and literature?
+\usepackage[table]{xcolor} % for alternating colors
+\rowcolors{2}{gray!25}{white}
+\renewcommand\arraystretch{1.3}
+
+%%% reset bibliography distances %%%
+\let\oldthebibliography\thebibliography
+\let\endoldthebibliography\endthebibliography
+\renewenvironment{thebibliography}[1]{
+  \begin{oldthebibliography}{#1}
+    \RaggedRight % remove if justification is desired
+    \setlength{\itemsep}{0em}
+    \setlength{\parskip}{0em}
+}
+{
+  \end{oldthebibliography}
+}
+%%% --- %%%
+
+%%%%%%%%%%%%%%%%%%%%%definitions%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\newcommand{\eps}{\varepsilon}
+\renewcommand{\d}{\mathrm{d}}
+\newcommand{\T}{\mathrm{T}}
+\renewcommand{\vec}[1]{{\mathbf{#1}}}
+\newcommand{\dx}{\,\mathrm{d}x}
+%\newcommand{\dA}{\,\mathrm{d}(x,y)}
+%\newcommand{\dV}{\mathrm{d}^3{x}\,}
+\newcommand{\dA}{\,\mathrm{dA}}
+\newcommand{\dV}{\mathrm{dV}\,}
+\newcommand{\Eins}{\mathbf{1}}
+\newcommand{\ExB}{$\bm{E}\times\bm{B} \,$}
+\newcommand{\GKI}{\int d^6 \bm{Z} \BSP}
+\newcommand{\GKIV}{\int dv_{\|} d \mu d \theta \BSP}
+\newcommand{\BSP}{B_{\|}^*}
+\newcommand{\GA}[1]{\langle #1   \rangle}
+\newcommand{\Abar}{\langle A_\parallel \rangle}
+%Vectors
+\newcommand{\bhat}{\bm{\hat{b}}}
+\newcommand{\bbar}{\overline{\bm{b}}}
+\newcommand{\chat}{\bm{\hat{c}}}
+\newcommand{\ahat}{\bm{\hat{a}}}
+\newcommand{\xhat}{\bm{\hat{x}}}
+\newcommand{\yhat}{\bm{\hat{y}}}
+\newcommand{\zhat}{\bm{\hat{z}}}
+\newcommand{\Xbar}{\bar{\vec{X}}}
+\newcommand{\phat}{\bm{\hat{\perp}}}
+\newcommand{\that}{\bm{\hat{\theta}}}
+\newcommand{\eI}{\bm{\hat{e}}_1}
+\newcommand{\eII}{\bm{\hat{e}}_2}
+\newcommand{\ud}{\mathrm{d}}
+%Derivatives etc.
+\newcommand{\pfrac}[2]{\frac{\partial#1}{\partial#2}}
+\newcommand{\ffrac}[2]{\frac{\delta#1}{\delta#2}}
+\newcommand{\fixd}[1]{\Big{\arrowvert}_{#1}}
+\newcommand{\curl}[1]{\nabla \times #1}
+\newcommand{\np}{\nabla_{\perp}}
+\newcommand{\npc}{\nabla_{\perp} \cdot }
+\newcommand{\nc}{\nabla\cdot }
+\newcommand{\GAI}{\Gamma_{1}^{\dagger}}
+\newcommand{\GAII}{\Gamma_{1}^{\dagger -1}}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%DOCUMENT%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{document}
+
+\title{The toefl project}
+\maketitle
+
+\begin{abstract}
+This is a program for 2d isothermal blob simulations.
+\end{abstract}
+
+\section{Equations}
+$N_e$ is the electron density, $N_p$ is the positron density. 
+ $\phi$ is the electric potential. We use Cartesian coordinates $x$, $y$. 
+
+\subsection{Model}
+
+"global"
+\begin{subequations}
+\begin{align}
+B(x)^{-1} = \kappa x +1-\kappa X\quad \Gamma_1 = ( 1- 0.5\mu\tau\nabla^2)^{-1}\\
+ -\nabla\cdot \left( \varepsilon + \frac{z_e\mu_e N_e + z_p\mu_pN_p}{B^2} \nabla \phi\right) = z_p \Gamma_{1p} N_p +z_e\Gamma_{1e} N_e, \quad
+\psi = \Gamma_1 \phi - \frac{1}{2} \mu \frac{(\nabla\phi)^2}{B^2}\\
+  \frac{\partial N}{\partial t} =
+  \frac{1}{B}\{ N, \psi\} 
+  + \kappa N\frac{\partial \psi}{\partial y} 
+  + \tau \kappa\frac{\partial N}{\partial y} +\nu\nabla^2N
+\end{align}
+\end{subequations}
+
+\subsection{Initialization}
+We define a Gaussian 
+\begin{align}
+    n(x,y) = 1 + A\exp\left( -\frac{(x-X)^2 + (y-Y)^2}{2\sigma^2}\right)
+    \label{}
+\end{align}
+where $X = p_x l_x$ and $Y=p_yl_y$ are the initial centre of mass position coordinates, $A$ is the amplitude and $\sigma$ the
+radius of the blob.
+We initialize 
+\begin{align}
+    N_s = \Gamma_{1s}^{-1} n \quad \phi = 0 \\
+    \rho = \phi = 0
+    \label{}
+\end{align}
+\subsection{Diagnostics}
+\begin{align}
+    M(t) = \int n-1 \\
+    \Lambda_n = \nu \int \Delta n  \\
+    \label{}
+\end{align}
+\section{Numerical methods}
+discontinuouos Galerkin on structured grid
+\begin{longtable}{ll>{\RaggedRight}p{7cm}}
+\toprule
+\rowcolor{gray!50}\textbf{Term} &  \textbf{Method} & \textbf{Description}  \\ \midrule
+coordinate system & Cartesian 2D & equidistant discretization of $[0,l_x] \times [0,l_y]$, equal number of Gaussian nodes in x and y \\
+matrix inversions & conjugate gradient & Use previous two solutions to extrapolate initial guess and $1/\chi$ as preconditioner \\
+\ExB advection & Arakawa & s.a. \cite{Einkemmer2013} \\
+curvature terms & direct & flux conserving \\
+time &  Karniadakis multistep & $3rd$ order explicit, diffusion $2nd$ order implicit \\
+\bottomrule
+\end{longtable}
+
+\section{Compilation and useage}
+There are two programs toeflR.cu, and toefl\_mpi.cu . Compilation with 
+\begin{verbatim}
+make device = <omp or gpu>
+\end{verbatim}
+Run with
+\begin{verbatim}
+path/to/feltor/src/ep/toeflR input.json
+echo np_x np_y | mpirun -n np_x*np_y path/to/feltor/src/ep/toefl_mpi\
+    input.json output.nc
+\end{verbatim}
+All programs write performance informations to std::cout.
+The first opens a terminal window with life simulation results the
+other two write the results to disc. 
+The second for distributed
+memory systems, which expects the distribution of processes in the
+x and y directions.
+
+\subsection{Input file structure}
+Input file format: json
+
+%%This is a booktabs table
+\begin{longtable}{llll>{\RaggedRight}p{7cm}}
+\toprule
+\rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Example} & \textbf{Default} & \textbf{Description}  \\ \midrule
+n      & integer & 3 & - &\# Gaussian nodes in x and y \\
+Nx     & integer &100& - &\# grid points in x \\
+Ny     & integer &100& - &\# grid points in y \\
+dt     & integer &3.0& - &time step in units of $c_s/\rho_s$ \\
+n\_out  & integer &3  & - &\# Gaussian nodes in x and y in output \\
+Nx\_out & integer &100& - &\# grid points in x in output fields \\
+Ny\_out & integer &100& - &\# grid points in y in output fields \\
+itstp  & integer &2  & - &   steps between outputs \\
+maxout & integer &100& - &      \# outputs excluding first \\
+eps\_pol   & float &1e-6    & - &  accuracy of polarisation solver \\
+eps\_gamma & float &1e-7    & - & accuracy of $\Gamma_1$ (only in gyrofluid model) \\
+eps\_time  & float &1e-10   & - & accuracy of implicit time-stepper \\
+curvature  & float &0.00015& - & magnetic curvature $\kappa$ \\
+tau        & float array &[-1., 1.]  & - & $\tau_s = T_s/Z_s T_e$
+mu         & float array &[-1., 1.]  & - & $\mu_s = m_s/Z_s m_e$
+z          & integer array  & [-1,1]  & - & charge number
+nu\_perp    & float &5e-3   & - & pependicular viscosity $\nu$ \\
+amplitude  & float &1.0    & - & amplitude $A$ of the blob \\
+sigma      & float &10     & - & blob radius $\sigma$ \\
+posX       & float &0.3    & - & blob x-position in units of $l_x$, i.e. $X = p_x l_x$\\
+posY       & float &0.5    & - & blob y-position in units of $l_y$, i.e. $Y = p_y l_y$ \\
+lx         & float &200    & - & $l_x$  \\
+ly         & float &200    & - & $l_y$  \\
+bc\_x   & char & "DIR"      & - & boundary condition in x (one of PER, DIR, NEU, DIR\_NEU or NEU\_DIR) \\
+bc\_y   & char & "PER"      & - & boundary condition in y (one of PER, DIR, NEU, DIR\_NEU or NEU\_DIR) \\
+debye  & float & 0 & 0 & debye length parameter
+
+\bottomrule
+\end{longtable}
+
+The default value is taken if the value name is not found in the input file. If there is no default and
+the value is not found,
+the program exits with an error message.
+
+\subsection{Structure of output file}
+Output file format: netcdf-4/hdf5
+%
+%Name | Type | Dimensionality | Description
+%---|---|---|---|
+\begin{longtable}{lll>{\RaggedRight}p{7cm}}
+\toprule
+\rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Dimension} & \textbf{Description}  \\ \midrule
+inputfile  &             text attribute & 1 & verbose input file as a string \\
+energy\_time             & Dataset & 1 & timesteps at which 1d variables are written \\
+time                     & Dataset & 1 & time at which fields are written \\
+x                        & Dataset & 1 & x-coordinate  \\
+y                        & Dataset & 1 & y-coordinate \\
+electrons                & Dataset & 3 (time, y, x) & electon density $n$ \\
+positrons                & Dataset & 3 (time, y, x) & positron density \\
+potential                & Dataset & 3 (time, y, x) & electric potential $\phi$  \\
+vorticity                & Dataset & 3 (time, y, x) & Laplacian of potential $\nabla^2\phi$  \\
+dEdt                     & Dataset & 1 (energy\_time) & change of energy per time  \\
+dissipation              & Dataset & 1 (energy\_time) & diffusion integrals  \\
+energy                   & Dataset & 1 (energy\_time) & total energy integral  \\
+mass                     & Dataset & 1 (energy\_time) & mass integral   \\
+\bottomrule
+\end{longtable}
+\section{Diagnostics toeflRdiag.cu}
+There only is a shared memory version available
+\begin{verbatim}
+cd path/to/feltor/diag
+make toeflRdiag 
+path/to/feltor/diag/toeflRdiag input.nc output.nc
+\end{verbatim}
+
+Input file format: netcdf-4/hdf5
+%
+%Name | Type | Dimensionality | Description
+%---|---|---|---|
+\begin{longtable}{lll>{\RaggedRight}p{7cm}}
+\toprule
+\rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Dimension} & \textbf{Description}  \\ \midrule
+inputfile  &             text attribute & 1 & verbose input file as a string \\
+electrons                & Dataset & 3 & electon density (time, y, x) \\
+ions                     & Dataset & 3 & ion density (time, y, x) \\
+potential                & Dataset & 3 & electric potential (time, y, x) \\
+\bottomrule
+\end{longtable}
+
+Output file format: netcdf-4/hdf5
+%
+%Name | Type | Dimensionality | Description
+%---|---|---|---|
+\begin{longtable}{lll>{\RaggedRight}p{7cm}}
+\toprule
+\rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Dimension} & \textbf{Description}  \\ \midrule
+ inputfile & text attribute & 1 & copy of inputfile attribute of the input file (the json string of the simulation input file) \\
+ time & Dataset & 1 & the time steps at which variables are written \\
+ posX & Dataset & 1 (time) & centre of mass (COM) position x-coordinate \\
+ posY & Dataset & 1 (time) &COM y-position \\
+ velX & Dataset & 1 (time)& COM x-velocity \\
+ velY & Dataset & 1 (time)& COM y-velocity \\
+ accX & Dataset & 1 (time)& COM x-acceleration \\
+ accY & Dataset & 1 (time)& COM y-acceleration \\
+ velCOM & Dataset & 1 (time)&absolute value of the COM velocity \\
+ posXmax& Dataset & 1 (time)&maximum amplitude x-position \\
+ posYmax& Dataset & 1 (time)&maximum amplitude y-position \\
+ velXmax& Dataset & 1 (time)&maximum amplitude x-velocity \\
+ velYmax& Dataset & 1 (time)&maximum amplitude y-velocity \\
+ maxamp & Dataset & 1 (time)&value of the maximum amplitude  \\
+  compactness\_ne& Dataset & 1 (time) &compactness of the density field \\
+ Ue& Dataset&  1 (time) &entropy electrons \\
+ Ui &Dataset& 1 (time) & entropy ions \\
+ Uphi& Dataset& 1 (time) &  exb energy \\
+ mass& Dataset & 1 (time) & mass of the blob without background \\
+\bottomrule
+\end{longtable}
+%..................................................................
+\begin{thebibliography}{1}
+    \bibitem{Kendl2017}
+    A. Kendl, G. Danler, M. Wiesenberger, and M. Held, "Interchange instability in matter-antimatter plasmas" Phys. Rev. Letters (2017) https://arxiv.org/abs/1611.07836
+\end{thebibliography}
+%..................................................................
+
+
+\end{document}
-- 
GitLab


From df0ab979619327c64dbadf491680b6ddb51b04cb Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sun, 27 Aug 2017 21:02:23 +0200
Subject: [PATCH 220/453] small corrections to toefl.tex and ep.tex

---
 src/ep/ep.tex       | 27 +++++++++++++--------------
 src/toefl/toefl.tex |  3 ++-
 2 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/src/ep/ep.tex b/src/ep/ep.tex
index 241ed57b4..829465d3d 100644
--- a/src/ep/ep.tex
+++ b/src/ep/ep.tex
@@ -84,7 +84,7 @@
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%DOCUMENT%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \begin{document}
 
-\title{The toefl project}
+\title{The electron-positron plasma project}
 \maketitle
 
 \begin{abstract}
@@ -120,24 +120,24 @@ where $X = p_x l_x$ and $Y=p_yl_y$ are the initial centre of mass position coord
 radius of the blob.
 We initialize 
 \begin{align}
-    N_s = \Gamma_{1s}^{-1} n \quad \phi = 0 \\
-    \rho = \phi = 0
+    N_s = \Gamma_{1s}^{-1} n \quad \phi = 0 
     \label{}
 \end{align}
 \subsection{Diagnostics}
 \begin{align}
     M(t) = \int n-1 \\
     \Lambda_n = \nu \int \Delta n  \\
+    ...
     \label{}
 \end{align}
 \section{Numerical methods}
-discontinuouos Galerkin on structured grid
+discontinuous Galerkin on structured grid
 \begin{longtable}{ll>{\RaggedRight}p{7cm}}
 \toprule
 \rowcolor{gray!50}\textbf{Term} &  \textbf{Method} & \textbf{Description}  \\ \midrule
 coordinate system & Cartesian 2D & equidistant discretization of $[0,l_x] \times [0,l_y]$, equal number of Gaussian nodes in x and y \\
 matrix inversions & conjugate gradient & Use previous two solutions to extrapolate initial guess and $1/\chi$ as preconditioner \\
-\ExB advection & Arakawa & s.a. \cite{Einkemmer2013} \\
+\ExB advection & Arakawa & - \\
 curvature terms & direct & flux conserving \\
 time &  Karniadakis multistep & $3rd$ order explicit, diffusion $2nd$ order implicit \\
 \bottomrule
@@ -181,9 +181,9 @@ eps\_pol   & float &1e-6    & - &  accuracy of polarisation solver \\
 eps\_gamma & float &1e-7    & - & accuracy of $\Gamma_1$ (only in gyrofluid model) \\
 eps\_time  & float &1e-10   & - & accuracy of implicit time-stepper \\
 curvature  & float &0.00015& - & magnetic curvature $\kappa$ \\
-tau        & float array &[-1., 1.]  & - & $\tau_s = T_s/Z_s T_e$
-mu         & float array &[-1., 1.]  & - & $\mu_s = m_s/Z_s m_e$
-z          & integer array  & [-1,1]  & - & charge number
+tau        & float array &[-1., 1.]  & - & $\tau_s = T_s/Z_s T_e$\\
+mu         & float array &[-1., 1.]  & - & $\mu_s = m_s/Z_s m_e$\\
+z          & integer array  & [-1,1]  & - & charge number\\
 nu\_perp    & float &5e-3   & - & pependicular viscosity $\nu$ \\
 amplitude  & float &1.0    & - & amplitude $A$ of the blob \\
 sigma      & float &10     & - & blob radius $\sigma$ \\
@@ -193,8 +193,7 @@ lx         & float &200    & - & $l_x$  \\
 ly         & float &200    & - & $l_y$  \\
 bc\_x   & char & "DIR"      & - & boundary condition in x (one of PER, DIR, NEU, DIR\_NEU or NEU\_DIR) \\
 bc\_y   & char & "PER"      & - & boundary condition in y (one of PER, DIR, NEU, DIR\_NEU or NEU\_DIR) \\
-debye  & float & 0 & 0 & debye length parameter
-
+debye  & float & 0 & 0 & debye length parameter\\
 \bottomrule
 \end{longtable}
 
@@ -225,12 +224,12 @@ energy                   & Dataset & 1 (energy\_time) & total energy integral  \
 mass                     & Dataset & 1 (energy\_time) & mass integral   \\
 \bottomrule
 \end{longtable}
-\section{Diagnostics toeflRdiag.cu}
+\section{Diagnostics toeflEPdiag.cu}
 There only is a shared memory version available
 \begin{verbatim}
 cd path/to/feltor/diag
-make toeflRdiag 
-path/to/feltor/diag/toeflRdiag input.nc output.nc
+make toeflEPdiag 
+path/to/feltor/diag/toeflEPdiag input.nc output.nc
 \end{verbatim}
 
 Input file format: netcdf-4/hdf5
@@ -242,7 +241,7 @@ Input file format: netcdf-4/hdf5
 \rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Dimension} & \textbf{Description}  \\ \midrule
 inputfile  &             text attribute & 1 & verbose input file as a string \\
 electrons                & Dataset & 3 & electon density (time, y, x) \\
-ions                     & Dataset & 3 & ion density (time, y, x) \\
+positrons                & Dataset & 3 & positron density (time, y, x) \\
 potential                & Dataset & 3 & electric potential (time, y, x) \\
 \bottomrule
 \end{longtable}
diff --git a/src/toefl/toefl.tex b/src/toefl/toefl.tex
index f909988ae..20f2abd43 100644
--- a/src/toefl/toefl.tex
+++ b/src/toefl/toefl.tex
@@ -191,10 +191,11 @@ We initialize
 \begin{align}
     M(t) = \int n-1 \\
     \Lambda_n = \nu \int \Delta n  \\
+    ...
     \label{}
 \end{align}
 \section{Numerical methods}
-discontinuouos Galerkin on structured grid
+discontinuous Galerkin on structured grid
 \begin{longtable}{ll>{\RaggedRight}p{7cm}}
 \toprule
 \rowcolor{gray!50}\textbf{Term} &  \textbf{Method} & \textbf{Description}  \\ \midrule
-- 
GitLab


From 7f7389c015a51c2c59aff3fce2c227efd05ed08b Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sun, 27 Aug 2017 22:09:02 +0200
Subject: [PATCH 221/453] implemented globalbcx and globalbcy

---
 inc/dg/backend/interpolation.cuh | 55 +++++++++------------
 inc/geometries/fieldaligned.h    | 83 +++++++++++++-------------------
 2 files changed, 57 insertions(+), 81 deletions(-)

diff --git a/inc/dg/backend/interpolation.cuh b/inc/dg/backend/interpolation.cuh
index dcf1f4d5b..9d465c4fa 100644
--- a/inc/dg/backend/interpolation.cuh
+++ b/inc/dg/backend/interpolation.cuh
@@ -119,13 +119,14 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const thrust::ho
  * @param x X-coordinates of interpolation points
  * @param y Y-coordinates of interpolation points
  * @param g The Grid on which to operate
- * @param boundary determines what to do when a point
- lies exactly on the boundary:  DIR generates zeroes in the interpolation matrix, NEU and other boundaries interpolate the inner side polynomial.
+ * @param bcx determines what to do when a point lies exactly on the boundary in x:  DIR generates zeroes in the interpolation matrix, 
+ NEU and PER interpolate the inner side polynomial. (DIR_NEU and NEU_DIR apply NEU / DIR to the respective left or right boundary )
+ * @param bcy determines what to do when a point lies exactly on the boundary in y. Behaviour correponds to bcx.
  * @attention all points (x,y) must lie within or on the boundaries of g.
  *
  * @return interpolation matrix
  */
-cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const thrust::host_vector<double>& x, const thrust::host_vector<double>& y, const aTopology2d& g , dg::bc boundary = dg::NEU)
+cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const thrust::host_vector<double>& x, const thrust::host_vector<double>& y, const aTopology2d& g , dg::bc bcx = dg::NEU, dg::bc bcy = dg::NEU)
 {
     assert( x.size() == y.size());
     std::vector<double> gauss_nodes = g.dlt().abscissas(); 
@@ -189,16 +190,15 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const thrust::ho
             for(unsigned k=0; k<pyF.size(); k++)
                 for( unsigned l=0; l<pxF.size(); l++)
                     pxy[k*px.size()+l]= pyF[k]*pxF[l];
-            if (boundary == dg::DIR)
+            if (  (x[i] == g.x0() && (bcx==dg::DIR || bcx==dg::DIR_NEU) )
+                ||(x[i] == g.x1() && (bcx==dg::DIR || bcx==dg::NEU_DIR) )
+                ||(y[i] == g.y0() && (bcy==dg::DIR || bcy==dg::DIR_NEU) )
+                ||(y[i] == g.y1() && (bcy==dg::DIR || bcy==dg::NEU_DIR) ))
             {
-                if ( x[i]==g.x0() || x[i]==g.x1()  || y[i]==g.y0()  || y[i]==g.y1())
-    //             if ( fabs(x[i]-g.x0())<1e-10 || fabs(x[i]-g.x1())<1e-10  || fabs(y[i]-g.y0())<1e-10  || fabs(y[i]-g.y1())<1e-10)
-                {
-                    //zeroe boundary values 
-                    for(unsigned k=0; k<py.size(); k++)
-                    for( unsigned l=0; l<px.size(); l++)
-                        pxy[k*px.size()+l]= 0; 
-                }
+                //zeroe boundary values 
+                for(unsigned k=0; k<py.size(); k++)
+                for( unsigned l=0; l<px.size(); l++)
+                    pxy[k*px.size()+l]= 0; 
             }
             for( unsigned k=0; k<g.n(); k++)
                 for( unsigned l=0; l<g.n(); l++)
@@ -247,10 +247,6 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const thrust::ho
     cusp::coo_matrix<int, double, cusp::host_memory> A( x.size(), g.size(), values.size());
     A.row_indices = row_indices; A.column_indices = column_indices; A.values = values;
 
-    if (boundary == DIR_NEU ) std::cerr << "DIR_NEU NOT IMPLEMENTED "<<std::endl;
-    if (boundary == NEU_DIR ) std::cerr << "NEU_DIR NOT IMPLEMENTED "<<std::endl;
-    if (boundary == dg::PER ) std::cerr << "PER NOT IMPLEMENTED "<<std::endl;
-    
     return A;
 }
 
@@ -264,13 +260,14 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const thrust::ho
  * @param y Y-coordinates of interpolation points
  * @param z Z-coordinates of interpolation points
  * @param g The Grid on which to operate
- * @param boundary determines what to do when a point
- lies exactly on the boundary:  DIR generates zeroes in the interpolation matrix, NEU and other boundaries interpolate the inner side polynomial.
+ * @param bcx determines what to do when a point lies exactly on the boundary in x:  DIR generates zeroes in the interpolation matrix, 
+ NEU and PER interpolate the inner side polynomial. (DIR_NEU and NEU_DIR apply NEU / DIR to the respective left or right boundary )
+ * @param bcy determines what to do when a point lies exactly on the boundary in y. Behaviour correponds to bcx.
  *
  * @return interpolation matrix
  * @attention all points (x, y, z) must lie within or on the boundaries of g
  */
-cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const thrust::host_vector<double>& x, const thrust::host_vector<double>& y, const thrust::host_vector<double>& z, const aTopology3d& g, dg::bc boundary= dg::NEU)
+cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const thrust::host_vector<double>& x, const thrust::host_vector<double>& y, const thrust::host_vector<double>& z, const aTopology3d& g, dg::bc bcx = dg::NEU, dg::bc bcy = dg::NEU)
 {
     assert( x.size() == y.size());
     assert( y.size() == z.size());
@@ -341,15 +338,15 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const thrust::ho
             for(unsigned k=0; k<pyF.size(); k++)
                 for( unsigned l=0; l<pxF.size(); l++)
                     pxyz[k*g.n()+l]= 1.*pyF[k]*pxF[l];
-            if (boundary == dg::DIR)
+            if (  (x[i] == g.x0() && (bcx==dg::DIR || bcx==dg::DIR_NEU) )
+                ||(x[i] == g.x1() && (bcx==dg::DIR || bcx==dg::NEU_DIR) )
+                ||(y[i] == g.y0() && (bcy==dg::DIR || bcy==dg::DIR_NEU) )
+                ||(y[i] == g.y1() && (bcy==dg::DIR || bcy==dg::NEU_DIR) ))
             {
-                if ( x[i]==g.x0() || x[i]==g.x1()  ||y[i]==g.y0()  || y[i]==g.y1())
-                {
-                    //zeroe boundary values 
-                    for(unsigned k=0; k<g.n(); k++)
-                    for(unsigned l=0; l<g.n(); l++)
-                        pxyz[k*g.n()+l]= 0; 
-                }
+                //zeroe boundary values 
+                for(unsigned k=0; k<g.n(); k++)
+                for(unsigned l=0; l<g.n(); l++)
+                    pxyz[k*g.n()+l]= 0; 
             }
             for( unsigned k=0; k<g.n(); k++)
                 for( unsigned l=0; l<g.n(); l++)
@@ -398,10 +395,6 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const thrust::ho
     cusp::coo_matrix<int, double, cusp::host_memory> A( x.size(), g.size(), values.size());
     A.row_indices = row_indices; A.column_indices = column_indices; A.values = values;
 
-    if (boundary == DIR_NEU ) std::cerr << "DIR_NEU NOT IMPLEMENTED "<<std::endl;
-    if (boundary == NEU_DIR ) std::cerr << "NEU_DIR NOT IMPLEMENTED "<<std::endl;
-    if (boundary == dg::PER ) std::cerr << "PER NOT IMPLEMENTED "<<std::endl;
-    
     return A;
 }
 /**
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index 16209686e..c3594ab52 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -198,13 +198,12 @@ struct BoxIntegrator
  * @param coords1 The resulting points (write only) guaranteed to lie inside the grid
  * @param phi1 The angle (read/write) contains maximum phi on input and resulting phi on output
  * @param eps error
- * @param globalbcz boundary condition  (DIR or NEU)
  */
 template< class Field, class Grid>
 void boxintegrator( Field& field, const Grid& grid, 
         const thrust::host_vector<double>& coords0, 
         thrust::host_vector<double>& coords1, 
-        double& phi1, double eps, dg::bc globalbcz)
+        double& phi1, double eps)
 {
     dg::integrateRK4( field, coords0, coords1, phi1, eps); //integration
     //First catch periodic domain
@@ -215,34 +214,29 @@ void boxintegrator( Field& field, const Grid& grid,
         std::cerr << "point "<<coords1[0]<<" "<<coords1[1]<<" is somewhere else!\n";
 #endif //DG_DEBUG
         double deltaS = coords1[2];
-        if( globalbcz == dg::DIR || globalbcz == dg::NEU)
+        BoxIntegrator<Field, Grid> boxy( field, grid, eps);
+        boxy.set_coords( coords0); //nimm alte koordinaten
+        if( phi1 > 0)
         {
-            BoxIntegrator<Field, Grid> boxy( field, grid, eps);
-            boxy.set_coords( coords0); //nimm alte koordinaten
-            if( phi1 > 0)
-            {
-                double dPhiMin = 0, dPhiMax = phi1;
-                dg::bisection1d( boxy, dPhiMin, dPhiMax,eps); //suche 0 stelle 
-                phi1 = (dPhiMin+dPhiMax)/2.;
-                dg::integrateRK4( field, coords0, coords1, dPhiMax, eps); //integriere bis über 0 stelle raus damit unten Wert neu gesetzt wird
-            }
-            else // phi1 < 0 
-            {
-                double dPhiMin = phi1, dPhiMax = 0;
-                dg::bisection1d( boxy, dPhiMin, dPhiMax,eps);
-                phi1 = (dPhiMin+dPhiMax)/2.;
-                dg::integrateRK4( field, coords0, coords1, dPhiMin, eps);
-            }
-            if (!(coords1[0] > grid.x0())) { coords1[0]=grid.x0();}
-            if (!(coords1[0] < grid.x1())) { coords1[0]=grid.x1();}
-            if (!(coords1[1] > grid.y0())) { coords1[1]=grid.y0();}
-            if (!(coords1[1] < grid.y1())) { coords1[1]=grid.y1();}
-            //idea: maybe we should take the "wrong" long Delta s instead of the short one to avoid deteriorated CFL condition:
-            coords1[2] = deltaS;
+            double dPhiMin = 0, dPhiMax = phi1;
+            dg::bisection1d( boxy, dPhiMin, dPhiMax,eps); //suche 0 stelle 
+            phi1 = (dPhiMin+dPhiMax)/2.;
+            dg::integrateRK4( field, coords0, coords1, dPhiMax, eps); //integriere bis über 0 stelle raus damit unten Wert neu gesetzt wird
         }
-        else if (globalbcz == DIR_NEU )std::cerr << "DIR_NEU NOT IMPLEMENTED "<<std::endl;
-        else if (globalbcz == NEU_DIR )std::cerr << "NEU_DIR NOT IMPLEMENTED "<<std::endl;
-        else if (globalbcz == dg::PER )std::cerr << "PER NOT IMPLEMENTED "<<std::endl;
+        else // phi1 < 0 
+        {
+            double dPhiMin = phi1, dPhiMax = 0;
+            dg::bisection1d( boxy, dPhiMin, dPhiMax,eps);
+            phi1 = (dPhiMin+dPhiMax)/2.;
+            dg::integrateRK4( field, coords0, coords1, dPhiMin, eps);
+        }
+        if (!(coords1[0] > grid.x0())) { coords1[0]=grid.x0();}
+        if (!(coords1[0] < grid.x1())) { coords1[0]=grid.x1();}
+        if (!(coords1[1] > grid.y0())) { coords1[1]=grid.y0();}
+        if (!(coords1[1] < grid.y1())) { coords1[1]=grid.y1();}
+        // we should take the "wrong" long Delta s instead of the short one to avoid deteriorated CFL condition:
+        coords1[2] = deltaS;
+        else std::cerr << "PER NOT IMPLEMENTED "<<std::endl;
     }
 }
 //////////////////////////////FieldAlignedCLASS////////////////////////////////////////////
@@ -267,25 +261,23 @@ struct FieldAligned
     /**
     * @brief Construct from a field and a grid
     *
-    * @tparam Field The Fieldlines to be integrated: 
-        Has to provide void operator()( const std::vector<dg::HVec>&, std::vector<dg::HVec>&) 
-        where the first index is R, the second Z and the last s (the length of the field line)
     * @tparam Limiter Class that can be evaluated on a 2d grid, returns 1 if there
         is a limiter and 0 if there isn't. 
         If a field line crosses the limiter in the plane \f$ \phi=0\f$ then the limiter boundary conditions apply. 
-    * @param field The field to integrate
+    * @param vec The field to integrate
     * @param grid The grid on which to operate
     * @param eps Desired accuracy of runge kutta
     * @param limit Instance of the limiter class (Default is a limiter everywhere, 
         note that if bcz is periodic it doesn't matter if there is a limiter or not)
-    * @param globalbcz Choose NEU or DIR. Defines BC in parallel on bounding box
+    * @param globalbcx Choose NEU or DIR. Defines the interpolation behaviour when a fieldline intersects the boundary box in the perpendicular direction
+    * @param globalbcy Choose NEU or DIR. Defines the interpolation behaviour when a fieldline intersects the boundary box in the perpendicular direction
     * @param deltaPhi Is either <0 (then it's ignored), may differ from hz() only if Nz() == 1
     * @note If there is a limiter, the boundary condition on the first/last plane is set 
         by the bcz variable from the grid and can be changed by the set_boundaries function. 
         If there is no limiter the boundary condition is periodic.
     */
     template <class Limiter>
-    FieldAligned(const dg::geo::BinaryVectorLvl0& vec, const Geometry& grid, unsigned multiplyX, unsigned multiplyY, double eps = 1e-4, Limiter limit = DefaultLimiter(), dg::bc globalbcz = dg::DIR, bool dependsOnX=true, bool dependsOnY=true, double deltaPhi = -1);
+    FieldAligned(const dg::geo::BinaryVectorLvl0& vec, const Geometry& grid, unsigned multiplyX, unsigned multiplyY, double eps = 1e-4, Limiter limit = DefaultLimiter(), dg::bc globalbcx = dg::DIR, dg::bc globalbcy = dg::DIR, bool dependsOnX=true, bool dependsOnY=true, double deltaPhi = -1);
 
     /**
     * @brief Set boundary conditions in the limiter region
@@ -304,15 +296,7 @@ struct FieldAligned
         right_ = dg::evaluate( dg::CONSTANT(right),g2d);
     }
 
-    /**
-     * @brief Set boundary conditions in the limiter region
-     *
-     * if Dirichlet boundaries are used the left value is the left function
-     value, if Neumann boundaries are used the left value is the left derivative value
-     * @param bcz boundary condition
-     * @param left left boundary value
-     * @param right right boundary value
-    */
+    ///@copydoc set_boundaries()
     void set_boundaries( dg::bc bcz, const container& left, const container& right)
     {
         bcz_ = bcz;
@@ -418,7 +402,7 @@ struct FieldAligned
 
 template<class Geometry, class IMatrix, class container>
 template <class Limiter>
-FieldAligned<Geometry, IMatrix, container>::FieldAligned(const dg::geo::BinaryVectorLvl0& mag, const Geometry& grid, unsigned mx, unsigned my, double eps, Limiter limit, dg::bc globalbcz, bool dependsOnX, bool dependsOnY, double deltaPhi):
+FieldAligned<Geometry, IMatrix, container>::FieldAligned(const dg::geo::BinaryVectorLvl0& mag, const Geometry& grid, unsigned mx, unsigned my, double eps, Limiter limit, dg::bc globalbcx, dg::bc globalbcy, bool dependsOnX, bool dependsOnY, double deltaPhi):
         hz_( dg::evaluate( dg::zero, grid)), hp_( hz_), hm_( hz_), 
         Nz_(grid.Nz()), bcz_(grid.bcz()), g_(grid)
 {
@@ -452,17 +436,17 @@ FieldAligned<Geometry, IMatrix, container>::FieldAligned(const dg::geo::BinaryVe
         thrust::host_vector<double> coords(3), coordsP(3), coordsM(3);
         coords[0] = y[0][i], coords[1] = y[1][i], coords[2] = y[2][i]; //x,y,s
         double phi1 = deltaPhi;
-        boxintegrator( field, g2dFine, coords, coordsP, phi1, eps, globalbcz);
+        boxintegrator( field, g2dFine, coords, coordsP, phi1, eps);
         phi1 =  - deltaPhi;
-        boxintegrator( field, g2dFine, coords, coordsM, phi1, eps, globalbcz);
+        boxintegrator( field, g2dFine, coords, coordsM, phi1, eps);
         yp[0][i] = coordsP[0], yp[1][i] = coordsP[1], yp[2][i] = coordsP[2];
         ym[0][i] = coordsM[0], ym[1][i] = coordsM[1], ym[2][i] = coordsM[2];
     }
     t.toc(); 
     std::cout << "Fieldline integration took "<<t.diff()<<"s\n";
     t.tic();
-    IMatrix plusFine  = dg::create::interpolation( yp[0], yp[1], g2dCoarse, globalbcz);
-    IMatrix minusFine = dg::create::interpolation( ym[0], ym[1], g2dCoarse, globalbcz);
+    IMatrix plusFine  = dg::create::interpolation( yp[0], yp[1], g2dCoarse, globalbcx, globalbcy);
+    IMatrix minusFine = dg::create::interpolation( ym[0], ym[1], g2dCoarse, globalbcx, globalbcy);
     IMatrix interpolation = dg::create::interpolation( g2dFine, g2dCoarse);
     IMatrix projection = dg::create::projection( g2dCoarse, g2dFine);
     t.toc();
@@ -475,8 +459,7 @@ FieldAligned<Geometry, IMatrix, container>::FieldAligned(const dg::geo::BinaryVe
     //Transposed matrices work only for csr_matrix due to bad matrix form for ell_matrix!!!
     cusp::transpose( plus, plusT);
     cusp::transpose( minus, minusT);     
-//     copy into h vectors
-    
+    //project h and copy into h vectors
     thrust::host_vector<double> hp( perp_size_), hm(hp);
     dg::blas2::symv( projection, yp[2], hp);
     dg::blas2::symv( projection, ym[2], hm);
-- 
GitLab


From db04d0b3d84e0ad5b9cb63e65c347157bb8cd3da Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 28 Aug 2017 05:24:45 -0700
Subject: [PATCH 222/453] introduced type if else for fieldaligned

---
 inc/dg/geometry/base_geometry.h |   4 +
 inc/geometries/fieldaligned.h   | 181 +++++++++++++++++---------------
 2 files changed, 100 insertions(+), 85 deletions(-)

diff --git a/inc/dg/geometry/base_geometry.h b/inc/dg/geometry/base_geometry.h
index e97c68fca..8409900d1 100644
--- a/inc/dg/geometry/base_geometry.h
+++ b/inc/dg/geometry/base_geometry.h
@@ -192,6 +192,7 @@ struct CartesianGrid2d: public dg::aGeometry2d
  */
 struct CartesianGrid3d: public dg::aGeometry3d
 {
+    typedef CartesianGrid2d perpendicular_grid;
     ///@copydoc Grid3d::Grid3d()
     CartesianGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = PER, bc bcy = PER, bc bcz = PER): dg::aGeometry3d(x0,x1,y0,y1,z0,z1,n,Nx,Ny,Nz,bcx,bcy,bcz){}
     /**
@@ -200,6 +201,7 @@ struct CartesianGrid3d: public dg::aGeometry3d
      */
     CartesianGrid3d( const dg::Grid3d& g):dg::aGeometry3d(g.x0(), g.x1(), g.y0(), g.y1(), g.z0(), g.z1(),g.n(),g.Nx(),g.Ny(),g.Nz(),g.bcx(),g.bcy(),g.bcz()){}
     virtual CartesianGrid3d* clone()const{return new CartesianGrid3d(*this);}
+    CartesianGrid2d perp_grid() const{ return CartesianGrid2d(x0(),x1(),y0(),y1(),n(),Nx(),Ny(),bcx(),bcy());}
     private:
     virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz){
         aTopology3d::do_set(new_n,new_Nx,new_Ny,new_Nz);
@@ -211,8 +213,10 @@ struct CartesianGrid3d: public dg::aGeometry3d
  */
 struct CylindricalGrid3d: public dg::aGeometry3d
 {
+    typedef CartesianGrid2d perpendicular_grid;
     CylindricalGrid3d( double R0, double R1, double Z0, double Z1, double phi0, double phi1, unsigned n, unsigned NR, unsigned NZ, unsigned Nphi, bc bcR, bc bcZ, bc bcphi = PER): dg::aGeometry3d(R0,R1,Z0,Z1,phi0,phi1,n,NR,NZ,Nphi,bcR,bcZ,bcphi){}
     virtual CylindricalGrid3d* clone()const{return new CylindricalGrid3d(*this);}
+    CartesianGrid2d perp_grid() const{ return CartesianGrid2d(x0(),x1(),y0(),y1(),n(),Nx(),Ny(),bcx(),bcy());}
     private:
     virtual SparseTensor<thrust::host_vector<double> > do_compute_metric()const{
         SparseTensor<thrust::host_vector<double> > metric(1);
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index c3594ab52..0a48d2d2b 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -53,25 +53,18 @@ struct DefaultField
 
 struct DSFieldCylindrical
 {
-    DSFieldCylindrical( const dg::geo::TokamakMagneticField& c):c_(c), invB_(c), R_0_(c.R0()) { }
-    /**
-     * @brief \f[ \frac{d \hat{R} }{ d \varphi}  = \frac{\hat{R}}{\hat{I}} \frac{\partial\hat{\psi}_p}{\partial \hat{Z}}, \hspace {3 mm}
-     \frac{d \hat{Z} }{ d \varphi}  =- \frac{\hat{R}}{\hat{I}} \frac{\partial \hat{\psi}_p}{\partial \hat{R}} , \hspace {3 mm}
-     \frac{d \hat{l} }{ d \varphi}  =\frac{\hat{R}^2 \hat{B}}{\hat{I}  \hat{R}_0}  \f]
-     */ 
-    void operator()( const dg::HVec& y, dg::HVec& yp) const
-    {
-        double ipol = c_.ipol()(y[0],y[1]);
-        yp[2] =  y[0]*y[0]/invB_(y[0],y[1])/ipol/R_0_;       //ds/dphi =  R^2 B/I/R_0_hat
-        yp[0] =  y[0]*c_.psipZ()(y[0],y[1])/ipol;            //dR/dphi =  R/I Psip_Z
-        yp[1] = -y[0]*c_.psipR()(y[0],y[1])/ipol ;           //dZ/dphi = -R/I Psip_R
+    DSFieldCylindrical( const dg::geo::BinaryVectorLvl0& v):v_(v) { }
+    void operator()( const dg::HVec& y, dg::HVec& yp) const {
+        double vz = v_.z()(y[0], y[1]);
+        yp[0] = v_.x()(y[0], y[1])/vz; 
+        yp[1] = v_.y()(y[0], y[1])/vz;
+        yp[2] = 1./vz;
     }
 
-    double error( const dg::HVec& x0, const dg::HVec& x1)
-    {
+    double error( const dg::HVec& x0, const dg::HVec& x1)const {
         return sqrt( (x0[0]-x1[0])*(x0[0]-x1[0]) +(x0[1]-x1[1])*(x0[1]-x1[1])+(x0[2]-x1[2])*(x0[2]-x1[2]));
     }
-    bool monitor( const dg::HVec& end){ 
+    bool monitor( const dg::HVec& end)const{ 
         if ( std::isnan(end[0]) || std::isnan(end[1]) || std::isnan(end[2]) ) 
         {
             return false;
@@ -85,9 +78,7 @@ struct DSFieldCylindrical
     }
     
     private:
-    dg::geo::TokamakMagneticField c_;
-    dg::geo::InvB invB_;
-    double R_0_;
+    dg::geo::BinaryVectorLvl0 v_;
 };
 
 struct DSField
@@ -99,13 +90,17 @@ struct DSField
         thrust::host_vector<double> b_zeta, b_eta;
         dg::pushForwardPerp( v.x(), v.y(), b_zeta, b_eta, g);
         thrust::host_vector<double> b_phi = dg::pullback( v.z(), g);
+        dg::blas1::pointwiseDivide(b_zeta, b_phi, b_zeta);
+        dg::blas1::pointwiseDivide(b_eta, b_phi, b_eta);
+        dg::blas1::transform(b_phi, b_phi, dg::INVERT());
         dzetadphi_ = dg::create::forward_transform( b_zeta, g );
         detadphi_ = dg::create::forward_transform( b_eta, g );
         dsdphi_ = dg::create::forward_transform( b_phi, g );
+
     }
     //interpolate the vectors given in the constructor on the given point
     //if point lies outside of grid boundaries zero is returned
-    void operator()(const thrust::host_vector<double>& y, thrust::host_vector<double>& yp)
+    void operator()(const thrust::host_vector<double>& y, thrust::host_vector<double>& yp) const
     {
         yp[0] = y[0], yp[1] = y[1];
         g_.get().shift_topologic( y[0], y[1], yp[0], yp[1]); //shift points onto domain
@@ -119,11 +114,10 @@ struct DSField
         }
     }
 
-    double error( const dg::HVec& x0, const dg::HVec& x1)
-    {
+    double error( const dg::HVec& x0, const dg::HVec& x1) const {
         return sqrt( (x0[0]-x1[0])*(x0[0]-x1[0]) +(x0[1]-x1[1])*(x0[1]-x1[1])+(x0[2]-x1[2])*(x0[2]-x1[2]));
     }
-    bool monitor( const dg::HVec& end){ 
+    bool monitor( const dg::HVec& end)const{ 
         if ( std::isnan(end[0]) || std::isnan(end[1]) || std::isnan(end[2]) ) 
         {
             return false;
@@ -141,15 +135,15 @@ struct DSField
 
 };
 
-///@brief Default Limiter means there is a limiter everywhere
-typedef ONE DefaultLimiter;
+///@brief Full Limiter means there is a limiter everywhere
+typedef ONE FullLimiter;
 
 ///@brief No Limiter 
 typedef ZERO NoLimiter;
 
 /**
  * @brief Integrate a field line to find whether the result lies inside or outside of the box
- * @tparam Field Must be usable in the integrateRK4 function
+ * @tparam Field Must be usable in the integrateRK() functions
  * @tparam Grid must provide 2d contains function
  */
 template < class Field, class Grid>
@@ -213,7 +207,7 @@ void boxintegrator( Field& field, const Grid& grid,
 #ifdef DG_DEBUG
         std::cerr << "point "<<coords1[0]<<" "<<coords1[1]<<" is somewhere else!\n";
 #endif //DG_DEBUG
-        double deltaS = coords1[2];
+        double deltaPhi = phi1;
         BoxIntegrator<Field, Grid> boxy( field, grid, eps);
         boxy.set_coords( coords0); //nimm alte koordinaten
         if( phi1 > 0)
@@ -234,11 +228,55 @@ void boxintegrator( Field& field, const Grid& grid,
         if (!(coords1[0] < grid.x1())) { coords1[0]=grid.x1();}
         if (!(coords1[1] > grid.y0())) { coords1[1]=grid.y0();}
         if (!(coords1[1] < grid.y1())) { coords1[1]=grid.y1();}
-        // we should take the "wrong" long Delta s instead of the short one to avoid deteriorated CFL condition:
-        coords1[2] = deltaS;
-        else std::cerr << "PER NOT IMPLEMENTED "<<std::endl;
+        //now assume the rest is purely toroidal
+        double deltaS = coords1[2];
+        field(coords1, coords0); //we are just interested in coords0[2]
+        coords1[2] = deltaS + (deltaPhi-phi1)*coords0[2]; // ds + dphi*f[2]
     }
 }
+
+
+namespace detail{
+
+//used in constructor of FieldAligned
+void integrate_all_fieldlines2d( const dg::geo::BinaryVectorLvl0& vec, const aGeometry2d* g2dFine_ptr, std::vector<thrust::host_vector<double>& yp_result, std::vector<thrust::host_vector<double>& ym_result )
+{
+    std::vector<thrust::host_vector<double> > y( 3, dg::evaluate( dg::cooX2d, *g2dFine_ptr)); //x
+    y[1] = dg::evaluate( dg::cooY2d, *g2dFine_ptr); //y
+    y[2] = dg::evaluate( dg::zero, *g2dFine_ptr); //s
+    std::vector<thrust::host_vector<double> > yp( 3, y[0]), ym(yp); 
+    //construct field on high polynomial grid, then integrate it
+    aGeometry2d* g2dField_ptr = g2dCoarse_ptr->clone();
+    g2dField_ptr->set( 11, g2dField_ptr->Nx(), g2dField_ptr->Ny());
+    dg::DSField field( vec, *g2dField_ptr);
+    //field in case of cartesian grid
+    dg::DSFieldCylindrical cyl_field(vec);
+#ifdef _OPENMP
+#pragma omp parallel for shared(field, cyl_field)
+#endif //_OPENMP
+    for( unsigned i=0; i<g2dFine.size(); i++)
+    {
+        thrust::host_vector<double> coords(3), coordsP(3), coordsM(3);
+        coords[0] = y[0][i], coords[1] = y[1][i], coords[2] = y[2][i]; //x,y,s
+        double phi1 = deltaPhi;
+        if( dynamic_cast<dg::CartesianGrid2d*>( g2dFine_ptr))
+            boxintegrator( cyl_field, *g2dFine_ptr, coords, coordsP, phi1, eps);
+        else 
+            boxintegrator( field, *g2dFine_ptr, coords, coordsP, phi1, eps);
+        phi1 =  - deltaPhi;
+        if( dynamic_cast<dg::CartesianGrid2d*>( g2dFine_ptr))
+            boxintegrator( cyl_field, *g2dFine_ptr, coords, coordsM, phi1, eps);
+        else 
+            boxintegrator( field, *g2dFine_ptr, coords, coordsM, phi1, eps);
+        yp[0][i] = coordsP[0], yp[1][i] = coordsP[1], yp[2][i] = coordsP[2];
+        ym[0][i] = coordsM[0], ym[1][i] = coordsM[1], ym[2][i] = coordsM[2];
+    }
+    yp_result=yp;
+    ym_result=ym;
+}
+
+}//namespace detail
+
 //////////////////////////////FieldAlignedCLASS////////////////////////////////////////////
 /**
 * @brief Class for the evaluation of a parallel derivative
@@ -265,19 +303,23 @@ struct FieldAligned
         is a limiter and 0 if there isn't. 
         If a field line crosses the limiter in the plane \f$ \phi=0\f$ then the limiter boundary conditions apply. 
     * @param vec The field to integrate
-    * @param grid The grid on which to operate
+    * @param grid The grid on which to operate defines the parallel boundary condition in case there is a limiter.
+    * @param multiplyX defines the resolution in X of the fine grid relative to grid
+    * @param multiplyY defines the resolution in Y of the fine grid relative to grid
     * @param eps Desired accuracy of runge kutta
     * @param limit Instance of the limiter class (Default is a limiter everywhere, 
-        note that if bcz is periodic it doesn't matter if there is a limiter or not)
-    * @param globalbcx Choose NEU or DIR. Defines the interpolation behaviour when a fieldline intersects the boundary box in the perpendicular direction
-    * @param globalbcy Choose NEU or DIR. Defines the interpolation behaviour when a fieldline intersects the boundary box in the perpendicular direction
+        note that if grid.bcz() is periodic it doesn't matter if there is a limiter or not)
+    * @param globalbcx Defines the interpolation behaviour when a fieldline intersects the boundary box in the perpendicular direction
+    * @param globalbcy Defines the interpolation behaviour when a fieldline intersects the boundary box in the perpendicular direction
+    * @param dependsOnX performance indicator for the fine 2 coarse operations (elements of vec depend on first coordinate yes or no)
+    * @param dependsOnY performance indicator for the fine 2 coarse operations (elements of vec depend on second coordinate yes or no)
     * @param deltaPhi Is either <0 (then it's ignored), may differ from hz() only if Nz() == 1
     * @note If there is a limiter, the boundary condition on the first/last plane is set 
-        by the bcz variable from the grid and can be changed by the set_boundaries function. 
+        by the grid.bcz() variable and can be changed by the set_boundaries function. 
         If there is no limiter the boundary condition is periodic.
     */
     template <class Limiter>
-    FieldAligned(const dg::geo::BinaryVectorLvl0& vec, const Geometry& grid, unsigned multiplyX, unsigned multiplyY, double eps = 1e-4, Limiter limit = DefaultLimiter(), dg::bc globalbcx = dg::DIR, dg::bc globalbcy = dg::DIR, bool dependsOnX=true, bool dependsOnY=true, double deltaPhi = -1);
+    FieldAligned(const dg::geo::BinaryVectorLvl0& vec, const Geometry& grid, unsigned multiplyX, unsigned multiplyY, double eps = 1e-4, Limiter limit = FullLimiter(), dg::bc globalbcx = dg::DIR, dg::bc globalbcy = dg::DIR, bool dependsOnX=true, bool dependsOnY=true, double deltaPhi = -1);
 
     /**
     * @brief Set boundary conditions in the limiter region
@@ -349,43 +391,33 @@ struct FieldAligned
     template< class BinaryOp, class UnaryOp>
     container evaluate( BinaryOp f, UnaryOp g, unsigned p0, unsigned rounds) const;
 
-    void operator()(enum whichMatrix which, const container& in, container& out);
-    /**
-    * @brief Applies the interpolation to the next planes 
-    *
-    * @param which specify what interpolation should be applied
-    * @param in input 
-    * @param out output may not equal intpu
-    */
-    void einsPlus( enum whichMatrix which, const container& in, container& out);
     /**
-    * @brief Applies the interpolation to the previous planes
+    * @brief Applies the interpolation 
     *
     * @param which specify what interpolation should be applied
     * @param in input 
-    * @param out output may not equal intpu
+    * @param out output may not equal input
     */
-    void einsMinus(enum whichMatrix which, const container& in, container& out);
+    void operator()(enum whichMatrix which, const container& in, container& out);
 
     /**
     * @brief hz is the distance between the plus and minus planes
-    *
     * @return three-dimensional vector
     */
     const container& hz()const {return hz_;}
     /**
     * @brief hp is the distance between the plus and current planes
-    *
     * @return three-dimensional vector
     */
     const container& hp()const {return hp_;}
     /**
     * @brief hm is the distance between the current and minus planes
-    *
     * @return three-dimensional vector
     */
     const container& hm()const {return hm_;}
     private:
+    void einsPlus( enum whichMatrix which, const container& in, container& out);
+    void einsMinus(enum whichMatrix which, const container& in, container& out);
     typedef cusp::array1d_view< typename container::iterator> View;
     typedef cusp::array1d_view< typename container::const_iterator> cView;
     IMatrix plus, minus, plusT, minusT; //interpolation matrices
@@ -402,53 +434,32 @@ struct FieldAligned
 
 template<class Geometry, class IMatrix, class container>
 template <class Limiter>
-FieldAligned<Geometry, IMatrix, container>::FieldAligned(const dg::geo::BinaryVectorLvl0& mag, const Geometry& grid, unsigned mx, unsigned my, double eps, Limiter limit, dg::bc globalbcx, dg::bc globalbcy, bool dependsOnX, bool dependsOnY, double deltaPhi):
+FieldAligned<Geometry, IMatrix, container>::FieldAligned(const dg::geo::BinaryVectorLvl0& vec, const Geometry& grid, unsigned mx, unsigned my, double eps, Limiter limit, dg::bc globalbcx, dg::bc globalbcy, bool dependsOnX, bool dependsOnY, double deltaPhi):
         hz_( dg::evaluate( dg::zero, grid)), hp_( hz_), hm_( hz_), 
         Nz_(grid.Nz()), bcz_(grid.bcz()), g_(grid)
 {
     if( deltaPhi <=0) deltaPhi = grid.hz();
     else assert( grid.Nz() == 1 || grid.hz()==deltaPhi);
     //Resize vector to 2D grid size
-
-    typename Geometry::perpendicular_grid g2dCoarse = grid.perp_grid( );
-    perp_size_ = g2dCoarse.size();
-    limiter_ = dg::evaluate( limit, g2dCoarse);
-    right_ = left_ = dg::evaluate( zero, g2dCoarse);
+    aGeometry2d* g2dCoarse_ptr = grid.perp_grid( );
+    perp_size_ = g2dCoarse_ptr->size();
+    limiter_ = dg::evaluate( limit, *g2dCoarse_ptr);
+    right_ = left_ = dg::evaluate( zero, *g2dCoarse_ptr);
     ghostM.resize( perp_size_); ghostP.resize( perp_size_);
-    //Set starting points
-    typename Geometry::perpendicular_grid g2dFine = g2dCoarse.muliplyCellNumbers( (double)mx, (double)my);
-    
-    std::vector<thrust::host_vector<double> > y( 3, dg::evaluate( dg::cooX2d, g2dFine)); //x
-    y[1] = dg::evaluate( dg::cooY2d, g2dFine); //y
-    y[2] = dg::evaluate( dg::zero, g2dFine); //s
-    std::vector<thrust::host_vector<double> > yp( 3, dg::evaluate(dg::zero, g2dFine)), ym(yp); 
+    //Set starting points and integrate field lines
+    aGeometry2d* g2dFine_ptr = g2dCoarse_ptr.clone();
+    g2dFine_ptr->muliplyCellNumbers( (double)mx, (double)my);
+    std::vector<thrust::host_vector<double> > yp( 3), ym(yp); 
     dg::Timer t;
     t.tic();
-    //construct field on high polynomial grid, then integrate it
-    typename Geometry::perpendicular_grid g2dField = g2dCoarse;
-    g2dField.set( 11, g2dField.Nx(), g2dField.Ny());
-    dg::DSField field( mag, g2dField);
-#ifdef _OPENMP
-#pragma omp parallel for shared(field)
-#endif //_OPENMP
-    for( unsigned i=0; i<g2dFine.size(); i++)
-    {
-        thrust::host_vector<double> coords(3), coordsP(3), coordsM(3);
-        coords[0] = y[0][i], coords[1] = y[1][i], coords[2] = y[2][i]; //x,y,s
-        double phi1 = deltaPhi;
-        boxintegrator( field, g2dFine, coords, coordsP, phi1, eps);
-        phi1 =  - deltaPhi;
-        boxintegrator( field, g2dFine, coords, coordsM, phi1, eps);
-        yp[0][i] = coordsP[0], yp[1][i] = coordsP[1], yp[2][i] = coordsP[2];
-        ym[0][i] = coordsM[0], ym[1][i] = coordsM[1], ym[2][i] = coordsM[2];
-    }
+    detail::integrate_all_fieldlines2d( g2dFine_ptr, yp, ym);
     t.toc(); 
     std::cout << "Fieldline integration took "<<t.diff()<<"s\n";
     t.tic();
-    IMatrix plusFine  = dg::create::interpolation( yp[0], yp[1], g2dCoarse, globalbcx, globalbcy);
-    IMatrix minusFine = dg::create::interpolation( ym[0], ym[1], g2dCoarse, globalbcx, globalbcy);
-    IMatrix interpolation = dg::create::interpolation( g2dFine, g2dCoarse);
-    IMatrix projection = dg::create::projection( g2dCoarse, g2dFine);
+    IMatrix plusFine  = dg::create::interpolation( yp[0], yp[1], *g2dCoarse_ptr, globalbcx, globalbcy);
+    IMatrix minusFine = dg::create::interpolation( ym[0], ym[1], *g2dCoarse_ptr, globalbcx, globalbcy);
+    IMatrix interpolation = dg::create::interpolation( *g2dFine_ptr, *g2dCoarse_ptr);
+    IMatrix projection = dg::create::projection( *g2dCoarse_ptr, *g2dFine_ptr);
     t.toc();
     std::cout <<"Creation of interpolation/projection took "<<t.diff()<<"s\n";
     t.tic();
-- 
GitLab


From e350f4ff764f559974f528b684cbfe35d4a31fde Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 28 Aug 2017 08:49:14 -0700
Subject: [PATCH 223/453] debugging fieldaligned and ds.h

---
 inc/dg/backend/cusp_matrix_blas.cuh |  10 +--
 inc/dg/backend/grid.h               |   6 +-
 inc/dg/backend/mpi_grid.h           |  10 +--
 inc/dg/runge_kutta.h                |  11 +--
 inc/geometries/Makefile             |   2 +-
 inc/geometries/ds.h                 |  84 ++++++++++-----------
 inc/geometries/fieldaligned.h       | 113 ++++++++++++++++++----------
 inc/geometries/flux_t.cu            |   7 +-
 8 files changed, 133 insertions(+), 110 deletions(-)

diff --git a/inc/dg/backend/cusp_matrix_blas.cuh b/inc/dg/backend/cusp_matrix_blas.cuh
index 01d328707..5be2c0396 100644
--- a/inc/dg/backend/cusp_matrix_blas.cuh
+++ b/inc/dg/backend/cusp_matrix_blas.cuh
@@ -33,9 +33,8 @@ inline void doSymv( Matrix& m,
                     ThrustVectorTag  )
 {
 #ifdef DG_DEBUG
-    assert( x.size() == y.size() );
-    assert( m.num_rows == m.num_cols );
-    assert( m.num_rows == x.size() );
+    assert( m.num_rows == y.size() );
+    assert( m.num_cols == x.size() );
 #endif //DG_DEBUG
     cusp::array1d_view< typename Vector::const_iterator> cx( x.cbegin(), x.cend());
     cusp::array1d_view< typename Vector::iterator> cy( y.begin(), y.end());
@@ -51,9 +50,8 @@ inline void doSymv( Matrix& m,
                     CuspVectorTag  )
 {
 #ifdef DG_DEBUG
-    assert( x.size() == y.size() );
-    assert( m.num_rows == m.num_cols );
-    assert( m.num_rows == x.size() );
+    assert( m.num_rows == y.size() );
+    assert( m.num_cols == x.size() );
 #endif //DG_DEBUG
     cusp::multiply( m, x, y);
 }
diff --git a/inc/dg/backend/grid.h b/inc/dg/backend/grid.h
index a9b8d01d2..9c0676dbf 100644
--- a/inc/dg/backend/grid.h
+++ b/inc/dg/backend/grid.h
@@ -292,7 +292,7 @@ struct aTopology2d
     * @param fx new number of cells is the nearest integer to fx*Nx()
     * @param fy new number of cells is the nearest integer to fy*Ny()
     */
-    void multiplyCellNumber( double fx, double fy){
+    void multiplyCellNumbers( double fx, double fy){
         do_set(n(), round(fx*(double)Nx()), round(fy*(double)Ny()));
     }
     /**
@@ -419,8 +419,8 @@ struct aTopology3d
 {
     typedef SharedTag memory_category;
     typedef ThreeDimensionalTag dimensionality;
-    ///@copydoc aTopology2d::multiplyCellNumber()
-    void multiplyCellNumber( double fx, double fy){
+    ///@copydoc aTopology2d::multiplyCellNumbers()
+    void multiplyCellNumbers( double fx, double fy){
         set(n(), round(fx*(double)Nx()), round(fy*(double)Ny()), Nz());
     }
     /**
diff --git a/inc/dg/backend/mpi_grid.h b/inc/dg/backend/mpi_grid.h
index 2b1aef09b..220dd6a34 100644
--- a/inc/dg/backend/mpi_grid.h
+++ b/inc/dg/backend/mpi_grid.h
@@ -35,13 +35,13 @@ struct aMPITopology2d
     * @param fx new global number of cells is fx*global().Nx()
     * @param fy new global number of cells is fy*global().Ny()
     */
-    void multiplyCellNumber( double fx, double fy){
+    void multiplyCellNumbers( double fx, double fy){
         set(g.n(), floor(fx*(double)g.Nx()+0.5), floor(fy*(double)g.Ny()+0.5));
     }
     /**
     * @copydoc Grid2d::set(unsigned,unsigned,unsigned)
     * @attention these are global parameters, i.e. set( g.n(), 2*g.Nx(), 2*g.Ny()) is NOT(!) what you want
-    *           use the multiplyCellNumber function instead, or set( g.n(), 2*g.global().Nx(), 2*g.global().Ny())
+    *           use the multiplyCellNumbers function instead, or set( g.n(), 2*g.global().Nx(), 2*g.global().Ny())
     */
     void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny) {
         check_division( new_Nx, new_Ny, g.bcx(), g.bcy());
@@ -342,14 +342,14 @@ struct aMPITopology3d
 {
     typedef MPITag memory_category;
     typedef ThreeDimensionalTag dimensionality;
-    ///@copydoc aMPITopology2d::multiplyCellNumber()
-    void multiplyCellNumber( double fx, double fy){
+    ///@copydoc aMPITopology2d::multiplyCellNumbers()
+    void multiplyCellNumbers( double fx, double fy){
         set(g.n(), round(fx*(double)g.Nx()), round(fy*(double)g.Ny()), g.Nz());
     }
     /**
      * @copydoc Grid3d::set(unsigned,unsigned,unsigned,unsigned)
      * @attention these are global parameters, i.e. set( g.n(), 2*g.Nx(), 2*g.Ny(), 2*g.Nz()) is NOT(!) what you want
-     *           use the multiply function instead, or set( g.n(), 2*g.global().Nx(), 2*g.global().Ny(), 2*g.global().Nz())
+     *           use the multiplyCellNumbers function instead, or set( g.n(), 2*g.global().Nx(), 2*g.global().Ny(), 2*g.global().Nz())
      */
     void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz) {
         check_division( new_Nx,new_Ny,new_Nz,g.bcx(),g.bcy(),g.bcz());
diff --git a/inc/dg/runge_kutta.h b/inc/dg/runge_kutta.h
index e84b20fd0..a5bef9075 100644
--- a/inc/dg/runge_kutta.h
+++ b/inc/dg/runge_kutta.h
@@ -554,7 +554,7 @@ void stepperRK17(RHS& rhs, const container& begin, container& end, double T_min,
  * @param begin initial condition (size 3)
  * @param end (write-only) contains solution on output
  * @param T_max time difference
- * @param eps_abs desired absolute accuracy
+ * @param eps_abs desired accuracy in the error function between end and end_old
  * @return 0 if converged, -1 and a warning to std::cerr when isnan appears, -2 if failed to reach eps_abs
  */
 template< class RHS, class container, unsigned s>
@@ -590,16 +590,7 @@ int integrateRK(RHS& rhs, const container& begin, container& end, double T_max,
         }  
         error = rhs.error( end, old_end);
         old_end = end;
-#ifdef DG_DEBUG
-#ifdef MPI_VERSION
-        int rank;
-        MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-        if(rank==0)
-#endif //MPI
-        std::cout << "NT "<<NT<<" dt "<<dt<<" error "<<error<<"\n";
-#endif //DG_DEBUG
     }
-
     if( std::isnan( error) )
     {
         std::cerr << "ATTENTION: Runge Kutta failed to converge. Error is NAN! "<<std::endl;
diff --git a/inc/geometries/Makefile b/inc/geometries/Makefile
index 012d6556a..9c03a18af 100644
--- a/inc/geometries/Makefile
+++ b/inc/geometries/Makefile
@@ -18,7 +18,7 @@ geometry_diag: geometry_diag.cu solovev.h
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(LIBS) $(INCLUDE) $(JSONLIB) -g
 
 %_t: %_t.cu %.h
-	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -g 
+	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -DDG_DEBUG -g 
 
 %_mpit: %_mpit.cu 
 	$(MPICC) $(INCLUDE) $(MPICFLAGS)  $< -o $@ -g $(LIBS) $(JSONLIB)
diff --git a/inc/geometries/ds.h b/inc/geometries/ds.h
index 83196195e..8625feb2f 100644
--- a/inc/geometries/ds.h
+++ b/inc/geometries/ds.h
@@ -29,21 +29,20 @@ arbitrary coordinates
 * @tparam Matrix The matrix class of the jump matrix
 * @tparam container The container-class on which the interpolation matrix operates on (does not need to be dg::HVec)
 */
-template< class IMatrix, class Matrix, class container >
+template< class Geometry, class IMatrix, class Matrix, class container >
 struct DS
 {
     /**
     * @brief Construct from a field and a grid
     *
-    * @tparam InvB The inverse magnitude of the magnetic field \f$ \frac{1}{B}\f$
-    * @param field The fieldaligned object containing interpolation matrices
-    * @param invB The inverse magentic field strength
+    * @param mag Take the magnetic field as 
     * @param no norm or not_normed affects the behaviour of the symv function
     * @param dir the direction affects both the operator() and the symv function
     @param jumpX determines if a jump matrix is added in X-direction
+    * @param dependsOnX performance indicator for the fine 2 coarse operations (elements of vec depend on first coordinate yes or no)
+    * @param dependsOnY performance indicator for the fine 2 coarse operations (elements of vec depend on second coordinate yes or no)
     */
-    template<class Geometry>
-    DS(const dg::geo::TokamakMagneticField& field, const Geometry& g, dg::norm no=dg::normed, dg::direction dir = dg::centered, bool dependsOnX = true, bool dependsOnY=true, bool dependsOnZ=false, unsigned mx=1, unsigned my=1);
+    DS(const dg::geo::TokamakMagneticField& mag, const Geometry& g, dg::norm no=dg::normed, dg::direction dir = dg::centered, bool dependsOnX = true, bool dependsOnY=true, unsigned mx=1, unsigned my=1);
 
     /**
     * @brief Apply the forward derivative on a 3d vector
@@ -196,9 +195,9 @@ struct DS
     *
     * @return acces to fieldaligned object
     */
-    const FieldAligned<IMatrix, container>& fieldaligned() const{return f_;}
+    const FieldAligned<Geometry, IMatrix, container>& fieldaligned() const{return f_;}
     private:
-    FieldAligned<IMatrix,container> f_;
+    FieldAligned<Geometry,IMatrix,container> f_;
     Matrix jumpX, jumpY;
     container tempP, temp0, tempM;
     container f, dsf;
@@ -208,29 +207,28 @@ struct DS
     dg::norm no_;
     dg::direction dir_;
     bool apply_jumpX_;
-    container volume_;
+    dg::SparseElement<container> volume_;
 };
 
 ///@cond
 ////////////////////////////////////DEFINITIONS////////////////////////////////////////
 
-template<class I, class M, class container>
-template <class Geometry>
-DS< I, M,container>::DS(const BinaryVectorLvl1& vector, Geometry grid, dg::norm no, dg::direction dir, bool jumpX, bool jumpY, unsigned mx, unsigned my):
+template<class Geometry, class I, class M, class container>
+DS<Geometry, I, M,container>::DS(const dg::geo::TokamakMagneticField& mag, const Geometry& grid, dg::norm no, dg::direction dir, bool jumpX, bool jumpY, unsigned mx, unsigned my):
+        f_( dg::geo::BinaryVectorLvl0( dg::geo::BHatR(mag), dg::geo::BHatZ(mag), dg::geo::BHatP(mag)), grid, mx, my, 1e-5, FullLimiter()),
         jumpX( dg::create::jumpX( grid)),
         jumpY( dg::create::jumpY( grid)),
-        tempP( dg::evaluate( dg::zero, grid())), temp0( tempP), tempM( tempP), 
+        tempP( dg::evaluate( dg::zero, grid)), temp0( tempP), tempM( tempP), 
         f(tempP), dsf(tempP),
         vol3d( dg::create::volume( grid)), inv3d( dg::create::inv_volume( grid)),
-        invB(dg::pullback(dg::geo::InvB(mag),grid)), 
+        invB(dg::pullback(dg::geo::InvB(mag), grid)), 
         no_(no), dir_(dir), apply_jumpX_(jumpX)
 {
-    volume_ = dg::evaluate( dg::one, grid);
-    dg::geo::multiplyVolume( volume_, grid);
+    volume_ = dg::tensor::volume(grid.metric());
 }
 
-template<class I, class M, class container>
-inline void DS<I,M,container>::operator()( const container& f, container& dsf) { 
+template<class G, class I, class M, class container>
+inline void DS<G,I,M,container>::operator()( const container& f, container& dsf) { 
     if( dir_ == dg::centered)
         return centered( f, dsf);
     else if( dir_ == dg::forward)
@@ -240,8 +238,8 @@ inline void DS<I,M,container>::operator()( const container& f, container& dsf) {
 }
 
 
-template<class I, class M, class container>
-void DS<I,M,container>::centered( const container& f, container& dsf)
+template<class G, class I, class M, class container>
+void DS<G, I,M,container>::centered( const container& f, container& dsf)
 {
     //direct discretisation
     assert( &f != &dsf);
@@ -251,8 +249,8 @@ void DS<I,M,container>::centered( const container& f, container& dsf)
     dg::blas1::pointwiseDivide( tempM, f_.hz(), dsf);
 }
 
-template<class I, class M, class container>
-void DS<I,M,container>::centeredAdj( const container& f, container& dsf)
+template<class G, class I, class M, class container>
+void DS<G, I,M,container>::centeredAdj( const container& f, container& dsf)
 {               
     //adjoint discretisation
     assert( &f != &dsf);    
@@ -264,8 +262,8 @@ void DS<I,M,container>::centeredAdj( const container& f, container& dsf)
     dg::blas1::pointwiseDot( inv3d, dsf, dsf); 
 }
 
-template<class I, class M, class container>
-void DS<I,M,container>::centeredAdjDir( const container& f, container& dsf)
+template<class G, class I, class M, class container>
+void DS<G,I,M,container>::centeredAdjDir( const container& f, container& dsf)
 {       
 //     Direct discretisation
     assert( &f != &dsf);    
@@ -277,8 +275,8 @@ void DS<I,M,container>::centeredAdjDir( const container& f, container& dsf)
     dg::blas1::pointwiseDivide( dsf, invB, dsf);
 }
 
-template<class I, class M, class container>
-void DS<I,M,container>::forward( const container& f, container& dsf)
+template<class G, class I, class M, class container>
+void DS<G,I,M,container>::forward( const container& f, container& dsf)
 {
     //direct
     assert( &f != &dsf);
@@ -287,8 +285,8 @@ void DS<I,M,container>::forward( const container& f, container& dsf)
     dg::blas1::pointwiseDivide( tempP, f_.hp(), dsf);
 }
 
-template<class I, class M, class container>
-void DS<I,M,container>::forwardAdj( const container& f, container& dsf)
+template<class G, class I, class M, class container>
+void DS<G,I,M,container>::forwardAdj( const container& f, container& dsf)
 {    
     //adjoint discretisation
     assert( &f != &dsf);
@@ -299,8 +297,8 @@ void DS<I,M,container>::forwardAdj( const container& f, container& dsf)
     dg::blas1::pointwiseDot( inv3d, dsf, dsf); 
 }
 
-template<class I, class M, class container>
-void DS<I,M,container>::forwardAdjDir( const container& f, container& dsf)
+template<class G, class I, class M, class container>
+void DS<G,I,M,container>::forwardAdjDir( const container& f, container& dsf)
 {
     //direct discretisation
     assert( &f != &dsf);
@@ -311,8 +309,8 @@ void DS<I,M,container>::forwardAdjDir( const container& f, container& dsf)
     dg::blas1::pointwiseDivide( dsf, invB, dsf);
 }
 
-template<class I, class M, class container>
-void DS<I,M,container>::backward( const container& f, container& dsf)
+template<class G,class I, class M, class container>
+void DS<G,I,M,container>::backward( const container& f, container& dsf)
 {
     //direct
     assert( &f != &dsf);
@@ -321,8 +319,8 @@ void DS<I,M,container>::backward( const container& f, container& dsf)
     dg::blas1::pointwiseDivide( tempM, f_.hm(), dsf);
 }
 
-template<class I, class M, class container>
-void DS<I,M,container>::backwardAdj( const container& f, container& dsf)
+template<class G,class I, class M, class container>
+void DS<G,I,M,container>::backwardAdj( const container& f, container& dsf)
 {    
     //adjoint discretisation
     assert( &f != &dsf);
@@ -333,8 +331,8 @@ void DS<I,M,container>::backwardAdj( const container& f, container& dsf)
     dg::blas1::pointwiseDot( inv3d, dsf, dsf); 
 }
 
-template<class I, class M, class container>
-void DS<I,M,container>::backwardAdjDir( const container& f, container& dsf)
+template<class G,class I, class M, class container>
+void DS<G,I,M,container>::backwardAdjDir( const container& f, container& dsf)
 {
     //direct
     assert( &f != &dsf);
@@ -345,8 +343,8 @@ void DS<I,M,container>::backwardAdjDir( const container& f, container& dsf)
     dg::blas1::pointwiseDivide( dsf, invB, dsf);
 }
 
-template<class I, class M, class container>
-void DS<I,M,container>::symv( const container& f, container& dsTdsf)
+template<class G,class I, class M, class container>
+void DS<G,I,M,container>::symv( const container& f, container& dsTdsf)
 {
     if(dir_ == dg::centered)
     {
@@ -366,11 +364,11 @@ void DS<I,M,container>::symv( const container& f, container& dsTdsf)
     if(apply_jumpX_)
     {
         dg::blas2::symv( jumpX, f, temp0);
-        dg::blas1::pointwiseDivide( temp0, volume_, temp0);
-        dg::blas1::axpby( -1., temp0c, 1., dsTdsf, dsTdsf);
+        dg::tensor::pointwiseDivide( temp0, volume_, temp0);
+        dg::blas1::axpby( -1., temp0, 1., dsTdsf, dsTdsf);
     }
     dg::blas2::symv( jumpY, f, temp0);
-    dg::blas1::pointwiseDivide( temp0, volume_, temp0);
+    dg::tensor::pointwiseDivide( temp0, volume_, temp0);
     dg::blas1::axpby( -1., temp0, 1., dsTdsf, dsTdsf);
     if( no_ == not_normed)
     {
@@ -379,8 +377,8 @@ void DS<I,M,container>::symv( const container& f, container& dsTdsf)
 }
 
 //enables the use of the dg::blas2::symv function 
-template< class I, class M, class V>
-struct MatrixTraits< DS<I,M, V> >
+template< class G, class I, class M, class V>
+struct MatrixTraits< DS<G,I,M, V> >
 {
     typedef double value_type;
     typedef SelfMadeMatrixTag matrix_category;
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index 0a48d2d2b..ea47e8f09 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -85,45 +85,47 @@ struct DSField
 {
     
     //z component of v may not vanish
-    DSField( const dg::geo::BinaryVectorLvl0& v, const aGeometry2d& g)
+    DSField( const dg::geo::BinaryVectorLvl0& v, const aGeometry2d& g): g_(g)
     {
-        thrust::host_vector<double> b_zeta, b_eta;
-        dg::pushForwardPerp( v.x(), v.y(), b_zeta, b_eta, g);
-        thrust::host_vector<double> b_phi = dg::pullback( v.z(), g);
-        dg::blas1::pointwiseDivide(b_zeta, b_phi, b_zeta);
-        dg::blas1::pointwiseDivide(b_eta, b_phi, b_eta);
-        dg::blas1::transform(b_phi, b_phi, dg::INVERT());
-        dzetadphi_ = dg::create::forward_transform( b_zeta, g );
-        detadphi_ = dg::create::forward_transform( b_eta, g );
-        dsdphi_ = dg::create::forward_transform( b_phi, g );
+        thrust::host_vector<double> v_zeta, v_eta;
+        dg::pushForwardPerp( v.x(), v.y(), v_zeta, v_eta, g);
+        thrust::host_vector<double> v_phi = dg::pullback( v.z(), g);
+        dg::blas1::pointwiseDivide(v_zeta, v_phi, v_zeta);
+        dg::blas1::pointwiseDivide(v_eta, v_phi, v_eta);
+        dg::blas1::transform(v_phi, v_phi, dg::INVERT<double>());
+        dzetadphi_  = dg::create::forward_transform( v_zeta, g );
+        detadphi_   = dg::create::forward_transform( v_eta, g );
+        dsdphi_     = dg::create::forward_transform( v_phi, g );
 
     }
     //interpolate the vectors given in the constructor on the given point
     //if point lies outside of grid boundaries zero is returned
     void operator()(const thrust::host_vector<double>& y, thrust::host_vector<double>& yp) const
     {
-        yp[0] = y[0], yp[1] = y[1];
-        g_.get().shift_topologic( y[0], y[1], yp[0], yp[1]); //shift points onto domain
-        if( !g_.get().contains( y[0], y[1])) yp[0] = yp[1]= yp[2] = 0;
+        double R = y[0], Z = y[1];
+        g_.get().shift_topologic( y[0], y[1], R, Z); //shift R,Z onto domain
+        if( !g_.get().contains( R, Z)) yp[0] = yp[1]= yp[2] = 0;
         else
         {
             //else interpolate
-            yp[0] = interpolate( y[0], y[1], dzetadphi_, g_.get());
-            yp[1] = interpolate( y[0], y[1], detadphi_, g_.get());
-            yp[2] = interpolate( y[0], y[1], dsdphi_, g_.get());
+            yp[0] = interpolate( R, Z, dzetadphi_, g_.get());
+            yp[1] = interpolate( R, Z, detadphi_,  g_.get());
+            yp[2] = interpolate( R, Z, dsdphi_,    g_.get());
         }
     }
 
+    ///take the sum of the absolute errors perp and parallel
     double error( const dg::HVec& x0, const dg::HVec& x1) const {
-        return sqrt( (x0[0]-x1[0])*(x0[0]-x1[0]) +(x0[1]-x1[1])*(x0[1]-x1[1])+(x0[2]-x1[2])*(x0[2]-x1[2]));
+        //here, we don't need to shift coordinates since x0 and x1 are both end points
+        return sqrt( (x0[0]-x1[0])*(x0[0]-x1[0]) +(x0[1]-x1[1])*(x0[1]-x1[1]))+sqrt((x0[2]-x1[2])*(x0[2]-x1[2]));
     }
     bool monitor( const dg::HVec& end)const{ 
         if ( std::isnan(end[0]) || std::isnan(end[1]) || std::isnan(end[2]) ) 
         {
             return false;
         }
-        //if new integrated point outside domain
-        if ((10*g_.get().x0() > end[0]  ) || (10*g_.get().x1() < end[0])  ||(10*g_.get().y0()  > end[1]  ) || (10*g_.get().y1() < end[1])||(-1e10 > end[2]  ) || (1e10 < end[2])  )
+        //if new integrated point far outside domain
+        if ((end[0] < g_.get().x0()-1e4 ) || (g_.get().x1()+1e4 < end[0])  ||(end[1] < g_.get().y0()-1e4 ) || (g_.get().y1()+1e4 < end[1])||(-1e10 > end[2]  ) || (1e10 < end[2])  )
         {
             return false;
         }
@@ -184,7 +186,7 @@ struct BoxIntegrator
 /**
  * @brief Integrate one field line in a given box, Result is guaranteed to lie inside the box
  *
- * @tparam Field Must be usable in the integrateRK4 function
+ * @tparam Field Must be usable in the integrateRK function
  * @tparam Grid must provide 2d contains function
  * @param field The field to use
  * @param grid instance of the Grid class 
@@ -230,8 +232,9 @@ void boxintegrator( Field& field, const Grid& grid,
         if (!(coords1[1] < grid.y1())) { coords1[1]=grid.y1();}
         //now assume the rest is purely toroidal
         double deltaS = coords1[2];
-        field(coords1, coords0); //we are just interested in coords0[2]
-        coords1[2] = deltaS + (deltaPhi-phi1)*coords0[2]; // ds + dphi*f[2]
+        thrust::host_vector<double> temp=coords0;
+        field(coords1, temp); //we are just interested in temp[2]
+        coords1[2] = deltaS + (deltaPhi-phi1)*temp[2]; // ds + dphi*f[2]
     }
 }
 
@@ -239,32 +242,37 @@ void boxintegrator( Field& field, const Grid& grid,
 namespace detail{
 
 //used in constructor of FieldAligned
-void integrate_all_fieldlines2d( const dg::geo::BinaryVectorLvl0& vec, const aGeometry2d* g2dFine_ptr, std::vector<thrust::host_vector<double>& yp_result, std::vector<thrust::host_vector<double>& ym_result )
+void integrate_all_fieldlines2d( const dg::geo::BinaryVectorLvl0& vec, const aGeometry2d* g2dFine_ptr, std::vector<thrust::host_vector<double> >& yp_result, std::vector<thrust::host_vector<double> >& ym_result , double deltaPhi, double eps)
 {
     std::vector<thrust::host_vector<double> > y( 3, dg::evaluate( dg::cooX2d, *g2dFine_ptr)); //x
     y[1] = dg::evaluate( dg::cooY2d, *g2dFine_ptr); //y
     y[2] = dg::evaluate( dg::zero, *g2dFine_ptr); //s
     std::vector<thrust::host_vector<double> > yp( 3, y[0]), ym(yp); 
     //construct field on high polynomial grid, then integrate it
-    aGeometry2d* g2dField_ptr = g2dCoarse_ptr->clone();
-    g2dField_ptr->set( 11, g2dField_ptr->Nx(), g2dField_ptr->Ny());
+    Timer t;
+    t.tic();
+    aGeometry2d* g2dField_ptr = g2dFine_ptr->clone();
+    //initial tests show that higher order polynomial might not be needed...
+    g2dField_ptr->set( g2dField_ptr->n(), g2dField_ptr->Nx(), g2dField_ptr->Ny());
     dg::DSField field( vec, *g2dField_ptr);
+    t.toc();
+    std::cout << "Generation of fine grid took "<<t.diff()<<"s\n";
     //field in case of cartesian grid
     dg::DSFieldCylindrical cyl_field(vec);
 #ifdef _OPENMP
 #pragma omp parallel for shared(field, cyl_field)
 #endif //_OPENMP
-    for( unsigned i=0; i<g2dFine.size(); i++)
+    for( unsigned i=0; i<g2dFine_ptr->size(); i++)
     {
         thrust::host_vector<double> coords(3), coordsP(3), coordsM(3);
         coords[0] = y[0][i], coords[1] = y[1][i], coords[2] = y[2][i]; //x,y,s
         double phi1 = deltaPhi;
-        if( dynamic_cast<dg::CartesianGrid2d*>( g2dFine_ptr))
+        if( dynamic_cast<const dg::CartesianGrid2d*>( g2dFine_ptr))
             boxintegrator( cyl_field, *g2dFine_ptr, coords, coordsP, phi1, eps);
         else 
             boxintegrator( field, *g2dFine_ptr, coords, coordsP, phi1, eps);
         phi1 =  - deltaPhi;
-        if( dynamic_cast<dg::CartesianGrid2d*>( g2dFine_ptr))
+        if( dynamic_cast<const dg::CartesianGrid2d*>( g2dFine_ptr))
             boxintegrator( cyl_field, *g2dFine_ptr, coords, coordsM, phi1, eps);
         else 
             boxintegrator( field, *g2dFine_ptr, coords, coordsM, phi1, eps);
@@ -273,6 +281,7 @@ void integrate_all_fieldlines2d( const dg::geo::BinaryVectorLvl0& vec, const aGe
     }
     yp_result=yp;
     ym_result=ym;
+    delete g2dField_ptr;
 }
 
 }//namespace detail
@@ -311,15 +320,13 @@ struct FieldAligned
         note that if grid.bcz() is periodic it doesn't matter if there is a limiter or not)
     * @param globalbcx Defines the interpolation behaviour when a fieldline intersects the boundary box in the perpendicular direction
     * @param globalbcy Defines the interpolation behaviour when a fieldline intersects the boundary box in the perpendicular direction
-    * @param dependsOnX performance indicator for the fine 2 coarse operations (elements of vec depend on first coordinate yes or no)
-    * @param dependsOnY performance indicator for the fine 2 coarse operations (elements of vec depend on second coordinate yes or no)
     * @param deltaPhi Is either <0 (then it's ignored), may differ from hz() only if Nz() == 1
     * @note If there is a limiter, the boundary condition on the first/last plane is set 
         by the grid.bcz() variable and can be changed by the set_boundaries function. 
         If there is no limiter the boundary condition is periodic.
     */
     template <class Limiter>
-    FieldAligned(const dg::geo::BinaryVectorLvl0& vec, const Geometry& grid, unsigned multiplyX, unsigned multiplyY, double eps = 1e-4, Limiter limit = FullLimiter(), dg::bc globalbcx = dg::DIR, dg::bc globalbcy = dg::DIR, bool dependsOnX=true, bool dependsOnY=true, double deltaPhi = -1);
+    FieldAligned(const dg::geo::BinaryVectorLvl0& vec, const Geometry& grid, unsigned multiplyX, unsigned multiplyY, double eps = 1e-4, Limiter limit = FullLimiter(), dg::bc globalbcx = dg::DIR, dg::bc globalbcy = dg::DIR, double deltaPhi = -1);
 
     /**
     * @brief Set boundary conditions in the limiter region
@@ -434,31 +441,57 @@ struct FieldAligned
 
 template<class Geometry, class IMatrix, class container>
 template <class Limiter>
-FieldAligned<Geometry, IMatrix, container>::FieldAligned(const dg::geo::BinaryVectorLvl0& vec, const Geometry& grid, unsigned mx, unsigned my, double eps, Limiter limit, dg::bc globalbcx, dg::bc globalbcy, bool dependsOnX, bool dependsOnY, double deltaPhi):
+FieldAligned<Geometry, IMatrix, container>::FieldAligned(const dg::geo::BinaryVectorLvl0& vec, const Geometry& grid, unsigned mx, unsigned my, double eps, Limiter limit, dg::bc globalbcx, dg::bc globalbcy, double deltaPhi):
         hz_( dg::evaluate( dg::zero, grid)), hp_( hz_), hm_( hz_), 
         Nz_(grid.Nz()), bcz_(grid.bcz()), g_(grid)
 {
     if( deltaPhi <=0) deltaPhi = grid.hz();
     else assert( grid.Nz() == 1 || grid.hz()==deltaPhi);
+    //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+    //%%%%%%%%%%%downcast grid since we don't have a virtual function perp_grid%%%%%%%%%%%%%
+    const aGeometry3d* grid_ptr = &grid;
+    aGeometry2d* g2dCoarse_ptr;
+    const dg::CartesianGrid3d* grid_cart = dynamic_cast<const dg::CartesianGrid3d*>(grid_ptr);
+    const dg::CylindricalGrid3d* grid_cyl = dynamic_cast<const dg::CylindricalGrid3d*>(grid_ptr);
+    const dg::CurvilinearProductGrid3d*  grid_curvi = dynamic_cast<const dg::CurvilinearProductGrid3d*>(grid_ptr);
+    if( grid_cart) 
+    {
+        dg::CartesianGrid2d cart = grid_cart->perp_grid();
+        g2dCoarse_ptr = cart.clone();
+    }
+    else if( grid_cyl) 
+    {
+        dg::CartesianGrid2d cart = grid_cyl->perp_grid();
+        g2dCoarse_ptr = cart.clone();
+    }
+    else if( grid_curvi) 
+    {
+        dg::CurvilinearGrid2d curv = grid_curvi->perp_grid();
+        g2dCoarse_ptr = curv.clone();
+    }
+    else
+        throw dg::Error( dg::Message(_ping_)<<"Grid class not recognized!");
+    //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     //Resize vector to 2D grid size
-    aGeometry2d* g2dCoarse_ptr = grid.perp_grid( );
     perp_size_ = g2dCoarse_ptr->size();
     limiter_ = dg::evaluate( limit, *g2dCoarse_ptr);
     right_ = left_ = dg::evaluate( zero, *g2dCoarse_ptr);
     ghostM.resize( perp_size_); ghostP.resize( perp_size_);
     //Set starting points and integrate field lines
-    aGeometry2d* g2dFine_ptr = g2dCoarse_ptr.clone();
-    g2dFine_ptr->muliplyCellNumbers( (double)mx, (double)my);
+    aGeometry2d* g2dFine_ptr = g2dCoarse_ptr->clone();
+    g2dFine_ptr->multiplyCellNumbers( (double)mx, (double)my);
     std::vector<thrust::host_vector<double> > yp( 3), ym(yp); 
+    std::cout << "Start fieldline integration!\n";
     dg::Timer t;
     t.tic();
-    detail::integrate_all_fieldlines2d( g2dFine_ptr, yp, ym);
+    detail::integrate_all_fieldlines2d( vec, g2dFine_ptr, yp, ym, deltaPhi, eps);
     t.toc(); 
     std::cout << "Fieldline integration took "<<t.diff()<<"s\n";
+    //%%%%%%%%%%%%%%%%%%Create interpolation and projection%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     t.tic();
     IMatrix plusFine  = dg::create::interpolation( yp[0], yp[1], *g2dCoarse_ptr, globalbcx, globalbcy);
     IMatrix minusFine = dg::create::interpolation( ym[0], ym[1], *g2dCoarse_ptr, globalbcx, globalbcy);
-    IMatrix interpolation = dg::create::interpolation( *g2dFine_ptr, *g2dCoarse_ptr);
+    //IMatrix interpolation = dg::create::interpolation( *g2dFine_ptr, *g2dCoarse_ptr);
     IMatrix projection = dg::create::projection( *g2dCoarse_ptr, *g2dFine_ptr);
     t.toc();
     std::cout <<"Creation of interpolation/projection took "<<t.diff()<<"s\n";
@@ -470,7 +503,7 @@ FieldAligned<Geometry, IMatrix, container>::FieldAligned(const dg::geo::BinaryVe
     //Transposed matrices work only for csr_matrix due to bad matrix form for ell_matrix!!!
     cusp::transpose( plus, plusT);
     cusp::transpose( minus, minusT);     
-    //project h and copy into h vectors
+    //%%%%%%%%%%%%%%%%%%%%%%%project h and copy into h vectors%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     thrust::host_vector<double> hp( perp_size_), hm(hp);
     dg::blas2::symv( projection, yp[2], hp);
     dg::blas2::symv( projection, ym[2], hm);
@@ -480,7 +513,9 @@ FieldAligned<Geometry, IMatrix, container>::FieldAligned(const dg::geo::BinaryVe
         thrust::copy( hm.begin(), hm.end(), hm_.begin() + i*perp_size_);
     }
     dg::blas1::scal( hm_, -1.);
-    dg::blas1::axpby(  1., hp_, +1., hm_, hz_);    //
+    dg::blas1::axpby(  1., hp_, +1., hm_, hz_);
+    delete g2dCoarse_ptr;
+    delete g2dFine_ptr;
  
 }
 
diff --git a/inc/geometries/flux_t.cu b/inc/geometries/flux_t.cu
index 3750050c9..d1317d044 100644
--- a/inc/geometries/flux_t.cu
+++ b/inc/geometries/flux_t.cu
@@ -15,6 +15,7 @@
 #include "solovev.h"
 #include "flux.h"
 #include "fieldaligned.h"
+#include "ds.h"
 #include "init.h"
 
 #include "file/nc_utilities.h"
@@ -65,7 +66,7 @@ int main( int argc, char* argv[])
     GeomParameters gp(js);
     Psip psip( gp); 
     std::cout << "Psi min "<<psip(gp.R_0, 0)<<"\n";
-    std::cout << "Type psi_0 and psi_1\n";
+    std::cout << "Type psi_0 (-20) and psi_1 (-4)\n";
     double psi_0, psi_1;
     std::cin >> psi_0>> psi_1;
     gp.display( std::cout);
@@ -167,8 +168,8 @@ int main( int argc, char* argv[])
     dg::geo::BHatP bhatP(c);
     //dg::geo::BinaryVectorLvl0 bhat( dg::geo::BHatR(c), dg::geo::BHatZ(c), dg::geo::BHatP(c));
     dg::geo::BinaryVectorLvl0 bhat( bhatR, bhatZ, bhatP);
-    dg::FieldAligned<dg::aGeometry3d, dg::IHMatrix, dg::HVec> fieldaligned( bhat, g3d, 1,1,gp.rk4eps, dg::NoLimiter() ); 
-    //dg::DS<dg::IHMatrix, dg::HMatrix, dg::HVec> ds( fieldaligned, flux::Field(gp, g3d.x(), g3d.f_x()), dg::normed, dg::centered);
+    dg::FieldAligned<dg::aGeometry3d, dg::IHMatrix, dg::HVec> fieldaligned( bhat, g3d, 1, 4, gp.rk4eps, dg::NoLimiter() ); 
+    dg::DS<dg::aGeometry3d, dg::IHMatrix, dg::HMatrix, dg::HVec> ds( c, g3d, dg::normed, dg::centered);
 
     
     t.toc();
-- 
GitLab


From 4a2d16f5be5be4c5ff9d5b0a5d53396a0fe7f950 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Tue, 29 Aug 2017 02:59:51 -0700
Subject: [PATCH 224/453] move  stuff in detail namesapce in fieldaligned and
 write clip_to_boundary function

---
 inc/geometries/ds.h           | 10 ++++----
 inc/geometries/fieldaligned.h | 43 ++++++++++++++++++++++-------------
 inc/geometries/flux_t.cu      | 39 ++++++++++++++++---------------
 3 files changed, 54 insertions(+), 38 deletions(-)

diff --git a/inc/geometries/ds.h b/inc/geometries/ds.h
index 8625feb2f..48bd02ede 100644
--- a/inc/geometries/ds.h
+++ b/inc/geometries/ds.h
@@ -35,12 +35,14 @@ struct DS
     /**
     * @brief Construct from a field and a grid
     *
-    * @param mag Take the magnetic field as 
+    * @param mag Take the magnetic field as vector field
+    * @param g  the boundary conditions are also taken from here
     * @param no norm or not_normed affects the behaviour of the symv function
     * @param dir the direction affects both the operator() and the symv function
-    @param jumpX determines if a jump matrix is added in X-direction
-    * @param dependsOnX performance indicator for the fine 2 coarse operations (elements of vec depend on first coordinate yes or no)
+    * @param dependsOnX performance indicator for the fine 2 coarse operations (elements of vec depend on first coordinate yes or no) also determines if a jump matrix is added in the x-direction
     * @param dependsOnY performance indicator for the fine 2 coarse operations (elements of vec depend on second coordinate yes or no)
+    * @param mx Multiplication factor in x
+    * @param my Multiplication factor in y
     */
     DS(const dg::geo::TokamakMagneticField& mag, const Geometry& g, dg::norm no=dg::normed, dg::direction dir = dg::centered, bool dependsOnX = true, bool dependsOnY=true, unsigned mx=1, unsigned my=1);
 
@@ -215,7 +217,7 @@ struct DS
 
 template<class Geometry, class I, class M, class container>
 DS<Geometry, I, M,container>::DS(const dg::geo::TokamakMagneticField& mag, const Geometry& grid, dg::norm no, dg::direction dir, bool jumpX, bool jumpY, unsigned mx, unsigned my):
-        f_( dg::geo::BinaryVectorLvl0( dg::geo::BHatR(mag), dg::geo::BHatZ(mag), dg::geo::BHatP(mag)), grid, mx, my, 1e-5, FullLimiter()),
+        f_( dg::geo::BinaryVectorLvl0( dg::geo::BHatR(mag), dg::geo::BHatZ(mag), dg::geo::BHatP(mag)), grid, mx, my, 1e-5, FullLimiter(), grid.bcx(), grid.bcy()),
         jumpX( dg::create::jumpX( grid)),
         jumpY( dg::create::jumpY( grid)),
         tempP( dg::evaluate( dg::zero, grid)), temp0( tempP), tempM( tempP), 
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index ea47e8f09..c26591d3b 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -143,6 +143,19 @@ typedef ONE FullLimiter;
 ///@brief No Limiter 
 typedef ZERO NoLimiter;
 
+namespace detail{
+
+
+void clip_to_boundary( thrust::host_vector<double>& x, const aTopology2d* grid)
+{
+    if (!(x[0] > grid->x0())) { x[0]=grid->x0();}
+    if (!(x[0] < grid->x1())) { x[0]=grid->x1();}
+    if (!(x[1] > grid->y0())) { x[1]=grid->y0();}
+    if (!(x[1] < grid->y1())) { x[1]=grid->y1();}
+
+}
+
+
 /**
  * @brief Integrate a field line to find whether the result lies inside or outside of the box
  * @tparam Field Must be usable in the integrateRK() functions
@@ -226,10 +239,7 @@ void boxintegrator( Field& field, const Grid& grid,
             phi1 = (dPhiMin+dPhiMax)/2.;
             dg::integrateRK4( field, coords0, coords1, dPhiMin, eps);
         }
-        if (!(coords1[0] > grid.x0())) { coords1[0]=grid.x0();}
-        if (!(coords1[0] < grid.x1())) { coords1[0]=grid.x1();}
-        if (!(coords1[1] > grid.y0())) { coords1[1]=grid.y0();}
-        if (!(coords1[1] < grid.y1())) { coords1[1]=grid.y1();}
+        detail::clip_to_boundary( coords1, &grid);
         //now assume the rest is purely toroidal
         double deltaS = coords1[2];
         thrust::host_vector<double> temp=coords0;
@@ -238,9 +248,6 @@ void boxintegrator( Field& field, const Grid& grid,
     }
 }
 
-
-namespace detail{
-
 //used in constructor of FieldAligned
 void integrate_all_fieldlines2d( const dg::geo::BinaryVectorLvl0& vec, const aGeometry2d* g2dFine_ptr, std::vector<thrust::host_vector<double> >& yp_result, std::vector<thrust::host_vector<double> >& ym_result , double deltaPhi, double eps)
 {
@@ -256,7 +263,7 @@ void integrate_all_fieldlines2d( const dg::geo::BinaryVectorLvl0& vec, const aGe
     g2dField_ptr->set( g2dField_ptr->n(), g2dField_ptr->Nx(), g2dField_ptr->Ny());
     dg::DSField field( vec, *g2dField_ptr);
     t.toc();
-    std::cout << "Generation of fine grid took "<<t.diff()<<"s\n";
+    std::cout << "Generation of interpolate grid took "<<t.diff()<<"s\n";
     //field in case of cartesian grid
     dg::DSFieldCylindrical cyl_field(vec);
 #ifdef _OPENMP
@@ -283,9 +290,10 @@ void integrate_all_fieldlines2d( const dg::geo::BinaryVectorLvl0& vec, const aGe
     ym_result=ym;
     delete g2dField_ptr;
 }
-
 }//namespace detail
 
+
+
 //////////////////////////////FieldAlignedCLASS////////////////////////////////////////////
 /**
 * @brief Class for the evaluation of a parallel derivative
@@ -423,8 +431,8 @@ struct FieldAligned
     */
     const container& hm()const {return hm_;}
     private:
-    void einsPlus( enum whichMatrix which, const container& in, container& out);
-    void einsMinus(enum whichMatrix which, const container& in, container& out);
+    void ePlus( enum whichMatrix which, const container& in, container& out);
+    void eMinus(enum whichMatrix which, const container& in, container& out);
     typedef cusp::array1d_view< typename container::iterator> View;
     typedef cusp::array1d_view< typename container::const_iterator> cView;
     IMatrix plus, minus, plusT, minusT; //interpolation matrices
@@ -478,11 +486,14 @@ FieldAligned<Geometry, IMatrix, container>::FieldAligned(const dg::geo::BinaryVe
     right_ = left_ = dg::evaluate( zero, *g2dCoarse_ptr);
     ghostM.resize( perp_size_); ghostP.resize( perp_size_);
     //Set starting points and integrate field lines
+    dg::Timer t;
+    t.tic();
     aGeometry2d* g2dFine_ptr = g2dCoarse_ptr->clone();
     g2dFine_ptr->multiplyCellNumbers( (double)mx, (double)my);
+    t.toc();
+    std::cout << "Create fine grid took "<<t.diff()<<"s\n";
     std::vector<thrust::host_vector<double> > yp( 3), ym(yp); 
     std::cout << "Start fieldline integration!\n";
-    dg::Timer t;
     t.tic();
     detail::integrate_all_fieldlines2d( vec, g2dFine_ptr, yp, ym, deltaPhi, eps);
     t.toc(); 
@@ -609,12 +620,12 @@ container FieldAligned<G, I,container>::evaluate( BinaryOp binary, UnaryOp unary
 template<class G, class I, class container>
 void FieldAligned<G, I, container >::operator()(enum whichMatrix which, const container& f, container& fe)
 {
-    if(which == einsPlus || which == einsMinusT) einsPlus( which, f, fe);
-    if(which == einsMinus || which == einsPlusT) einsMinus( which, f, fe);
+    if(which == einsPlus || which == einsMinusT) ePlus( which, f, fe);
+    if(which == einsMinus || which == einsPlusT) eMinus( which, f, fe);
 }
 
 template< class G, class I, class container>
-void FieldAligned<G, I, container>::einsPlus( enum whichMatrix which, const container& f, container& fpe)
+void FieldAligned<G, I, container>::ePlus( enum whichMatrix which, const container& f, container& fpe)
 {
     View ghostPV( ghostP.begin(), ghostP.end());
     View ghostMV( ghostM.begin(), ghostM.end());
@@ -649,7 +660,7 @@ void FieldAligned<G, I, container>::einsPlus( enum whichMatrix which, const cont
 }
 
 template< class G, class I, class container>
-void FieldAligned<G, I, container>::einsMinus( enum whichMatrix which, const container& f, container& fme)
+void FieldAligned<G, I, container>::eMinus( enum whichMatrix which, const container& f, container& fme)
 {
     //note that thrust functions don't work on views
     View ghostPV( ghostP.begin(), ghostP.end());
diff --git a/inc/geometries/flux_t.cu b/inc/geometries/flux_t.cu
index d1317d044..a01edbf97 100644
--- a/inc/geometries/flux_t.cu
+++ b/inc/geometries/flux_t.cu
@@ -77,7 +77,7 @@ int main( int argc, char* argv[])
     dg::geo::TokamakMagneticField c = dg::geo::createSolovevField( gp);
     dg::geo::FluxGenerator flux( c.get_psip(), c.get_ipol(), psi_0, psi_1, gp.R_0, 0., 1);
     dg::CurvilinearProductGrid3d g3d(flux, n, Nx, Ny,Nz, dg::DIR);
-    dg::CurvilinearGrid2d g2d(flux, n, Nx,Ny, dg::DIR);
+    dg::CurvilinearGrid2d g2d(flux, n, Nx,Ny, dg::NEU);
     dg::Grid2d g2d_periodic(g2d.x0(), g2d.x1(), g2d.y0(), g2d.y1(), g2d.n(), g2d.Nx(), g2d.Ny()+1); 
     t.toc();
     std::cout << "Construction took "<<t.diff()<<"s"<<std::endl;
@@ -168,8 +168,11 @@ int main( int argc, char* argv[])
     dg::geo::BHatP bhatP(c);
     //dg::geo::BinaryVectorLvl0 bhat( dg::geo::BHatR(c), dg::geo::BHatZ(c), dg::geo::BHatP(c));
     dg::geo::BinaryVectorLvl0 bhat( bhatR, bhatZ, bhatP);
-    dg::FieldAligned<dg::aGeometry3d, dg::IHMatrix, dg::HVec> fieldaligned( bhat, g3d, 1, 4, gp.rk4eps, dg::NoLimiter() ); 
-    dg::DS<dg::aGeometry3d, dg::IHMatrix, dg::HMatrix, dg::HVec> ds( c, g3d, dg::normed, dg::centered);
+    unsigned mx, my;
+    std::cout << "Type multipleX and multipleY!\n";
+    std::cin >> mx >> my;
+    //dg::FieldAligned<dg::aGeometry3d, dg::IHMatrix, dg::HVec> fieldaligned( bhat, g3d, 1, 4, gp.rk4eps, dg::NoLimiter() ); 
+    dg::DS<dg::aGeometry3d, dg::IHMatrix, dg::HMatrix, dg::HVec> ds( c, g3d, dg::normed, dg::centered, false, true, mx, my);
 
     
     t.toc();
@@ -179,21 +182,21 @@ int main( int argc, char* argv[])
     dg::HVec gradLnB = dg::pullback( dg::geo::GradLnB(c), g3d);
     dg::blas1::pointwiseDivide( ones3d, B, B);
     dg::HVec function = dg::pullback( dg::geo::FuncNeu(c), g3d), derivative(function);
-    //ds( function, derivative);
-
-    //ds.centeredT( B, divB);
-    //double norm =  sqrt( dg::blas2::dot(divB, vol3d, divB));
-    //std::cout << "Divergence of B is "<<norm<<"\n";
-
-    //ds.centered( lnB, gradB);
-    //std::cout << "num. norm of gradLnB is "<<sqrt( dg::blas2::dot( gradB,vol3d, gradB))<<"\n";
-    //norm = sqrt( dg::blas2::dot( gradLnB, vol3d, gradLnB) );
-    //std::cout << "ana. norm of gradLnB is "<<norm<<"\n";
-    //dg::blas1::axpby( 1., gradB, -1., gradLnB, gradLnB);
-    //X = divB;
-    //err = nc_put_var_double( ncid, varID[4], periodify(X, g2d_periodic).data());
-    //double norm2 = sqrt(dg::blas2::dot(gradLnB, vol3d,gradLnB));
-    //std::cout << "rel. error of lnB is    "<<norm2/norm<<"\n";
+    ds( function, derivative);
+
+    ds.centeredAdj( B, divB);
+    double norm =  sqrt( dg::blas2::dot(divB, vol3d, divB));
+    std::cout << "Divergence of B is "<<norm<<"\n";
+
+    ds.centered( lnB, gradB);
+    std::cout << "num. norm of gradLnB is "<<sqrt( dg::blas2::dot( gradB,vol3d, gradB))<<"\n";
+    norm = sqrt( dg::blas2::dot( gradLnB, vol3d, gradLnB) );
+    std::cout << "ana. norm of gradLnB is "<<norm<<"\n";
+    dg::blas1::axpby( 1., gradB, -1., gradLnB, gradLnB);
+    X = divB;
+    err = nc_put_var_double( ncid, varID[4], periodify(X, g2d_periodic).data());
+    double norm2 = sqrt(dg::blas2::dot(gradLnB, vol3d, gradLnB));
+    std::cout << "rel. error of lnB is    "<<norm2/norm<<"\n";
 
     err = nc_close( ncid);
     return 0;
-- 
GitLab


From 5a53c24c83166a1d0085ad03d742aaa4008c6925 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Tue, 29 Aug 2017 07:05:27 -0700
Subject: [PATCH 225/453] added some documentation

---
 inc/dg/cg.h                     |  3 +++
 inc/geometries/ds_b.cu          | 20 ++++++++++----------
 inc/geometries/magnetic_field.h |  4 ++--
 3 files changed, 15 insertions(+), 12 deletions(-)

diff --git a/inc/dg/cg.h b/inc/dg/cg.h
index 0e82f1de0..ca58faa80 100644
--- a/inc/dg/cg.h
+++ b/inc/dg/cg.h
@@ -137,6 +137,7 @@ class CG
     NOTE: the same comparison hold for A with the result that A contains 
     significantly more elements than z whence ddot(r,A,r) is far slower than ddot(r,z)
 */
+///@cond
 template< class container>
 template< class Matrix, class Preconditioner>
 unsigned CG< container>::operator()( Matrix& A, container& x, const container& b, Preconditioner& P, value_type eps, value_type nrmb_correction)
@@ -254,6 +255,8 @@ unsigned CG< container>::operator()( Matrix& A, container& x, const container& b
     }
     return max_iter;
 }
+///@endcond
+
 
 
 /**
diff --git a/inc/geometries/ds_b.cu b/inc/geometries/ds_b.cu
index 2e3288308..9ee8ff0d9 100644
--- a/inc/geometries/ds_b.cu
+++ b/inc/geometries/ds_b.cu
@@ -28,14 +28,14 @@ int main()
     unsigned n, Nx, Ny, Nz;
     std::cin >> n>> Nx>>Ny>>Nz;
     std::cout << "You typed "<<n<<" "<<Nx<<" "<<Ny<<" "<<Nz<<std::endl;
-    dg::CylindricalGrid<dg::DVec> g3d( R_0 - 1, R_0+1, -1, 1, 0, 2.*M_PI, n, Nx, Ny, Nz, dg::NEU, dg::NEU, dg::PER);
-    dg::CylindricalGrid<dg::DVec> g3d_fine( R_0 - 1, R_0+1, -1, 1, 0, 2.*M_PI, 3*n, 2*Nx, 2*Ny, Nz, dg::NEU, dg::NEU, dg::PER);
-    const dg::DVec w3d = dg::create::volume( g3d);
+    dg::CylindricalGrid3d g3d( R_0 - 1, R_0+1, -1, 1, 0, 2.*M_PI, n, Nx, Ny, Nz, dg::NEU, dg::NEU, dg::PER);
+    const dg::DVec vol3d = dg::create::volume( g3d);
     dg::Timer t;
     t.tic();
-    dg::DDS::FieldAligned dsFA( field, g3d_fine, 1e-10, dg::DefaultLimiter(), dg::NEU);
+    dg::geo::BinaryVectorLvl0 bhat( bhatR, bhatZ, bhatP);
+    dg::FieldAligned dsFA( field, g3d_fine, 1e-10, dg::DefaultLimiter(), dg::NEU);
 
-    dg::DDS ds ( dsFA, g3d, field, dg::not_normed, dg::centered);
+    dg::DS<dg::aGeometry3d, dg::IDMatrix, dg::DMatrix, dg::DVec> ds ( dsFA, g3d, field, dg::not_normed, dg::centered);
     t.toc();
     std::cout << "Creation of parallel Derivative took     "<<t.diff()<<"s\n";
 
@@ -46,9 +46,9 @@ int main()
     t.toc();
     std::cout << "Application of parallel Derivative took  "<<t.diff()<<"s\n";
     dg::blas1::axpby( 1., solution, -1., derivative);
-    double norm = dg::blas2::dot( w3d, solution);
+    double norm = dg::blas2::dot( vol3d, solution);
     std::cout << "Norm Solution "<<sqrt( norm)<<"\n";
-    std::cout << "Relative Difference Is "<< sqrt( dg::blas2::dot( derivative, w3d, derivative)/norm )<<"\n";
+    std::cout << "Relative Difference Is "<< sqrt( dg::blas2::dot( derivative, vol3d, derivative)/norm )<<"\n";
     std::cout << "Error is from the parallel derivative only if n>2\n"; //since the function is a parabola
     dg::Gaussian init0(R_0+0.5, 0, 0.2, 0.2, 1);
     dg::GaussianZ modulate(0., M_PI/3., 1);
@@ -57,13 +57,13 @@ int main()
     t.toc();
     std::cout << "Fieldaligned initialization took "<<t.diff()<<"s\n";
     ds( function, derivative);
-    norm = dg::blas2::dot(w3d, derivative);
+    norm = dg::blas2::dot(vol3d, derivative);
     std::cout << "Norm Centered Derivative "<<sqrt( norm)<<" (compare with that of ds_mpib)\n";
     ds.forward( function, derivative);
-    norm = dg::blas2::dot(w3d, derivative);
+    norm = dg::blas2::dot(vol3d, derivative);
     std::cout << "Norm Forward  Derivative "<<sqrt( norm)<<" (compare with that of ds_mpib)\n";
     ds.backward( function, derivative);
-    norm = dg::blas2::dot(w3d, derivative);
+    norm = dg::blas2::dot(vol3d, derivative);
     std::cout << "Norm Backward Derivative "<<sqrt( norm)<<" (compare with that of ds_mpib)\n";
     
     return 0;
diff --git a/inc/geometries/magnetic_field.h b/inc/geometries/magnetic_field.h
index b43d2d011..055c741c8 100644
--- a/inc/geometries/magnetic_field.h
+++ b/inc/geometries/magnetic_field.h
@@ -19,9 +19,9 @@ namespace geo
 
  This is the representation of toroidally axisymmetric magnetic fields that can be modeled in the form
  \f[
- \vec B = \frac{R_0}{R} \left( I \hat e_\varphi + \nabla \psi_p \times \hat e_\varphi\right)
+ \vec B = \frac{R_0}{R} \left( I(\psi_p) \hat e_\varphi + \nabla \psi_p \times \hat e_\varphi\right)
  \f]
- where \f$ R_0\f$ is a normalization constant, \f$ I\f$ the current 
+ where \f$ R_0\f$ is a normalization constant, \f$ I\f$ the poloidal current 
  and \f$ \psi_p\f$ the poloidal flux function.
  @note We implicitly also assume the toroidal field line approximation, i.e. all curvature
  and other perpendicular terms created with this field will assume that the perpendicular 
-- 
GitLab


From c2a36c39f66ca1c5f2940b36f73b98f66ca0af44 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <Matthias.Wiesenberger@uibk.ac.at>
Date: Wed, 30 Aug 2017 13:39:28 +0200
Subject: [PATCH 226/453] added 3 pointwiseDot functions in arakawa_b.cu

---
 config/devices/devices.mk |  4 +--
 config/marconi.mk         |  3 +-
 inc/dg/arakawa.h          | 60 +++++++++++++++++++++++++++++++++------
 3 files changed, 55 insertions(+), 12 deletions(-)

diff --git a/config/devices/devices.mk b/config/devices/devices.mk
index c5d0b1c2f..7694ee689 100644
--- a/config/devices/devices.mk
+++ b/config/devices/devices.mk
@@ -24,9 +24,9 @@ CFLAGS+= $(OMPFLAG)
 MPICFLAGS+=$(CFLAGS) #includes values in CFLAGS defined later
 endif #device=omp
 ifeq ($(strip $(device)),mic)
-CFLAGS+=-Wall -x c++
+CFLAGS+=-Wall -x c++ 
 CFLAGS+= -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_OMP
 CFLAGS+= $(OMPFLAG) #-mmic 
 MPICFLAGS+=$(CFLAGS) #includes values in CFLAGS defined later
-OPT=-O3 -xMIC-AVX512 -fma -finline-functions -align -restrict
+OPT=-O3 -xMIC-AVX512 -fma -finline-functions -align
 endif #device=mic
diff --git a/config/marconi.mk b/config/marconi.mk
index 3101f515b..43abd6b68 100644
--- a/config/marconi.mk
+++ b/config/marconi.mk
@@ -2,12 +2,13 @@
 ifeq ($(strip $(HPC_SYSTEM)),marconi)
 INCLUDE += -I$(HOME)/include # cusp, thrust
 INCLUDE += -I$(NETCDF_INC) -I$(HDF5_INC)
-GLFLAGS  = -lm
+GLFLAGS  = -lm 
 CC=icc
 MPICC=mpiicc
 OPT=-O3 -xHost -restrict # overwritten for mic in devices.mk
 #MPICFLAGS+= -DMPICH_IGNORE_CXX_SEEK
 OMPFLAG=-qopenmp
+CFLAGS=-restrict
 JSONLIB=-L$(HOME)/include/json/../../src/lib_json -ljsoncpp # json library for input parameters
 LIBS    +=-L$(HDF5_LIB) -lhdf5 -lhdf5_hl
 LIBS    +=-L$(NETCDF_LIB) -lnetcdf -lcurl
diff --git a/inc/dg/arakawa.h b/inc/dg/arakawa.h
index 986451512..afe1ff6bb 100644
--- a/inc/dg/arakawa.h
+++ b/inc/dg/arakawa.h
@@ -53,6 +53,42 @@ void pointwiseDot( double alpha, const std::vector<const container* >& x1, const
     }
 }
 }
+template<class container>
+void pointwiseDot( double alpha, const container& x1, const container& y1, 
+                   double beta,  const container& x2, const container& y2, 
+                   double gamma, container & z)
+{
+    unsigned K=x1.size();
+    const double * RESTRICT x1_ptr; 
+    const double * RESTRICT y1_ptr; 
+    const double * RESTRICT x2_ptr; 
+    const double * RESTRICT y2_ptr; 
+    double * RESTRICT z_ptr;
+    //const double *  x1_ptr; 
+    //const double *  y1_ptr; 
+    //const double *  x2_ptr; 
+    //const double *  y2_ptr; 
+    //double *  z_ptr;
+    {
+        x1_ptr = thrust::raw_pointer_cast( &(x1.data()[0]));
+        x2_ptr = thrust::raw_pointer_cast( &(x2.data()[0]));
+        y1_ptr = thrust::raw_pointer_cast( &(y1.data()[0]));
+        y2_ptr = thrust::raw_pointer_cast( &(y2.data()[0]));
+         z_ptr = thrust::raw_pointer_cast( &(z.data()[0]));
+    }
+    unsigned size = x1.size();
+#pragma omp parallel
+{
+    double temp;
+#pragma omp for simd
+    for( unsigned i=0; i<size; i++)
+    {
+        z_ptr[i] = alpha*x1_ptr[i]*y1_ptr[i] 
+                  +beta*x2_ptr[i]*y2_ptr[i]
+                  +gamma*z_ptr[i];
+    }
+}
+}
 
 /*! @file 
   
@@ -163,13 +199,19 @@ void ArakawaX< Geometry, Matrix, container>::operator()( const container& lhs, c
     blas2::symv( bdxf, rhs, dxrhs);
     blas2::symv( bdyf, rhs, dyrhs);
 
-    std::vector<const container* > s0(3), t0(3), s1(3), t1(3); 
-    std::vector<container* > s2(3);
-    s0[0] = &dxlhs, t0[0] = &dyrhs, s1[0] = &dylhs, t1[0] = &dxrhs;
-    s0[1] =   &lhs, t0[1] = &dyrhs, s1[1] = &dylhs, t1[1] =   &rhs;
-    s0[2] = &dxlhs, t0[2] =   &rhs, s1[2] =   &lhs, t1[2] = &dxrhs;
-    s2[0] = &result, s2[1] = &dxlhs, s2[2] = &dxrhs;
-    pointwiseDot( 1./3., s0, t0, -1./3., s1, t1, 0., s2);
+    //std::vector<const container* > s0(3), t0(3), s1(3), t1(3); 
+    //std::vector<container* > s2(3);
+    //s0[0] = &dxlhs, t0[0] = &dyrhs, s1[0] = &dylhs, t1[0] = &dxrhs;
+    //s0[1] =   &lhs, t0[1] = &dyrhs, s1[1] = &dylhs, t1[1] =   &rhs;
+    //s0[2] = &dxlhs, t0[2] =   &rhs, s1[2] =   &lhs, t1[2] = &dxrhs;
+    //s2[0] = &result, s2[1] = &dxlhs, s2[2] = &dxrhs;
+    //pointwiseDot( 1./3., s0, t0, -1./3., s1, t1, 0., s2);
+
+    pointwiseDot( 1./3., dxlhs, dyrhs, -1./3., dylhs, dxrhs, 0., result);
+    pointwiseDot( 1./3.,   lhs, dyrhs, -1./3., dylhs,   rhs, 0., helper_);
+    pointwiseDot( 1./3., dxlhs,   rhs, -1./3.,   lhs, dxrhs, 0., dylhs);
+    blas2::symv( 1., bdxf, helper_, 1., result);
+    blas2::symv( 1., bdyf, dylhs, 1., result);
     
     //// order is important now
     //// +x (1) -> result und (2) -> blhs
@@ -195,8 +237,8 @@ void ArakawaX< Geometry, Matrix, container>::operator()( const container& lhs, c
     ////blas1::axpby( 1., dxlhs,  -0., helper); //x+ - +x
     ////blas1::axpby( 0., result, -1., dylhs);  //+x - x+
 
-    blas2::symv( 1., bdxf, dxlhs, 1., result);
-    blas2::symv( 1., bdyf, dxrhs, 1., result);
+    //blas2::symv( 1., bdxf, dxlhs, 1., result);
+    //blas2::symv( 1., bdyf, dxrhs, 1., result);
     geo::dividePerpVolume( result, grid);
 }
 
-- 
GitLab


From e46a0773985d527fef9d3156a298ec072d3c9f55 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Wed, 30 Aug 2017 05:06:00 -0700
Subject: [PATCH 227/453] added case gamma==0 in arakawa pointwiseDot function

---
 inc/dg/arakawa.h | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/inc/dg/arakawa.h b/inc/dg/arakawa.h
index afe1ff6bb..e7e6aa91e 100644
--- a/inc/dg/arakawa.h
+++ b/inc/dg/arakawa.h
@@ -58,7 +58,6 @@ void pointwiseDot( double alpha, const container& x1, const container& y1,
                    double beta,  const container& x2, const container& y2, 
                    double gamma, container & z)
 {
-    unsigned K=x1.size();
     const double * RESTRICT x1_ptr; 
     const double * RESTRICT y1_ptr; 
     const double * RESTRICT x2_ptr; 
@@ -77,10 +76,9 @@ void pointwiseDot( double alpha, const container& x1, const container& y1,
          z_ptr = thrust::raw_pointer_cast( &(z.data()[0]));
     }
     unsigned size = x1.size();
-#pragma omp parallel
+if(gamma!=0)
 {
-    double temp;
-#pragma omp for simd
+#pragma omp parallel for simd
     for( unsigned i=0; i<size; i++)
     {
         z_ptr[i] = alpha*x1_ptr[i]*y1_ptr[i] 
@@ -88,6 +86,15 @@ void pointwiseDot( double alpha, const container& x1, const container& y1,
                   +gamma*z_ptr[i];
     }
 }
+else
+{
+#pragma omp parallel for simd
+    for( unsigned i=0; i<size; i++)
+    {
+        z_ptr[i] = alpha*x1_ptr[i]*y1_ptr[i] 
+                  +beta*x2_ptr[i]*y2_ptr[i];
+    }
+}
 }
 
 /*! @file 
-- 
GitLab


From 98d895cb0e422f43d4bb4f4b0488d35a99028811 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Wed, 30 Aug 2017 07:01:24 -0700
Subject: [PATCH 228/453] try to implement trivial interpolation integration

---
 inc/geometries/fieldaligned.h | 45 +++++++++++++++++++++++++----------
 inc/geometries/flux_t.cu      |  2 +-
 2 files changed, 33 insertions(+), 14 deletions(-)

diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index c26591d3b..22a69cf6b 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -146,13 +146,16 @@ typedef ZERO NoLimiter;
 namespace detail{
 
 
+void clip_to_boundary( double& x, double& y, const aTopology2d* grid)
+{
+    if (!(x > grid->x0())) { x=grid->x0();}
+    if (!(x < grid->x1())) { x=grid->x1();}
+    if (!(y > grid->y0())) { y=grid->y0();}
+    if (!(y < grid->y1())) { y=grid->y1();}
+}
 void clip_to_boundary( thrust::host_vector<double>& x, const aTopology2d* grid)
 {
-    if (!(x[0] > grid->x0())) { x[0]=grid->x0();}
-    if (!(x[0] < grid->x1())) { x[0]=grid->x1();}
-    if (!(x[1] > grid->y0())) { x[1]=grid->y0();}
-    if (!(x[1] < grid->y1())) { x[1]=grid->y1();}
-
+    clip_to_boundary(x[0], x[1], grid);
 }
 
 
@@ -260,7 +263,7 @@ void integrate_all_fieldlines2d( const dg::geo::BinaryVectorLvl0& vec, const aGe
     t.tic();
     aGeometry2d* g2dField_ptr = g2dFine_ptr->clone();
     //initial tests show that higher order polynomial might not be needed...
-    g2dField_ptr->set( g2dField_ptr->n(), g2dField_ptr->Nx(), g2dField_ptr->Ny());
+    g2dField_ptr->set( 3*g2dField_ptr->n(), g2dField_ptr->Nx(), g2dField_ptr->Ny());
     dg::DSField field( vec, *g2dField_ptr);
     t.toc();
     std::cout << "Generation of interpolate grid took "<<t.diff()<<"s\n";
@@ -486,16 +489,32 @@ FieldAligned<Geometry, IMatrix, container>::FieldAligned(const dg::geo::BinaryVe
     right_ = left_ = dg::evaluate( zero, *g2dCoarse_ptr);
     ghostM.resize( perp_size_); ghostP.resize( perp_size_);
     //Set starting points and integrate field lines
+    std::cout << "Start fieldline integration!\n";
     dg::Timer t;
     t.tic();
-    aGeometry2d* g2dFine_ptr = g2dCoarse_ptr->clone();
-    g2dFine_ptr->multiplyCellNumbers( (double)mx, (double)my);
+    //aGeometry2d* g2dFine_ptr = g2dCoarse_ptr->clone();
+    //g2dFine_ptr->multiplyCellNumbers( (double)mx, (double)my);
     t.toc();
     std::cout << "Create fine grid took "<<t.diff()<<"s\n";
-    std::vector<thrust::host_vector<double> > yp( 3), ym(yp); 
-    std::cout << "Start fieldline integration!\n";
+    std::vector<thrust::host_vector<double> > yp_coarse( 3), ym_coarse(yp_coarse); 
     t.tic();
-    detail::integrate_all_fieldlines2d( vec, g2dFine_ptr, yp, ym, deltaPhi, eps);
+    detail::integrate_all_fieldlines2d( vec, g2dCoarse_ptr, yp_coarse, ym_coarse, deltaPhi, eps);
+    dg::Grid2d g2dFine((dg::Grid2d(*g2dCoarse_ptr)));
+    g2dFine.multiplyCellNumbers((double)mx, (double)my);
+    IMatrix interpolate = dg::create::interpolation( g2dFine, *g2dCoarse_ptr);
+    std::vector<thrust::host_vector<double> > yp( 3, dg::evaluate(dg::zero, g2dFine)), ym(yp); 
+    for( unsigned i=0; i<3; i++)
+    {
+        dg::blas2::symv( interpolate, yp_coarse[i], yp[i]);
+        dg::blas2::symv( interpolate, ym_coarse[i], ym[i]);
+    }
+    for( unsigned i=0; i<yp[0].size(); i++)
+    {
+        detail::clip_to_boundary( yp[0][i], yp[1][i], &g2dFine);
+        detail::clip_to_boundary( ym[0][i], ym[1][i], &g2dFine);
+    }
+
+
     t.toc(); 
     std::cout << "Fieldline integration took "<<t.diff()<<"s\n";
     //%%%%%%%%%%%%%%%%%%Create interpolation and projection%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -503,7 +522,7 @@ FieldAligned<Geometry, IMatrix, container>::FieldAligned(const dg::geo::BinaryVe
     IMatrix plusFine  = dg::create::interpolation( yp[0], yp[1], *g2dCoarse_ptr, globalbcx, globalbcy);
     IMatrix minusFine = dg::create::interpolation( ym[0], ym[1], *g2dCoarse_ptr, globalbcx, globalbcy);
     //IMatrix interpolation = dg::create::interpolation( *g2dFine_ptr, *g2dCoarse_ptr);
-    IMatrix projection = dg::create::projection( *g2dCoarse_ptr, *g2dFine_ptr);
+    IMatrix projection = dg::create::projection( *g2dCoarse_ptr, g2dFine);
     t.toc();
     std::cout <<"Creation of interpolation/projection took "<<t.diff()<<"s\n";
     t.tic();
@@ -526,7 +545,7 @@ FieldAligned<Geometry, IMatrix, container>::FieldAligned(const dg::geo::BinaryVe
     dg::blas1::scal( hm_, -1.);
     dg::blas1::axpby(  1., hp_, +1., hm_, hz_);
     delete g2dCoarse_ptr;
-    delete g2dFine_ptr;
+    //delete g2dFine_ptr;
  
 }
 
diff --git a/inc/geometries/flux_t.cu b/inc/geometries/flux_t.cu
index a01edbf97..3e8554b4b 100644
--- a/inc/geometries/flux_t.cu
+++ b/inc/geometries/flux_t.cu
@@ -47,7 +47,7 @@ double cosineY( double x, double y) {return sin(x)*cos(y);}
 
 int main( int argc, char* argv[])
 {
-    std::cout << "Type n, Nx, Ny, Nz\n";
+    std::cout << "Type n(3), Nx(8), Ny(80), Nz(20)\n";
     unsigned n, Nx, Ny, Nz;
     std::cin >> n>> Nx>>Ny>>Nz;   
     Json::Reader reader;
-- 
GitLab


From 24e4cd477cf4098af25baa30ec2156cc57a1bc9e Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Wed, 30 Aug 2017 07:28:27 -0700
Subject: [PATCH 229/453] found bug in fieldaligned interpolation

---
 inc/geometries/fieldaligned.h | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index 22a69cf6b..d6f31c95b 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -200,7 +200,7 @@ struct BoxIntegrator
 };
 
 /**
- * @brief Integrate one field line in a given box, Result is guaranteed to lie inside the box
+ * @brief Integrate one field line in a given box, Result is guaranteed to lie inside the box modulo periodic boundary conditions
  *
  * @tparam Field Must be usable in the integrateRK function
  * @tparam Grid must provide 2d contains function
@@ -218,9 +218,10 @@ void boxintegrator( Field& field, const Grid& grid,
         double& phi1, double eps)
 {
     dg::integrateRK4( field, coords0, coords1, phi1, eps); //integration
-    //First catch periodic domain
-    grid.shift_topologic( coords0[0], coords0[1], coords1[0], coords1[1]);
-    if ( !grid.contains( coords1[0], coords1[1]))   //Punkt liegt immer noch außerhalb 
+    double R = coords1[0], Z=coords1[1];
+    //First, catch periodic domain
+    grid.shift_topologic( coords0[0], coords0[1], R, Z);
+    if ( !grid.contains( R, Z))   //point still outside domain
     {
 #ifdef DG_DEBUG
         std::cerr << "point "<<coords1[0]<<" "<<coords1[1]<<" is somewhere else!\n";
@@ -510,8 +511,10 @@ FieldAligned<Geometry, IMatrix, container>::FieldAligned(const dg::geo::BinaryVe
     }
     for( unsigned i=0; i<yp[0].size(); i++)
     {
-        detail::clip_to_boundary( yp[0][i], yp[1][i], &g2dFine);
-        detail::clip_to_boundary( ym[0][i], ym[1][i], &g2dFine);
+        g2dFine.shift_topologic( yp[0][i], yp[1][i], yp[0][i], yp[1][i]);
+        g2dFine.shift_topologic( ym[0][i], ym[1][i], ym[0][i], ym[1][i]);
+        //detail::clip_to_boundary( yp[0][i], yp[1][i], &g2dFine);
+        //detail::clip_to_boundary( ym[0][i], ym[1][i], &g2dFine);
     }
 
 
-- 
GitLab


From fbd7b32fc4f4d552fcad19fcbb4ec669ca3c44c2 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Wed, 30 Aug 2017 08:01:13 -0700
Subject: [PATCH 230/453] cleaned up fieldaligned and ds_geom_t a bit

---
 inc/geometries/ds_geom_t.cu   | 32 ++++++++++++++++----------------
 inc/geometries/fieldaligned.h | 10 +---------
 2 files changed, 17 insertions(+), 25 deletions(-)

diff --git a/inc/geometries/ds_geom_t.cu b/inc/geometries/ds_geom_t.cu
index 9eafc7b00..ca176214a 100644
--- a/inc/geometries/ds_geom_t.cu
+++ b/inc/geometries/ds_geom_t.cu
@@ -103,22 +103,22 @@ int main( int argc, char* argv[])
 
     MagneticField c(gp);
         
-    dg::geo::Field<MagneticField> field(c, gp.R_0);
-    dg::geo::InvB<MagneticField> invB(c, gp.R_0);
-    dg::geo::LnB<MagneticField> lnB(c, gp.R_0);
-    dg::geo::BR<MagneticField> bR(c, gp.R_0);
-    dg::geo::BZ<MagneticField> bZ(c, gp.R_0);
-    dg::geo::CurvatureNablaBR<MagneticField> curvatureR(c, gp.R_0);
-    dg::geo::CurvatureNablaBZ<MagneticField> curvatureZ(c, gp.R_0);
-    dg::geo::GradLnB<MagneticField> gradLnB(c, gp.R_0);
+    dg::geo::Field> field(c, gp.R_0);
+    dg::geo::InvB> invB(c, gp.R_0);
+    dg::geo::LnB> lnB(c, gp.R_0);
+    dg::geo::BR> bR(c, gp.R_0);
+    dg::geo::BZ> bZ(c, gp.R_0);
+    dg::geo::CurvatureNablaBR> curvatureR(c, gp.R_0);
+    dg::geo::CurvatureNablaBZ> curvatureZ(c, gp.R_0);
+    dg::geo::GradLnB> gradLnB(c, gp.R_0);
     dg::geo::Pupil<Psip> pupil(c.psip, gp.psipmaxcut);
     InvNormR invnormr(gp);
-    dg::geo::FieldR<MagneticField> fieldR(c, gp.R_0);
-    dg::geo::FieldZ<MagneticField> fieldZ(c, gp.R_0);
-    dg::geo::FieldP<MagneticField> fieldP(c, gp.R_0);
-    dg::geo::BHatR<MagneticField> bhatR(c, gp.R_0);
-    dg::geo::BHatZ<MagneticField> bhatZ(c, gp.R_0);
-    dg::geo::BHatP<MagneticField> bhatP(c, gp.R_0);
+    dg::geo::FieldR> fieldR(c, gp.R_0);
+    dg::geo::FieldZ> fieldZ(c, gp.R_0);
+    dg::geo::FieldP> fieldP(c, gp.R_0);
+    dg::geo::BHatR> bhatR(c, gp.R_0);
+    dg::geo::BHatZ> bhatZ(c, gp.R_0);
+    dg::geo::BHatP> bhatP(c, gp.R_0);
     dg::Grid3d grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI,p.n, p.Nx, p.Ny,p.Nz);
     dg::HVec vecR = dg::evaluate( fieldR, grid);
     dg::HVec vecZ = dg::evaluate( fieldZ, grid);
@@ -175,8 +175,8 @@ int main( int argc, char* argv[])
 
                 std::cout <<"---------------------------------------------------------------------------------------------" << "\n";
                 std::cout <<"-----(1a) test with testfunction  (works for DIR)" << "\n";
-                dg::geo::TestFunction<MagneticField> func(c, gp.R_0);
-                dg::geo::DeriTestFunction<MagneticField> derifunc(c, gp.R_0);
+                dg::geo::TestFunction> func(c, gp.R_0);
+                dg::geo::DeriTestFunction> derifunc(c, gp.R_0);
                 std::cout << "Construct parallel  derivative\n";
                 dg::Timer t;
                 t.tic();
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index d6f31c95b..b0eda749f 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -264,7 +264,7 @@ void integrate_all_fieldlines2d( const dg::geo::BinaryVectorLvl0& vec, const aGe
     t.tic();
     aGeometry2d* g2dField_ptr = g2dFine_ptr->clone();
     //initial tests show that higher order polynomial might not be needed...
-    g2dField_ptr->set( 3*g2dField_ptr->n(), g2dField_ptr->Nx(), g2dField_ptr->Ny());
+    g2dField_ptr->set( 1*g2dField_ptr->n(), g2dField_ptr->Nx(), g2dField_ptr->Ny());
     dg::DSField field( vec, *g2dField_ptr);
     t.toc();
     std::cout << "Generation of interpolate grid took "<<t.diff()<<"s\n";
@@ -492,11 +492,6 @@ FieldAligned<Geometry, IMatrix, container>::FieldAligned(const dg::geo::BinaryVe
     //Set starting points and integrate field lines
     std::cout << "Start fieldline integration!\n";
     dg::Timer t;
-    t.tic();
-    //aGeometry2d* g2dFine_ptr = g2dCoarse_ptr->clone();
-    //g2dFine_ptr->multiplyCellNumbers( (double)mx, (double)my);
-    t.toc();
-    std::cout << "Create fine grid took "<<t.diff()<<"s\n";
     std::vector<thrust::host_vector<double> > yp_coarse( 3), ym_coarse(yp_coarse); 
     t.tic();
     detail::integrate_all_fieldlines2d( vec, g2dCoarse_ptr, yp_coarse, ym_coarse, deltaPhi, eps);
@@ -524,7 +519,6 @@ FieldAligned<Geometry, IMatrix, container>::FieldAligned(const dg::geo::BinaryVe
     t.tic();
     IMatrix plusFine  = dg::create::interpolation( yp[0], yp[1], *g2dCoarse_ptr, globalbcx, globalbcy);
     IMatrix minusFine = dg::create::interpolation( ym[0], ym[1], *g2dCoarse_ptr, globalbcx, globalbcy);
-    //IMatrix interpolation = dg::create::interpolation( *g2dFine_ptr, *g2dCoarse_ptr);
     IMatrix projection = dg::create::projection( *g2dCoarse_ptr, g2dFine);
     t.toc();
     std::cout <<"Creation of interpolation/projection took "<<t.diff()<<"s\n";
@@ -548,8 +542,6 @@ FieldAligned<Geometry, IMatrix, container>::FieldAligned(const dg::geo::BinaryVe
     dg::blas1::scal( hm_, -1.);
     dg::blas1::axpby(  1., hp_, +1., hm_, hz_);
     delete g2dCoarse_ptr;
-    //delete g2dFine_ptr;
- 
 }
 
 template<class G, class I, class container>
-- 
GitLab


From ad6cbde3e1847edf2cd78dd36d69b08630740a23 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Wed, 30 Aug 2017 09:52:49 -0700
Subject: [PATCH 231/453] debugging ds_geom_t.cu

---
 inc/geometries/ds_geom_t.cu   | 60 ++++++++++++++++-------------------
 inc/geometries/fieldaligned.h |  2 ++
 inc/geometries/testfunctors.h |  4 +--
 3 files changed, 32 insertions(+), 34 deletions(-)

diff --git a/inc/geometries/ds_geom_t.cu b/inc/geometries/ds_geom_t.cu
index ca176214a..32b33ac91 100644
--- a/inc/geometries/ds_geom_t.cu
+++ b/inc/geometries/ds_geom_t.cu
@@ -8,9 +8,6 @@
 
 #include <cusp/print.h>
 
-// #define DG_DEBUG
-
-
 #include "file/nc_utilities.h"
 #include "draw/host_window.h"
 #include "dg/backend/timer.cuh"
@@ -24,8 +21,7 @@
 #include "init.h"
 #include "testfunctors.h"
 #include "magnetic_field.h"
-
-using namespace dg::geo::solovev;
+#include "ds.h"
 
 struct Parameters
 {
@@ -59,8 +55,8 @@ struct Parameters
 
 struct InvNormR
 {
-    InvNormR( GeomParameters gp): R_0(gp.R_0){}
-    double operator()( double R, double Z, double phi)
+    InvNormR( dg::geo::solovev::GeomParameters gp): R_0(gp.R_0){}
+    double operator()( double R, double Z, double phi)const
     {
         return R_0/R;
     }
@@ -90,7 +86,7 @@ int main( int argc, char* argv[])
         reader.parse( isG, geom_js, false);
     }
     const Parameters p(input_js);
-    const GeomParameters gp(geom_js);
+    const dg::geo::solovev::GeomParameters gp(geom_js);
     p.display( std::cout);
     gp.display( std::cout);
 
@@ -101,24 +97,24 @@ int main( int argc, char* argv[])
     std::cout << "The grid parameters" <<"\n";
     std::cout  << Rmin<<" rho_s " << Rmax <<" rho_s " << Zmin <<" rho_s " <<Zmax <<" rho_s " <<"\n";
 
-    MagneticField c(gp);
+    dg::geo::TokamakMagneticField c = dg::geo::createSolovevField(gp);
         
-    dg::geo::Field> field(c, gp.R_0);
-    dg::geo::InvB> invB(c, gp.R_0);
-    dg::geo::LnB> lnB(c, gp.R_0);
-    dg::geo::BR> bR(c, gp.R_0);
-    dg::geo::BZ> bZ(c, gp.R_0);
-    dg::geo::CurvatureNablaBR> curvatureR(c, gp.R_0);
-    dg::geo::CurvatureNablaBZ> curvatureZ(c, gp.R_0);
-    dg::geo::GradLnB> gradLnB(c, gp.R_0);
-    dg::geo::Pupil<Psip> pupil(c.psip, gp.psipmaxcut);
+    dg::geo::InvB invB(c);
+    dg::geo::LnB lnB(c);
+    dg::geo::BR bR(c);
+    dg::geo::BZ bZ(c);
+    dg::geo::CurvatureNablaBR curvatureR(c);
+    dg::geo::CurvatureNablaBZ curvatureZ(c);
+    dg::geo::GradLnB gradLnB(c);
+    dg::geo::Pupil pupil(c.psip(), gp.psipmaxcut);
     InvNormR invnormr(gp);
-    dg::geo::FieldR> fieldR(c, gp.R_0);
-    dg::geo::FieldZ> fieldZ(c, gp.R_0);
-    dg::geo::FieldP> fieldP(c, gp.R_0);
-    dg::geo::BHatR> bhatR(c, gp.R_0);
-    dg::geo::BHatZ> bhatZ(c, gp.R_0);
-    dg::geo::BHatP> bhatP(c, gp.R_0);
+    dg::geo::FieldR fieldR(c);
+    dg::geo::FieldZ fieldZ(c);
+    dg::geo::FieldP fieldP(c);
+    dg::geo::BHatR bhatR(c);
+    dg::geo::BHatZ bhatZ(c);
+    dg::geo::BHatP bhatP(c);
+    dg::DSFieldCylindrical field(dg::geo::BinaryVectorLvl0(bhatR, bhatZ, bhatP));
     dg::Grid3d grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI,p.n, p.Nx, p.Ny,p.Nz);
     dg::HVec vecR = dg::evaluate( fieldR, grid);
     dg::HVec vecZ = dg::evaluate( fieldZ, grid);
@@ -169,19 +165,19 @@ int main( int argc, char* argv[])
             {
                 std::cout << "n = " << k*n << " Nx = " <<pow(2,i)* Nx << " Ny = " <<pow(2,i)* Ny << " Nz = "<<pow(2,zz)* Nz <<"\n";
                 //Similar to feltor grid
-                dg::CylindricalGrid3d<dg::DVec> g3d( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI,k*n,pow(2,i)* Nx,pow(2,i)* Ny, pow(2,zz)*Nz,dg::NEU, dg::NEU, dg::PER);
+                dg::CylindricalGrid3d g3d( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI,k*n,pow(2,i)* Nx,pow(2,i)* Ny, pow(2,zz)*Nz,dg::NEU, dg::NEU, dg::PER);
                 const dg::DVec w3d = dg::create::volume( g3d);
                 dg::DVec pupilongrid = dg::evaluate( pupil, g3d);
 
                 std::cout <<"---------------------------------------------------------------------------------------------" << "\n";
                 std::cout <<"-----(1a) test with testfunction  (works for DIR)" << "\n";
-                dg::geo::TestFunction> func(c, gp.R_0);
-                dg::geo::DeriTestFunction> derifunc(c, gp.R_0);
+                dg::geo::TestFunction func(c);
+                dg::geo::DeriTestFunction derifunc(c);
                 std::cout << "Construct parallel  derivative\n";
                 dg::Timer t;
                 t.tic();
-                dg::DDS::FieldAligned dsFA( field, g3d, gp.rk4eps, dg::geo::PsiLimiter<Psip>(c.psip, gp.psipmaxlim), g3d.bcx()); 
-                dg::DDS ds( dsFA, field, dg::normed, dg::centered); //choose bc of grid
+                dg::FieldAligned<dg::aGeometry3d, dg::IDMatrix, dg::DVec > dsFA( field, g3d, gp.rk4eps, dg::geo::PsiLimiter(c.psip(), gp.psipmaxlim), g3d.bcx()); 
+                dg::DS<dg::aGeometry3d, dg::IDMatrix, dg::DMatrix, dg::DVec>  ds( dsFA, field, dg::normed, dg::centered); //choose bc of grid
                 t.toc();
                 std::cout << "-----> Creation of parallel Derivative took"<<t.diff()<<"s\n";
 
@@ -304,10 +300,10 @@ int main( int argc, char* argv[])
                 std::cout << "Rel Diff = "<<reldiff2b <<"\n";
                 std::cout <<"---------------------------------------------------------------------------------------------" << "\n";
                 std::cout <<"-----(3) test with gradlnb and with (a) Arakawa and (b) Poisson discretization" << "\n";    
-                dg::ArakawaX< dg::CylindricalGrid3d<dg::DVec>, dg::DMatrix, dg::DVec>    arakawa(g3d); 
-                dg::Poisson< dg::CylindricalGrid3d<dg::DVec>, dg::DMatrix, dg::DVec>     poiss(g3d);
+                dg::ArakawaX< dg::CylindricalGrid3d, dg::DMatrix, dg::DVec> arakawa(g3d); 
+                dg::Poisson<  dg::CylindricalGrid3d, dg::DMatrix, dg::DVec> poiss(g3d);
                 dg::DVec invBongrid = dg::evaluate( invB, g3d);
-                dg::DVec psipongrid = dg::evaluate( c.psip, g3d);
+                dg::DVec psipongrid = dg::evaluate( c.psip(), g3d);
                 dg::DVec invnormrongrid = dg::evaluate( invnormr, g3d);
                 dg::DVec arakawasolution(g3d.size());
                 dg::DVec poisssolution(g3d.size());
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index b0eda749f..a7bf24669 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -9,11 +9,13 @@
 #include "dg/backend/projection.cuh"
 #include "dg/backend/functions.h"
 
+#include "dg/geometry/geometry.h"
 #include "dg/functors.h"
 #include "dg/nullstelle.h"
 #include "dg/runge_kutta.h"
 #include "magnetic_field.h"
 #include "fluxfunctions.h"
+#include "curvilinear.h"
 
 namespace dg{
 
diff --git a/inc/geometries/testfunctors.h b/inc/geometries/testfunctors.h
index 96372eb6d..990bf28b2 100644
--- a/inc/geometries/testfunctors.h
+++ b/inc/geometries/testfunctors.h
@@ -345,7 +345,7 @@ struct TestFunction
     /**
      * @brief \f[ f(R,Z,\varphi) = -\frac{\cos(\varphi)}{R\hat b_\varphi} \f]
      */ 
-    double operator()( double R, double Z, double phi)
+    double operator()( double R, double Z, double phi)const
     {
 //         return psip_(R,Z,phi)*sin(phi);
 //         double Rmin = gp_.R_0-(p_.boxscaleRm)*gp_.a;
@@ -381,7 +381,7 @@ struct DeriTestFunction
 /**
  * @brief \f[ \nabla_\parallel f = \frac{\sin(\varphi)}{R}\f]
  */ 
-    double operator()( double R, double Z, double phi)
+    double operator()( double R, double Z, double phi)const 
     {
 //         double Rmin = gp_.R_0-(p_.boxscaleRm)*gp_.a;
 //         double Rmax = gp_.R_0+(p_.boxscaleRp)*gp_.a;
-- 
GitLab


From 273c7f0ea2d1e07166df34819decd7f55356382a Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 31 Aug 2017 22:05:08 +0200
Subject: [PATCH 232/453] updated marconi.mk and devices.mk

---
 config/devices/devices.mk | 4 ++--
 config/marconi.mk         | 6 ++++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/config/devices/devices.mk b/config/devices/devices.mk
index c5d0b1c2f..7694ee689 100644
--- a/config/devices/devices.mk
+++ b/config/devices/devices.mk
@@ -24,9 +24,9 @@ CFLAGS+= $(OMPFLAG)
 MPICFLAGS+=$(CFLAGS) #includes values in CFLAGS defined later
 endif #device=omp
 ifeq ($(strip $(device)),mic)
-CFLAGS+=-Wall -x c++
+CFLAGS+=-Wall -x c++ 
 CFLAGS+= -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_OMP
 CFLAGS+= $(OMPFLAG) #-mmic 
 MPICFLAGS+=$(CFLAGS) #includes values in CFLAGS defined later
-OPT=-O3 -xMIC-AVX512 -fma -finline-functions -align -restrict
+OPT=-O3 -xMIC-AVX512 -fma -finline-functions -align
 endif #device=mic
diff --git a/config/marconi.mk b/config/marconi.mk
index 58eacc638..43abd6b68 100644
--- a/config/marconi.mk
+++ b/config/marconi.mk
@@ -2,12 +2,13 @@
 ifeq ($(strip $(HPC_SYSTEM)),marconi)
 INCLUDE += -I$(HOME)/include # cusp, thrust
 INCLUDE += -I$(NETCDF_INC) -I$(HDF5_INC)
-GLFLAGS  = -lm
+GLFLAGS  = -lm 
 CC=icc
 MPICC=mpiicc
 OPT=-O3 -xHost -restrict # overwritten for mic in devices.mk
 #MPICFLAGS+= -DMPICH_IGNORE_CXX_SEEK
 OMPFLAG=-qopenmp
+CFLAGS=-restrict
 JSONLIB=-L$(HOME)/include/json/../../src/lib_json -ljsoncpp # json library for input parameters
 LIBS    +=-L$(HDF5_LIB) -lhdf5 -lhdf5_hl
 LIBS    +=-L$(NETCDF_LIB) -lnetcdf -lcurl
@@ -24,7 +25,8 @@ endif
 
 ###########configure mic jobs with#########################
 #mcdram=cache:numa=quadrant
-#export KMP_AFFINITY=scatter
+#export KMP_AFFINITY=scatter #important
 #export OM_NUM_THREADS=68
+#qsub -I -qxfuaknldebug -A FUA21_FELTOR -l select=1:ncpus=68:mcdram=cach:numa=quadrant -l walltime=0:20:00
 
 
-- 
GitLab


From 07adcf8fb098cd085246f5482628cb5ecb3b898e Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Fri, 1 Sep 2017 07:30:30 -0700
Subject: [PATCH 233/453] corrected perp in toefl.tex

---
 src/toefl/toefl.tex | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/toefl/toefl.tex b/src/toefl/toefl.tex
index f909988ae..11a0da1fc 100644
--- a/src/toefl/toefl.tex
+++ b/src/toefl/toefl.tex
@@ -118,18 +118,18 @@ use Cartesian coordinates $x$, $y$.
 \begin{subequations}
 \begin{align}
 B(x)^{-1} = \kappa x +1-\kappa X\quad \Gamma_1 = ( 1- 0.5\tau\nabla^2)^{-1}\\
- -\nabla\cdot \left(\frac{N}{B^2} \nabla \phi\right) = \Gamma_1 N-n, \quad
- \text{Boussinesq:}\quad -\nabla^2 \phi = \frac{B^2}{N} (\Gamma_1 N -n) \\
+ -\nabla\cdot \left(\frac{N}{B^2} \nabla_\perp \phi\right) = \Gamma_1 N-n, \quad
+ \text{Boussinesq:}\quad -\nabla_\perp^2 \phi = \frac{B^2}{N} (\Gamma_1 N -n) \\
 \psi = \Gamma_1 \phi - \frac{1}{2} \frac{(\nabla\phi)^2}{B^2}\\
  \frac{\partial n}{\partial t}     = 
     \frac{1}{B}\{ n, \phi\} 
   + \kappa n\frac{\partial \phi}{\partial y} 
   -\kappa \frac{\partial n}{\partial y}
-  + \nu \nabla^2 n  \\
+  + \nu \nabla_\perp^2 n  \\
   \frac{\partial N}{\partial t} =
   \frac{1}{B}\{ N, \psi\} 
   + \kappa N\frac{\partial \psi}{\partial y} 
-  + \tau \kappa\frac{\partial N}{\partial y} +\nu\nabla^2N
+  + \tau \kappa\frac{\partial N}{\partial y} +\nu\nabla_\perp^2N
 \end{align}
 \end{subequations}
 
-- 
GitLab


From ebce276908412e0316d4017a50aec35fa3988a82 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sat, 2 Sep 2017 23:24:12 +0200
Subject: [PATCH 234/453] introduced optimized blas1 functions for thrust omp
 backend

---
 inc/dg/arakawa.h                            | 126 +--------
 inc/dg/backend/sparseblockmat_omp_kernels.h |  11 +-
 inc/dg/backend/thrust_vector_blas.cuh       | 278 +++++++++++++++++++-
 inc/dg/backend/vector_categories.h          |  10 +
 inc/dg/blas1.h                              |  43 +++
 5 files changed, 322 insertions(+), 146 deletions(-)

diff --git a/inc/dg/arakawa.h b/inc/dg/arakawa.h
index e7e6aa91e..cc4c12bc5 100644
--- a/inc/dg/arakawa.h
+++ b/inc/dg/arakawa.h
@@ -11,92 +11,6 @@
 #include "backend/mpi_evaluation.h"
 #endif
 
-template<class container>
-void pointwiseDot( double alpha, const std::vector<const container* >& x1, const std::vector<const container* >& y1, 
-                   double beta,  const std::vector<const container* >& x2, const std::vector<const container* >& y2, 
-                   double gamma, std::vector<container* > & z)
-{
-    unsigned K=x1.size();
-    const double * RESTRICT x1_ptr[K]; 
-    const double * RESTRICT y1_ptr[K]; 
-    const double * RESTRICT x2_ptr[K]; 
-    const double * RESTRICT y2_ptr[K]; 
-    double * RESTRICT z_ptr[K];
-    //const double *  x1_ptr[K]; 
-    //const double *  y1_ptr[K]; 
-    //const double *  x2_ptr[K]; 
-    //const double *  y2_ptr[K]; 
-    //double *  z_ptr[K];
-    for(unsigned i=0; i<K; i++)
-    {
-        x1_ptr[i] = thrust::raw_pointer_cast( &(x1[i]->data()[0]));
-        x2_ptr[i] = thrust::raw_pointer_cast( &(x2[i]->data()[0]));
-        y1_ptr[i] = thrust::raw_pointer_cast( &(y1[i]->data()[0]));
-        y2_ptr[i] = thrust::raw_pointer_cast( &(y2[i]->data()[0]));
-         z_ptr[i] = thrust::raw_pointer_cast( &(z[i]->data()[0]));
-    }
-    unsigned size = x1[0]->size();
-#pragma omp parallel
-{
-    double temp[K];
-#pragma omp for simd
-    for( unsigned i=0; i<size; i++)
-    {
-        for( unsigned l=0; l<K; l++)
-        {
-            temp[l] = alpha*x1_ptr[l][i]*y1_ptr[l][i] 
-                      +beta*x2_ptr[l][i]*y2_ptr[l][i]
-                      +gamma*z_ptr[l][i];
-        }
-        for( unsigned l=0; l<K; l++)
-            z_ptr[l][i] = temp[l];
-    }
-}
-}
-template<class container>
-void pointwiseDot( double alpha, const container& x1, const container& y1, 
-                   double beta,  const container& x2, const container& y2, 
-                   double gamma, container & z)
-{
-    const double * RESTRICT x1_ptr; 
-    const double * RESTRICT y1_ptr; 
-    const double * RESTRICT x2_ptr; 
-    const double * RESTRICT y2_ptr; 
-    double * RESTRICT z_ptr;
-    //const double *  x1_ptr; 
-    //const double *  y1_ptr; 
-    //const double *  x2_ptr; 
-    //const double *  y2_ptr; 
-    //double *  z_ptr;
-    {
-        x1_ptr = thrust::raw_pointer_cast( &(x1.data()[0]));
-        x2_ptr = thrust::raw_pointer_cast( &(x2.data()[0]));
-        y1_ptr = thrust::raw_pointer_cast( &(y1.data()[0]));
-        y2_ptr = thrust::raw_pointer_cast( &(y2.data()[0]));
-         z_ptr = thrust::raw_pointer_cast( &(z.data()[0]));
-    }
-    unsigned size = x1.size();
-if(gamma!=0)
-{
-#pragma omp parallel for simd
-    for( unsigned i=0; i<size; i++)
-    {
-        z_ptr[i] = alpha*x1_ptr[i]*y1_ptr[i] 
-                  +beta*x2_ptr[i]*y2_ptr[i]
-                  +gamma*z_ptr[i];
-    }
-}
-else
-{
-#pragma omp parallel for simd
-    for( unsigned i=0; i<size; i++)
-    {
-        z_ptr[i] = alpha*x1_ptr[i]*y1_ptr[i] 
-                  +beta*x2_ptr[i]*y2_ptr[i];
-    }
-}
-}
-
 /*! @file 
   
   object for computation of Poisson bracket
@@ -206,46 +120,12 @@ void ArakawaX< Geometry, Matrix, container>::operator()( const container& lhs, c
     blas2::symv( bdxf, rhs, dxrhs);
     blas2::symv( bdyf, rhs, dyrhs);
 
-    //std::vector<const container* > s0(3), t0(3), s1(3), t1(3); 
-    //std::vector<container* > s2(3);
-    //s0[0] = &dxlhs, t0[0] = &dyrhs, s1[0] = &dylhs, t1[0] = &dxrhs;
-    //s0[1] =   &lhs, t0[1] = &dyrhs, s1[1] = &dylhs, t1[1] =   &rhs;
-    //s0[2] = &dxlhs, t0[2] =   &rhs, s1[2] =   &lhs, t1[2] = &dxrhs;
-    //s2[0] = &result, s2[1] = &dxlhs, s2[2] = &dxrhs;
-    //pointwiseDot( 1./3., s0, t0, -1./3., s1, t1, 0., s2);
+    blas1::pointwiseDot( 1./3., dxlhs, dyrhs, -1./3., dylhs, dxrhs, 0., result);
+    blas1::pointwiseDot( 1./3.,   lhs, dyrhs, -1./3., dylhs,   rhs, 0., helper_);
+    blas1::pointwiseDot( 1./3., dxlhs,   rhs, -1./3.,   lhs, dxrhs, 0., dylhs);
 
-    pointwiseDot( 1./3., dxlhs, dyrhs, -1./3., dylhs, dxrhs, 0., result);
-    pointwiseDot( 1./3.,   lhs, dyrhs, -1./3., dylhs,   rhs, 0., helper_);
-    pointwiseDot( 1./3., dxlhs,   rhs, -1./3.,   lhs, dxrhs, 0., dylhs);
     blas2::symv( 1., bdxf, helper_, 1., result);
     blas2::symv( 1., bdyf, dylhs, 1., result);
-    
-    //// order is important now
-    //// +x (1) -> result und (2) -> blhs
-    //blas1::pointwiseDot( lhs, dyrhs, result);
-    //blas1::pointwiseDot( lhs, dxrhs, helper_);
-
-    //// ++ (1) -> dyrhs and (2) -> dxrhs
-    //blas1::pointwiseDot( dxlhs, dyrhs, dyrhs);
-    //blas1::pointwiseDot( dylhs, dxrhs, dxrhs);
-
-    //// x+ (1) -> dxlhs and (2) -> dylhs
-    //blas1::pointwiseDot( dxlhs, rhs, dxlhs);
-    //blas1::pointwiseDot( dylhs, rhs, dylhs);
-
-    //blas1::axpby( 1./3., dyrhs, -1./3., dxrhs);  //dxl*dyr - dyl*dxr -> dxrhs
-    ////everything which needs a dx 
-    //blas1::axpby( 1./3., dxlhs, -1./3., helper_);   //dxl*r - l*dxr     -> helper 
-    ////everything which needs a dy
-    //blas1::axpby( 1./3., result, -1./3., dylhs); //l*dyr - dyl*r     -> dylhs
-
-    ////blas1::axpby( 0., dyrhs,  -0., dxrhs); //++
-    //////for testing purposes (note that you need to set criss-cross)
-    ////blas1::axpby( 1., dxlhs,  -0., helper); //x+ - +x
-    ////blas1::axpby( 0., result, -1., dylhs);  //+x - x+
-
-    //blas2::symv( 1., bdxf, dxlhs, 1., result);
-    //blas2::symv( 1., bdyf, dxrhs, 1., result);
     geo::dividePerpVolume( result, grid);
 }
 
diff --git a/inc/dg/backend/sparseblockmat_omp_kernels.h b/inc/dg/backend/sparseblockmat_omp_kernels.h
index 838e588a8..5f04add8a 100644
--- a/inc/dg/backend/sparseblockmat_omp_kernels.h
+++ b/inc/dg/backend/sparseblockmat_omp_kernels.h
@@ -1,13 +1,4 @@
-
-#if defined(__INTEL_COMPILER)
-  // On Intel compiler, you need to pass the -restrict compiler flag in addition to your own compiler flags.
-# define RESTRICT restrict
-#elif defined(__GNUG__)
-# define RESTRICT __restrict__
-#else
-# warning Missing restrict keyword for this compiler
-# define RESTRICT
-#endif
+#include "vector_categories.h"
 
 namespace dg{
 
diff --git a/inc/dg/backend/thrust_vector_blas.cuh b/inc/dg/backend/thrust_vector_blas.cuh
index 1cd8009ce..e65cb68a5 100644
--- a/inc/dg/backend/thrust_vector_blas.cuh
+++ b/inc/dg/backend/thrust_vector_blas.cuh
@@ -84,6 +84,8 @@ inline void doScal(  Vector& x,
               typename Vector::value_type alpha, 
               ThrustVectorTag)
 {
+    if( alpha == 1.) 
+        return;
     thrust::transform( x.begin(), x.end(), x.begin(), 
             detail::Axpby_Functor<typename Vector::value_type>( 0, alpha));
 }
@@ -108,14 +110,49 @@ inline void doAxpby( typename Vector::value_type alpha,
 #endif //DG_DEBUG
     if( alpha == 0)
     {
-        if( beta == 1) 
-            return;
-        thrust::transform( y.begin(), y.end(), y.begin(), 
-                detail::Axpby_Functor<typename Vector::value_type>( 0, beta));
+        doScal( y, beta, ThrustVectorTag());
+        return;
+    }
+    if( &x == &y)
+    {
+        doScal( y, (alpha+beta), ThrustVectorTag());
         return;
     }
-    thrust::transform( x.begin(), x.end(), y.begin(), y.begin(), 
+    if( alpha==1. && beta == 0) 
+    {
+        thrust::copy( x.begin(), x.end(), y.begin());
+        return; 
+    }
+#if THRUST_DEVICE_SYSTEM!=THRUST_DEVICE_SYSTEM_CUDA
+    const double * RESTRICT x_ptr = thrust::raw_pointer_cast( &x.data()[0]);
+    double * RESTRICT y_ptr = thrust::raw_pointer_cast( &y.data()[0]);
+    unsigned size = x.size();
+    if( beta == 1.)
+    {
+#pragma omp parallel for simd
+        for( unsigned i=0; i<size; i++)
+            y_ptr[i] += alpha*x_ptr[i];
+    }
+    else if (beta == 0)
+    {
+#pragma omp parallel for simd
+        for( unsigned i=0; i<size; i++)
+            y_ptr[i] = alpha*x_ptr[i];
+    }
+    else
+    {
+#pragma omp parallel for simd
+        for( unsigned i=0; i<size; i++)
+            y_ptr[i] = alpha*x_ptr[i] + beta*y_ptr[i];
+    }
+#else
+    if( beta != 0)
+        thrust::transform( x.begin(), x.end(), y.begin(), y.begin(), 
             detail::Axpby_Functor< typename Vector::value_type>( alpha, beta) );
+    else 
+        thrust::transform( x.begin(), x.end(), y.begin(),
+            detail::Axpby_Functor< typename Vector::value_type>( 0., alpha) );
+#endif
 }
 
 template< class Vector>
@@ -123,6 +160,7 @@ inline void doAxpby( typename Vector::value_type alpha,
               const Vector& x, 
               typename Vector::value_type beta, 
               const Vector& y, 
+              typename Vector::value_type gamma, 
               Vector& z, 
               ThrustVectorTag)
 {
@@ -132,18 +170,76 @@ inline void doAxpby( typename Vector::value_type alpha,
 #endif //DG_DEBUG
     if( alpha == 0)
     {
-        thrust::transform( y.begin(), y.end(), z.begin(), 
-                detail::Axpby_Functor<typename Vector::value_type>( 0, beta));
+        doAxpby( beta, y, gamma, z, ThrustVectorTag());
+        return;
+    }
+    else if( beta == 0)
+    {
+        doAxpby( alpha, x, gamma, z, ThrustVectorTag());
         return;
     }
-    if( beta == 0)
+    if( &x==&y)
     {
-        thrust::transform( x.begin(), x.end(), z.begin(), 
-                detail::Axpby_Functor<typename Vector::value_type>( 0, alpha));
+        doAxpby( alpha+beta, x, gamma, z);
         return;
     }
-    thrust::transform( x.begin(), x.end(), y.begin(), z.begin(), 
+    else if( &x==&z)
+    {
+        doAxpby( beta, y, alpha+gamma, z);
+        return;
+    }
+    else if( &y==&z)
+    {
+        doAxpby( alpha, x, beta+gamma, z);
+        return;
+    }
+#if THRUST_DEVICE_SYSTEM!=THRUST_DEVICE_SYSTEM_CUDA
+    const double * RESTRICT x_ptr = thrust::raw_pointer_cast( &x.data()[0]);
+    const double * RESTRICT y_ptr = thrust::raw_pointer_cast( &y.data()[0]);
+    double * RESTRICT z_ptr = thrust::raw_pointer_cast( &z.data()[0]);
+    unsigned size = x.size();
+    if( gamma == 1.)
+    {
+#pragma omp parallel for simd
+        for( unsigned i=0; i<size; i++)
+            z_ptr[i] += alpha*x_ptr[i]+beta*y_ptr[i];
+    }
+    else if (gamma == 0)
+    {
+#pragma omp parallel for simd
+        for( unsigned i=0; i<size; i++)
+            z_ptr[i] = alpha*x_ptr[i]+beta*y_ptr[i];
+    }
+    else
+    {
+#pragma omp parallel for simd
+        for( unsigned i=0; i<size; i++)
+            z_ptr[i] = alpha*x_ptr[i] + beta*y_ptr[i] + gamma*z_ptr[i];
+    }
+#else
+    if( gamma==0)
+    {
+        thrust::transform( x.begin(), x.end(), y.begin(), z.begin(), 
             detail::Axpby_Functor< typename Vector::value_type>( alpha, beta) );
+    }
+    else 
+    {
+        doAxpby( alpha, x, gamma, z);
+        doAxpby( beta, y, 1., z);
+    }
+
+#endif
+}
+
+template< class Vector>
+inline void doAxpby( typename Vector::value_type alpha, 
+              const Vector& x, 
+              typename Vector::value_type beta, 
+              const Vector& y, 
+              Vector& z, 
+              ThrustVectorTag)
+{
+    doAxpby( alpha, x, beta, y, 0., z, ThrustVectorTag());
 }
 
 template< class Vector>
@@ -188,17 +284,78 @@ inline void doPointwiseDot(
 #endif //DG_DEBUG
     if( alpha == 0)
     {
-        if( beta == 1) 
-            return;
         dg::blas1::detail::doScal(y, beta, dg::ThrustVectorTag());
         return;
     }
+#if THRUST_DEVICE_SYSTEM!=THRUST_DEVICE_SYSTEM_CUDA
+    const double * x1_ptr = thrust::raw_pointer_cast( &(x1.data()[0]));
+    const double * x2_ptr = thrust::raw_pointer_cast( &(x2.data()[0]));
+     double * y_ptr = thrust::raw_pointer_cast( &(y.data()[0]));
+    if( y_ptr != x1_ptr && y_ptr != x2_ptr)
+    {
+        double * RESTRICT yr_ptr = thrust::raw_pointer_cast( &(y.data()[0]));
+        unsigned size = x1.size();
+        if( beta == 0)
+        {
+#pragma omp parallel for simd
+            for( unsigned i=0; i<size; i++)
+            {
+                yr_ptr[i] = alpha*x1_ptr[i]*x2_ptr[i];
+            }
+        }
+        else if ( beta == 1)
+        {
+#pragma omp parallel for simd
+            for( unsigned i=0; i<size; i++)
+            {
+                yr_ptr[i] += alpha*x1_ptr[i]*x2_ptr[i];
+            }
+        }
+        else
+        {
+#pragma omp parallel for simd
+            for( unsigned i=0; i<size; i++)
+            {
+                yr_ptr[i] = alpha*x1_ptr[i]*x2_ptr[i]+beta*yr_ptr[i];
+            }
+        }
+    }
+    else
+    {
+        unsigned size = x1.size();
+        if( beta == 0)
+        {
+#pragma omp parallel for simd
+            for( unsigned i=0; i<size; i++)
+            {
+                y_ptr[i] = alpha*x1_ptr[i]*x2_ptr[i];
+            }
+        }
+        else if ( beta == 1)
+        {
+#pragma omp parallel for simd
+            for( unsigned i=0; i<size; i++)
+            {
+                y_ptr[i] += alpha*x1_ptr[i]*x2_ptr[i];
+            }
+        }
+        else
+        {
+#pragma omp parallel for simd
+            for( unsigned i=0; i<size; i++)
+            {
+                y_ptr[i] = alpha*x1_ptr[i]*x2_ptr[i]+beta*y_ptr[i];
+            }
+        }
+    }
+#else
     thrust::transform( 
         y.begin(), y.end(),
         thrust::make_zip_iterator( thrust::make_tuple( x1.begin(), x2.begin() )),  
         y.begin(),
         detail::ThrustVectorDoSymv<Vector>( alpha, beta)
     ); 
+#endif
 }
 
 template< class Vector>
@@ -213,6 +370,101 @@ inline void doPointwiseDivide( const Vector& x1, const Vector& x2, Vector& y, Th
 }
 
 
+template<class Vector>
+inline void doPointwiseDot(  
+              typename Vector::value_type alpha, 
+              const Vector& x1,
+              const Vector& y1, 
+              typename Vector::value_type beta, 
+              const Vector& x2,
+              const Vector& y2, 
+              typename Vector::value_type gamma, 
+              Vector& z, 
+              ThrustVectorTag)
+{
+    if( alpha==0){ 
+        doPointwiseDot( beta, x2,y2, gamma, z, ThrustVectorTag());
+        return;
+    }
+    else if( beta==0){
+        doPointwiseDot( alpha, x1,y1, gamma, z, ThrustVectorTag());
+        return;
+    }
+#if THRUST_DEVICE_SYSTEM!=THRUST_DEVICE_SYSTEM_CUDA
+    const double *x1_ptr = thrust::raw_pointer_cast( &(x1.data()[0]));
+    const double *x2_ptr = thrust::raw_pointer_cast( &(x2.data()[0]));
+    const double *y1_ptr = thrust::raw_pointer_cast( &(y1.data()[0]));
+    const double *y2_ptr = thrust::raw_pointer_cast( &(y2.data()[0]));
+        double * z_ptr = thrust::raw_pointer_cast( &(z.data()[0]));
+    unsigned size = x1.size();
+    if( z_ptr != x1_ptr && z_ptr != x2_ptr && z_ptr != y1_ptr && z_ptr != y2_ptr)
+    {
+        double * RESTRICT zr_ptr = thrust::raw_pointer_cast( &(z.data()[0]));
+        if(gamma==0)
+        {
+#pragma omp parallel for simd
+            for( unsigned i=0; i<size; i++)
+            {
+                zr_ptr[i] = alpha*x1_ptr[i]*y1_ptr[i] 
+                            +beta*x2_ptr[i]*y2_ptr[i];
+            }
+        }
+        else if(gamma==1.)
+        {
+#pragma omp parallel for simd
+            for( unsigned i=0; i<size; i++)
+            {
+                zr_ptr[i] += alpha*x1_ptr[i]*y1_ptr[i] 
+                            +beta*x2_ptr[i]*y2_ptr[i];
+            }
+        }
+        else
+        {
+#pragma omp parallel for simd
+            for( unsigned i=0; i<size; i++)
+            {
+                zr_ptr[i] = alpha*x1_ptr[i]*y1_ptr[i] 
+                            +beta*x2_ptr[i]*y2_ptr[i]
+                            +gamma*zr_ptr[i];
+            }
+        }
+    }
+    else
+    {
+        if(gamma==0)
+        {
+#pragma omp parallel for simd
+            for( unsigned i=0; i<size; i++)
+            {
+                z_ptr[i] = alpha*x1_ptr[i]*y1_ptr[i] 
+                            +beta*x2_ptr[i]*y2_ptr[i];
+            }
+        }
+        else if(gamma==1.)
+        {
+#pragma omp parallel for simd
+            for( unsigned i=0; i<size; i++)
+            {
+                z_ptr[i] += alpha*x1_ptr[i]*y1_ptr[i] 
+                            +beta*x2_ptr[i]*y2_ptr[i];
+            }
+        }
+        else
+        {
+#pragma omp parallel for simd
+            for( unsigned i=0; i<size; i++)
+            {
+                z_ptr[i] = alpha*x1_ptr[i]*y1_ptr[i] 
+                            +beta*x2_ptr[i]*y2_ptr[i]
+                            +gamma*z_ptr[i];
+            }
+        }
+    }
+#else
+    doPointwiseDot( alpha, x1, y1, gamma, z, ThrustVectorTag());
+    doPointwiseDot( beta,  x2, y2, 1., z, ThrustVectorTag());
+#endif
+}
 
 } //namespace detail
 ///@endcond
diff --git a/inc/dg/backend/vector_categories.h b/inc/dg/backend/vector_categories.h
index 86032d55c..2d7b69eab 100644
--- a/inc/dg/backend/vector_categories.h
+++ b/inc/dg/backend/vector_categories.h
@@ -1,6 +1,16 @@
 #ifndef _DG_VECTOR_CATEGORIES_
 #define _DG_VECTOR_CATEGORIES_
 
+#if defined(__INTEL_COMPILER)
+  // On Intel compiler, you need to pass the -restrict compiler flag in addition to your own compiler flags.
+# define RESTRICT restrict
+#elif defined(__GNUG__)
+# define RESTRICT __restrict__
+#else
+# warning Missing restrict keyword for this compiler
+# define RESTRICT
+#endif
+
 namespace dg{
 
 struct AnyVectorTag{};
diff --git a/inc/dg/blas1.h b/inc/dg/blas1.h
index 555719ed7..bfc233eb9 100644
--- a/inc/dg/blas1.h
+++ b/inc/dg/blas1.h
@@ -116,6 +116,26 @@ inline void axpby( typename VectorTraits<Vector>::value_type alpha, const Vector
     dg::blas1::detail::doAxpby( alpha, x, beta, y, result, typename dg::VectorTraits<Vector>::vector_category() );
     return;
 }
+
+/*! @brief Modified BLAS 1 routine axpy
+ *
+ * This routine computes \f[ z_i =  \alpha x_i + \beta y_i + \gamma z_i \f] 
+ * @param alpha Scalar  
+ * @param x Vector x may equal result
+ * @param beta Scalar
+ * @param y Vector y may equal result
+ * @param gamma Scalar
+ * @param z Vector contains solution on output
+ * @note If DG_DEBUG is defined a range check shall be performed
+ * @attention If a thrust::device_vector is used then this routine is NON-BLOCKING!
+ */
+template< class Vector>
+inline void axpby( typename VectorTraits<Vector>::value_type alpha, const Vector& x, typename VectorTraits<Vector>::value_type beta, const Vector& y, typename VectorTraits<Vector>::value_type gamma, Vector& z)
+{
+    dg::blas1::detail::doAxpby( alpha, x, beta, y, gamma, z, typename dg::VectorTraits<Vector>::vector_category() );
+    return;
+}
+
 /*! @brief "new" BLAS 1 routine transform
  *
  * This routine computes \f[ y_i = op(x_i) \f] 
@@ -174,6 +194,7 @@ inline void pointwiseDot( const Vector& x1, const Vector& x2, Vector& y)
     dg::blas1::detail::doPointwiseDot( x1, x2, y, typename dg::VectorTraits<Vector>::vector_category() );
     return;
 }
+
 /**
 * @brief A 'new' BLAS 1 routine. 
 *
@@ -206,6 +227,28 @@ inline void pointwiseDivide( const Vector& x1, const Vector& x2, Vector& y)
     dg::blas1::detail::doPointwiseDivide( x1, x2, y, typename dg::VectorTraits<Vector>::vector_category() );
     return;
 }
+
+/**
+* @brief A 'new' fused multiply-add BLAS 1 routine. 
+*
+* Multiplies and adds vectors element by element: \f[ z_i = \alpha x_{1i}y_{1i} + \beta x_{2i]y_{2i} + \gamma z_i\f]
+* @param alpha scalar
+* @param x1 Vector x1  
+* @param y1 Vector y1 
+* @param beta scalar
+* @param x2 Vector x1  
+* @param y2 Vector y1 
+* @param z  Vector z contains result on output 
+* @note aliases are allowed: we perform an alias analysis on the given references to detect possible performance optimizations
+*/
+template<class Vector>
+void pointwiseDot(  typename VectorTraits<Vector>::value_type alpha, const Vector& x1, const Vector& y1, 
+                    typename VectorTraits<Vector>::value_type beta,  const Vector& x2, const Vector& y2, 
+                    typename VectorTraits<Vector>::value_type gamma, Vector & z)
+{
+    dg::blas1::detail::doPointwiseDot( alpha, x1, y1, beta, x2, y2, gamma, z, typename dg::VectorTraits<Vector>::vector_category() );
+    return;
+}
 ///@}
 }//namespace blas1
 } //namespace dg
-- 
GitLab


From a426ec241e29422885e6977648d7b8517b5df042 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sun, 3 Sep 2017 00:18:47 +0200
Subject: [PATCH 235/453] introduced new blas1 functions in multistep.h

---
 inc/dg/backend/std_vector_blas.cuh    | 36 +++++++++++++++++++++++++++
 inc/dg/backend/thrust_vector_blas.cuh | 10 ++++----
 inc/dg/blas1_t.cu                     |  8 +++---
 inc/dg/multistep.h                    | 26 ++++++++-----------
 inc/dg/multistep_t.cu                 |  4 +--
 inc/dg/poisson.h                      |  4 +--
 6 files changed, 60 insertions(+), 28 deletions(-)

diff --git a/inc/dg/backend/std_vector_blas.cuh b/inc/dg/backend/std_vector_blas.cuh
index 8c58e651a..e2e777108 100644
--- a/inc/dg/backend/std_vector_blas.cuh
+++ b/inc/dg/backend/std_vector_blas.cuh
@@ -57,6 +57,24 @@ inline void doAxpby( typename VectorTraits<Vector>::value_type alpha,
         
 }
 
+template< class Vector>
+inline void doAxpby( typename VectorTraits<Vector>::value_type alpha, 
+              const std::vector<Vector>& x, 
+              typename VectorTraits<Vector>::value_type beta, 
+              const std::vector<Vector>& y, 
+              typename VectorTraits<Vector>::value_type gamma, 
+              std::vector<Vector>& z, 
+              StdVectorTag)
+{
+#ifdef DG_DEBUG
+    assert( !x.empty());
+    assert( x.size() == y.size() );
+#endif //DG_DEBUG
+    for( unsigned i=0; i<x.size(); i++)
+        doAxpby( alpha, x[i], beta, y[i], gamma, z[i], typename VectorTraits<Vector>::vector_category());
+        
+}
+
 template<class container>
 inline void doCopy( const std::vector<container>& x, std::vector<container>& y, StdVectorTag)
 {
@@ -122,6 +140,24 @@ std::vector<Vector>& y, StdVectorTag)
     for( unsigned i=0; i<x1.size(); i++)
         doPointwiseDot( alpha, x1[i], x2[i], beta, y[i], typename VectorTraits<Vector>::vector_category() );
 }
+template< class Vector>
+inline void doPointwiseDot( typename VectorTraits<Vector>::value_type alpha, 
+const std::vector<Vector>& x1, const std::vector<Vector>& y1, 
+typename VectorTraits<Vector>::value_type beta, 
+const std::vector<Vector>& x2, const std::vector<Vector>& y2, 
+typename VectorTraits<Vector>::value_type gamma, 
+std::vector<Vector>& z, StdVectorTag)
+{
+#ifdef DG_DEBUG
+    assert( !x1.empty());
+    assert( x1.size() == y1.size() );
+    assert( x1.size() == x2.size() );
+    assert( x1.size() == y2.size() );
+    assert( x1.size() == z.size() );
+#endif //DG_DEBUG
+    for( unsigned i=0; i<x1.size(); i++)
+        doPointwiseDot( alpha, x1[i], y1[i], beta, x2[i], y2[i], gamma,z[i], typename VectorTraits<Vector>::vector_category() );
+}
 
 template< class Vector>
 inline void doPointwiseDivide( const std::vector<Vector>& x1, const std::vector<Vector>& x2, std::vector<Vector>& y, StdVectorTag)
diff --git a/inc/dg/backend/thrust_vector_blas.cuh b/inc/dg/backend/thrust_vector_blas.cuh
index e65cb68a5..1e7d64605 100644
--- a/inc/dg/backend/thrust_vector_blas.cuh
+++ b/inc/dg/backend/thrust_vector_blas.cuh
@@ -180,17 +180,17 @@ inline void doAxpby( typename Vector::value_type alpha,
     }
     if( &x==&y)
     {
-        doAxpby( alpha+beta, x, gamma, z);
+        doAxpby( alpha+beta, x, gamma, z, ThrustVectorTag());
         return;
     }
     else if( &x==&z)
     {
-        doAxpby( beta, y, alpha+gamma, z);
+        doAxpby( beta, y, alpha+gamma, z, ThrustVectorTag());
         return;
     }
     else if( &y==&z)
     {
-        doAxpby( alpha, x, beta+gamma, z);
+        doAxpby( alpha, x, beta+gamma, z, ThrustVectorTag());
         return;
     }
 #if THRUST_DEVICE_SYSTEM!=THRUST_DEVICE_SYSTEM_CUDA
@@ -224,8 +224,8 @@ inline void doAxpby( typename Vector::value_type alpha,
     }
     else 
     {
-        doAxpby( alpha, x, gamma, z);
-        doAxpby( beta, y, 1., z);
+        doAxpby( alpha, x, gamma, z, ThrustVectorTag());
+        doAxpby( beta, y, 1., z, ThrustVectorTag());
     }
 
 #endif
diff --git a/inc/dg/blas1_t.cu b/inc/dg/blas1_t.cu
index 8dd2e7452..94c19009b 100644
--- a/inc/dg/blas1_t.cu
+++ b/inc/dg/blas1_t.cu
@@ -1,4 +1,4 @@
-#define CUSP_DEVICE_BLAS_SYSTEM CUSP_DEVICE_BLAS_CUBLAS
+//#define CUSP_DEVICE_BLAS_SYSTEM CUSP_DEVICE_BLAS_CUBLAS
 #include <iostream>
 #include <vector>
 
@@ -9,8 +9,8 @@ struct EXP{ __host__ __device__ double operator()(double x){return exp(x);}};
 
 //test program that (should ) call every blas1 function for every specialization
 
-//typedef thrust::host_vector<double>  Vector;
-typedef cusp::array1d<double, cusp::device_memory>  Vector;
+typedef thrust::host_vector<double>  Vector;
+//typedef cusp::array1d<double, cusp::device_memory>  Vector;
 int main()
 {
     Vector v1( 5, 2), v2( 5, 3);
@@ -31,6 +31,8 @@ int main()
     std::cout << "2*2+ 3*3 = " << v2[0] <<" (13)\n";
     dg::blas1::axpby( 2.5, v1, 0., v2);
     std::cout << "2.5*2+ 0 = " << v2[0] <<" (5)\n";
+    dg::blas1::axpby( 2.5, v1, 2., v2, 3, v3);
+    std::cout << "2.5*2+ 2.*5-3*12 = " << v3[0] <<" (-21)\n";
     dg::blas1::copy( v2, v1);
     std::cout << "5 = " << v1[0] <<" (5)"<< std::endl;
     dg::blas1::scal( v1, 0.4);
diff --git a/inc/dg/multistep.h b/inc/dg/multistep.h
index 6b6b6f1fc..8b4c2a4b4 100644
--- a/inc/dg/multistep.h
+++ b/inc/dg/multistep.h
@@ -104,14 +104,14 @@ template< size_t k, class Vector>
 template< class Functor>
 void AB<k, Vector>::operator()( Functor& f, Vector& u)
 {
-    blas1::axpby( 1., u_, 0, u);
+    blas1::copy(  u_, u);
     f( u, f_[0]);
     for( unsigned i=0; i<k; i++)
         blas1::axpby( dt_*ab_coeff<k>::b[i], f_[i], 1., u_);
     //permute f_[k-1]  to be the new f_[0]
     for( unsigned i=k-1; i>0; i--)
         f_[i-1].swap( f_[i]);
-    blas1::axpby( 1., u_, 0, u);
+    blas1::copy( u_, u);
 }
 
 ///@cond
@@ -305,24 +305,22 @@ template< class Functor, class Diffusion>
 void Karniadakis<Vector>::operator()( Functor& f, Diffusion& diff, Vector& u)
 {
 
-    blas1::axpby( 1., u_[0], 0, u); //save u_[0]
+    blas1::copy( u_[0], u); //save u_[0]
     f( u, f_[0]);
-    blas1::axpby( dt_*b[1], f_[1], dt_*b[2], f_[2], f_[2]);
-    blas1::axpby( dt_*b[0], f_[0],       1., f_[2], f_[2]);
-    blas1::axpby( a[1], u_[1], a[2], u_[2], u_[2]);
-    blas1::axpby( a[0], u_[0],   1., u_[2], u_[2]);
-    blas1::axpby( 1., u_[2], 1., f_[2], u);
+    blas1::axpby( dt_*b[0], f_[0], dt_*b[1], f_[1], dt_*b[2], f_[2]);
+    blas1::axpby( a[0], u_[0], a[1], u_[1], a[2], u_[2]);
     //permute f_[2], u_[2]  to be the new f_[0], u_[0]
     for( unsigned i=2; i>0; i--)
     {
         f_[i-1].swap( f_[i]);
         u_[i-1].swap( u_[i]);
     }
+    blas1::axpby( 1., f_[0], 1., u_[0]);
+    blas2::symv( diff.weights(), u_[0], u_[0]);
     //compute implicit part
     double alpha[2] = {2., -1.};
     //double alpha[2] = {1., 0.};
-    blas1::axpby( alpha[0], u_[1], alpha[1],  u_[2], u_[0]); //extrapolate previous solutions
-    blas2::symv( diff.weights(), u, u);
+    blas1::axpby( alpha[0], u_[1], alpha[1],  u_[2], u); //extrapolate previous solutions
     detail::Implicit<Diffusion, Vector> implicit( -dt_/11.*6., diff, f_[0]);
 #ifdef DG_BENCHMARK
 #ifdef MPI_VERSION
@@ -331,18 +329,16 @@ void Karniadakis<Vector>::operator()( Functor& f, Diffusion& diff, Vector& u)
 #endif//MPI
     Timer t;
     t.tic(); 
-    unsigned number = pcg( implicit, u_[0], u, diff.precond(), eps_);
+    unsigned number = pcg( implicit, u, u_[0], diff.precond(), eps_);
     t.toc();
 #ifdef MPI_VERSION
     if(rank==0)
 #endif//MPI
     std::cout << "# of pcg iterations for timestep: "<<number<<"/"<<pcg.get_max()<<" took "<<t.diff()<<"s\n";
 #else
-    pcg( implicit, u_[0], u, diff.precond(), eps_);
+    pcg( implicit, u, u_[0], diff.precond(), eps_);
 #endif //BENCHMARK
-    blas1::axpby( 1., u_[0], 0, u); //save u_[0]
-
-
+    blas1::copy( u, u_[0]); //store result
 }
 ///@endcond
 
diff --git a/inc/dg/multistep_t.cu b/inc/dg/multistep_t.cu
index d3bb4c53a..819af8963 100644
--- a/inc/dg/multistep_t.cu
+++ b/inc/dg/multistep_t.cu
@@ -83,8 +83,8 @@ int main()
     //thrust::swap(y0, y1);
     for( unsigned i=0; i<NT; i++)
     {
-        //tvb( rhs, diffusion, y0);
-        sirk( rhs, diffusion, y0, y1, dt);
+        tvb( rhs, diffusion, y0);
+        //sirk( rhs, diffusion, y0, y1, dt);
         y0.swap(y1);
     }
     double norm_y0 = dg::blas2::dot( w2d, y0[0]);
diff --git a/inc/dg/poisson.h b/inc/dg/poisson.h
index d6fa7ad2d..770d1a022 100644
--- a/inc/dg/poisson.h
+++ b/inc/dg/poisson.h
@@ -160,9 +160,7 @@ void Poisson< Geometry, Matrix, container>::operator()( const container& lhs, co
     blas2::symv(  dxrhs_, rhs,  dxrhsrhs_); //dx_rhs rhs
     blas2::symv(  dyrhs_, rhs,  dyrhsrhs_); //dy_rhs rhs
     
-    blas1::pointwiseDot( dxlhslhs_, dyrhsrhs_, result);   //dx_lhs lhs * dy_rhs rhs
-    blas1::pointwiseDot( -1., dylhslhs_, dxrhsrhs_, 1., result);    //- dy_lhs lhs * dx_rhs rhs
-
+    blas1::pointwiseDot( 1., dxlhslhs_, dyrhsrhs_, -1., dylhslhs_, dxrhsrhs_, 0., result);   //dx_lhs lhs * dy_rhs rhs
     geo::dividePerpVolume( result, g_);
 }
 
-- 
GitLab


From 5f92da73c544cff5456f36fb0f01621fb6f486a9 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sun, 3 Sep 2017 00:23:45 +0200
Subject: [PATCH 236/453] renamed new blas1 function axpbygz

---
 inc/dg/blas1.h     | 2 +-
 inc/dg/multistep.h | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/inc/dg/blas1.h b/inc/dg/blas1.h
index bfc233eb9..0261fd758 100644
--- a/inc/dg/blas1.h
+++ b/inc/dg/blas1.h
@@ -130,7 +130,7 @@ inline void axpby( typename VectorTraits<Vector>::value_type alpha, const Vector
  * @attention If a thrust::device_vector is used then this routine is NON-BLOCKING!
  */
 template< class Vector>
-inline void axpby( typename VectorTraits<Vector>::value_type alpha, const Vector& x, typename VectorTraits<Vector>::value_type beta, const Vector& y, typename VectorTraits<Vector>::value_type gamma, Vector& z)
+inline void axpbygz( typename VectorTraits<Vector>::value_type alpha, const Vector& x, typename VectorTraits<Vector>::value_type beta, const Vector& y, typename VectorTraits<Vector>::value_type gamma, Vector& z)
 {
     dg::blas1::detail::doAxpby( alpha, x, beta, y, gamma, z, typename dg::VectorTraits<Vector>::vector_category() );
     return;
diff --git a/inc/dg/multistep.h b/inc/dg/multistep.h
index 8b4c2a4b4..eb6741ab9 100644
--- a/inc/dg/multistep.h
+++ b/inc/dg/multistep.h
@@ -307,8 +307,8 @@ void Karniadakis<Vector>::operator()( Functor& f, Diffusion& diff, Vector& u)
 
     blas1::copy( u_[0], u); //save u_[0]
     f( u, f_[0]);
-    blas1::axpby( dt_*b[0], f_[0], dt_*b[1], f_[1], dt_*b[2], f_[2]);
-    blas1::axpby( a[0], u_[0], a[1], u_[1], a[2], u_[2]);
+    blas1::axpbygz( dt_*b[0], f_[0], dt_*b[1], f_[1], dt_*b[2], f_[2]);
+    blas1::axpbygz( a[0], u_[0], a[1], u_[1], a[2], u_[2]);
     //permute f_[2], u_[2]  to be the new f_[0], u_[0]
     for( unsigned i=2; i>0; i--)
     {
-- 
GitLab


From 82f370908188bc33de84142af2e06f322a185d66 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <Matthias.Wiesenberger@uibk.ac.at>
Date: Sun, 3 Sep 2017 10:43:52 +0200
Subject: [PATCH 237/453] some benchmarks on Marconi

---
 config/marconi.mk                     |  4 +--
 inc/dg/arakawa.h                      | 12 +++++---
 inc/dg/backend/thrust_vector_blas.cuh | 42 +++++++++++++--------------
 inc/dg/blas_b.cu                      | 28 +++++++++++++++---
 4 files changed, 55 insertions(+), 31 deletions(-)

diff --git a/config/marconi.mk b/config/marconi.mk
index 43abd6b68..afee829bf 100644
--- a/config/marconi.mk
+++ b/config/marconi.mk
@@ -26,7 +26,7 @@ endif
 ###########configure mic jobs with#########################
 #mcdram=cache:numa=quadrant
 #export KMP_AFFINITY=scatter #important
-#export OM_NUM_THREADS=68
-#qsub -I -qxfuaknldebug -A FUA21_FELTOR -l select=1:ncpus=68:mcdram=cach:numa=quadrant -l walltime=0:20:00
+#export OMP_NUM_THREADS=68
+#qsub -I -qxfuaknldebug -A FUA21_FELTOR -l select=1:ncpus=68:mcdram=cache:numa=quadrant -l walltime=0:29:00
 
 
diff --git a/inc/dg/arakawa.h b/inc/dg/arakawa.h
index cc4c12bc5..8e7f094a2 100644
--- a/inc/dg/arakawa.h
+++ b/inc/dg/arakawa.h
@@ -121,11 +121,15 @@ void ArakawaX< Geometry, Matrix, container>::operator()( const container& lhs, c
     blas2::symv( bdyf, rhs, dyrhs);
 
     blas1::pointwiseDot( 1./3., dxlhs, dyrhs, -1./3., dylhs, dxrhs, 0., result);
-    blas1::pointwiseDot( 1./3.,   lhs, dyrhs, -1./3., dylhs,   rhs, 0., helper_);
-    blas1::pointwiseDot( 1./3., dxlhs,   rhs, -1./3.,   lhs, dxrhs, 0., dylhs);
+    //blas1::pointwiseDot( 1./3.,   lhs, dyrhs, -1./3., dylhs,   rhs, 0., helper_);
+    //blas1::pointwiseDot( 1./3., dxlhs,   rhs, -1./3.,   lhs, dxrhs, 0., dylhs);
+    blas1::pointwiseDot( 1./3.,   lhs, dyrhs, -1./3., dylhs,   rhs, 0., dylhs);
+    blas1::pointwiseDot( 1./3., dxlhs,   rhs, -1./3.,   lhs, dxrhs, 0., dxrhs);
 
-    blas2::symv( 1., bdxf, helper_, 1., result);
-    blas2::symv( 1., bdyf, dylhs, 1., result);
+    //blas2::symv( 1., bdxf, helper_, 1., result);
+    //blas2::symv( 1., bdyf, dylhs, 1., result);
+    blas2::symv( 1., bdxf, dylhs, 1., result);
+    blas2::symv( 1., bdyf, dxrhs, 1., result);
     geo::dividePerpVolume( result, grid);
 }
 
diff --git a/inc/dg/backend/thrust_vector_blas.cuh b/inc/dg/backend/thrust_vector_blas.cuh
index 1e7d64605..1527cd38d 100644
--- a/inc/dg/backend/thrust_vector_blas.cuh
+++ b/inc/dg/backend/thrust_vector_blas.cuh
@@ -124,8 +124,8 @@ inline void doAxpby( typename Vector::value_type alpha,
         return; 
     }
 #if THRUST_DEVICE_SYSTEM!=THRUST_DEVICE_SYSTEM_CUDA
-    const double * RESTRICT x_ptr = thrust::raw_pointer_cast( &x.data()[0]);
-    double * RESTRICT y_ptr = thrust::raw_pointer_cast( &y.data()[0]);
+    const typename Vector::value_type * RESTRICT x_ptr = thrust::raw_pointer_cast( &x.data()[0]);
+    typename Vector::value_type * RESTRICT y_ptr = thrust::raw_pointer_cast( &y.data()[0]);
     unsigned size = x.size();
     if( beta == 1.)
     {
@@ -194,9 +194,9 @@ inline void doAxpby( typename Vector::value_type alpha,
         return;
     }
 #if THRUST_DEVICE_SYSTEM!=THRUST_DEVICE_SYSTEM_CUDA
-    const double * RESTRICT x_ptr = thrust::raw_pointer_cast( &x.data()[0]);
-    const double * RESTRICT y_ptr = thrust::raw_pointer_cast( &y.data()[0]);
-    double * RESTRICT z_ptr = thrust::raw_pointer_cast( &z.data()[0]);
+    const typename Vector::value_type * RESTRICT x_ptr = thrust::raw_pointer_cast( &x.data()[0]);
+    const typename Vector::value_type * RESTRICT y_ptr = thrust::raw_pointer_cast( &y.data()[0]);
+    typename Vector::value_type * RESTRICT z_ptr = thrust::raw_pointer_cast( &z.data()[0]);
     unsigned size = x.size();
     if( gamma == 1.)
     {
@@ -288,12 +288,12 @@ inline void doPointwiseDot(
         return;
     }
 #if THRUST_DEVICE_SYSTEM!=THRUST_DEVICE_SYSTEM_CUDA
-    const double * x1_ptr = thrust::raw_pointer_cast( &(x1.data()[0]));
-    const double * x2_ptr = thrust::raw_pointer_cast( &(x2.data()[0]));
-     double * y_ptr = thrust::raw_pointer_cast( &(y.data()[0]));
+    const typename Vector::value_type * x1_ptr = thrust::raw_pointer_cast( &(x1.data()[0]));
+    const typename Vector::value_type * x2_ptr = thrust::raw_pointer_cast( &(x2.data()[0]));
+     typename Vector::value_type * y_ptr = thrust::raw_pointer_cast( &(y.data()[0]));
     if( y_ptr != x1_ptr && y_ptr != x2_ptr)
     {
-        double * RESTRICT yr_ptr = thrust::raw_pointer_cast( &(y.data()[0]));
+        typename Vector::value_type * RESTRICT yr_ptr = thrust::raw_pointer_cast( &(y.data()[0]));
         unsigned size = x1.size();
         if( beta == 0)
         {
@@ -325,7 +325,7 @@ inline void doPointwiseDot(
         unsigned size = x1.size();
         if( beta == 0)
         {
-#pragma omp parallel for simd
+#pragma omp parallel for 
             for( unsigned i=0; i<size; i++)
             {
                 y_ptr[i] = alpha*x1_ptr[i]*x2_ptr[i];
@@ -333,7 +333,7 @@ inline void doPointwiseDot(
         }
         else if ( beta == 1)
         {
-#pragma omp parallel for simd
+#pragma omp parallel for 
             for( unsigned i=0; i<size; i++)
             {
                 y_ptr[i] += alpha*x1_ptr[i]*x2_ptr[i];
@@ -341,7 +341,7 @@ inline void doPointwiseDot(
         }
         else
         {
-#pragma omp parallel for simd
+#pragma omp parallel for 
             for( unsigned i=0; i<size; i++)
             {
                 y_ptr[i] = alpha*x1_ptr[i]*x2_ptr[i]+beta*y_ptr[i];
@@ -391,15 +391,15 @@ inline void doPointwiseDot(
         return;
     }
 #if THRUST_DEVICE_SYSTEM!=THRUST_DEVICE_SYSTEM_CUDA
-    const double *x1_ptr = thrust::raw_pointer_cast( &(x1.data()[0]));
-    const double *x2_ptr = thrust::raw_pointer_cast( &(x2.data()[0]));
-    const double *y1_ptr = thrust::raw_pointer_cast( &(y1.data()[0]));
-    const double *y2_ptr = thrust::raw_pointer_cast( &(y2.data()[0]));
-        double * z_ptr = thrust::raw_pointer_cast( &(z.data()[0]));
+    const typename Vector::value_type *x1_ptr = thrust::raw_pointer_cast( &(x1.data()[0]));
+    const typename Vector::value_type *x2_ptr = thrust::raw_pointer_cast( &(x2.data()[0]));
+    const typename Vector::value_type *y1_ptr = thrust::raw_pointer_cast( &(y1.data()[0]));
+    const typename Vector::value_type *y2_ptr = thrust::raw_pointer_cast( &(y2.data()[0]));
+        typename Vector::value_type * z_ptr = thrust::raw_pointer_cast( &(z.data()[0]));
     unsigned size = x1.size();
     if( z_ptr != x1_ptr && z_ptr != x2_ptr && z_ptr != y1_ptr && z_ptr != y2_ptr)
     {
-        double * RESTRICT zr_ptr = thrust::raw_pointer_cast( &(z.data()[0]));
+        typename Vector::value_type * RESTRICT zr_ptr = thrust::raw_pointer_cast( &(z.data()[0]));
         if(gamma==0)
         {
 #pragma omp parallel for simd
@@ -433,7 +433,7 @@ inline void doPointwiseDot(
     {
         if(gamma==0)
         {
-#pragma omp parallel for simd
+#pragma omp parallel for 
             for( unsigned i=0; i<size; i++)
             {
                 z_ptr[i] = alpha*x1_ptr[i]*y1_ptr[i] 
@@ -442,7 +442,7 @@ inline void doPointwiseDot(
         }
         else if(gamma==1.)
         {
-#pragma omp parallel for simd
+#pragma omp parallel for 
             for( unsigned i=0; i<size; i++)
             {
                 z_ptr[i] += alpha*x1_ptr[i]*y1_ptr[i] 
@@ -451,7 +451,7 @@ inline void doPointwiseDot(
         }
         else
         {
-#pragma omp parallel for simd
+#pragma omp parallel for 
             for( unsigned i=0; i<size; i++)
             {
                 z_ptr[i] = alpha*x1_ptr[i]*y1_ptr[i] 
diff --git a/inc/dg/blas_b.cu b/inc/dg/blas_b.cu
index bbc163043..d6b48932d 100644
--- a/inc/dg/blas_b.cu
+++ b/inc/dg/blas_b.cu
@@ -48,7 +48,7 @@ int main()
         norm += dg::blas1::dot( w2d, x);
     t.toc();
     std::cout<<"DOT took                         " <<t.diff()/20.<<"s\t"<<gbytes*20/t.diff()<<"GB/s\n";
-    Vector y(x);
+    Vector y(x), z(x), u(x), v(x);
     Matrix M;
     dg::blas2::transfer(dg::create::dx( grid, dg::centered), M);
     t.tic();
@@ -89,13 +89,33 @@ int main()
     for( int i=0; i<20; i++)
         dg::blas1::axpby( 1., y, -1., x);
     t.toc();
-    std::cout<<"AXPBY took                       "<<t.diff()/20<<"s\t"<<gbytes*20/t.diff()<<"GB/s\n";
+    std::cout<<"AXPBY (1*y-1*x=x)                "<<t.diff()/20<<"s\t"<<gbytes*20/t.diff()<<"GB/s\n";
+    t.tic();
+    for( int i=0; i<20; i++)
+        dg::blas1::axpbygz( 1., x, -1., y, 2., z);
+    t.toc();
+    std::cout<<"AXPBYGZ (1*x-1*y+2*z=z)          "<<t.diff()/20<<"s\t"<<gbytes*20/t.diff()<<"GB/s\n";
+    t.tic();
+    for( int i=0; i<20; i++)
+        dg::blas1::axpbygz( 1., x, -1., y, 3., x);
+    t.toc();
+    std::cout<<"AXPBYGZ (1*x-1.*y+3*x=x)         "<<t.diff()/20<<"s\t"<<gbytes*20/t.diff()<<"GB/s\n";
 
     t.tic();
     for( int i=0; i<20; i++)
-        dg::blas1::pointwiseDot( y, x, x);
+        dg::blas1::pointwiseDot( 1., y, x, 2., x);
+    t.toc();
+    std::cout<<"pointwiseDot (1*yx+2*x=x)        "<<t.diff()/20<<"s\t" <<gbytes*20/t.diff()<<"GB/s\n";
+    t.tic();
+    for( int i=0; i<20; i++)
+        dg::blas1::pointwiseDot( 1., y, x, 2.,u,v,0.,  z);
+    t.toc();
+    std::cout<<"pointwiseDot (1*yx+2*uv=z)       "<<t.diff()/20<<"s\t" <<gbytes*20/t.diff()<<"GB/s\n";
+    t.tic();
+    for( int i=0; i<20; i++)
+        dg::blas1::pointwiseDot( 1., y, x, 2.,u,v,0.,  v);
     t.toc();
-    std::cout<<"pointwiseDot took                "<<t.diff()/20<<"s\t" <<gbytes*20/t.diff()<<"GB/s\n";
+    std::cout<<"pointwiseDot (1*yx+2*uv=v)       "<<t.diff()/20<<"s\t" <<gbytes*20/t.diff()<<"GB/s\n";
     t.tic();
     for( int i=0; i<20; i++)
         norm += dg::blas2::dot( w2d, y);
-- 
GitLab


From 3b92223ae06d8f6c53eaef354ce2741f6e00367c Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <Matthias.Wiesenberger@uibk.ac.at>
Date: Sun, 3 Sep 2017 11:25:11 +0200
Subject: [PATCH 238/453] introduced simd in thrust omp backend

---
 inc/dg/backend/thrust_vector_blas.cuh | 28 ++++++-------
 inc/dg/blas_b.cu                      | 59 ++++++++++++++-------------
 2 files changed, 42 insertions(+), 45 deletions(-)

diff --git a/inc/dg/backend/thrust_vector_blas.cuh b/inc/dg/backend/thrust_vector_blas.cuh
index 1527cd38d..cdb56bcef 100644
--- a/inc/dg/backend/thrust_vector_blas.cuh
+++ b/inc/dg/backend/thrust_vector_blas.cuh
@@ -242,16 +242,6 @@ inline void doAxpby( typename Vector::value_type alpha,
     doAxpby( alpha, x, beta, y, 0., z, ThrustVectorTag());
 }
 
-template< class Vector>
-inline void doPointwiseDot( const Vector& x1, const Vector& x2, Vector& y, ThrustVectorTag)
-{
-#ifdef DG_DEBUG
-    assert( x1.size() == x2.size() );
-    assert( x1.size() == y.size() );
-#endif //DG_DEBUG
-    thrust::transform( x1.begin(), x1.end(), x2.begin(), y.begin(), 
-                        thrust::multiplies<typename VectorTraits<Vector>::value_type>());
-}
 
 template < class Vector>
 struct ThrustVectorDoSymv
@@ -325,7 +315,7 @@ inline void doPointwiseDot(
         unsigned size = x1.size();
         if( beta == 0)
         {
-#pragma omp parallel for 
+#pragma omp parallel for simd
             for( unsigned i=0; i<size; i++)
             {
                 y_ptr[i] = alpha*x1_ptr[i]*x2_ptr[i];
@@ -333,7 +323,7 @@ inline void doPointwiseDot(
         }
         else if ( beta == 1)
         {
-#pragma omp parallel for 
+#pragma omp parallel for simd
             for( unsigned i=0; i<size; i++)
             {
                 y_ptr[i] += alpha*x1_ptr[i]*x2_ptr[i];
@@ -341,7 +331,7 @@ inline void doPointwiseDot(
         }
         else
         {
-#pragma omp parallel for 
+#pragma omp parallel for simd
             for( unsigned i=0; i<size; i++)
             {
                 y_ptr[i] = alpha*x1_ptr[i]*x2_ptr[i]+beta*y_ptr[i];
@@ -358,6 +348,12 @@ inline void doPointwiseDot(
 #endif
 }
 
+template< class Vector>
+inline void doPointwiseDot( const Vector& x1, const Vector& x2, Vector& y, ThrustVectorTag)
+{
+    doPointwiseDot( 1., x1, x2, 0., y, ThrustVectorTag());
+}
+
 template< class Vector>
 inline void doPointwiseDivide( const Vector& x1, const Vector& x2, Vector& y, ThrustVectorTag)
 {
@@ -433,7 +429,7 @@ inline void doPointwiseDot(
     {
         if(gamma==0)
         {
-#pragma omp parallel for 
+#pragma omp parallel for simd
             for( unsigned i=0; i<size; i++)
             {
                 z_ptr[i] = alpha*x1_ptr[i]*y1_ptr[i] 
@@ -442,7 +438,7 @@ inline void doPointwiseDot(
         }
         else if(gamma==1.)
         {
-#pragma omp parallel for 
+#pragma omp parallel for simd
             for( unsigned i=0; i<size; i++)
             {
                 z_ptr[i] += alpha*x1_ptr[i]*y1_ptr[i] 
@@ -451,7 +447,7 @@ inline void doPointwiseDot(
         }
         else
         {
-#pragma omp parallel for 
+#pragma omp parallel for simd
             for( unsigned i=0; i<size; i++)
             {
                 z_ptr[i] = alpha*x1_ptr[i]*y1_ptr[i] 
diff --git a/inc/dg/blas_b.cu b/inc/dg/blas_b.cu
index d6b48932d..45a494200 100644
--- a/inc/dg/blas_b.cu
+++ b/inc/dg/blas_b.cu
@@ -42,93 +42,94 @@ int main()
     std::cout << "Sizeof value type is "<<sizeof(value_type)<<"\n";
     value_type gbytes=(value_type)x.size()*sizeof(value_type)/1e9;
     std::cout << "Sizeof vectors is "<<gbytes<<" GB\n";
+    unsigned multi=200;
     t.tic();
     value_type norm=0;
-    for( unsigned i=0; i<20; i++)
+    for( unsigned i=0; i<multi; i++)
         norm += dg::blas1::dot( w2d, x);
     t.toc();
-    std::cout<<"DOT took                         " <<t.diff()/20.<<"s\t"<<gbytes*20/t.diff()<<"GB/s\n";
+    std::cout<<"DOT took                         " <<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
     Vector y(x), z(x), u(x), v(x);
     Matrix M;
     dg::blas2::transfer(dg::create::dx( grid, dg::centered), M);
     t.tic();
-    for( int i=0; i<20; i++)
+    for( int i=0; i<multi; i++)
         dg::blas2::symv( M, x, y);
     t.toc();
-    std::cout<<"centered x derivative took       "<<t.diff()/20.<<"s\t"<<gbytes*20/t.diff()<<"GB/s\n";
+    std::cout<<"centered x derivative took       "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
 
     dg::blas2::transfer(dg::create::dx( grid, dg::backward), M);
     t.tic();
-    for( int i=0; i<20; i++)
+    for( int i=0; i<multi; i++)
         dg::blas2::symv( M, x, y);
     t.toc();
-    std::cout<<"forward x derivative took        "<<t.diff()/20<<"s\t"<<gbytes*20/t.diff()<<"GB/s\n";
+    std::cout<<"forward x derivative took        "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
 
     dg::blas2::transfer(dg::create::dy( grid, dg::backward), M);
     t.tic();
-    for( int i=0; i<20; i++)
+    for( int i=0; i<multi; i++)
         dg::blas2::symv( M, x, y);
     t.toc();
-    std::cout<<"forward y derivative took        "<<t.diff()/20<<"s\t"<<gbytes*20/t.diff()<<"GB/s\n";
+    std::cout<<"forward y derivative took        "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
 
     dg::blas2::transfer(dg::create::dy( grid, dg::centered), M);
     t.tic();
-    for( int i=0; i<20; i++)
+    for( int i=0; i<multi; i++)
         dg::blas2::symv( M, x, y);
     t.toc();
-    std::cout<<"centered y derivative took       "<<t.diff()/20<<"s\t"<<gbytes*20/t.diff()<<"GB/s\n";
+    std::cout<<"centered y derivative took       "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
 
     dg::blas2::transfer(dg::create::jumpX( grid), M);
     t.tic();
-    for( int i=0; i<20; i++)
+    for( int i=0; i<multi; i++)
         dg::blas2::symv( M, x, y);
     t.toc();
-    std::cout<<"jump X took                      "<<t.diff()/20<<"s\t"<<gbytes*20/t.diff()<<"GB/s\n";
+    std::cout<<"jump X took                      "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
 
     t.tic();
-    for( int i=0; i<20; i++)
+    for( int i=0; i<multi; i++)
         dg::blas1::axpby( 1., y, -1., x);
     t.toc();
-    std::cout<<"AXPBY (1*y-1*x=x)                "<<t.diff()/20<<"s\t"<<gbytes*20/t.diff()<<"GB/s\n";
+    std::cout<<"AXPBY (1*y-1*x=x)                "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
     t.tic();
-    for( int i=0; i<20; i++)
+    for( int i=0; i<multi; i++)
         dg::blas1::axpbygz( 1., x, -1., y, 2., z);
     t.toc();
-    std::cout<<"AXPBYGZ (1*x-1*y+2*z=z)          "<<t.diff()/20<<"s\t"<<gbytes*20/t.diff()<<"GB/s\n";
+    std::cout<<"AXPBYGZ (1*x-1*y+2*z=z)          "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
     t.tic();
-    for( int i=0; i<20; i++)
+    for( int i=0; i<multi; i++)
         dg::blas1::axpbygz( 1., x, -1., y, 3., x);
     t.toc();
-    std::cout<<"AXPBYGZ (1*x-1.*y+3*x=x)         "<<t.diff()/20<<"s\t"<<gbytes*20/t.diff()<<"GB/s\n";
+    std::cout<<"AXPBYGZ (1*x-1.*y+3*x=x)         "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
 
     t.tic();
-    for( int i=0; i<20; i++)
-        dg::blas1::pointwiseDot( 1., y, x, 2., x);
+    for( int i=0; i<multi; i++)
+        dg::blas1::pointwiseDot(  y, x, x);
     t.toc();
-    std::cout<<"pointwiseDot (1*yx+2*x=x)        "<<t.diff()/20<<"s\t" <<gbytes*20/t.diff()<<"GB/s\n";
+    std::cout<<"pointwiseDot (yx=x)              "<<t.diff()/multi<<"s\t" <<gbytes*multi/t.diff()<<"GB/s\n";
     t.tic();
-    for( int i=0; i<20; i++)
+    for( int i=0; i<multi; i++)
         dg::blas1::pointwiseDot( 1., y, x, 2.,u,v,0.,  z);
     t.toc();
-    std::cout<<"pointwiseDot (1*yx+2*uv=z)       "<<t.diff()/20<<"s\t" <<gbytes*20/t.diff()<<"GB/s\n";
+    std::cout<<"pointwiseDot (1*yx+2*uv=z)       "<<t.diff()/multi<<"s\t" <<gbytes*multi/t.diff()<<"GB/s\n";
     t.tic();
-    for( int i=0; i<20; i++)
+    for( int i=0; i<multi; i++)
         dg::blas1::pointwiseDot( 1., y, x, 2.,u,v,0.,  v);
     t.toc();
-    std::cout<<"pointwiseDot (1*yx+2*uv=v)       "<<t.diff()/20<<"s\t" <<gbytes*20/t.diff()<<"GB/s\n";
+    std::cout<<"pointwiseDot (1*yx+2*uv=v)       "<<t.diff()/multi<<"s\t" <<gbytes*multi/t.diff()<<"GB/s\n";
     t.tic();
-    for( int i=0; i<20; i++)
+    for( int i=0; i<multi; i++)
         norm += dg::blas2::dot( w2d, y);
     t.toc();
-    std::cout<<"DOT(w,y) took                    " <<t.diff()/20.<<"s\t"<<gbytes*20/t.diff()<<"GB/s\n";
+    std::cout<<"DOT(w,y) took                    " <<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
 
     t.tic();
-    for( int i=0; i<20; i++)
+    for( int i=0; i<multi; i++)
     {
         norm += dg::blas2::dot( x, w2d, y);
     }
     t.toc();
-    std::cout<<"DOT(x,w,y) took                  " <<t.diff()/20<<"s\t"<<gbytes*20/t.diff()<<"GB/s\n";
+    std::cout<<"DOT(x,w,y) took                  " <<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
     std::cout<<norm<<std::endl;
 
     return 0;
-- 
GitLab


From 331cd155bf7d336bdc8ccc2c69a4a7d775b3b1c5 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sun, 3 Sep 2017 11:55:19 +0200
Subject: [PATCH 239/453] added restrict to CUDA kernels as well

---
 config/devices/devices.mk                     |  2 +-
 inc/dg/backend/sparseblockmat_gpu_kernels.cuh | 30 +++++++++----------
 inc/dg/blas_b.cu                              |  4 +--
 3 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/config/devices/devices.mk b/config/devices/devices.mk
index 7694ee689..9359f7398 100644
--- a/config/devices/devices.mk
+++ b/config/devices/devices.mk
@@ -1,7 +1,7 @@
 ifeq ($(strip $(device)),gpu)
 ccc_:=$(CC)
 CC = nvcc --compiler-bindir $(ccc_)
-OPT=-O2
+OPT=-O3
 CFLAGS+=-D_FORCE_INLINES
 CFLAGS+= --compiler-options -Wall $(NVCCARCH)
 CFLAGS+= -Xcompiler $(OMPFLAG)
diff --git a/inc/dg/backend/sparseblockmat_gpu_kernels.cuh b/inc/dg/backend/sparseblockmat_gpu_kernels.cuh
index 7fc4fa482..7bbd23853 100644
--- a/inc/dg/backend/sparseblockmat_gpu_kernels.cuh
+++ b/inc/dg/backend/sparseblockmat_gpu_kernels.cuh
@@ -6,12 +6,12 @@ namespace dg
 // general multiply kernel
 template<class value_type>
  __global__ void ell_multiply_kernel( value_type alpha, value_type beta,
-         const value_type* data, const int* cols_idx, const int* data_idx, 
+         const value_type* __restrict__  data, const int* __restrict__  cols_idx, const int* __restrict__  data_idx, 
          const int num_rows, const int num_cols, const int blocks_per_line,
          const int n, const int size,
          const int right, 
-         const int* right_range,
-         const value_type* x, value_type *y
+         const int* __restrict__  right_range,
+         const value_type* __restrict__  x, value_type * __restrict__ y
          )
 {
     const int thread_id = blockDim.x * blockIdx.x + threadIdx.x;
@@ -42,12 +42,12 @@ template<class value_type>
 // multiply kernel, n=3, 3 blocks per line
 template<class value_type>
  __global__ void ell_multiply_kernel33(value_type alpha, value_type beta,
-         const value_type* data, const int* cols_idx, const int* data_idx, 
+         const value_type* __restrict__  data, const int* __restrict__  cols_idx, const int* __restrict__  data_idx, 
          const int num_rows, const int num_cols,
          const int size,
          const int right, 
-         const int* right_range,
-         const value_type* x, value_type *y
+         const int* __restrict__  right_range,
+         const value_type* __restrict__  x, value_type * __restrict__ y
          )
 {
     const int thread_id = blockDim.x * blockIdx.x + threadIdx.x;
@@ -87,12 +87,12 @@ template<class value_type>
 // multiply kernel, n=3, 2 blocks per line
 template<class value_type>
  __global__ void ell_multiply_kernel32(value_type alpha, value_type beta,
-         const value_type* data, const int* cols_idx, const int* data_idx, 
+         const value_type* __restrict__  data, const int* __restrict__  cols_idx, const int* __restrict__  data_idx, 
          const int num_rows, const int num_cols, 
          const int size,
          const int right, 
-         const int* right_range,
-         const value_type* x, value_type *y
+         const int* __restrict__  right_range,
+         const value_type* __restrict__  x, value_type * __restrict__ y
          )
 {
     //int size = left*num_rows*n*right;
@@ -128,10 +128,10 @@ template<class value_type>
 // multiply kernel, n=3, 3 blocks per line, right = 1
 template<class value_type>
  __global__ void ell_multiply_kernel33x(value_type alpha, value_type beta,
-         const value_type* data, const int* cols_idx, const int* data_idx, 
+         const value_type* __restrict__  data, const int* __restrict__  cols_idx, const int* __restrict__  data_idx, 
          const int num_rows, const int num_cols,
          const int size,
-         const value_type* x, value_type *y
+         const value_type* __restrict__  x, value_type * __restrict__ y
          )
 {
     const int thread_id = blockDim.x * blockIdx.x + threadIdx.x;
@@ -167,10 +167,10 @@ template<class value_type>
 // multiply kernel, n=3, 2 blocks per line, right = 1
 template<class value_type>
  __global__ void ell_multiply_kernel32x(value_type alpha, value_type beta,
-         const value_type* data, const int* cols_idx, const int* data_idx, 
+         const value_type* __restrict__  data, const int* __restrict__  cols_idx, const int* __restrict__  data_idx, 
          const int num_rows, const int num_cols,
          const int size,
-         const value_type* x, value_type *y
+         const value_type* __restrict__  x, value_type * __restrict__ y
          )
 {
     const int thread_id = blockDim.x * blockIdx.x + threadIdx.x;
@@ -203,11 +203,11 @@ template<class value_type>
 // multiply kernel
 template<class value_type>
  __global__ void coo_multiply_kernel(
-         const value_type* data, const int* rows_idx, const int* cols_idx, const int* data_idx, 
+         const value_type* __restrict__  data, const int* __restrict__  rows_idx, const int* __restrict__  cols_idx, const int* __restrict__  data_idx, 
          const int num_rows, const int num_cols, const int entry,
          const int n, 
          const int left, const int right, 
-         value_type alpha, const value_type* x, value_type beta, value_type *y
+         value_type alpha, const value_type* __restrict__  x, value_type beta, value_type * __restrict__ y
          )
 {
     int size = left*n*right;
diff --git a/inc/dg/blas_b.cu b/inc/dg/blas_b.cu
index 45a494200..487d0a343 100644
--- a/inc/dg/blas_b.cu
+++ b/inc/dg/blas_b.cu
@@ -42,10 +42,10 @@ int main()
     std::cout << "Sizeof value type is "<<sizeof(value_type)<<"\n";
     value_type gbytes=(value_type)x.size()*sizeof(value_type)/1e9;
     std::cout << "Sizeof vectors is "<<gbytes<<" GB\n";
-    unsigned multi=200;
+    int multi=200;
     t.tic();
     value_type norm=0;
-    for( unsigned i=0; i<multi; i++)
+    for( int i=0; i<multi; i++)
         norm += dg::blas1::dot( w2d, x);
     t.toc();
     std::cout<<"DOT took                         " <<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
-- 
GitLab


From ad1471980c157d79b8dcebdf8a8acef46e80753a Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sun, 3 Sep 2017 12:29:54 +0200
Subject: [PATCH 240/453] maded gpu pointwiseDot kernel

---
 inc/dg/backend/thrust_vector_blas.cuh | 29 +++++++++++++++++++++++----
 inc/dg/blas1_t.cu                     |  4 ++--
 2 files changed, 27 insertions(+), 6 deletions(-)

diff --git a/inc/dg/backend/thrust_vector_blas.cuh b/inc/dg/backend/thrust_vector_blas.cuh
index cdb56bcef..25c91ef6b 100644
--- a/inc/dg/backend/thrust_vector_blas.cuh
+++ b/inc/dg/backend/thrust_vector_blas.cuh
@@ -366,6 +366,24 @@ inline void doPointwiseDivide( const Vector& x1, const Vector& x2, Vector& y, Th
 }
 
 
+#if THRUST_DEVICE_SYSTEM==THRUST_DEVICE_SYSTEM_CUDA
+template<class value_type>
+ __global__ void pointwiseDot_kernel( value_type alpha, value_type beta, value_type gamma,
+         const value_type*  x1, const value_type* y1, const value_type* x2, 
+         const value_type*  y2, value_type* z,  
+         const int size
+         )
+{
+    const int thread_id = blockDim.x * blockIdx.x + threadIdx.x;
+    const int grid_size = gridDim.x*blockDim.x;
+    //every thread takes num_rows/grid_size rows
+    for( int row = thread_id; row<size; row += grid_size)
+    {
+        z[row]=alpha*x1[row]*y1[row]+beta*x2[row]*y2[row]+gamma*z[row];
+    }
+}
+#endif
+
 template<class Vector>
 inline void doPointwiseDot(  
               typename Vector::value_type alpha, 
@@ -386,13 +404,13 @@ inline void doPointwiseDot(
         doPointwiseDot( alpha, x1,y1, gamma, z, ThrustVectorTag());
         return;
     }
-#if THRUST_DEVICE_SYSTEM!=THRUST_DEVICE_SYSTEM_CUDA
     const typename Vector::value_type *x1_ptr = thrust::raw_pointer_cast( &(x1.data()[0]));
     const typename Vector::value_type *x2_ptr = thrust::raw_pointer_cast( &(x2.data()[0]));
     const typename Vector::value_type *y1_ptr = thrust::raw_pointer_cast( &(y1.data()[0]));
     const typename Vector::value_type *y2_ptr = thrust::raw_pointer_cast( &(y2.data()[0]));
         typename Vector::value_type * z_ptr = thrust::raw_pointer_cast( &(z.data()[0]));
     unsigned size = x1.size();
+#if THRUST_DEVICE_SYSTEM!=THRUST_DEVICE_SYSTEM_CUDA
     if( z_ptr != x1_ptr && z_ptr != x2_ptr && z_ptr != y1_ptr && z_ptr != y2_ptr)
     {
         typename Vector::value_type * RESTRICT zr_ptr = thrust::raw_pointer_cast( &(z.data()[0]));
@@ -457,12 +475,15 @@ inline void doPointwiseDot(
         }
     }
 #else
-    doPointwiseDot( alpha, x1, y1, gamma, z, ThrustVectorTag());
-    doPointwiseDot( beta,  x2, y2, 1., z, ThrustVectorTag());
+    //set up kernel parameters
+    const size_t BLOCK_SIZE = 256; 
+    const size_t NUM_BLOCKS = std::min<size_t>((size-1)/BLOCK_SIZE+1, 65000);
+    pointwiseDot_kernel<typename Vector::value_type><<<NUM_BLOCKS, BLOCK_SIZE>>>( alpha, beta, gamma, x1_ptr, y1_ptr, x2_ptr, y2_ptr, z_ptr, size);
 #endif
 }
 
-} //namespace detail
+}//namespace detail
+
 ///@endcond
 } //namespace blas1
 } //namespace dg
diff --git a/inc/dg/blas1_t.cu b/inc/dg/blas1_t.cu
index 94c19009b..7a0428f41 100644
--- a/inc/dg/blas1_t.cu
+++ b/inc/dg/blas1_t.cu
@@ -9,7 +9,7 @@ struct EXP{ __host__ __device__ double operator()(double x){return exp(x);}};
 
 //test program that (should ) call every blas1 function for every specialization
 
-typedef thrust::host_vector<double>  Vector;
+typedef thrust::device_vector<double>  Vector;
 //typedef cusp::array1d<double, cusp::device_memory>  Vector;
 int main()
 {
@@ -31,7 +31,7 @@ int main()
     std::cout << "2*2+ 3*3 = " << v2[0] <<" (13)\n";
     dg::blas1::axpby( 2.5, v1, 0., v2);
     std::cout << "2.5*2+ 0 = " << v2[0] <<" (5)\n";
-    dg::blas1::axpby( 2.5, v1, 2., v2, 3, v3);
+    dg::blas1::axpbygz( 2.5, v1, 2., v2, 3., v3);
     std::cout << "2.5*2+ 2.*5-3*12 = " << v3[0] <<" (-21)\n";
     dg::blas1::copy( v2, v1);
     std::cout << "5 = " << v1[0] <<" (5)"<< std::endl;
-- 
GitLab


From 71c95b70efb9a55af3884b338eb320a44b5c0554 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sun, 3 Sep 2017 21:54:48 +0200
Subject: [PATCH 241/453] added multiply_b file

---
 inc/dg/geometry/multiply.h    | 66 +++++++++++++++--------------------
 inc/dg/geometry/multiply_b.cu | 38 ++++++++++++++++++++
 inc/dg/geometry/multiply_t.cu |  8 ++---
 inc/dg/geometry/tensor.h      |  4 +--
 4 files changed, 73 insertions(+), 43 deletions(-)
 create mode 100644 inc/dg/geometry/multiply_b.cu

diff --git a/inc/dg/geometry/multiply.h b/inc/dg/geometry/multiply.h
index 2ddfb18b9..35fba4cf7 100644
--- a/inc/dg/geometry/multiply.h
+++ b/inc/dg/geometry/multiply.h
@@ -119,6 +119,26 @@ void pointwiseDivide( const container& in, const SparseElement<container>& mu, c
         out=in;
 }
 
+///@cond
+namespace detail
+{
+//i0 must be the diagonal index, out0 may alias in0 but not in1
+template<class container>
+void multiply2d_helper( const SparseTensor<container>& t, const container& in0, const container& in1, container& out0, int i0[2], int i1[2])
+{
+    if( t.isSet(i0[0],i0[1]) && t.isSet(i1[0],i1[1]) ) 
+        dg::blas1::pointwiseDot( 1. , t.value(i0[0],i0[1]), in0, 1., t.value(i1[0], i1[1]), in1, 0., out0); 
+    else if( t.isSet(i0[0],i0[1]) && !t.isSet(i1[0],i1[1]) ) 
+        dg::blas1::pointwiseDot( t.value(i0[0], i0[1]), in0, out0);
+    else 
+    {
+        out0=in0;
+        if( t.isSet(i1[0], i1[1]))
+            dg::blas1::pointwiseDot( 1.,  t.value(i1[0], i1[1]), in1, 1., out0);
+    }
+}
+}//namespace detail
+///@endcond
 /**
  * @brief Multiply a tensor with a vector in 2d
  *
@@ -135,29 +155,10 @@ void pointwiseDivide( const container& in, const SparseElement<container>& mu, c
 template<class container>
 void multiply2d( const SparseTensor<container>& t, const container& in0, const container& in1, container& out0, container& out1)
 {
-    if(!t.isSet(0,1))//lower triangular
-    {
-        if(t.isSet(1,1))  dg::blas1::pointwiseDot( t.value(1,1), in1, out1);
-        else out1=in1;
-        if(t.isSet(1,0)) dg::blas1::pointwiseDot( 1., t.value(1,0), in0, 1., out1);
-        if( t.isSet(0,0)) dg::blas1::pointwiseDot( t.value(0,0), in0, out0);
-        else out0=in0;
-        return;
-    }
-    //upper triangular and default
-    if( t.isSet(0,0) ) 
-        dg::blas1::pointwiseDot( t.value(0,0), in0, out0); 
-    else 
-        out0=in0;
-    if(t.isSet(0,1)) //true
-        dg::blas1::pointwiseDot( 1.,  t.value(0,1), in1, 1., out0);
-
-    if( t.isSet(1,1) )
-        dg::blas1::pointwiseDot( t.value(1,1), in1, out1);
-    else 
-        out1=in1;
-    if(t.isSet(1,0)) //if aliasing happens this is wrong
-        dg::blas1::pointwiseDot( 1.,  t.value(1,0), in0, 1., out1);
+    int i0[2] = {0,0}, i1[2] = {1,0};
+    int i3[2] = {1,1}, i2[2] = {1,0};
+    detail::multiply2d_helper( t, in0, in1, out0, i0, i1);
+    detail::multiply2d_helper( t, in1, in0, out1, i3, i2);
 }
 
 /**
@@ -175,20 +176,11 @@ void multiply2d( const SparseTensor<container>& t, const container& in0, const c
 template<class container>
 void multiply2d_inplace( const SparseTensor<container>& t, container& inout0, container& inout1, container& workspace)
 {
-    if( t.isSet(0,1) ) dg::blas1::pointwiseDot( t.value(0,1), inout1, workspace);
-    //compute out1 inplace
-    if( t.isSet(1,1)) dg::blas1::pointwiseDot( t.value(1,1), inout1, inout1);
-    if( t.isSet(1,0)) dg::blas1::pointwiseDot( 1., t.value(1,0), inout0, 1., inout1);
-
-    if(t.isSet(0,1)) //workspace is filled
-    {
-        if( !t.isSet(0,0)) dg::blas1::axpby( 1., inout0, 1., workspace, workspace);
-        else dg::blas1::pointwiseDot( 1., t.value(0,0), inout0, 1., workspace); 
-        workspace.swap( inout0);
-    }
-    else
-        if( t.isSet(0,0)) dg::blas1::pointwiseDot( t.value(0,0), inout0, inout0); 
-
+    int i0[2] = {0,0}, i1[2] = {1,0};
+    int i3[2] = {1,1}, i2[2] = {1,0};
+    detail::multiply2d_helper( t, inout0, inout1, workspace, i0, i1);
+    detail::multiply2d_helper( t, inout1, inout0, inout1, i3, i2);
+    workspace.swap( inout0);
     //needs to load a vector         11 times if every element is set (5 is the optimal symmetric inplace algorithm)
     //if triangular                  7 (5 optimal)
     //if diagonal                    4 (optimal)
diff --git a/inc/dg/geometry/multiply_b.cu b/inc/dg/geometry/multiply_b.cu
new file mode 100644
index 000000000..194e156aa
--- /dev/null
+++ b/inc/dg/geometry/multiply_b.cu
@@ -0,0 +1,38 @@
+
+#include <iostream>
+#include <cmath>
+
+#include "tensor.h"
+#include "multiply.h"
+#include "dg/backend/timer.cuh"
+
+typedef thrust::device_vector<double> Vector;
+
+int main()
+{
+    dg::Timer t;
+    unsigned n, Nx, Ny, Nz; 
+    std::cout << "Type n, Nx, Ny and Nz\n";
+    std::cin >> n >> Nx >> Ny >> Nz;
+    dg::Grid3d grid( 0., 2.*M_PI, 0, 2.*M_PI, 0, 2.*M_PI, n, Nx, Ny, Nz);
+    Vector w2d;
+    dg::blas1::transfer( dg::create::weights(grid), w2d);
+    dg::SparseTensor<Vector> g(3);
+    g.idx(0,0) = 0, g.idx(0,1) = g.idx(1,0) = 1, g.idx(1,1) = 2; 
+    g.value(0) = g.value(1) = g.value(2) = w2d;
+    Vector v_x = dg::evaluate( dg::CONSTANT(2), grid), w_x(v_x), temp(v_x);
+    Vector v_y = dg::evaluate( dg::CONSTANT(5), grid), w_y(v_y);
+    double gbytes=(double)v_x.size()*sizeof(double)/1e9;
+    int multi=20;
+    t.tic();
+    for( int i=0; i<multi; i++)
+        dg::tensor::multiply2d_inplace( g, v_x, v_y, temp);
+    t.toc();
+    std::cout<<"multiply_inplace(g,v_x,v_y,temp) took   "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
+    t.tic();
+    for( int i=0; i<multi; i++)
+        dg::tensor::multiply2d( g, v_x, v_y, w_x, w_y);
+    t.toc();
+    std::cout<<"multiply2d(g,v_x,v_y,w_x,w_y) took      "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
+    return 0;
+}
diff --git a/inc/dg/geometry/multiply_t.cu b/inc/dg/geometry/multiply_t.cu
index e5ca51019..7d560c94a 100644
--- a/inc/dg/geometry/multiply_t.cu
+++ b/inc/dg/geometry/multiply_t.cu
@@ -66,16 +66,16 @@ int main()
     std::cout << "Test Tensor multiplies \n";
     std::cout << "Multiply T with [8,9]\n";
     dg::tensor::multiply2d(t, inout0, inout1, work0, work1);
-    std::cout << "Result         is ["<<work0[0]<<" "<<work1[0]<<"]\n";
+    std::cout << "Result         is ["<<work0[0]<<" "<<work1[0]<<"] ([86 120])\n";
     dg::tensor::multiply2d_inplace(t, inout0, inout1, work1);
-    std::cout << "Result inplace is ["<<inout0[0]<<" "<<inout1[0]<<"]\n T is \n";
+    std::cout << "Result inplace is ["<<inout0[0]<<" "<<inout1[0]<<"] ([86 120])\n T is \n";
     t.idx(0,2) = 2; std::swap( t.idx(1,1), t.idx(2,1));  print(t);
     std::cout << "Multiply T with [8,9,2]\n";
     dg::tensor::multiply3d(t, eight, nine,two, work0, work1, work2);
-    std::cout << "Result         is ["<<work0[0]<<" "<<work1[0]<<" "<<work2[0]<<"]\n";
+    std::cout << "Result         is ["<<work0[0]<<" "<<work1[0]<<" "<<work2[0]<<"] ([102 57 76])\n";
     inout0=eight, inout1=nine, inout2=two;
     dg::tensor::multiply3d_inplace(t, inout0, inout1,inout2, work1,work2);
-    std::cout << "Result inplace is ["<<inout0[0]<<" "<<inout1[0]<<" "<<inout2[0]<<"]\n";
+    std::cout << "Result inplace is ["<<inout0[0]<<" "<<inout1[0]<<" "<<inout2[0]<<"] ([102 57 76])\n";
     std::cout << "Determinant of T: "<<dg::tensor::determinant(t).value()[0]<<" (320)\n";
     std::cout << "Perp Determinant of T: "<<dg::tensor::determinant(t.perp()).value()[0]<<" (-32)\n";
     std::swap(t.idx(2,1), t.idx(2,0)); 
diff --git a/inc/dg/geometry/tensor.h b/inc/dg/geometry/tensor.h
index 4acfa4d72..857fe4cdf 100644
--- a/inc/dg/geometry/tensor.h
+++ b/inc/dg/geometry/tensor.h
@@ -88,14 +88,14 @@ struct SparseTensor
     SparseTensor( ):mat_idx_(3,-1.) {}
 
     /**
-     * @brief reserve space for Ts in the values array
+     * @brief reserve space for value_size Ts in the values array
      * @param value_size reserve space for this number of Ts (default constructor) 
      */
     SparseTensor( unsigned value_size): mat_idx_(3,-1.), values_(value_size){}
 
     /**
     * @brief pass array of Ts
-    * @param values The contained Ts must all have the same size
+    * @param values The contained Ts are stored in the object
     */
     SparseTensor( const std::vector<T>& values ): mat_idx_(3,-1.), values_(values){}
 
-- 
GitLab


From 13209acf28062e03a2fdff6686a5e7a403af44e9 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <Matthias.Wiesenberger@uibk.ac.at>
Date: Sun, 3 Sep 2017 22:45:27 +0200
Subject: [PATCH 242/453] got rid of multiply_inplace

---
 inc/dg/elliptic.h             |   8 +--
 inc/dg/geometry/multiply.h    | 104 +++-------------------------------
 inc/dg/geometry/multiply_b.cu |   2 +-
 inc/dg/geometry/multiply_t.cu |   6 +-
 4 files changed, 15 insertions(+), 105 deletions(-)

diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index eb5c5f587..9b614fdab 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -143,11 +143,11 @@ class Elliptic
     void symv( const container& x, container& y) 
     {
         //compute gradient
-        dg::blas2::gemv( rightx, x, gradx); //R_x*f 
-        dg::blas2::gemv( righty, x, y); //R_y*f
+        dg::blas2::gemv( rightx, x, tempx); //R_x*f 
+        dg::blas2::gemv( righty, x, tempy); //R_y*f
 
-        //multiply with tensor
-        dg::tensor::multiply2d_inplace(chi_, gradx, y, tempx);
+        //multiply with tensor (note the alias)
+        dg::tensor::multiply2d(chi_, tempx, tempy, gradx, tempy);
 
         //now take divergence
         dg::blas2::symv( lefty, tempy, y);  
diff --git a/inc/dg/geometry/multiply.h b/inc/dg/geometry/multiply.h
index 35fba4cf7..c95981318 100644
--- a/inc/dg/geometry/multiply.h
+++ b/inc/dg/geometry/multiply.h
@@ -145,46 +145,23 @@ void multiply2d_helper( const SparseTensor<container>& t, const container& in0,
  * Compute \f$ w^i = t^{ij}v_j\f$ for \f$ i,j\in \{1,2\}\f$ in the first two dimensions (ignores the 3rd dimension in t)
  * @copydoc hide_container_lvl1
  * @param t input Tensor
- * @param in0 (input) first component 
- * @param in1 (input) second component
- * @param out0 (output) first component 
- * @param out1 (output) second component 
+ * @param in0 (input) first component    (restricted)
+ * @param in1 (input) second component   (restricted)
+ * @param out0 (output) first component  (restricted)
+ * @param out1 (output) second component (may alias in1)
  * @note this version keeps the input intact 
- * @attention aliasing only allowed if tensor is either lower, or upper triangular 
+ * @attention aliasing only allowed between out1 and in1
  */
 template<class container>
 void multiply2d( const SparseTensor<container>& t, const container& in0, const container& in1, container& out0, container& out1)
 {
     int i0[2] = {0,0}, i1[2] = {1,0};
     int i3[2] = {1,1}, i2[2] = {1,0};
+    //order is important because out1 may alias in1
     detail::multiply2d_helper( t, in0, in1, out0, i0, i1);
     detail::multiply2d_helper( t, in1, in0, out1, i3, i2);
-}
-
-/**
- * @brief Multiply a tensor with a vector in 2d inplace
- *
- * Compute \f$ v^i = t^{ij}v_j\f$ for \f$ i,j\in \{1,2\}\f$ in the first two dimensions (ignores the 3rd dimension in t)
- * @copydoc hide_container_lvl1
- * @param t input Tensor
- * @param inout0 (input/output) first component 
- * @param inout1 (input/output) second component
- * @param workspace (write) optional workspace
- * @note this version overwrites the input and the workspace
- * @attention aliasing not allowed
- */
-template<class container>
-void multiply2d_inplace( const SparseTensor<container>& t, container& inout0, container& inout1, container& workspace)
-{
-    int i0[2] = {0,0}, i1[2] = {1,0};
-    int i3[2] = {1,1}, i2[2] = {1,0};
-    detail::multiply2d_helper( t, inout0, inout1, workspace, i0, i1);
-    detail::multiply2d_helper( t, inout1, inout0, inout1, i3, i2);
-    workspace.swap( inout0);
-    //needs to load a vector         11 times if every element is set (5 is the optimal symmetric inplace algorithm)
-    //if triangular                  7 (5 optimal)
-    //if diagonal                    4 (optimal)
-    //if unity                       0 (optimal)
+    //needs to load a vector         10 times if every element is set (7 is the optimal algorithm for symmetric t)
+    //(the ideal algorithm also only needs 70% of the time (tested on marconi))
 }
 
 /**
@@ -245,71 +222,6 @@ void multiply3d( const SparseTensor<container>& t, const container& in0, const c
         dg::blas1::pointwiseDot( 1., t.value(2,0), in0, 1., out2);
 }
 
-/**
- * @brief Multiply a tensor with a vector in 2d inplace
- *
- * Compute \f$ v^i = t^{ij}v_j\f$ for \f$ i,j\in \{1,2,3\}\f$
- * @copydoc hide_container_lvl1
- * @param t input Tensor
- * @param inout0 (input/output) first component 
- * @param inout1 (input/output) second component
- * @param inout2 (input/output) second component
- * @param workspace0 (write) optional workspace
- * @param workspace1 (write) optional workspace
- * @note this version overwrites the input and may or may not write into the workspace
- * @attention aliasing not allowed
- */
-template<class container>
-void multiply3d_inplace( const SparseTensor<container>& t, container& inout0, container& inout1, container& inout2, container& workspace0, container& workspace1)
-{
-    //first: store off-diagonals of first two rows
-    if( t.isSet(0,1) ) {
-        dg::blas1::pointwiseDot( t.value(0,1), inout1, workspace0);
-        if( t.isSet(0,2) ) dg::blas1::pointwiseDot( 1.,t.value(0,2), inout2, 1.,workspace0);
-    }
-    if(!t.isSet(0,1) && t.isSet(0,2))
-    {
-        dg::blas1::pointwiseDot( t.value(0,2), inout2, workspace0);
-    }
-    //else workspace0 is empty
-    //
-    if( t.isSet(1,0) ) {
-        dg::blas1::pointwiseDot( t.value(1,0), inout0, workspace1);
-        if( t.isSet(1,2) ) dg::blas1::pointwiseDot( 1.,t.value(1,2), inout2, 1.,workspace1);
-    }
-    if(!t.isSet(1,0) && t.isSet(1,2))
-    {
-        dg::blas1::pointwiseDot( t.value(1,2), inout2, workspace1);
-    }
-    //else workspace1 is empty
-    //
-    //second: compute out2 inplace
-    if( t.isSet(2,2)) dg::blas1::pointwiseDot( t.value(2,2), inout2, inout2);
-    if( t.isSet(2,1)) dg::blas1::pointwiseDot( 1., t.value(2,1), inout1, 1., inout2);
-    if( t.isSet(2,0)) dg::blas1::pointwiseDot( 1., t.value(2,0), inout0, 1., inout2);
-
-    //third add values stored in workspaces
-    if(t.isSet(0,1) ||t.isSet(0,2) ) //workspace0 is filled
-    {
-        if( !t.isSet(0,0)) dg::blas1::axpby( 1., inout0, 1., workspace0, workspace0);
-        else dg::blas1::pointwiseDot( 1., t.value(0,0), inout0, 1., workspace0); 
-        workspace0.swap( inout0);
-    }
-    else
-        if( t.isSet(0,0)) dg::blas1::pointwiseDot( t.value(0,0), inout0, inout0); 
-    if(t.isSet(1,0) ||t.isSet(1,2) ) //workspace1 is filled
-    {
-        if( !t.isSet(1,1)) dg::blas1::axpby( 1., inout1, 1., workspace1, workspace1);
-        else dg::blas1::pointwiseDot( 1., t.value(1,1), inout1, 1., workspace1); 
-        workspace1.swap( inout1);
-    }
-    else
-        if( t.isSet(1,1)) dg::blas1::pointwiseDot( t.value(1,1), inout1, inout1); 
-    //if everything is set: 28 loads (12 optimal, 9 optimal symmetric)
-    //if effective 2d:      12 (same as 2d algorithm, 5 optimal symmetric)
-
-}
-
 /**
 * @brief Compute the determinant of a tensor
 * @copydoc hide_container_lvl1
diff --git a/inc/dg/geometry/multiply_b.cu b/inc/dg/geometry/multiply_b.cu
index 194e156aa..f6269fdb6 100644
--- a/inc/dg/geometry/multiply_b.cu
+++ b/inc/dg/geometry/multiply_b.cu
@@ -31,7 +31,7 @@ int main()
     std::cout<<"multiply_inplace(g,v_x,v_y,temp) took   "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
     t.tic();
     for( int i=0; i<multi; i++)
-        dg::tensor::multiply2d( g, v_x, v_y, w_x, w_y);
+        dg::tensor::multiply2d( g, v_x, v_y, w_x, v_y);
     t.toc();
     std::cout<<"multiply2d(g,v_x,v_y,w_x,w_y) took      "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
     return 0;
diff --git a/inc/dg/geometry/multiply_t.cu b/inc/dg/geometry/multiply_t.cu
index 7d560c94a..0a25c275f 100644
--- a/inc/dg/geometry/multiply_t.cu
+++ b/inc/dg/geometry/multiply_t.cu
@@ -67,15 +67,13 @@ int main()
     std::cout << "Multiply T with [8,9]\n";
     dg::tensor::multiply2d(t, inout0, inout1, work0, work1);
     std::cout << "Result         is ["<<work0[0]<<" "<<work1[0]<<"] ([86 120])\n";
-    dg::tensor::multiply2d_inplace(t, inout0, inout1, work1);
-    std::cout << "Result inplace is ["<<inout0[0]<<" "<<inout1[0]<<"] ([86 120])\n T is \n";
+    dg::tensor::multiply2d(t, inout0, inout1, work0, inout1);
+    std::cout << "Result inplace is ["<<work0[0]<<" "<<inout1[0]<<"] ([86 120])\n T is \n";
     t.idx(0,2) = 2; std::swap( t.idx(1,1), t.idx(2,1));  print(t);
     std::cout << "Multiply T with [8,9,2]\n";
     dg::tensor::multiply3d(t, eight, nine,two, work0, work1, work2);
     std::cout << "Result         is ["<<work0[0]<<" "<<work1[0]<<" "<<work2[0]<<"] ([102 57 76])\n";
     inout0=eight, inout1=nine, inout2=two;
-    dg::tensor::multiply3d_inplace(t, inout0, inout1,inout2, work1,work2);
-    std::cout << "Result inplace is ["<<inout0[0]<<" "<<inout1[0]<<" "<<inout2[0]<<"] ([102 57 76])\n";
     std::cout << "Determinant of T: "<<dg::tensor::determinant(t).value()[0]<<" (320)\n";
     std::cout << "Perp Determinant of T: "<<dg::tensor::determinant(t.perp()).value()[0]<<" (-32)\n";
     std::swap(t.idx(2,1), t.idx(2,0)); 
-- 
GitLab


From fc937e01c913bf61ac07cdc9f74c6619a1c8c3b5 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sun, 3 Sep 2017 23:44:24 +0200
Subject: [PATCH 243/453] rewrite multiply3d

---
 inc/dg/backend/thrust_vector_blas.cuh | 52 ++++++++++++++------
 inc/dg/blas1_t.cu                     |  2 +
 inc/dg/geometry/multiply.h            | 71 ++++++++-------------------
 inc/dg/geometry/multiply_t.cu         |  5 +-
 4 files changed, 63 insertions(+), 67 deletions(-)

diff --git a/inc/dg/backend/thrust_vector_blas.cuh b/inc/dg/backend/thrust_vector_blas.cuh
index 25c91ef6b..bbc675343 100644
--- a/inc/dg/backend/thrust_vector_blas.cuh
+++ b/inc/dg/backend/thrust_vector_blas.cuh
@@ -384,16 +384,16 @@ template<class value_type>
 }
 #endif
 
-template<class Vector>
+template<class value_type>
 inline void doPointwiseDot(  
-              typename Vector::value_type alpha, 
-              const Vector& x1,
-              const Vector& y1, 
-              typename Vector::value_type beta, 
-              const Vector& x2,
-              const Vector& y2, 
-              typename Vector::value_type gamma, 
-              Vector& z, 
+              value_type alpha, 
+              const thrust::device_vector<value_type>& x1,
+              const thrust::device_vector<value_type>& y1, 
+              value_type beta, 
+              const thrust::device_vector<value_type>& x2,
+              const thrust::device_vector<value_type>& y2, 
+              value_type gamma, 
+              thrust::device_vector<value_type>& z, 
               ThrustVectorTag)
 {
     if( alpha==0){ 
@@ -404,16 +404,16 @@ inline void doPointwiseDot(
         doPointwiseDot( alpha, x1,y1, gamma, z, ThrustVectorTag());
         return;
     }
-    const typename Vector::value_type *x1_ptr = thrust::raw_pointer_cast( &(x1.data()[0]));
-    const typename Vector::value_type *x2_ptr = thrust::raw_pointer_cast( &(x2.data()[0]));
-    const typename Vector::value_type *y1_ptr = thrust::raw_pointer_cast( &(y1.data()[0]));
-    const typename Vector::value_type *y2_ptr = thrust::raw_pointer_cast( &(y2.data()[0]));
-        typename Vector::value_type * z_ptr = thrust::raw_pointer_cast( &(z.data()[0]));
+    const value_type *x1_ptr = thrust::raw_pointer_cast( x1.data());
+    const value_type *x2_ptr = thrust::raw_pointer_cast( x2.data());
+    const value_type *y1_ptr = thrust::raw_pointer_cast( y1.data());
+    const value_type *y2_ptr = thrust::raw_pointer_cast( y2.data());
+          value_type * z_ptr = thrust::raw_pointer_cast( z.data());
     unsigned size = x1.size();
 #if THRUST_DEVICE_SYSTEM!=THRUST_DEVICE_SYSTEM_CUDA
     if( z_ptr != x1_ptr && z_ptr != x2_ptr && z_ptr != y1_ptr && z_ptr != y2_ptr)
     {
-        typename Vector::value_type * RESTRICT zr_ptr = thrust::raw_pointer_cast( &(z.data()[0]));
+        value_type * RESTRICT zr_ptr = thrust::raw_pointer_cast( &(z.data()[0]));
         if(gamma==0)
         {
 #pragma omp parallel for simd
@@ -478,9 +478,29 @@ inline void doPointwiseDot(
     //set up kernel parameters
     const size_t BLOCK_SIZE = 256; 
     const size_t NUM_BLOCKS = std::min<size_t>((size-1)/BLOCK_SIZE+1, 65000);
-    pointwiseDot_kernel<typename Vector::value_type><<<NUM_BLOCKS, BLOCK_SIZE>>>( alpha, beta, gamma, x1_ptr, y1_ptr, x2_ptr, y2_ptr, z_ptr, size);
+    pointwiseDot_kernel<value_type><<<NUM_BLOCKS, BLOCK_SIZE>>>( alpha, beta, gamma, x1_ptr, y1_ptr, x2_ptr, y2_ptr, z_ptr, size);
 #endif
 }
+template<class value_type>
+inline void doPointwiseDot(  
+              value_type alpha, 
+              const thrust::host_vector<value_type>& x1,
+              const thrust::host_vector<value_type>& y1, 
+              value_type beta, 
+              const thrust::host_vector<value_type>& x2,
+              const thrust::host_vector<value_type>& y2, 
+              value_type gamma, 
+              thrust::host_vector<value_type>& z, 
+              ThrustVectorTag)
+{
+    unsigned size=x1.size();
+    for( unsigned i=0; i<size; i++)
+    {
+        z[i] = alpha*x1[i]*y1[i] 
+                    +beta*x2[i]*y2[i]
+                    +gamma*z[i];
+    }
+}
 
 }//namespace detail
 
diff --git a/inc/dg/blas1_t.cu b/inc/dg/blas1_t.cu
index 7a0428f41..762d5a471 100644
--- a/inc/dg/blas1_t.cu
+++ b/inc/dg/blas1_t.cu
@@ -59,6 +59,8 @@ int main()
     std::cout << "2*3 = "<<w3[0][0]<<" (6)\n";
     dg::blas1::pointwiseDot( 2., w1, w2, -4., w3);
     std::cout << "2*2*3 -4*6 = "<<w3[0][0]<<" (-12)\n";
+    dg::blas1::pointwiseDot( 2., w1[0], w2[0], -4., v1, v2, 0., v2);
+    std::cout << "2*2*3 -4*2*3 = "<<v2[0]<<" (-12)\n";
     dg::blas1::axpby( 2., w1, 3., w2);
     std::cout << "2*2+ 3*3 = " << w2[0][0] <<" (13)\n";
     dg::blas1::axpby( 2.5, w1, 0., w2);
diff --git a/inc/dg/geometry/multiply.h b/inc/dg/geometry/multiply.h
index c95981318..22fb7996f 100644
--- a/inc/dg/geometry/multiply.h
+++ b/inc/dg/geometry/multiply.h
@@ -146,16 +146,15 @@ void multiply2d_helper( const SparseTensor<container>& t, const container& in0,
  * @copydoc hide_container_lvl1
  * @param t input Tensor
  * @param in0 (input) first component    (restricted)
- * @param in1 (input) second component   (restricted)
+ * @param in1 (input) second component   (may alias out1)
  * @param out0 (output) first component  (restricted)
  * @param out1 (output) second component (may alias in1)
- * @note this version keeps the input intact 
  * @attention aliasing only allowed between out1 and in1
  */
 template<class container>
 void multiply2d( const SparseTensor<container>& t, const container& in0, const container& in1, container& out0, container& out1)
 {
-    int i0[2] = {0,0}, i1[2] = {1,0};
+    int i0[2] = {0,0}, i1[2] = {0,1};
     int i3[2] = {1,1}, i2[2] = {1,0};
     //order is important because out1 may alias in1
     detail::multiply2d_helper( t, in0, in1, out0, i0, i1);
@@ -170,56 +169,26 @@ void multiply2d( const SparseTensor<container>& t, const container& in0, const c
  * Compute \f$ w^i = t^{ij}v_j\f$ for \f$ i,j\in \{1,2,3\}\f$
  * @copydoc hide_container_lvl1
  * @param t input Tensor
- * @param in0 (input)  first component 
- * @param in1 (input)  second component
- * @param in2 (input)  third component
- * @param out0 (output)  first component 
- * @param out1 (output)  second component 
- * @param out2 (output)  third component 
- * @note this version keeps the input intact 
- * @attention aliasing only allowed if tensor is either lower, or upper triangular 
+ * @param in0 (input)  first component  (restricted)
+ * @param in1 (input)  second component (restricted)
+ * @param in2 (input)  third component  (may alias out2)
+ * @param out0 (output)  first component  (restricted)
+ * @param out1 (output)  second component  (restricted)
+ * @param out2 (output)  third component (may alias in2)
+ * @attention aliasing only allowed between out2 and in2
  */
 template<class container>
 void multiply3d( const SparseTensor<container>& t, const container& in0, const container& in1, const container& in2, container& out0, container& out1, container& out2)
 {
-    if( !t.isSet(0,1)&&!t.isSet(0,2)&&!t.isSet(1,2))
-    {
-        //lower triangular
-        if(!t.isSet(2,2)) out2=in2;
-        else dg::blas1::pointwiseDot( t.value(2,2), in2, out2);
-        if(t.isSet(2,1))
-            dg::blas1::pointwiseDot( 1., t.value(2,1), in1, 1., out2);
-        if(t.isSet(2,0))
-            dg::blas1::pointwiseDot( 1., t.value(2,0), in0, 1., out2);
-        if( !t.isSet(1,1)) out1=in1;
-        else dg::blas1::pointwiseDot( t.value(1,1), in1, out1);
-        if(t.isSet(1,0))
-            dg::blas1::pointwiseDot( 1., t.value(1,0), in0, 1., out1);
-        if( !t.isSet(0,0)) out0=in0;
-        else dg::blas1::pointwiseDot( t.value(0,0), in0, out0); 
-        return;
-    }
-    //upper triangular and default
-    if( !t.isSet(0,0)) out0=in0;
-    else dg::blas1::pointwiseDot( t.value(0,0), in0, out0); 
-    if(t.isSet( 0,1))
-        dg::blas1::pointwiseDot( 1., t.value(0,1), in1, 1., out0);
-    if(t.isSet( 0,2))
-        dg::blas1::pointwiseDot( 1., t.value(0,2), in2, 1., out0);
-
-    if( !t.isSet(1,1)) out1=in1;
-    else dg::blas1::pointwiseDot( t.value(1,1), in1, out1);
-    if(t.isSet(1,2))
-        dg::blas1::pointwiseDot( 1., t.value(1,2), in2, 1., out1);
-    if(t.isSet(1,0)) //wrong if inplace
-        dg::blas1::pointwiseDot( 1., t.value(1,0), in0, 1., out1);
-
-    if(!t.isSet(2,2)) out2=in2;
-    else dg::blas1::pointwiseDot( t.value(2,2), in2, out2);
-    if(t.isSet(2,1))
-        dg::blas1::pointwiseDot( 1., t.value(2,1), in1, 1., out2);
-    if(t.isSet(2,0))
-        dg::blas1::pointwiseDot( 1., t.value(2,0), in0, 1., out2);
+    int i0[2] = {0,0}, i1[2] = {0,1};
+    int i3[2] = {1,1}, i2[2] = {1,0};
+    int i5[2] = {2,2}, i4[2] = {2,0};
+    detail::multiply2d_helper( t, in0, in1, out0, i0, i1);
+    if( t.isSet(0,2)) dg::blas1::pointwiseDot( 1., t.value(0,2), in2, 1., out0);
+    detail::multiply2d_helper( t, in1, in0, out1, i3, i2);
+    if( t.isSet(1,2)) dg::blas1::pointwiseDot( 1., t.value(1,2), in2, 1., out1);
+    detail::multiply2d_helper( t, in2, in0, out2, i5, i4);
+    if( t.isSet(2,1)) dg::blas1::pointwiseDot( 1., t.value(2,1), in1, 1., out2);
 }
 
 /**
@@ -292,7 +261,9 @@ void multiply3d( const CholeskyTensor<container>& ch, const container& in0, cons
 {
     multiply3d(ch.upper(),    in0,  in1, in2,  out0, out1, out2);
     multiply3d(ch.diagonal(), out0, out1,out2, out0, out1, out2);
-    multiply3d(ch.lower(),    out0, out1,out2, out0, out1, out2);
+    container temp(out1);
+    multiply3d(ch.lower(),    out0, out1,out2, out0, temp, out2);
+    temp.swap(out1);
 }
 
 template<class container>
diff --git a/inc/dg/geometry/multiply_t.cu b/inc/dg/geometry/multiply_t.cu
index 0a25c275f..9bec2458c 100644
--- a/inc/dg/geometry/multiply_t.cu
+++ b/inc/dg/geometry/multiply_t.cu
@@ -64,8 +64,9 @@ int main()
     dg::tensor::pointwiseDivide(inout0,mu,inout0); 
     std::cout << "Restore 8 = "<<inout0[0]<<"\n";
     std::cout << "Test Tensor multiplies \n";
+    print(t);
     std::cout << "Multiply T with [8,9]\n";
-    dg::tensor::multiply2d(t, inout0, inout1, work0, work1);
+    dg::tensor::multiply2d(t, eight, nine, work0, work1);
     std::cout << "Result         is ["<<work0[0]<<" "<<work1[0]<<"] ([86 120])\n";
     dg::tensor::multiply2d(t, inout0, inout1, work0, inout1);
     std::cout << "Result inplace is ["<<work0[0]<<" "<<inout1[0]<<"] ([86 120])\n T is \n";
@@ -74,6 +75,8 @@ int main()
     dg::tensor::multiply3d(t, eight, nine,two, work0, work1, work2);
     std::cout << "Result         is ["<<work0[0]<<" "<<work1[0]<<" "<<work2[0]<<"] ([102 57 76])\n";
     inout0=eight, inout1=nine, inout2=two;
+    dg::tensor::multiply3d(t, inout0, inout1, inout2, work0, work1, inout2);
+    std::cout << "Result inplace is ["<<work0[0]<<" "<<work1[0]<<" "<<inout2[0]<<"] ([102 57 76])\n";
     std::cout << "Determinant of T: "<<dg::tensor::determinant(t).value()[0]<<" (320)\n";
     std::cout << "Perp Determinant of T: "<<dg::tensor::determinant(t.perp()).value()[0]<<" (-32)\n";
     std::swap(t.idx(2,1), t.idx(2,0)); 
-- 
GitLab


From 89944b2609568ec40ac25101697b1574fd52a8c0 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sun, 3 Sep 2017 23:56:55 +0200
Subject: [PATCH 244/453] debugged mpi_matrix for new blas1 functions

---
 inc/dg/backend/mpi_matrix.h      |  6 +++---
 inc/dg/backend/mpi_vector_blas.h | 27 +++++++++++++++++++++++++++
 inc/dg/geometry/transform.h      | 10 +++++-----
 3 files changed, 35 insertions(+), 8 deletions(-)

diff --git a/inc/dg/backend/mpi_matrix.h b/inc/dg/backend/mpi_matrix.h
index 78d586804..6dbef6963 100644
--- a/inc/dg/backend/mpi_matrix.h
+++ b/inc/dg/backend/mpi_matrix.h
@@ -116,7 +116,7 @@ struct RowColDistMat
         //if(rank==0)std::cout << "Inner points took "<<t.diff()<<"s\n";
         //2. communicate outer points
         //t.tic();
-        const container& temp = c_.collect( x.data());
+        const container& temp = c_.global_gather( x.data());
         //t.toc();
         //if(rank==0)std::cout << "Collect      took "<<t.diff()<<"s\n";
         //3. compute and add outer points
@@ -259,7 +259,7 @@ struct RowDistMat
             return;
 
         }
-        container temp = c_.collect( x.data());
+        container temp = c_.global_gather( x.data());
         //t.toc();
         //if(rank==0)std::cout << "collect took "<<t.diff()<<"s\n";
         //t.tic();
@@ -372,7 +372,7 @@ struct ColDistMat
         dg::blas2::detail::doSymv( alpha, m_, x.data(), beta, temp, 
                        typename dg::MatrixTraits<LocalMatrix>::matrix_category(), 
                        typename dg::VectorTraits<container>::vector_category() );
-        c_.send_and_reduce( temp, y.data());
+        c_.global_scatter_reduce( temp, y.data());
     }
     template<class container> 
     void symv( const MPI_Vector<container>& x, MPI_Vector<container>& y)
diff --git a/inc/dg/backend/mpi_vector_blas.h b/inc/dg/backend/mpi_vector_blas.h
index 30cc342e8..3cde835d9 100644
--- a/inc/dg/backend/mpi_vector_blas.h
+++ b/inc/dg/backend/mpi_vector_blas.h
@@ -94,6 +94,19 @@ inline void doAxpby( typename VectorTraits<Vector>::value_type alpha,
     doAxpby( alpha,x.data(),beta, y.data(), z.data(), typename VectorTraits<container>::vector_category());
 }
 
+template< class Vector>
+inline void doAxpby( typename VectorTraits<Vector>::value_type alpha, 
+              const Vector& x, 
+              typename VectorTraits<Vector>::value_type beta, 
+              const Vector& y, 
+              typename VectorTraits<Vector>::value_type gamma, 
+              Vector& z, 
+              MPIVectorTag)
+{
+    typedef typename Vector::container_type container;
+    doAxpby( alpha,x.data(),beta, y.data(), gamma, z.data(), typename VectorTraits<container>::vector_category());
+}
+
 template< class Vector>
 inline void doPointwiseDot( const Vector& x1, const Vector& x2, Vector& y, MPIVectorTag)
 {
@@ -119,6 +132,20 @@ inline void doPointwiseDivide( const Vector& x1, const Vector& x2, Vector& y, MP
     typedef typename Vector::container_type container;
     doPointwiseDivide( x1.data(), x2.data(), y.data(), typename VectorTraits<container>::vector_category());
 }
+
+template< class Vector>
+inline void doPointwiseDot( typename VectorTraits<Vector>::value_type alpha, 
+        const Vector& x1, const Vector& x2, 
+        typename VectorTraits<Vector>::value_type beta,
+        const Vector& y1, const Vector& y2, 
+        typename VectorTraits<Vector>::value_type gamma,
+        Vector& z, 
+        MPIVectorTag)
+{
+    typedef typename Vector::container_type container;
+    doPointwiseDot( alpha, x1.data(), x2.data(), beta, y1.data(), y2.data(), gamma, z.data(), typename VectorTraits<container>::vector_category());
+
+}
         
 
 }//namespace detail
diff --git a/inc/dg/geometry/transform.h b/inc/dg/geometry/transform.h
index d030ad551..5c7225a8d 100644
--- a/inc/dg/geometry/transform.h
+++ b/inc/dg/geometry/transform.h
@@ -120,8 +120,8 @@ void pushForwardPerp( const Functor1& vR, const Functor2& vZ,
     typedef typename GeometryTraits< Geometry>::host_vector host_vec;
     host_vec out1 = pullback( vR, g), temp1(out1);
     host_vec out2 = pullback( vZ, g);
-    dg::tensor::multiply2d_inplace(g.jacobian(), out1, out2, temp1);
-    dg::blas1::transfer( out1, vx);
+    dg::tensor::multiply2d(g.jacobian(), out1, out2, temp1, out2);
+    dg::blas1::transfer( temp1, vx);
     dg::blas1::transfer( out2, vy);
 }
 
@@ -152,9 +152,9 @@ void pushForward( const Functor1& vR, const Functor2& vZ, const Functor3& vPhi,
     host_vec out1 = pullback( vR, g), temp1(out1);
     host_vec out2 = pullback( vZ, g), temp2(out2);
     host_vec out3 = pullback( vPhi, g);
-    dg::tensor::multiply3d_inplace(g.jacobian(), out1, out2, out3, temp1, temp2);
-    dg::blas1::transfer( out1, vx);
-    dg::blas1::transfer( out2, vy);
+    dg::tensor::multiply3d(g.jacobian(), out1, out2, out3, temp1, temp2, out3);
+    dg::blas1::transfer( temp1, vx);
+    dg::blas1::transfer( temp2, vy);
     dg::blas1::transfer( out3, vz);
 }
 
-- 
GitLab


From 1823384fa940ac1d52e039bd65e88b92ab64503f Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sun, 3 Sep 2017 23:59:16 +0200
Subject: [PATCH 245/453] added mpi version for new blas functions

---
 inc/dg/backend/mpi_vector_blas.h | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/inc/dg/backend/mpi_vector_blas.h b/inc/dg/backend/mpi_vector_blas.h
index 30cc342e8..3cde835d9 100644
--- a/inc/dg/backend/mpi_vector_blas.h
+++ b/inc/dg/backend/mpi_vector_blas.h
@@ -94,6 +94,19 @@ inline void doAxpby( typename VectorTraits<Vector>::value_type alpha,
     doAxpby( alpha,x.data(),beta, y.data(), z.data(), typename VectorTraits<container>::vector_category());
 }
 
+template< class Vector>
+inline void doAxpby( typename VectorTraits<Vector>::value_type alpha, 
+              const Vector& x, 
+              typename VectorTraits<Vector>::value_type beta, 
+              const Vector& y, 
+              typename VectorTraits<Vector>::value_type gamma, 
+              Vector& z, 
+              MPIVectorTag)
+{
+    typedef typename Vector::container_type container;
+    doAxpby( alpha,x.data(),beta, y.data(), gamma, z.data(), typename VectorTraits<container>::vector_category());
+}
+
 template< class Vector>
 inline void doPointwiseDot( const Vector& x1, const Vector& x2, Vector& y, MPIVectorTag)
 {
@@ -119,6 +132,20 @@ inline void doPointwiseDivide( const Vector& x1, const Vector& x2, Vector& y, MP
     typedef typename Vector::container_type container;
     doPointwiseDivide( x1.data(), x2.data(), y.data(), typename VectorTraits<container>::vector_category());
 }
+
+template< class Vector>
+inline void doPointwiseDot( typename VectorTraits<Vector>::value_type alpha, 
+        const Vector& x1, const Vector& x2, 
+        typename VectorTraits<Vector>::value_type beta,
+        const Vector& y1, const Vector& y2, 
+        typename VectorTraits<Vector>::value_type gamma,
+        Vector& z, 
+        MPIVectorTag)
+{
+    typedef typename Vector::container_type container;
+    doPointwiseDot( alpha, x1.data(), x2.data(), beta, y1.data(), y2.data(), gamma, z.data(), typename VectorTraits<container>::vector_category());
+
+}
         
 
 }//namespace detail
-- 
GitLab


From 1fb75e0dca04fb38693ea5b46fa580a0c00b3cc8 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 4 Sep 2017 06:42:37 -0700
Subject: [PATCH 246/453] work on fieldaligned and feltor a bit

---
 inc/dg/blas1.h                    |   2 +-
 inc/dg/dg_doc.h                   |   1 -
 inc/geometries/ds_b.cu            |  63 ++++++++++++++--
 inc/geometries/fieldaligned.h     | 101 +++++++++++++-------------
 inc/geometries/flux_t.cu          |  42 +----------
 inc/geometries/mpi_fieldaligned.h |   2 +-
 inc/geometries/solovev_doc.h      |   3 +-
 src/feltor/feltor.cu              |  15 ++--
 src/feltor/feltor.cuh             | 116 +++++++++++-------------------
 src/feltor/feltor_hpc.cu          |  12 ++--
 src/feltor/feltor_mpi.cu          |  10 +--
 11 files changed, 174 insertions(+), 193 deletions(-)

diff --git a/inc/dg/blas1.h b/inc/dg/blas1.h
index a2c64a758..35464b738 100644
--- a/inc/dg/blas1.h
+++ b/inc/dg/blas1.h
@@ -230,7 +230,7 @@ inline void pointwiseDivide( const Vector& x1, const Vector& x2, Vector& y)
 /**
 * @brief A 'new' fused multiply-add BLAS 1 routine. 
 *
-* Multiplies and adds vectors element by element: \f[ z_i = \alpha x_{1i}y_{1i} + \beta x_{2i]y_{2i} + \gamma z_i\f]
+* Multiplies and adds vectors element by element: \f[ z_i = \alpha x_{1i}y_{1i} + \beta x_{2i}y_{2i} + \gamma z_i \f]
 * @param alpha scalar
 * @param x1 Vector x1  
 * @param y1 Vector y1 
diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index 8db2f518c..f97b51bd9 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -83,7 +83,6 @@
  * @{
  *     @defgroup arakawa Discretization of Poisson bracket
  *     @defgroup matrixoperators Elliptic and Helmholtz operators
- *     @defgroup fieldaligned Fieldaligned derivatives
  * @}
  * @defgroup templates Level 99: Template models
  * Documentation for template models
diff --git a/inc/geometries/ds_b.cu b/inc/geometries/ds_b.cu
index 9ee8ff0d9..1bf614496 100644
--- a/inc/geometries/ds_b.cu
+++ b/inc/geometries/ds_b.cu
@@ -5,10 +5,12 @@
 #include "blas.h"
 #include "ds.h"
 #include "functors.h"
+#include "solovev.h"
+#include "flux.h"
+#include "geometry.h"
 
 #include "backend/functions.h"
 #include "backend/timer.cuh"
-#include "geometry.h"
 
 double func(double R, double Z, double phi)
 {
@@ -23,17 +25,33 @@ double deri(double R, double Z, double phi)
 
 int main()
 {
-    Field field( R_0, I_0);
+    std::cout << "First test the cylindrical version\n";
     std::cout << "Type n, Nx, Ny, Nz\n";
     unsigned n, Nx, Ny, Nz;
     std::cin >> n>> Nx>>Ny>>Nz;
     std::cout << "You typed "<<n<<" "<<Nx<<" "<<Ny<<" "<<Nz<<std::endl;
-    dg::CylindricalGrid3d g3d( R_0 - 1, R_0+1, -1, 1, 0, 2.*M_PI, n, Nx, Ny, Nz, dg::NEU, dg::NEU, dg::PER);
+    Json::Reader reader;
+    Json::Value js;
+    if( argc==1) {
+        std::ifstream is("geometry_params_Xpoint.js");
+        reader.parse(is,js,false);
+    }
+    else {
+        std::ifstream is(argv[1]);
+        reader.parse(is,js,false);
+    }
+    dg::geo::solovev::GeomParameters gp(js);
+    dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField( gp);
+    dg::geo::BHatR bhatR(mag);
+    dg::geo::BHatZ bhatZ(mag);
+    dg::geo::BHatP bhatP(mag);
+    dg::geo::BinaryVectorLvl0 bhat( bhatR, bhatZ, bhatP);
+    dg::CylindricalGrid3d g3d( gp.R_0 - 1, gp.R_0+1, -1, 1, 0, 2.*M_PI, n, Nx, Ny, Nz, dg::NEU, dg::NEU, dg::PER);
     const dg::DVec vol3d = dg::create::volume( g3d);
     dg::Timer t;
     t.tic();
     dg::geo::BinaryVectorLvl0 bhat( bhatR, bhatZ, bhatP);
-    dg::FieldAligned dsFA( field, g3d_fine, 1e-10, dg::DefaultLimiter(), dg::NEU);
+    dg::FieldAligned dsFA( bhat, g3d, 1e-10, dg::DefaultLimiter(), dg::NEU);
 
     dg::DS<dg::aGeometry3d, dg::IDMatrix, dg::DMatrix, dg::DVec> ds ( dsFA, g3d, field, dg::not_normed, dg::centered);
     t.toc();
@@ -65,6 +83,43 @@ int main()
     ds.backward( function, derivative);
     norm = dg::blas2::dot(vol3d, derivative);
     std::cout << "Norm Backward Derivative "<<sqrt( norm)<<" (compare with that of ds_mpib)\n";
+
+    /////////////////////////TEST 3d flux grid//////////////////////////////////////
+    std::cout << "Start DS test on flux grid!"<<std::endl;
+    t.tic();
+    //dg::geo::BinaryVectorLvl0 bhat( dg::geo::BHatR(c), dg::geo::BHatZ(c), dg::geo::BHatP(c));
+    unsigned mx, my;
+    std::cout << "Type multipleX and multipleY!\n";
+    std::cin >> mx >> my;
+
+    dg::geo::FluxGenerator flux( mag.get_psip(), mag.get_ipol(), psi_0, psi_1, gp.R_0, 0., 1);
+    dg::CurvilinearProductGrid3d g3d(flux, n, Nx, Ny,Nz, dg::DIR);
+    //dg::FieldAligned<dg::aGeometry3d, dg::IHMatrix, dg::HVec> fieldaligned( bhat, g3d, 1, 4, gp.rk4eps, dg::NoLimiter() ); 
+    dg::DS<dg::aGeometry3d, dg::IHMatrix, dg::HMatrix, dg::HVec> ds( mag, g3d, dg::normed, dg::centered, false, true, mx, my);
+
+    
+    t.toc();
+    std::cout << "Construction took "<<t.diff()<<"s\n";
+    dg::HVec B = dg::pullback( dg::geo::InvB(mag), g3d), divB(B);
+    dg::HVec lnB = dg::pullback( dg::geo::LnB(mag), g3d), gradB(B);
+    dg::HVec gradLnB = dg::pullback( dg::geo::GradLnB(mag), g3d);
+    dg::blas1::pointwiseDivide( ones3d, B, B);
+    dg::HVec function = dg::pullback( dg::geo::FuncNeu(mag), g3d), derivative(function);
+    ds( function, derivative);
+
+    ds.centeredAdj( B, divB);
+    double norm =  sqrt( dg::blas2::dot(divB, vol3d, divB));
+    std::cout << "Divergence of B is "<<norm<<"\n";
+
+    ds.centered( lnB, gradB);
+    std::cout << "num. norm of gradLnB is "<<sqrt( dg::blas2::dot( gradB,vol3d, gradB))<<"\n";
+    norm = sqrt( dg::blas2::dot( gradLnB, vol3d, gradLnB) );
+    std::cout << "ana. norm of gradLnB is "<<norm<<"\n";
+    dg::blas1::axpby( 1., gradB, -1., gradLnB, gradLnB);
+    X = divB;
+    err = nc_put_var_double( ncid, varID[4], periodify(X, g2d_periodic).data());
+    double norm2 = sqrt(dg::blas2::dot(gradLnB, vol3d, gradLnB));
+    std::cout << "rel. error of lnB is    "<<norm2/norm<<"\n";
     
     return 0;
 }
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index a7bf24669..556355cc9 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -19,18 +19,27 @@
 
 namespace dg{
 
+///@brief Enum for the use in Fieldaligned
+///@ingroup fieldaligned
 enum whichMatrix
 {
-    einsPlus = 0,  
-    einsPlusT = 1,  
-    einsMinus = 2,  
-    einsMinusT = 3,  
+    einsPlus = 0,   /// plus interpolation in next plane
+    einsPlusT = 1,  /// transposed plus interpolation in previous plane
+    einsMinus = 2,  /// minus interpolation in previous plane
+    einsMinusT = 3, /// transposed minus interpolation in next plane
 };
 
-/**
- * @brief With the Default field ds becomes a dz
- */
-struct DefaultField
+///@brief Full Limiter means there is a limiter everywhere
+///@ingroup fieldaligned
+typedef ONE FullLimiter;
+
+///@brief No Limiter 
+///@ingroup fieldaligned
+typedef ZERO NoLimiter;
+///@cond
+namespace detail{
+
+struct DZField
 {
     void operator()( const dg::HVec& y, dg::HVec& yp)
     {
@@ -85,7 +94,6 @@ struct DSFieldCylindrical
 
 struct DSField
 {
-    
     //z component of v may not vanish
     DSField( const dg::geo::BinaryVectorLvl0& v, const aGeometry2d& g): g_(g)
     {
@@ -139,15 +147,6 @@ struct DSField
 
 };
 
-///@brief Full Limiter means there is a limiter everywhere
-typedef ONE FullLimiter;
-
-///@brief No Limiter 
-typedef ZERO NoLimiter;
-
-namespace detail{
-
-
 void clip_to_boundary( double& x, double& y, const aTopology2d* grid)
 {
     if (!(x > grid->x0())) { x=grid->x0();}
@@ -176,7 +175,7 @@ struct BoxIntegrator
      * @param g The 2d or 3d grid
      * @param eps the accuracy of the runge kutta integrator
      */
-    BoxIntegrator( Field field, const Grid& g, double eps): field_(field), g_(g), coords_(3), coordsp_(3), eps_(eps) {}
+    BoxIntegrator( const Field& field, const Grid& g, double eps): field_(field), g_(g), coords_(3), coordsp_(3), eps_(eps) {}
     /**
      * @brief Set the starting coordinates for next field line integration
      * @param coords the new coords (must have size = 3)
@@ -191,12 +190,12 @@ struct BoxIntegrator
     double operator()( double deltaPhi)
     {
         dg::integrateRK4( field_, coords_, coordsp_, deltaPhi, eps_);
-        if( !g_.get().contains( coordsp_[0], coordsp_[1]) ) return -1;
+        if( !g_.contains( coordsp_[0], coordsp_[1]) ) return -1;
         return +1;
     }
     private:
-    Field field_;
-    dg::Handle<Grid> g_;
+    const Field& field_;
+    const Grid& g_;
     thrust::host_vector<double> coords_, coordsp_;
     double eps_;
 };
@@ -214,7 +213,7 @@ struct BoxIntegrator
  * @param eps error
  */
 template< class Field, class Grid>
-void boxintegrator( Field& field, const Grid& grid, 
+void boxintegrator( const Field& field, const Grid& grid, 
         const thrust::host_vector<double>& coords0, 
         thrust::host_vector<double>& coords1, 
         double& phi1, double eps)
@@ -229,7 +228,7 @@ void boxintegrator( Field& field, const Grid& grid,
         std::cerr << "point "<<coords1[0]<<" "<<coords1[1]<<" is somewhere else!\n";
 #endif //DG_DEBUG
         double deltaPhi = phi1;
-        BoxIntegrator<Field, Grid> boxy( field, grid, eps);
+        BoxIntegrator<Field, Grid> boxy( field, grid, eps);//stores references to field and grid
         boxy.set_coords( coords0); //nimm alte koordinaten
         if( phi1 > 0)
         {
@@ -255,40 +254,37 @@ void boxintegrator( Field& field, const Grid& grid,
 }
 
 //used in constructor of FieldAligned
-void integrate_all_fieldlines2d( const dg::geo::BinaryVectorLvl0& vec, const aGeometry2d* g2dFine_ptr, std::vector<thrust::host_vector<double> >& yp_result, std::vector<thrust::host_vector<double> >& ym_result , double deltaPhi, double eps)
+void integrate_all_fieldlines2d( const dg::geo::BinaryVectorLvl0& vec, const aGeometry2d* g2dField_ptr, std::vector<thrust::host_vector<double> >& yp_result, std::vector<thrust::host_vector<double> >& ym_result , double deltaPhi, double eps)
 {
-    std::vector<thrust::host_vector<double> > y( 3, dg::evaluate( dg::cooX2d, *g2dFine_ptr)); //x
-    y[1] = dg::evaluate( dg::cooY2d, *g2dFine_ptr); //y
-    y[2] = dg::evaluate( dg::zero, *g2dFine_ptr); //s
+    std::vector<thrust::host_vector<double> > y( 3, dg::evaluate( dg::cooX2d, *g2dField_ptr)); //x
+    y[1] = dg::evaluate( dg::cooY2d, *g2dField_ptr); //y
+    y[2] = dg::evaluate( dg::zero, *g2dField_ptr); //s
     std::vector<thrust::host_vector<double> > yp( 3, y[0]), ym(yp); 
     //construct field on high polynomial grid, then integrate it
     Timer t;
     t.tic();
-    aGeometry2d* g2dField_ptr = g2dFine_ptr->clone();
-    //initial tests show that higher order polynomial might not be needed...
-    g2dField_ptr->set( 1*g2dField_ptr->n(), g2dField_ptr->Nx(), g2dField_ptr->Ny());
-    dg::DSField field( vec, *g2dField_ptr);
+    dg::detail::DSField field( vec, *g2dField_ptr);
     t.toc();
     std::cout << "Generation of interpolate grid took "<<t.diff()<<"s\n";
     //field in case of cartesian grid
-    dg::DSFieldCylindrical cyl_field(vec);
+    dg::detail::DSFieldCylindrical cyl_field(vec);
 #ifdef _OPENMP
 #pragma omp parallel for shared(field, cyl_field)
 #endif //_OPENMP
-    for( unsigned i=0; i<g2dFine_ptr->size(); i++)
+    for( unsigned i=0; i<g2dField_ptr->size(); i++)
     {
         thrust::host_vector<double> coords(3), coordsP(3), coordsM(3);
         coords[0] = y[0][i], coords[1] = y[1][i], coords[2] = y[2][i]; //x,y,s
         double phi1 = deltaPhi;
-        if( dynamic_cast<const dg::CartesianGrid2d*>( g2dFine_ptr))
-            boxintegrator( cyl_field, *g2dFine_ptr, coords, coordsP, phi1, eps);
+        if( dynamic_cast<const dg::CartesianGrid2d*>( g2dField_ptr))
+            boxintegrator( cyl_field, *g2dField_ptr, coords, coordsP, phi1, eps);
         else 
-            boxintegrator( field, *g2dFine_ptr, coords, coordsP, phi1, eps);
+            boxintegrator( field, *g2dField_ptr, coords, coordsP, phi1, eps);
         phi1 =  - deltaPhi;
-        if( dynamic_cast<const dg::CartesianGrid2d*>( g2dFine_ptr))
-            boxintegrator( cyl_field, *g2dFine_ptr, coords, coordsM, phi1, eps);
+        if( dynamic_cast<const dg::CartesianGrid2d*>( g2dField_ptr))
+            boxintegrator( cyl_field, *g2dField_ptr, coords, coordsM, phi1, eps);
         else 
-            boxintegrator( field, *g2dFine_ptr, coords, coordsM, phi1, eps);
+            boxintegrator( field, *g2dField_ptr, coords, coordsM, phi1, eps);
         yp[0][i] = coordsP[0], yp[1][i] = coordsP[1], yp[2][i] = coordsP[2];
         ym[0][i] = coordsM[0], ym[1][i] = coordsM[1], ym[2][i] = coordsM[2];
     }
@@ -297,7 +293,7 @@ void integrate_all_fieldlines2d( const dg::geo::BinaryVectorLvl0& vec, const aGe
     delete g2dField_ptr;
 }
 }//namespace detail
-
+///@endcond
 
 
 //////////////////////////////FieldAlignedCLASS////////////////////////////////////////////
@@ -307,7 +303,7 @@ void integrate_all_fieldlines2d( const dg::geo::BinaryVectorLvl0& vec, const aGe
 * This class discretizes the operators \f$ \nabla_\parallel = 
 \mathbf{b}\cdot \nabla = b_R\partial_R + b_Z\partial_Z + b_\phi\partial_\phi \f$, \f$\nabla_\parallel^\dagger\f$ and \f$\Delta_\parallel=\nabla_\parallel^\dagger\cdot\nabla_\parallel\f$ in
 cylindrical coordinates
-* @ingroup utilities
+* @ingroup fieldaligned
 * @tparam Geometry The Geometry class 
 * @tparam IMatrix The matrix class of the interpolation matrix
 * @tparam container The container-class on which the interpolation matrix operates on (does not need to be dg::HVec)
@@ -491,15 +487,20 @@ FieldAligned<Geometry, IMatrix, container>::FieldAligned(const dg::geo::BinaryVe
     limiter_ = dg::evaluate( limit, *g2dCoarse_ptr);
     right_ = left_ = dg::evaluate( zero, *g2dCoarse_ptr);
     ghostM.resize( perp_size_); ghostP.resize( perp_size_);
-    //Set starting points and integrate field lines
+    //%%%%%%%%%%%%%%%%%%%%%%%%%%Set starting points and integrate field lines%%%%%%%%%%%%%%
     std::cout << "Start fieldline integration!\n";
     dg::Timer t;
     std::vector<thrust::host_vector<double> > yp_coarse( 3), ym_coarse(yp_coarse); 
     t.tic();
-    detail::integrate_all_fieldlines2d( vec, g2dCoarse_ptr, yp_coarse, ym_coarse, deltaPhi, eps);
-    dg::Grid2d g2dFine((dg::Grid2d(*g2dCoarse_ptr)));
+    
+    dg::aGeometry2d* g2dField_ptr = g2dCoarse_ptr->clone();//INTEGRATE HIGH ORDER GRID
+    g2dField_ptr->set( 7, g2dField_ptr->Nx(), g2dField_ptr->Ny());
+    detail::integrate_all_fieldlines2d( vec, g2dField_ptr, yp_coarse, ym_coarse, deltaPhi, eps);
+    delete g2dField_ptr;
+
+    dg::Grid2d g2dFine((dg::Grid2d(*g2dCoarse_ptr)));//FINE GRID
     g2dFine.multiplyCellNumbers((double)mx, (double)my);
-    IMatrix interpolate = dg::create::interpolation( g2dFine, *g2dCoarse_ptr);
+    IMatrix interpolate = dg::create::interpolation( g2dFine, *g2dCoarse_ptr);  //INTERPOLATE TO FINE GRID
     std::vector<thrust::host_vector<double> > yp( 3, dg::evaluate(dg::zero, g2dFine)), ym(yp); 
     for( unsigned i=0; i<3; i++)
     {
@@ -510,11 +511,9 @@ FieldAligned<Geometry, IMatrix, container>::FieldAligned(const dg::geo::BinaryVe
     {
         g2dFine.shift_topologic( yp[0][i], yp[1][i], yp[0][i], yp[1][i]);
         g2dFine.shift_topologic( ym[0][i], ym[1][i], ym[0][i], ym[1][i]);
-        //detail::clip_to_boundary( yp[0][i], yp[1][i], &g2dFine);
-        //detail::clip_to_boundary( ym[0][i], ym[1][i], &g2dFine);
+        detail::clip_to_boundary( yp[0][i], yp[1][i], &g2dFine);
+        detail::clip_to_boundary( ym[0][i], ym[1][i], &g2dFine);
     }
-
-
     t.toc(); 
     std::cout << "Fieldline integration took "<<t.diff()<<"s\n";
     //%%%%%%%%%%%%%%%%%%Create interpolation and projection%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -529,7 +528,7 @@ FieldAligned<Geometry, IMatrix, container>::FieldAligned(const dg::geo::BinaryVe
     cusp::multiply( projection, minusFine, minus);
     t.toc();
     std::cout<< "Multiplication of P*I took: "<<t.diff()<<"s\n";
-    //Transposed matrices work only for csr_matrix due to bad matrix form for ell_matrix!!!
+    //%Transposed matrices work only for csr_matrix due to bad matrix form for ell_matrix!!!
     cusp::transpose( plus, plusT);
     cusp::transpose( minus, minusT);     
     //%%%%%%%%%%%%%%%%%%%%%%%project h and copy into h vectors%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
diff --git a/inc/geometries/flux_t.cu b/inc/geometries/flux_t.cu
index 3e8554b4b..6ef2612a3 100644
--- a/inc/geometries/flux_t.cu
+++ b/inc/geometries/flux_t.cu
@@ -20,8 +20,6 @@
 
 #include "file/nc_utilities.h"
 
-using namespace dg::geo::solovev;
-
 thrust::host_vector<double> periodify( const thrust::host_vector<double>& in, const dg::Grid2d& g)
 {
     thrust::host_vector<double> out(g.size());
@@ -63,7 +61,7 @@ int main( int argc, char* argv[])
         reader.parse(is,js,false);
     }
     //write parameters from file into variables
-    GeomParameters gp(js);
+    dg::geo::solovev::GeomParameters gp(js);
     Psip psip( gp); 
     std::cout << "Psi min "<<psip(gp.R_0, 0)<<"\n";
     std::cout << "Type psi_0 (-20) and psi_1 (-4)\n";
@@ -160,44 +158,6 @@ int main( int argc, char* argv[])
     std::cout << "relative difference in volume2d is "<<fabs(volumeRZ - volume2d)/volume2d<<std::endl;
     std::cout << "Note that the error might also come from the volume in RZP!\n"; //since integration of jacobian is fairly good probably
 
-    /////////////////////////TEST 3d grid//////////////////////////////////////
-    std::cout << "Start DS test!"<<std::endl;
-    t.tic();
-    dg::geo::BHatR bhatR(c);
-    dg::geo::BHatZ bhatZ(c);
-    dg::geo::BHatP bhatP(c);
-    //dg::geo::BinaryVectorLvl0 bhat( dg::geo::BHatR(c), dg::geo::BHatZ(c), dg::geo::BHatP(c));
-    dg::geo::BinaryVectorLvl0 bhat( bhatR, bhatZ, bhatP);
-    unsigned mx, my;
-    std::cout << "Type multipleX and multipleY!\n";
-    std::cin >> mx >> my;
-    //dg::FieldAligned<dg::aGeometry3d, dg::IHMatrix, dg::HVec> fieldaligned( bhat, g3d, 1, 4, gp.rk4eps, dg::NoLimiter() ); 
-    dg::DS<dg::aGeometry3d, dg::IHMatrix, dg::HMatrix, dg::HVec> ds( c, g3d, dg::normed, dg::centered, false, true, mx, my);
-
-    
-    t.toc();
-    std::cout << "Construction took "<<t.diff()<<"s\n";
-    dg::HVec B = dg::pullback( dg::geo::InvB(c), g3d), divB(B);
-    dg::HVec lnB = dg::pullback( dg::geo::LnB(c), g3d), gradB(B);
-    dg::HVec gradLnB = dg::pullback( dg::geo::GradLnB(c), g3d);
-    dg::blas1::pointwiseDivide( ones3d, B, B);
-    dg::HVec function = dg::pullback( dg::geo::FuncNeu(c), g3d), derivative(function);
-    ds( function, derivative);
-
-    ds.centeredAdj( B, divB);
-    double norm =  sqrt( dg::blas2::dot(divB, vol3d, divB));
-    std::cout << "Divergence of B is "<<norm<<"\n";
-
-    ds.centered( lnB, gradB);
-    std::cout << "num. norm of gradLnB is "<<sqrt( dg::blas2::dot( gradB,vol3d, gradB))<<"\n";
-    norm = sqrt( dg::blas2::dot( gradLnB, vol3d, gradLnB) );
-    std::cout << "ana. norm of gradLnB is "<<norm<<"\n";
-    dg::blas1::axpby( 1., gradB, -1., gradLnB, gradLnB);
-    X = divB;
-    err = nc_put_var_double( ncid, varID[4], periodify(X, g2d_periodic).data());
-    double norm2 = sqrt(dg::blas2::dot(gradLnB, vol3d, gradLnB));
-    std::cout << "rel. error of lnB is    "<<norm2/norm<<"\n";
-
     err = nc_close( ncid);
     return 0;
 }
diff --git a/inc/geometries/mpi_fieldaligned.h b/inc/geometries/mpi_fieldaligned.h
index ab501d5d4..4ba263551 100644
--- a/inc/geometries/mpi_fieldaligned.h
+++ b/inc/geometries/mpi_fieldaligned.h
@@ -88,7 +88,7 @@ struct ZShifter
 /**
  * @brief Class for the evaluation of a parallel derivative (MPI Version)
  *
- * @ingroup utilities
+ * @ingroup fieldaligned
  * @tparam LocalMatrix The matrix class of the interpolation matrix
  * @tparam Communicator The communicator used to exchange data in the RZ planes
  * @tparam LocalContainer The container-class to on which the interpolation matrix operates on (does not need to be dg::HVec)
diff --git a/inc/geometries/solovev_doc.h b/inc/geometries/solovev_doc.h
index 71dc5f19b..34385d400 100644
--- a/inc/geometries/solovev_doc.h
+++ b/inc/geometries/solovev_doc.h
@@ -14,7 +14,8 @@
       @defgroup magnetic 3.2 magnetic field and associated functors
       @defgroup profiles 3.3 miscellaneous functors based on flux functions
  * @}
- * @defgroup misc 4. Miscellaneous additions
+ * @defgroup fieldaligned 4. Fieldaligned derivatives
+ * @defgroup misc 5. Miscellaneous additions
  *
  * Objects that are used to define and integrate the magnetic field lines. 
  * All objects can be used in the evaluation() function.
diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index 1ee7ecda3..4a72261b3 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -17,12 +17,9 @@
 
 /*
    - reads parameters from input.txt or any other given file, 
-   - integrates the Feltor - functor and 
+   - integrates the Explicit - functor and 
    - directly visualizes results on the screen using parameters in window_params.txt
 */
-typedef dg::FieldAligned< dg::CylindricalGrid3d, dg::IDMatrix, dg::DVec> DFA;
-
-using namespace dg::geo::solovev;
 
 int main( int argc, char* argv[])
 {
@@ -48,7 +45,7 @@ int main( int argc, char* argv[])
         std::cerr << "ERROR: Too many arguments!\nUsage: "<< argv[0]<<" [inputfile] [geomfile] \n";
         return -1;
     }
-    const eule::Parameters p( js);
+    const feltor::Parameters p( js);
     const dg::geo::solovev::GeomParameters gp(gs);
     p.display( std::cout);
     gp.display( std::cout);
@@ -69,10 +66,10 @@ int main( int argc, char* argv[])
     dg::CylindricalGrid3d grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n, p.Nx, p.Ny, p.Nz, p.bc, p.bc, dg::PER);  
 
     //create RHS 
-    std::cout << "Constructing Feltor...\n";
-    eule::Feltor<dg::CylindricalGrid3d, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec> feltor( grid, p, gp); //initialize before rolkar!
-    std::cout << "Constructing Rolkar...\n";
-    eule::Rolkar<dg::CylindricalGrid3d, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec> rolkar( grid, p, gp, feltor.ds(), feltor.dsDIR());
+    std::cout << "Constructing Explicit...\n";
+    feltor::Explicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec> feltor( grid, mag, p, gp); //initialize before rolkar!
+    std::cout << "Constructing Implicit...\n";
+    feltor::Implicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec> rolkar( grid, p, gp, feltor.ds(), feltor.dsDIR());
     std::cout << "Done!\n";
 
     /////////////////////The initial field///////////////////////////////////////////
diff --git a/src/feltor/feltor.cuh b/src/feltor/feltor.cuh
index 4482bb33e..ec9b32367 100644
--- a/src/feltor/feltor.cuh
+++ b/src/feltor/feltor.cuh
@@ -13,12 +13,12 @@
   Contains the solvers 
   */
 
-namespace eule
+namespace feltor
 {
 ///@addtogroup solver
 ///@{
 /**
- * @brief Implicit (perpendicular diffusive) terms for Feltor solver
+ * @brief Implicit (perpendicular diffusive) terms for Explicit solver
  *
  \f[
     \begin{align}
@@ -26,12 +26,9 @@ namespace eule
     \frac{C}{\mu} (U_e - U_i) - \nu_\perp\Delta_\perp^2 U   
     \end{align}
 \f]
- * @tparam Matrix The Matrix class
- * @tparam container The Vector class 
- * @tparam container The container class
  */
-template<class Geometry, class DS, class Matrix, class container>
-struct Rolkar
+template<class Geometry, class IMatrix, class Matrix, class container>
+struct Implicit
 {
 
     /**
@@ -42,7 +39,7 @@ struct Rolkar
      * @param p the physics parameters
      * @param gp the geometry parameters
      */
-    Rolkar( const Geometry& g, eule::Parameters p, dg::geo::solovev::GeomParameters gp, DS& dsN, DS& dsDIR):
+    Implicit( const Geometry& g, feltor::Parameters p, dg::geo::solovev::GeomParameters gp, DS& dsN, DS& dsDIR):
         p(p),
         gp(gp),
         LaplacianM_perpN  ( g, g.bcx(), g.bcy(), dg::normed, dg::centered),
@@ -52,7 +49,7 @@ struct Rolkar
     {
         using dg::geo::solovev::Psip;
         dg::blas1::transfer( dg::evaluate( dg::zero, g), temp);
-        dg::blas1::transfer( dg::pullback( dg::geo::GaussianDamping<Psip>(Psip(gp), gp.psipmaxcut, gp.alpha), g), dampgauss_);
+        dg::blas1::transfer( dg::pullback( dg::geo::GaussianDamping(Psip(gp), gp.psipmaxcut, gp.alpha), g), dampgauss_);
     }
 
     /**
@@ -125,7 +122,7 @@ struct Rolkar
      */
     //const container& damping(){return dampprof_;}
   private:
-    const eule::Parameters p;
+    const feltor::Parameters p;
     const dg::geo::solovev::GeomParameters gp;
     container temp;
     container dampgauss_;
@@ -133,15 +130,8 @@ struct Rolkar
     DS& dsN_,dsDIR_;
 };
 
-/**
- * @brief compute explicit terms
- *
- * @tparam Matrix matrix class to use
- * @tparam container main container to hold the vectors
- * @tparam container class of the weights
- */
-template< class Geometry, class DS, class Matrix, class container >
-struct Feltor
+template< class Geometry, class IMatrix, class Matrix, class container >
+struct Explicit
 {
     /**
      * @brief Construct from parameters
@@ -151,7 +141,7 @@ struct Feltor
      * @param p the physics parameters
      * @param gp the geometry parameters
      */
-    Feltor( const Geometry& g, eule::Parameters p, dg::geo::solovev::GeomParameters gp);
+    Explicit( const Geometry& g, const dg::geo::TokamakMagneticField& mag, feltor::Parameters p, dg::geo::solovev::GeomParameters gp);
 
 
     /**
@@ -159,8 +149,8 @@ struct Feltor
      *
      * @return 
      */
-    DS& ds(){return dsN_;}
-    DS& dsDIR(){return dsDIR_;}
+    dg::DS<Geometry, IMatrix, Matrix, container>& ds(){return dsN_;}
+    dg::DS<Geometry, IMatrix, Matrix, container>& dsDIR(){return dsDIR_;}
 
     /**
      * @brief Returns phi and psi that belong to the last solve of the polarization equation
@@ -179,7 +169,7 @@ struct Feltor
     void initializene( const container& y, container& target);
 
     /**
-     * @brief Compute explicit rhs of Feltor equations
+     * @brief Compute explicit rhs of Explicit equations
      *
      * @param y y[0] := N_e - 1, y[1] := N_i - 1, y[2] := U_e, y[3] := U_i
      * @param yp Result
@@ -265,15 +255,14 @@ struct Feltor
     std::vector<container> dsy, curvy,curvkappay; 
 
     //matrices and solvers
-    DS dsDIR_;
-    DS dsN_;
+    dg::DS<Geometry, IMatrix, Matrix, container> dsDIR_, dsN_;
     dg::Poisson<   Geometry, Matrix, container> poissonN,poissonDIR; 
     dg::Elliptic<  Geometry, Matrix, container > pol,lapperpN,lapperpDIR;
     dg::Helmholtz< Geometry, Matrix, container > invgammaDIR, invgammaN;
 
     dg::Invert<container> invert_pol,invert_invgammaN,invert_invgammaPhi;
 
-    const eule::Parameters p;
+    const feltor::Parameters p;
     const dg::geo::solovev::GeomParameters gp;
 
     double mass_, energy_, diff_, ediff_, aligned_;
@@ -282,35 +271,20 @@ struct Feltor
 ///@}
 
 ///@cond
-template<class Grid, class DS, class Matrix, class container>
-Feltor<Grid, DS, Matrix, container>::Feltor( const Grid& g, eule::Parameters p, dg::geo::solovev::GeomParameters gp): 
-    dsDIR_( typename DS::FieldAligned( 
-                dg::geo::Field<dg::geo::solovev::MagneticField>(
-                    dg::geo::solovev::MagneticField(gp), gp.R_0
-                    ), 
-                g, gp.rk4eps, 
-                dg::geo::PsiLimiter<dg::geo::solovev::Psip>(
-                    dg::geo::solovev::Psip(gp), gp.psipmaxlim
-                    ), 
+template<class Grid, class IMatrix, class Matrix, class container>
+Explicit<Grid, IMatrix, Matrix, container>::Explicit( const Grid& g, const TokamakMagneticField& mag, feltor::Parameters p, dg::geo::solovev::GeomParameters gp): 
+    dsDIR_( dg::FieldAligned<Grid, IMatrix, container>( mag, g, gp.rk4eps, 
+                dg::geo::PsiLimiter(mag.psip(), gp.psipmaxlim), 
                 dg::DIR, (2*M_PI)/((double)p.Nz)
                 ), 
-            dg::geo::Field<dg::geo::solovev::MagneticField>(
-                dg::geo::solovev::MagneticField(gp), gp.R_0
-                ), 
+            dg::geo::InvB(mag), 
             dg::normed, dg::forward ),
-    dsN_( typename DS::FieldAligned(
-                dg::geo::Field<dg::geo::solovev::MagneticField>(
-                    dg::geo::solovev::MagneticField(gp), gp.R_0), 
-                g, gp.rk4eps, 
-                dg::geo::PsiLimiter<dg::geo::solovev::Psip>(
-                    dg::geo::solovev::Psip(gp), gp.psipmaxlim
-                    ), 
+    dsN_( dg::FieldAligned<Grid, IMatrix, container>( mag, g, gp.rk4eps, 
+                dg::geo::PsiLimiter(mag.psip(), gp.psipmaxlim), 
                 g.bcx(), (2*M_PI)/((double)p.Nz)
                 ), 
-          dg::geo::Field<dg::geo::solovev::MagneticField>(
-              dg::geo::solovev::MagneticField(gp), gp.R_0
-              ), 
-          dg::normed, dg::forward ),
+            dg::geo::InvB(mag), 
+            dg::normed, dg::forward ),
     //////////the poisson operators ////////////////////////////////////////
     poissonN(  g, g.bcx(), g.bcy(), dg::DIR, dg::DIR), //first N/U then phi BCC
     poissonDIR(g, dg::DIR, dg::DIR, dg::DIR, dg::DIR), //first N/U then phi BCC
@@ -336,18 +310,16 @@ Feltor<Grid, DS, Matrix, container>::Feltor( const Grid& g, eule::Parameters p,
     invert_invgammaN.construct(   omega, p.Nx*p.Ny*p.Nz*p.n*p.n, p.eps_gamma); 
     invert_invgammaPhi.construct( omega, p.Nx*p.Ny*p.Nz*p.n*p.n, p.eps_gamma); 
     //////////////////////////////init fields /////////////////////
-    using namespace dg::geo::solovev;
-    MagneticField mf(gp);
-    dg::blas1::transfer(  dg::pullback(dg::geo::Field<MagneticField>(mf, gp.R_0),                     g), binv);
-    dg::blas1::transfer(  dg::pullback(dg::geo::GradLnB<MagneticField>(mf, gp.R_0),                   g), gradlnB);
-    dg::blas1::transfer(  dg::pullback(dg::geo::TanhSource<Psip>(mf.psip, gp.psipmin, gp.alpha),      g), source);
-    dg::blas1::transfer(  dg::pullback(dg::geo::GaussianDamping<Psip>(mf.psip, gp.psipmax, gp.alpha), g), damping);
+    dg::blas1::transfer(  dg::pullback(dg::geo::InvB(mag),                                         g), binv);
+    dg::blas1::transfer(  dg::pullback(dg::geo::GradLnB(mag),                                      g), gradlnB);
+    dg::blas1::transfer(  dg::pullback(dg::geo::TanhSource(mf.psip, gp.psipmin, gp.alpha),         g), source);
+    dg::blas1::transfer(  dg::pullback(dg::geo::GaussianDamping(mag.psip(), gp.psipmax, gp.alpha), g), damping);
     ////////////////////////////transform curvature components////////
-    dg::geo::pushForwardPerp(dg::geo::CurvatureNablaBR<MagneticField>(mf, gp.R_0), dg::geo::CurvatureNablaBZ<MagneticField>(mf, gp.R_0), curvX, curvY, g);
-    dg::blas1::transfer(  dg::pullback(dg::geo::DivCurvatureKappa<MagneticField>(mf, gp.R_0), g), divCurvKappa);
+    dg::geo::pushForwardPerp(dg::geo::CurvatureNablaBR(mag), dg::geo::CurvatureNablaBZ(mag), curvX, curvY, g);
+    dg::blas1::transfer(  dg::pullback(dg::geo::DivCurvatureKappa(mag), g), divCurvKappa);
     if (p.curvmode==1) 
     {
-        dg::geo::pushForwardPerp(dg::geo::CurvatureKappaR(), dg::geo::CurvatureKappaZ<MagneticField>(mf, gp.R_0), curvKappaX, curvKappaY, g);
+        dg::geo::pushForwardPerp(dg::geo::CurvatureKappaR(), dg::geo::CurvatureKappaZ(mag), curvKappaX, curvKappaY, g);
         dg::blas1::axpby( 1.,curvX,1.,curvKappaX,curvX);
         dg::blas1::axpby( 1.,curvY,1.,curvKappaY,curvY);
     }
@@ -360,7 +332,7 @@ Feltor<Grid, DS, Matrix, container>::Feltor( const Grid& g, eule::Parameters p,
         dg::blas1::scal(divCurvKappa,0.);
     }
     ///////////////////init densities//////////////////////////////
-    dg::blas1::transfer( dg::pullback(dg::geo::Nprofile<Psip>(p.bgprofamp, p.nprofileamp, gp, mf.psip),g), profne);
+    dg::blas1::transfer( dg::pullback(dg::geo::Nprofile(p.bgprofamp, p.nprofileamp, gp, mag.psip()),g), profne);
     dg::blas1::transfer(  profne ,profNi);
     dg::blas1::plus( profNi, -1); 
     initializene(profNi, profne); //ne = Gamma N_i (needs Invert object)
@@ -372,9 +344,9 @@ Feltor<Grid, DS, Matrix, container>::Feltor( const Grid& g, eule::Parameters p,
 }
 
 template<class Geometry, class DS, class Matrix, class container>
-container& Feltor<Geometry, DS, Matrix, container>::polarisation( const std::vector<container>& y)
+container& Explicit<Geometry, DS, Matrix, container>::polarisation( const std::vector<container>& y)
 {
- dg::blas1::axpby( p.mu[1], y[1], 0, chi);       //chi =  \mu_i (n_i-1) 
+    dg::blas1::axpby( p.mu[1], y[1], 0, chi);       //chi =  \mu_i (n_i-1) 
     dg::blas1::plus( chi, p.mu[1]);
     dg::blas1::pointwiseDot( chi, binv, chi);
     dg::blas1::pointwiseDot( chi, binv, chi);       //chi = (\mu_i n_i ) /B^2
@@ -389,7 +361,7 @@ container& Feltor<Geometry, DS, Matrix, container>::polarisation( const std::vec
 }
 
 template< class Geometry, class DS, class Matrix, class container>
-container& Feltor<Geometry, DS, Matrix,container>::compute_psi( container& potential)
+container& Explicit<Geometry, DS, Matrix,container>::compute_psi( container& potential)
 {
     invert_invgammaPhi(invgammaDIR,chi,potential);                    //chi  Gamma phi
     poissonN.variationRHS(potential, omega);
@@ -400,14 +372,14 @@ container& Feltor<Geometry, DS, Matrix,container>::compute_psi( container& poten
 }
 
 template<class Geometry, class DS, class Matrix, class container>
-void Feltor<Geometry, DS, Matrix, container>::initializene( const container& src, container& target)
+void Explicit<Geometry, DS, Matrix, container>::initializene( const container& src, container& target)
 { 
     invert_invgammaN(invgammaN,target,src); //=ne-1 = Gamma (ni-1)    
 }
 
 
 template<class G, class DS, class M, class V>
-double Feltor<G, DS, M, V>::add_parallel_dynamics( std::vector<V>& y, std::vector<V>& yp)
+double Explicit<G, DS, M, V>::add_parallel_dynamics( std::vector<V>& y, std::vector<V>& yp)
 {
     double z[2]     = {-1.0,1.0};
     double Dpar[4]  = {0.0, 0.0,0.0,0.0};
@@ -526,7 +498,7 @@ double Feltor<G, DS, M, V>::add_parallel_dynamics( std::vector<V>& y, std::vecto
 
 
 template<class Geometry, class DS, class Matrix, class container>
-void Feltor<Geometry, DS, Matrix, container>::operator()( std::vector<container>& y, std::vector<container>& yp)
+void Explicit<Geometry, DS, Matrix, container>::operator()( std::vector<container>& y, std::vector<container>& yp)
 {
     /* y[0] := N_e - 1
        y[1] := N_i - 1
@@ -698,25 +670,23 @@ void Feltor<Geometry, DS, Matrix, container>::operator()( std::vector<container>
 
 //Computes curvature operator
 template<class Geometry, class DS, class Matrix, class container>
-void Feltor<Geometry, DS, Matrix, container>::vecdotnablaN(const container& vecX, const container& vecY, container& src, container& target)
+void Explicit<Geometry, DS, Matrix, container>::vecdotnablaN(const container& vecX, const container& vecY, container& src, container& target)
 {
     container temp1(src);
     dg::blas2::gemv( poissonN.dxlhs(), src, target); //d_R src
-    dg::blas1::pointwiseDot( vecX, target, target); // C^R d_R src
     dg::blas2::gemv( poissonN.dylhs(), src, temp1);  //d_Z src
-    dg::blas1::pointwiseDot( 1., vecY, temp1, 1., target);   // C^Z d_Z src + C^R d_R src
+    dg::blas1::pointwiseDot( 1., vecY, temp1, 1., vecX, target, 0, target);   // C^Z d_Z src + C^R d_R src
 }
 
 template<class Geometry, class DS, class Matrix, class container>
-void Feltor<Geometry, DS, Matrix, container>::vecdotnablaDIR(const container& vecX, const container& vecY,  container& src, container& target)
+void Explicit<Geometry, DS, Matrix, container>::vecdotnablaDIR(const container& vecX, const container& vecY,  container& src, container& target)
 {
     container temp1(src);
     dg::blas2::gemv( poissonDIR.dxrhs(), src, target); //d_R src
-    dg::blas1::pointwiseDot( vecX, target, target); // C^R d_R src
     dg::blas2::gemv( poissonDIR.dyrhs(), src, temp1);  //d_Z src
-    dg::blas1::pointwiseDot( 1., vecY, temp1, 1., target);// C^Z d_Z src + C^R d_R src
+    dg::blas1::pointwiseDot( 1., vecY, temp1, 1., vecX, target, 0, target);   // C^Z d_Z src + C^R d_R src
 }
 
 ///@endcond
-} //namespace eule
+} //namespace feltor
 
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index b82c117fc..a9079b613 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -18,7 +18,7 @@
 
 /*
    - reads parameters from input.txt or any other given file, 
-   - Initializes and integrates Feltor and 
+   - Initializes and integrates Explicit and 
    - writes outputs to a given outputfile using netcdf 
         density fields are the real densities in XSPACE ( not logarithmic values)
 
@@ -43,7 +43,7 @@ int main( int argc, char* argv[])
         reader.parse(is,js,false);
         reader.parse(ks,gs,false);
     }
-    const eule::Parameters p( js);
+    const feltor::Parameters p( js);
     const dg::geo::solovev::GeomParameters gp(gs);
     p.display( std::cout);
     gp.display( std::cout);
@@ -59,10 +59,10 @@ int main( int argc, char* argv[])
     dg::CylindricalGrid3d grid_out( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n_out, p.Nx_out, p.Ny_out, p.Nz_out, p.bc, p.bc, dg::PER);  
      
     //create RHS 
-    std::cout << "Constructing Feltor...\n";
-    eule::Feltor<dg::CylindricalGrid3d, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec> feltor( grid, p, gp); //initialize before rolkar!
-    std::cout << "Constructing Rolkar...\n";
-    eule::Rolkar< dg::CylindricalGrid3d, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec > rolkar( grid, p, gp, feltor.ds(), feltor.dsDIR());
+    std::cout << "Constructing Explicit...\n";
+    feltor::Explicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec> feltor( grid, p, gp); //initialize before rolkar!
+    std::cout << "Constructing Implicit...\n";
+    feltor::Implicit< dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec > rolkar( grid, p, gp, feltor.ds(), feltor.dsDIR());
     std::cout << "Done!\n";
 
     /////////////////////The initial field//////////////////////////////////////////
diff --git a/src/feltor/feltor_mpi.cu b/src/feltor/feltor_mpi.cu
index d118ee66e..91917f840 100644
--- a/src/feltor/feltor_mpi.cu
+++ b/src/feltor/feltor_mpi.cu
@@ -72,7 +72,7 @@ int main( int argc, char* argv[])
         reader.parse(is,js,false);
         reader.parse(ks,gs,false);
     }
-    const eule::Parameters p( js);
+    const feltor::Parameters p( js);
     const dg::geo::solovev::GeomParameters gp(gs);
     if(rank==0)p.display( std::cout);
     if(rank==0)gp.display( std::cout);
@@ -88,10 +88,10 @@ int main( int argc, char* argv[])
      dg::CylindricalMPIGrid3d grid_out( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n_out, p.Nx_out, p.Ny_out, p.Nz_out, p.bc, p.bc, dg::PER, comm);  
      
     //create RHS 
-    if(rank==0)std::cout << "Constructing Feltor...\n";
-    eule::Feltor<dg::CylindricalMPIGrid3d, dg::DS<DFA, dg::MDMatrix, dg::MDVec>, dg::MDMatrix, dg::MDVec> feltor( grid, p, gp); //initialize before rolkar!
-    if(rank==0)std::cout << "Constructing Rolkar...\n";
-    eule::Rolkar< dg::CylindricalMPIGrid3d, dg::DS<DFA, dg::MDMatrix, dg::MDVec>, dg::MDMatrix, dg::MDVec > rolkar( grid, p, gp, feltor.ds(), feltor.dsDIR());
+    if(rank==0)std::cout << "Constructing Explicit...\n";
+    feltor::Explicit<dg::CylindricalMPIGrid3d, dg::IDMatrix, dg::MDMatrix, dg::MDVec> feltor( grid, p, gp); //initialize before rolkar!
+    if(rank==0)std::cout << "Constructing Implicit...\n";
+    feltor::Implicit< dg::CylindricalMPIGrid3d, dg::IDMatrix, dg::MDMatrix, dg::MDVec > rolkar( grid, p, gp, feltor.ds(), feltor.dsDIR());
     if(rank==0)std::cout << "Done!\n";
 
     /////////////////////The initial field/////////////////////////////////////////
-- 
GitLab


From 53bda02db0f6f1ebdd55b78be3e6d4f4542f98c8 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 4 Sep 2017 07:13:21 -0700
Subject: [PATCH 247/453]  add omp versions for dot functions

---
 inc/dg/backend/thrust_matrix_blas.cuh | 52 +++++++++--------------
 inc/dg/backend/thrust_vector_blas.cuh | 59 +++++++++++++++++++--------
 inc/dg/blas1_t.cu                     |  2 +
 3 files changed, 63 insertions(+), 50 deletions(-)

diff --git a/inc/dg/backend/thrust_matrix_blas.cuh b/inc/dg/backend/thrust_matrix_blas.cuh
index dcb552b31..fcb5c9867 100644
--- a/inc/dg/backend/thrust_matrix_blas.cuh
+++ b/inc/dg/backend/thrust_matrix_blas.cuh
@@ -48,13 +48,23 @@ inline typename MatrixTraits<Matrix>::value_type doDot( const Vector& x, const M
     assert( x.size() == y.size() && x.size() == m.size() );
 #endif //DG_DEBUG
     typedef typename MatrixTraits<Matrix>::value_type value_type;
+#if THRUST_DEVICE_SYSTEM!=THRUST_DEVICE_SYSTEM_CUDA
+    value_type sum = 0;
+    unsigned size=x.size();
+    #pragma omp parallel reduction(+:sum) 
+    for( unsigned i=0; i<size; i++)
+        sum += x[i]*m[i]*y[i];
+    return sum;
+#else
     return thrust::inner_product(  x.begin(), x.end(), 
                             thrust::make_zip_iterator( thrust::make_tuple( y.begin(), m.begin())  ), 
                             value_type(0),
                             thrust::plus<value_type>(),
                             detail::ThrustVectorDoDot<Matrix>()
                             );
+#endif
 }
+
 template< class Matrix, class Vector>
 inline typename MatrixTraits<Matrix>::value_type doDot( const Matrix& m, const Vector& x, dg::ThrustMatrixTag, dg::ThrustVectorTag)
 {
@@ -62,31 +72,23 @@ inline typename MatrixTraits<Matrix>::value_type doDot( const Matrix& m, const V
     assert( m.size() == x.size());
 #endif //DG_DEBUG
     typedef typename MatrixTraits<Matrix>::value_type value_type;
+#if THRUST_DEVICE_SYSTEM!=THRUST_DEVICE_SYSTEM_CUDA
+    value_type sum = 0;
+    unsigned size=x.size();
+    #pragma omp parallel reduction(+:sum)
+    for( unsigned i=0; i<size; i++)
+        sum += x[i]*x[i]*m[i];
+    return sum;
+#else
     return thrust::inner_product( x.begin(), x.end(),
                                   m.begin(),
                                   value_type(0),
                                   thrust::plus<value_type>(),
                                   detail::ThrustVectorDoDot<Matrix>()
             ); //very fast
+#endif
 }
 
-template < class Vector>
-struct ThrustVectorDoSymv
-{
-    typedef typename VectorTraits<Vector>::value_type value_type;
-    typedef thrust::tuple< value_type, value_type> Pair; 
-    __host__ __device__
-        ThrustVectorDoSymv( value_type alpha, value_type beta): alpha_(alpha), beta_(beta){}
-
-    __host__ __device__
-        value_type operator()(const value_type& x, const Pair& p) 
-        {
-            return alpha_*thrust::get<0>(p)*x + beta_*thrust::get<1>(p);
-        }
-  private:
-    value_type alpha_, beta_;
-};
-
 template< class Matrix, class Vector>
 inline void doSymv(  
               typename MatrixTraits<Matrix>::value_type alpha, 
@@ -97,21 +99,7 @@ inline void doSymv(
               ThrustMatrixTag,
               ThrustVectorTag)
 {
-#ifdef DG_DEBUG
-    assert( x.size() == y.size() && x.size() == m.size() );
-#endif //DG_DEBUG
-    if( alpha == 0)
-    {
-        if( beta == 1) 
-            return;
-        dg::blas1::detail::doAxpby( 0., x, beta, y, dg::ThrustVectorTag());
-        return;
-    }
-    thrust::transform( x.begin(), x.end(), 
-                       thrust::make_zip_iterator( thrust::make_tuple( m.begin(), y.begin() )),  
-                       y.begin(),
-                       detail::ThrustVectorDoSymv<Matrix>( alpha, beta)
-                      ); 
+    dg::blas1::detail::doPointwiseDot( alpha, m, x, beta, y, ThrustVectorTag());
 }
 
 template< class Matrix, class Vector>
diff --git a/inc/dg/backend/thrust_vector_blas.cuh b/inc/dg/backend/thrust_vector_blas.cuh
index 25c91ef6b..e5750278e 100644
--- a/inc/dg/backend/thrust_vector_blas.cuh
+++ b/inc/dg/backend/thrust_vector_blas.cuh
@@ -68,7 +68,16 @@ typename Vector::value_type doDot( const Vector& x, const Vector& y, ThrustVecto
     assert( x.size() == y.size() );
 #endif //DG_DEBUG
     typedef typename Vector::value_type value_type;
+#if THRUST_DEVICE_SYSTEM!=THRUST_DEVICE_SYSTEM_CUDA
+    value_type sum = 0;
+    unsigned size=x.size();
+    #pragma omp parallel reduction(+:sum) 
+    for( unsigned i=0; i<size; i++)
+        sum += x[i]*y[i];
+    return sum;
+#else
     return thrust::inner_product( x.begin(), x.end(),  y.begin(), value_type(0));
+#endif
 }
 
 template< class Vector, class UnaryOp>
@@ -86,16 +95,30 @@ inline void doScal(  Vector& x,
 {
     if( alpha == 1.) 
         return;
+#if THRUST_DEVICE_SYSTEM!=THRUST_DEVICE_SYSTEM_CUDA
+    unsigned size=x.size();
+    #pragma omp parallel for simd
+    for( unsigned i=0; i<size; i++)
+        x[i]*=alpha;
+#else
     thrust::transform( x.begin(), x.end(), x.begin(), 
             detail::Axpby_Functor<typename Vector::value_type>( 0, alpha));
+#endif
 }
 template< class Vector>
 inline void doPlus(  Vector& x, 
               typename Vector::value_type alpha, 
               ThrustVectorTag)
 {
+#if THRUST_DEVICE_SYSTEM!=THRUST_DEVICE_SYSTEM_CUDA
+    unsigned size=x.size();
+    #pragma omp parallel for simd
+    for( unsigned i=0; i<size; i++)
+        x[i]+=alpha;
+#else
     thrust::transform( x.begin(), x.end(), x.begin(), 
             detail::Plus_Functor<typename Vector::value_type>( alpha));
+#endif
 }
 
 template< class Vector>
@@ -129,19 +152,19 @@ inline void doAxpby( typename Vector::value_type alpha,
     unsigned size = x.size();
     if( beta == 1.)
     {
-#pragma omp parallel for simd
+        #pragma omp parallel for simd
         for( unsigned i=0; i<size; i++)
             y_ptr[i] += alpha*x_ptr[i];
     }
     else if (beta == 0)
     {
-#pragma omp parallel for simd
+        #pragma omp parallel for simd
         for( unsigned i=0; i<size; i++)
             y_ptr[i] = alpha*x_ptr[i];
     }
     else
     {
-#pragma omp parallel for simd
+        #pragma omp parallel for simd
         for( unsigned i=0; i<size; i++)
             y_ptr[i] = alpha*x_ptr[i] + beta*y_ptr[i];
     }
@@ -200,19 +223,19 @@ inline void doAxpby( typename Vector::value_type alpha,
     unsigned size = x.size();
     if( gamma == 1.)
     {
-#pragma omp parallel for simd
+        #pragma omp parallel for simd
         for( unsigned i=0; i<size; i++)
             z_ptr[i] += alpha*x_ptr[i]+beta*y_ptr[i];
     }
     else if (gamma == 0)
     {
-#pragma omp parallel for simd
+        #pragma omp parallel for simd
         for( unsigned i=0; i<size; i++)
             z_ptr[i] = alpha*x_ptr[i]+beta*y_ptr[i];
     }
     else
     {
-#pragma omp parallel for simd
+        #pragma omp parallel for simd
         for( unsigned i=0; i<size; i++)
             z_ptr[i] = alpha*x_ptr[i] + beta*y_ptr[i] + gamma*z_ptr[i];
     }
@@ -287,7 +310,7 @@ inline void doPointwiseDot(
         unsigned size = x1.size();
         if( beta == 0)
         {
-#pragma omp parallel for simd
+            #pragma omp parallel for simd
             for( unsigned i=0; i<size; i++)
             {
                 yr_ptr[i] = alpha*x1_ptr[i]*x2_ptr[i];
@@ -295,7 +318,7 @@ inline void doPointwiseDot(
         }
         else if ( beta == 1)
         {
-#pragma omp parallel for simd
+            #pragma omp parallel for simd
             for( unsigned i=0; i<size; i++)
             {
                 yr_ptr[i] += alpha*x1_ptr[i]*x2_ptr[i];
@@ -303,7 +326,7 @@ inline void doPointwiseDot(
         }
         else
         {
-#pragma omp parallel for simd
+            #pragma omp parallel for simd
             for( unsigned i=0; i<size; i++)
             {
                 yr_ptr[i] = alpha*x1_ptr[i]*x2_ptr[i]+beta*yr_ptr[i];
@@ -315,7 +338,7 @@ inline void doPointwiseDot(
         unsigned size = x1.size();
         if( beta == 0)
         {
-#pragma omp parallel for simd
+            #pragma omp parallel for simd
             for( unsigned i=0; i<size; i++)
             {
                 y_ptr[i] = alpha*x1_ptr[i]*x2_ptr[i];
@@ -323,7 +346,7 @@ inline void doPointwiseDot(
         }
         else if ( beta == 1)
         {
-#pragma omp parallel for simd
+            #pragma omp parallel for simd
             for( unsigned i=0; i<size; i++)
             {
                 y_ptr[i] += alpha*x1_ptr[i]*x2_ptr[i];
@@ -331,7 +354,7 @@ inline void doPointwiseDot(
         }
         else
         {
-#pragma omp parallel for simd
+            #pragma omp parallel for simd
             for( unsigned i=0; i<size; i++)
             {
                 y_ptr[i] = alpha*x1_ptr[i]*x2_ptr[i]+beta*y_ptr[i];
@@ -416,7 +439,7 @@ inline void doPointwiseDot(
         typename Vector::value_type * RESTRICT zr_ptr = thrust::raw_pointer_cast( &(z.data()[0]));
         if(gamma==0)
         {
-#pragma omp parallel for simd
+            #pragma omp parallel for simd
             for( unsigned i=0; i<size; i++)
             {
                 zr_ptr[i] = alpha*x1_ptr[i]*y1_ptr[i] 
@@ -425,7 +448,7 @@ inline void doPointwiseDot(
         }
         else if(gamma==1.)
         {
-#pragma omp parallel for simd
+            #pragma omp parallel for simd
             for( unsigned i=0; i<size; i++)
             {
                 zr_ptr[i] += alpha*x1_ptr[i]*y1_ptr[i] 
@@ -434,7 +457,7 @@ inline void doPointwiseDot(
         }
         else
         {
-#pragma omp parallel for simd
+            #pragma omp parallel for simd
             for( unsigned i=0; i<size; i++)
             {
                 zr_ptr[i] = alpha*x1_ptr[i]*y1_ptr[i] 
@@ -447,7 +470,7 @@ inline void doPointwiseDot(
     {
         if(gamma==0)
         {
-#pragma omp parallel for simd
+            #pragma omp parallel for simd
             for( unsigned i=0; i<size; i++)
             {
                 z_ptr[i] = alpha*x1_ptr[i]*y1_ptr[i] 
@@ -456,7 +479,7 @@ inline void doPointwiseDot(
         }
         else if(gamma==1.)
         {
-#pragma omp parallel for simd
+            #pragma omp parallel for simd
             for( unsigned i=0; i<size; i++)
             {
                 z_ptr[i] += alpha*x1_ptr[i]*y1_ptr[i] 
@@ -465,7 +488,7 @@ inline void doPointwiseDot(
         }
         else
         {
-#pragma omp parallel for simd
+            #pragma omp parallel for simd
             for( unsigned i=0; i<size; i++)
             {
                 z_ptr[i] = alpha*x1_ptr[i]*y1_ptr[i] 
diff --git a/inc/dg/blas1_t.cu b/inc/dg/blas1_t.cu
index 7a0428f41..762d5a471 100644
--- a/inc/dg/blas1_t.cu
+++ b/inc/dg/blas1_t.cu
@@ -59,6 +59,8 @@ int main()
     std::cout << "2*3 = "<<w3[0][0]<<" (6)\n";
     dg::blas1::pointwiseDot( 2., w1, w2, -4., w3);
     std::cout << "2*2*3 -4*6 = "<<w3[0][0]<<" (-12)\n";
+    dg::blas1::pointwiseDot( 2., w1[0], w2[0], -4., v1, v2, 0., v2);
+    std::cout << "2*2*3 -4*2*3 = "<<v2[0]<<" (-12)\n";
     dg::blas1::axpby( 2., w1, 3., w2);
     std::cout << "2*2+ 3*3 = " << w2[0][0] <<" (13)\n";
     dg::blas1::axpby( 2.5, w1, 0., w2);
-- 
GitLab


From 5445db3d288c4bfa7c762f6e7233f104912a638d Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 4 Sep 2017 07:16:13 -0700
Subject: [PATCH 248/453] added host vector version for new pointwiseDot
 function

---
 inc/dg/backend/thrust_vector_blas.cuh | 53 +++++++++++++++++++--------
 1 file changed, 37 insertions(+), 16 deletions(-)

diff --git a/inc/dg/backend/thrust_vector_blas.cuh b/inc/dg/backend/thrust_vector_blas.cuh
index e5750278e..7084966ce 100644
--- a/inc/dg/backend/thrust_vector_blas.cuh
+++ b/inc/dg/backend/thrust_vector_blas.cuh
@@ -407,16 +407,16 @@ template<class value_type>
 }
 #endif
 
-template<class Vector>
+template<class value_type>
 inline void doPointwiseDot(  
-              typename Vector::value_type alpha, 
-              const Vector& x1,
-              const Vector& y1, 
-              typename Vector::value_type beta, 
-              const Vector& x2,
-              const Vector& y2, 
-              typename Vector::value_type gamma, 
-              Vector& z, 
+              value_type alpha, 
+              const thrust::device_vector<value_type>& x1,
+              const thrust::device_vector<value_type>& y1, 
+              value_type beta, 
+              const thrust::device_vector<value_type>& x2,
+              const thrust::device_vector<value_type>& y2, 
+              value_type gamma, 
+              thrust::device_vector<value_type>& z, 
               ThrustVectorTag)
 {
     if( alpha==0){ 
@@ -427,16 +427,16 @@ inline void doPointwiseDot(
         doPointwiseDot( alpha, x1,y1, gamma, z, ThrustVectorTag());
         return;
     }
-    const typename Vector::value_type *x1_ptr = thrust::raw_pointer_cast( &(x1.data()[0]));
-    const typename Vector::value_type *x2_ptr = thrust::raw_pointer_cast( &(x2.data()[0]));
-    const typename Vector::value_type *y1_ptr = thrust::raw_pointer_cast( &(y1.data()[0]));
-    const typename Vector::value_type *y2_ptr = thrust::raw_pointer_cast( &(y2.data()[0]));
-        typename Vector::value_type * z_ptr = thrust::raw_pointer_cast( &(z.data()[0]));
+    const value_type *x1_ptr = thrust::raw_pointer_cast( x1.data());
+    const value_type *x2_ptr = thrust::raw_pointer_cast( x2.data());
+    const value_type *y1_ptr = thrust::raw_pointer_cast( y1.data());
+    const value_type *y2_ptr = thrust::raw_pointer_cast( y2.data());
+          value_type * z_ptr = thrust::raw_pointer_cast( z.data());
     unsigned size = x1.size();
 #if THRUST_DEVICE_SYSTEM!=THRUST_DEVICE_SYSTEM_CUDA
     if( z_ptr != x1_ptr && z_ptr != x2_ptr && z_ptr != y1_ptr && z_ptr != y2_ptr)
     {
-        typename Vector::value_type * RESTRICT zr_ptr = thrust::raw_pointer_cast( &(z.data()[0]));
+        value_type * RESTRICT zr_ptr = thrust::raw_pointer_cast( &(z.data()[0]));
         if(gamma==0)
         {
             #pragma omp parallel for simd
@@ -501,9 +501,30 @@ inline void doPointwiseDot(
     //set up kernel parameters
     const size_t BLOCK_SIZE = 256; 
     const size_t NUM_BLOCKS = std::min<size_t>((size-1)/BLOCK_SIZE+1, 65000);
-    pointwiseDot_kernel<typename Vector::value_type><<<NUM_BLOCKS, BLOCK_SIZE>>>( alpha, beta, gamma, x1_ptr, y1_ptr, x2_ptr, y2_ptr, z_ptr, size);
+    pointwiseDot_kernel<value_type><<<NUM_BLOCKS, BLOCK_SIZE>>>( alpha, beta, gamma, x1_ptr, y1_ptr, x2_ptr, y2_ptr, z_ptr, size);
 #endif
 }
+template<class value_type>
+inline void doPointwiseDot(  
+              value_type alpha, 
+              const thrust::host_vector<value_type>& x1,
+              const thrust::host_vector<value_type>& y1, 
+              value_type beta, 
+              const thrust::host_vector<value_type>& x2,
+              const thrust::host_vector<value_type>& y2, 
+              value_type gamma, 
+              thrust::host_vector<value_type>& z, 
+              ThrustVectorTag)
+{
+    unsigned size=x1.size();
+    #pragma omp parallel for simd
+    for( unsigned i=0; i<size; i++)
+    {
+        z[i] = alpha*x1[i]*y1[i] 
+                    +beta*x2[i]*y2[i]
+                    +gamma*z[i];
+    }
+}
 
 }//namespace detail
 
-- 
GitLab


From c67368dabd9fdf9b478ca400cb40e1f59b1ffa02 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <Matthias.Wiesenberger@uibk.ac.at>
Date: Mon, 4 Sep 2017 21:05:52 +0200
Subject: [PATCH 249/453] added for simd to reduction operator

---
 inc/dg/backend/thrust_matrix_blas.cuh | 4 ++--
 inc/dg/backend/thrust_vector_blas.cuh | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/inc/dg/backend/thrust_matrix_blas.cuh b/inc/dg/backend/thrust_matrix_blas.cuh
index fcb5c9867..5429a5035 100644
--- a/inc/dg/backend/thrust_matrix_blas.cuh
+++ b/inc/dg/backend/thrust_matrix_blas.cuh
@@ -51,7 +51,7 @@ inline typename MatrixTraits<Matrix>::value_type doDot( const Vector& x, const M
 #if THRUST_DEVICE_SYSTEM!=THRUST_DEVICE_SYSTEM_CUDA
     value_type sum = 0;
     unsigned size=x.size();
-    #pragma omp parallel reduction(+:sum) 
+    #pragma omp parallel for simd reduction(+:sum) 
     for( unsigned i=0; i<size; i++)
         sum += x[i]*m[i]*y[i];
     return sum;
@@ -75,7 +75,7 @@ inline typename MatrixTraits<Matrix>::value_type doDot( const Matrix& m, const V
 #if THRUST_DEVICE_SYSTEM!=THRUST_DEVICE_SYSTEM_CUDA
     value_type sum = 0;
     unsigned size=x.size();
-    #pragma omp parallel reduction(+:sum)
+    #pragma omp parallel for simd reduction(+:sum)
     for( unsigned i=0; i<size; i++)
         sum += x[i]*x[i]*m[i];
     return sum;
diff --git a/inc/dg/backend/thrust_vector_blas.cuh b/inc/dg/backend/thrust_vector_blas.cuh
index 7084966ce..c19d5dff4 100644
--- a/inc/dg/backend/thrust_vector_blas.cuh
+++ b/inc/dg/backend/thrust_vector_blas.cuh
@@ -71,7 +71,7 @@ typename Vector::value_type doDot( const Vector& x, const Vector& y, ThrustVecto
 #if THRUST_DEVICE_SYSTEM!=THRUST_DEVICE_SYSTEM_CUDA
     value_type sum = 0;
     unsigned size=x.size();
-    #pragma omp parallel reduction(+:sum) 
+    #pragma omp parallel for simd reduction(+:sum) 
     for( unsigned i=0; i<size; i++)
         sum += x[i]*y[i];
     return sum;
-- 
GitLab


From c4c51d25f91d7a92aa9fb887d6ec64c1faa9e2eb Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Mon, 4 Sep 2017 22:14:40 +0200
Subject: [PATCH 250/453] adapted devices.mk file for CUDA

---
 config/devices/devices.mk | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/config/devices/devices.mk b/config/devices/devices.mk
index 9359f7398..332f034c3 100644
--- a/config/devices/devices.mk
+++ b/config/devices/devices.mk
@@ -4,16 +4,16 @@ CC = nvcc --compiler-bindir $(ccc_)
 OPT=-O3
 CFLAGS+=-D_FORCE_INLINES
 CFLAGS+= --compiler-options -Wall $(NVCCARCH)
-CFLAGS+= -Xcompiler $(OMPFLAG)
+#CFLAGS+= -Xcompiler $(OMPFLAG)
 #CFLAGS+= -DTHRUST_HOST_SYSTEM=THRUST_HOST_SYSTEM_OMP
-CFLAGS+= -DCUSP_DEVICE_BLAS_SYSTEM=CUSP_DEVICE_BLAS_CUBLAS -lcublas
-CFLAGS+= -DCUSP_USE_TEXTURE_MEMORY
+#CFLAGS+= -DCUSP_DEVICE_BLAS_SYSTEM=CUSP_DEVICE_BLAS_CUBLAS -lcublas
+#CFLAGS+= -DCUSP_USE_TEXTURE_MEMORY
 mpiccc_:=$(MPICC)
 MPICC=nvcc --compiler-bindir $(mpiccc_)
 MPICFLAGS+= -D_FORCE_INLINES
-MPICFLAGS+= -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_CUDA
-MPICFLAGS+= -DCUSP_DEVICE_BLAS_SYSTEM=CUSP_DEVICE_BLAS_CUBLAS -lcublas
-MPICFLAGS+= -DCUSP_USE_TEXTURE_MEMORY
+#MPICFLAGS+= -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_CUDA
+#MPICFLAGS+= -DCUSP_DEVICE_BLAS_SYSTEM=CUSP_DEVICE_BLAS_CUBLAS -lcublas
+#MPICFLAGS+= -DCUSP_USE_TEXTURE_MEMORY
 MPICFLAGS+= --compiler-options -Wall $(NVCCARCH)
 MPICFLAGS+= --compiler-options $(OPT)
 endif #device=gpu
-- 
GitLab


From 83445bac264fb58ba047f8c0ecb144c5e5ffa5df Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Mon, 4 Sep 2017 22:16:36 +0200
Subject: [PATCH 251/453] remove parallel for in host_vector version again

---
 inc/dg/backend/thrust_vector_blas.cuh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/inc/dg/backend/thrust_vector_blas.cuh b/inc/dg/backend/thrust_vector_blas.cuh
index c19d5dff4..e98d47a19 100644
--- a/inc/dg/backend/thrust_vector_blas.cuh
+++ b/inc/dg/backend/thrust_vector_blas.cuh
@@ -517,7 +517,6 @@ inline void doPointwiseDot(
               ThrustVectorTag)
 {
     unsigned size=x1.size();
-    #pragma omp parallel for simd
     for( unsigned i=0; i<size; i++)
     {
         z[i] = alpha*x1[i]*y1[i] 
-- 
GitLab


From dcd2dbe9108fa9ec347df980b2f31f5f062b959b Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Tue, 5 Sep 2017 02:26:56 +0200
Subject: [PATCH 252/453] first success of multigrid

---
 inc/dg/elliptic2d_b.cu | 61 +++++++++++++++++++++++++++++++++++++-----
 1 file changed, 54 insertions(+), 7 deletions(-)

diff --git a/inc/dg/elliptic2d_b.cu b/inc/dg/elliptic2d_b.cu
index 92e724b08..6be8691be 100644
--- a/inc/dg/elliptic2d_b.cu
+++ b/inc/dg/elliptic2d_b.cu
@@ -9,6 +9,7 @@
 #include "elliptic.h"
 #include "cg.h"
 #include "backend/timer.cuh"
+#include "backend/projection.cuh"
 
 //NOTE: IF DEVICE=CPU THEN THE POLARISATION ASSEMBLY IS NOT PARALLEL AS IT IS NOW 
 
@@ -63,17 +64,63 @@ int main()
     std::cout << "Create Polarisation object and set chi!\n";
     t.tic();
     {
-    dg::Elliptic<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> pol( grid, dg::not_normed, dg::centered, jfactor);
-    pol.set_chi( chi);
+    std::vector<dg::Grid2d> g( 4, grid);
+    g[1].multiplyCellNumbers( 0.5, 0.5);
+    g[2].multiplyCellNumbers( 0.25, 0.25);
+    g[3].multiplyCellNumbers( 0.125, 0.125);
+    dg::IDMatrix project3 = dg::create::projection( g[3], g[0]);
+    dg::IDMatrix project2 = dg::create::projection( g[2], g[0]);
+    dg::IDMatrix project1 = dg::create::projection( g[1], g[0]);
+    dg::IDMatrix inter01 = dg::create::interpolation( g[0], g[1]);
+    dg::IDMatrix inter12 = dg::create::interpolation( g[1], g[2]);
+    dg::IDMatrix inter23 = dg::create::interpolation( g[2], g[3]);
+    std::vector<dg::DVec> w2d_(4, w2d), v2d_(4,v2d);
+    dg::Elliptic<dg::CartesianGrid2d, dg::DMatrix, dg::DVec>  pol0(g[0], dg::not_normed,dg::centered, jfactor);
+    dg::Elliptic<dg::CartesianGrid2d, dg::DMatrix, dg::DVec>  pol1(g[1], dg::not_normed,dg::centered, jfactor);
+    dg::Elliptic<dg::CartesianGrid2d, dg::DMatrix, dg::DVec>  pol2(g[2], dg::not_normed,dg::centered, jfactor);
+    dg::Elliptic<dg::CartesianGrid2d, dg::DMatrix, dg::DVec>  pol3(g[3], dg::not_normed,dg::centered, jfactor);
+    for( unsigned i=0; i<4; i++)
+    {
+        w2d_[i] = dg::create::weights( g[i]);
+        v2d_[i] = dg::create::inv_weights( g[i]);
+    }
+    std::vector<dg::DVec> chi_(w2d_), chi_inv_(chi_), b_(chi_), x_(chi_);
+    dg::blas2::symv( project3, chi, chi_[3]);
+    dg::blas2::symv( project2, chi, chi_[2]);
+    dg::blas2::symv( project1, chi, chi_[1]);
+    dg::blas2::symv( project3, chi_inv, chi_inv_[3]);
+    dg::blas2::symv( project2, chi_inv, chi_inv_[2]);
+    dg::blas2::symv( project1, chi_inv, chi_inv_[1]);
+    dg::blas2::symv( project3, b, b_[3]);
+    dg::blas2::symv( project2, b, b_[2]);
+    dg::blas2::symv( project1, b, b_[1]);
+    dg::blas2::symv( project3, x, x_[3]);
+    dg::blas2::symv( project2, x, x_[2]);
+    dg::blas2::symv( project1, x, x_[1]);
+    pol0.set_chi( chi);
+    pol1.set_chi( chi_[1]);
+    pol2.set_chi( chi_[2]);
+    pol3.set_chi( chi_[3]);
     t.toc();
     std::cout << "Creation of polarisation object took: "<<t.diff()<<"s\n";
 
-    dg::Invert<dg::DVec > invert( x, n*n*Nx*Ny, eps);
-
-
-    std::cout << eps<<" ";
+    std::vector<dg::CG<dg::DVec > > cg(4);
+    cg[0].construct( x, g[0].size());
+    for( unsigned i=1; i<4; i++)
+        cg[i].construct(x_[i], g[i].size());
+    std::cout << eps<<" \n";
     t.tic();
-    std::cout << " "<< invert( pol, x, b, w2d, chi_inv, v2d);
+    dg::blas2::symv( w2d_[3], b_[3], b_[3]); 
+    std::cout << " # iterations grid "<< cg[3]( pol3, x_[3], b_[3], chi_inv_[3], v2d_[3], eps) << " \n";
+    dg::blas2::symv( inter23, x_[3], x_[2]);
+    dg::blas2::symv( w2d_[2], b_[2], b_[2]); 
+    std::cout << " # iterations grid "<< cg[2]( pol2, x_[2], b_[2], chi_inv_[2], v2d_[2], eps) << " \n";
+    dg::blas2::symv( inter12, x_[2], x_[1]);
+    dg::blas2::symv( w2d_[1], b_[1], b_[1]); 
+    std::cout << " # iterations grid "<< cg[1]( pol1, x_[1], b_[1], chi_inv_[1], v2d_[1], eps) << " \n";
+    dg::blas2::symv( inter01, x_[1], x);
+    dg::blas2::symv( w2d, b, b_[0]); 
+    std::cout << " # iterations fine grid "<< cg[0]( pol0, x, b_[0], chi_inv, v2d, eps)<<std::endl;
     t.toc();
     //std::cout << "Took "<<t.diff()<<"s\n";
     }
-- 
GitLab


From 45c7434d4bc8da4800050cc2ff2ff819f45cf83b Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Tue, 5 Sep 2017 11:49:53 +0200
Subject: [PATCH 253/453] introduce IMatrix blas_b

---
 inc/dg/blas_b.cu | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/inc/dg/blas_b.cu b/inc/dg/blas_b.cu
index 487d0a343..580832e0a 100644
--- a/inc/dg/blas_b.cu
+++ b/inc/dg/blas_b.cu
@@ -8,6 +8,7 @@
 #include "blas.h"
 #include "backend/derivatives.h"
 #include "backend/evaluation.cuh"
+#include "backend/projection.cuh"
 
 const double lx = 2.*M_PI;
 const double ly = 2.*M_PI;
@@ -22,14 +23,16 @@ typedef dg::DVec Vector;
 //typedef thrust::device_vector<double> Vector;
 typedef dg::DMatrix Matrix;
 //typedef cusp::array1d<double, cusp::device_memory> Vector;
+typedef dg::IDMatrix IMatrix;
 
 int main()
 {
     dg::Timer t;
     unsigned n, Nx, Ny, Nz; 
-    std::cout << "Type n, Nx, Ny and Nz\n";
+    std::cout << "Type n, Nx, Ny and Nz ( Nx and Ny shall be multiples of 2)\n";
     std::cin >> n >> Nx >> Ny >> Nz;
-    dg::Grid3d grid( 0., lx, 0, ly, 0, ly, n, Nx, Ny, Nz);
+    dg::Grid3d grid(      0., lx, 0, ly, 0, ly, n, Nx, Ny, Nz);
+    dg::Grid3d grid_half( 0., lx, 0, ly, 0, ly, n, Nx/2, Ny/2, Nz);
     Vector w2d;
     dg::blas1::transfer( dg::create::weights(grid), w2d);
 
@@ -85,6 +88,15 @@ int main()
         dg::blas2::symv( M, x, y);
     t.toc();
     std::cout<<"jump X took                      "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
+    IMatrix inter; 
+    dg::blas2::transfer(dg::create::interpolation( grid, grid_half), inter);
+    Vector x_half = dg::evaluate( dg::zero, grid_half);
+    t.tic();
+    for( int i=0; i<multi; i++)
+        dg::blas2::gemv( inter, x_half, x);
+    t.toc();
+    std::cout<<"Interpolation on original grid  "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
+    
 
     t.tic();
     for( int i=0; i<multi; i++)
-- 
GitLab


From 385d8eda5e91b7f87c9fdcdd85bc866790ff043e Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Tue, 5 Sep 2017 18:13:14 +0200
Subject: [PATCH 254/453] added own implementation of csr spmv multiply

---
 inc/dg/backend/cusp_matrix_blas.cuh | 68 ++++++++++++++++++++++-------
 inc/dg/blas_b.cu                    | 22 +++++++++-
 inc/dg/elliptic2d_b.cu              | 10 ++---
 3 files changed, 78 insertions(+), 22 deletions(-)

diff --git a/inc/dg/backend/cusp_matrix_blas.cuh b/inc/dg/backend/cusp_matrix_blas.cuh
index 5be2c0396..438388541 100644
--- a/inc/dg/backend/cusp_matrix_blas.cuh
+++ b/inc/dg/backend/cusp_matrix_blas.cuh
@@ -5,11 +5,13 @@
 #include <cassert>
 #endif //DG_DEBUG
 
+#include <typeinfo>
 #include <cusp/multiply.h>
 #include <cusp/convert.h>
 #include <cusp/array1d.h>
 
 #include "matrix_categories.h"
+#include "vector_categories.h"
 
 ///@cond
 namespace dg{
@@ -23,6 +25,54 @@ inline void doTransfer( const Matrix1& x, Matrix2& y, CuspMatrixTag, CuspMatrixT
 {
     cusp::convert(x,y);
 }
+template< class Matrix, class Vector>
+inline void doSymv( Matrix& m, 
+                    const Vector&x, 
+                    Vector& y, 
+                    CuspMatrixTag, 
+                    ThrustVectorTag,
+                    ThrustVectorTag, cusp::csr_format  )
+{
+#if THRUST_DEVICE_SYSTEM!=THRUST_DEVICE_SYSTEM_CUDA
+    typedef typename Vector::value_type value_type;
+    typedef typename Matrix::index_type index_type;
+    const value_type* RESTRICT val_ptr = thrust::raw_pointer_cast( &m.values[0]);
+    const value_type* RESTRICT x_ptr = thrust::raw_pointer_cast( &x[0]);
+    value_type* RESTRICT y_ptr = thrust::raw_pointer_cast( &y[0]);
+    const index_type* RESTRICT row_ptr = thrust::raw_pointer_cast( &m.row_offsets[0]);
+    const index_type* RESTRICT col_ptr = thrust::raw_pointer_cast( &m.column_indices[0]);
+    int size = x.size();
+    #pragma omp parallel for
+    for(int i = 0; i < size; i++)
+    {
+        value_type temp = 0.;
+        for (index_type jj = row_ptr[i]; jj < row_ptr[i+1]; jj++)
+        {
+            temp += val_ptr[jj]*x_ptr[ col_ptr[jj]];
+        }
+
+        y_ptr[i] = temp;
+    }
+#else
+    cusp::array1d_view< typename Vector::const_iterator> cx( x.cbegin(), x.cend());
+    cusp::array1d_view< typename Vector::iterator> cy( y.begin(), y.end());
+    cusp::multiply( m, cx, cy);
+#endif
+}
+
+template< class Matrix, class Vector>
+inline void doSymv( Matrix& m, 
+                    const Vector&x, 
+                    Vector& y, 
+                    CuspMatrixTag, 
+                    ThrustVectorTag,
+                    ThrustVectorTag, 
+                    cusp::sparse_format  )
+{
+    cusp::array1d_view< typename Vector::const_iterator> cx( x.cbegin(), x.cend());
+    cusp::array1d_view< typename Vector::iterator> cy( y.begin(), y.end());
+    cusp::multiply( m, cx, cy);
+}
 
 template< class Matrix, class Vector>
 inline void doSymv( Matrix& m, 
@@ -36,9 +86,7 @@ inline void doSymv( Matrix& m,
     assert( m.num_rows == y.size() );
     assert( m.num_cols == x.size() );
 #endif //DG_DEBUG
-    cusp::array1d_view< typename Vector::const_iterator> cx( x.cbegin(), x.cend());
-    cusp::array1d_view< typename Vector::iterator> cy( y.begin(), y.end());
-    cusp::multiply( m, cx, cy);
+    doSymv( m,x,y, CuspMatrixTag(), ThrustVectorTag(), ThrustVectorTag(), typename Matrix::format());
 }
 
 template< class Matrix, class Vector>
@@ -64,13 +112,7 @@ inline void doGemv( Matrix& m,
                     ThrustVectorTag,
                     ThrustVectorTag  )
 {
-#ifdef DG_DEBUG
-    assert( m.num_rows == y.size() );
-    assert( m.num_cols == x.size() );
-#endif //DG_DEBUG
-    cusp::array1d_view< typename Vector::const_iterator> cx( x.cbegin(), x.cend());
-    cusp::array1d_view< typename Vector::iterator> cy( y.begin(), y.end());
-    cusp::multiply( m, cx, cy);
+    doSymv( m, x, y, CuspMatrixTag(), ThrustVectorTag(), ThrustVectorTag());
 }
 
 template< class Matrix, class Vector>
@@ -81,11 +123,7 @@ inline void doGemv( Matrix& m,
                     CuspVectorTag,
                     CuspVectorTag  )
 {
-#ifdef DG_DEBUG
-    assert( m.num_rows == y.size() );
-    assert( m.num_cols == x.size() );
-#endif //DG_DEBUG
-    cusp::multiply( m, x, y);
+    doGemv( m,x,y,CuspMatrixTag(), CuspVectorTag(), CuspVectorTag());
 }
 
 } //namespace detail
diff --git a/inc/dg/blas_b.cu b/inc/dg/blas_b.cu
index 487d0a343..65291a233 100644
--- a/inc/dg/blas_b.cu
+++ b/inc/dg/blas_b.cu
@@ -8,6 +8,7 @@
 #include "blas.h"
 #include "backend/derivatives.h"
 #include "backend/evaluation.cuh"
+#include "backend/projection.cuh"
 
 const double lx = 2.*M_PI;
 const double ly = 2.*M_PI;
@@ -22,14 +23,16 @@ typedef dg::DVec Vector;
 //typedef thrust::device_vector<double> Vector;
 typedef dg::DMatrix Matrix;
 //typedef cusp::array1d<double, cusp::device_memory> Vector;
+typedef dg::IDMatrix IMatrix;
 
 int main()
 {
     dg::Timer t;
     unsigned n, Nx, Ny, Nz; 
-    std::cout << "Type n, Nx, Ny and Nz\n";
+    std::cout << "Type n, Nx, Ny and Nz ( Nx and Ny shall be multiples of 2)\n";
     std::cin >> n >> Nx >> Ny >> Nz;
-    dg::Grid3d grid( 0., lx, 0, ly, 0, ly, n, Nx, Ny, Nz);
+    dg::Grid3d grid(      0., lx, 0, ly, 0, ly, n, Nx, Ny, Nz);
+    dg::Grid3d grid_half( 0., lx, 0, ly, 0, ly, n, Nx/2, Ny/2, Nz);
     Vector w2d;
     dg::blas1::transfer( dg::create::weights(grid), w2d);
 
@@ -85,6 +88,21 @@ int main()
         dg::blas2::symv( M, x, y);
     t.toc();
     std::cout<<"jump X took                      "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
+    IMatrix inter, project; 
+    dg::blas2::transfer(dg::create::interpolation( grid, grid_half), inter);
+    dg::blas2::transfer(dg::create::projection( grid_half, grid), project);
+    Vector x_half = dg::evaluate( dg::zero, grid_half);
+    t.tic();
+    for( int i=0; i<multi; i++)
+        dg::blas2::gemv( inter, x_half, x);
+    t.toc();
+    std::cout<<"Interpolation on original grid  "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
+    t.tic();
+    for( int i=0; i<multi; i++)
+        dg::blas2::gemv( project, x, x_half);
+    t.toc();
+    std::cout<<"Projection onto half grid       "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
+    
 
     t.tic();
     for( int i=0; i<multi; i++)
diff --git a/inc/dg/elliptic2d_b.cu b/inc/dg/elliptic2d_b.cu
index 6be8691be..3b5aa1815 100644
--- a/inc/dg/elliptic2d_b.cu
+++ b/inc/dg/elliptic2d_b.cu
@@ -110,14 +110,14 @@ int main()
         cg[i].construct(x_[i], g[i].size());
     std::cout << eps<<" \n";
     t.tic();
-    dg::blas2::symv( w2d_[3], b_[3], b_[3]); 
-    std::cout << " # iterations grid "<< cg[3]( pol3, x_[3], b_[3], chi_inv_[3], v2d_[3], eps) << " \n";
-    dg::blas2::symv( inter23, x_[3], x_[2]);
+    //dg::blas2::symv( w2d_[3], b_[3], b_[3]); 
+    //std::cout << " # iterations grid "<< cg[3]( pol3, x_[3], b_[3], chi_inv_[3], v2d_[3], eps/10) << " \n";
+    //dg::blas2::symv( inter23, x_[3], x_[2]);
     dg::blas2::symv( w2d_[2], b_[2], b_[2]); 
-    std::cout << " # iterations grid "<< cg[2]( pol2, x_[2], b_[2], chi_inv_[2], v2d_[2], eps) << " \n";
+    std::cout << " # iterations grid "<< cg[2]( pol2, x_[2], b_[2], chi_inv_[2], v2d_[2], eps/10) << " \n";
     dg::blas2::symv( inter12, x_[2], x_[1]);
     dg::blas2::symv( w2d_[1], b_[1], b_[1]); 
-    std::cout << " # iterations grid "<< cg[1]( pol1, x_[1], b_[1], chi_inv_[1], v2d_[1], eps) << " \n";
+    std::cout << " # iterations grid "<< cg[1]( pol1, x_[1], b_[1], chi_inv_[1], v2d_[1], eps/10) << " \n";
     dg::blas2::symv( inter01, x_[1], x);
     dg::blas2::symv( w2d, b, b_[0]); 
     std::cout << " # iterations fine grid "<< cg[0]( pol0, x, b_[0], chi_inv, v2d, eps)<<std::endl;
-- 
GitLab


From 2223cbce7609916761de6e7e92703b5fff5c6a28 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <Matthias.Wiesenberger@uibk.ac.at>
Date: Tue, 5 Sep 2017 18:35:36 +0200
Subject: [PATCH 255/453] debug cusp_matrix_blas

---
 inc/dg/backend/cusp_matrix_blas.cuh |  9 +++++----
 inc/dg/blas_b.cu                    | 12 +++++++-----
 2 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/inc/dg/backend/cusp_matrix_blas.cuh b/inc/dg/backend/cusp_matrix_blas.cuh
index 438388541..48849e22e 100644
--- a/inc/dg/backend/cusp_matrix_blas.cuh
+++ b/inc/dg/backend/cusp_matrix_blas.cuh
@@ -41,14 +41,15 @@ inline void doSymv( Matrix& m,
     value_type* RESTRICT y_ptr = thrust::raw_pointer_cast( &y[0]);
     const index_type* RESTRICT row_ptr = thrust::raw_pointer_cast( &m.row_offsets[0]);
     const index_type* RESTRICT col_ptr = thrust::raw_pointer_cast( &m.column_indices[0]);
-    int size = x.size();
-    #pragma omp parallel for
-    for(int i = 0; i < size; i++)
+    int rows = m.num_rows;
+    #pragma omp parallel for 
+    for(int i = 0; i < rows; i++)
     {
         value_type temp = 0.;
         for (index_type jj = row_ptr[i]; jj < row_ptr[i+1]; jj++)
         {
-            temp += val_ptr[jj]*x_ptr[ col_ptr[jj]];
+            index_type j = col_ptr[jj];
+            temp += val_ptr[jj]*x_ptr[j];
         }
 
         y_ptr[i] = temp;
diff --git a/inc/dg/blas_b.cu b/inc/dg/blas_b.cu
index 65291a233..23c86de95 100644
--- a/inc/dg/blas_b.cu
+++ b/inc/dg/blas_b.cu
@@ -45,6 +45,11 @@ int main()
     std::cout << "Sizeof value type is "<<sizeof(value_type)<<"\n";
     value_type gbytes=(value_type)x.size()*sizeof(value_type)/1e9;
     std::cout << "Sizeof vectors is "<<gbytes<<" GB\n";
+    std::cout << "Generate interpolation and projection\n";
+    IMatrix inter, project; 
+    dg::blas2::transfer(dg::create::interpolation( grid, grid_half), inter);
+    dg::blas2::transfer(dg::create::projection( grid_half, grid), project);
+    std::cout << "Done...\n";
     int multi=200;
     t.tic();
     value_type norm=0;
@@ -88,20 +93,17 @@ int main()
         dg::blas2::symv( M, x, y);
     t.toc();
     std::cout<<"jump X took                      "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
-    IMatrix inter, project; 
-    dg::blas2::transfer(dg::create::interpolation( grid, grid_half), inter);
-    dg::blas2::transfer(dg::create::projection( grid_half, grid), project);
     Vector x_half = dg::evaluate( dg::zero, grid_half);
     t.tic();
     for( int i=0; i<multi; i++)
         dg::blas2::gemv( inter, x_half, x);
     t.toc();
-    std::cout<<"Interpolation on original grid  "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
+    std::cout<<"Interpolation half to full grid  "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
     t.tic();
     for( int i=0; i<multi; i++)
         dg::blas2::gemv( project, x, x_half);
     t.toc();
-    std::cout<<"Projection onto half grid       "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
+    std::cout<<"Projection full to half grid     "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
     
 
     t.tic();
-- 
GitLab


From 2bf6981f8df007fa75d741cd3a36300a60011551 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Tue, 5 Sep 2017 23:04:43 +0200
Subject: [PATCH 256/453] added fast interpolation and projection

---
 inc/dg/backend/fast_interpolation.h | 160 ++++++++++++++++++++++++++++
 inc/dg/backend/projection_t.cu      |  10 +-
 inc/dg/backend/sparseblockmat.h     |  10 +-
 3 files changed, 173 insertions(+), 7 deletions(-)
 create mode 100644 inc/dg/backend/fast_interpolation.h

diff --git a/inc/dg/backend/fast_interpolation.h b/inc/dg/backend/fast_interpolation.h
new file mode 100644
index 000000000..51ef9b714
--- /dev/null
+++ b/inc/dg/backend/fast_interpolation.h
@@ -0,0 +1,160 @@
+#pragma once
+
+#include <thrust/host_vector.h>
+#include "grid.h"
+#include "interpolation.cuh"
+#include "projection.cuh"
+#include "matrix_traits.h"
+#include "sparseblockmat.h"
+#include "memory.h"
+
+namespace dg
+{
+
+template <class Matrix, class container>
+struct MultiMatrix
+{
+    MultiMatrix(){}
+
+    MultiMatrix( int dimension): inter_(dimension), temp_(dimension-1 > 0 ? dimension-1 : 0 ){}
+    template<class OtherMatrix, class OtherContainer>
+    MultiMatrix( const MultiMatrix<OtherMatrix, OtherContainer>& src){
+            temp_.resize( src.get_temp().size());
+            inter_.resize( src.get_matrices().size());
+            for( int i=0; i<temp_.size(); i++)
+            {
+                temp_[i].data() = src.get_temp()[i].data();
+                inter_[i] = src.get_matrices()[i];
+            }
+    }
+
+    void symv( const container& x, container& y) const{gemv(x,y);}
+    void gemv( const container& x, container& y) const
+    {
+        int dims = inter_.size();
+        if( dims == 1) 
+        {
+            dg::blas2::symv( inter_[0], x, y);
+            return;
+        }
+        dg::blas2::symv( inter_[0], x,temp_[0].data());
+        for( int i=1; i<dims-1; i++)
+            dg::blas2::symv( inter_[i], temp_[i-1].data(), temp_[i].data());
+        dg::blas2::symv( inter_[dims-1], temp_[dims-2].data(), y);
+    }
+    std::vector<Buffer<container> >& get_temp(){ return temp_;}
+    const std::vector<Buffer<container> >& get_temp()const{ return temp_;}
+    std::vector<Matrix>& get_matrices(){ return inter_;}
+    const std::vector<Matrix>& get_matrices()const{ return inter_;}
+    private:
+    std::vector<Matrix > inter_;
+    std::vector<Buffer<container> > temp_;
+};
+
+template <class M, class V>
+struct MatrixTraits<MultiMatrix<M, V> >
+{
+    typedef typename VectorTraits<V>::value_type value_type;
+    typedef SelfMadeMatrixTag matrix_category;
+};
+template <class M, class V>
+struct MatrixTraits<const MultiMatrix<M, V> >
+{
+    typedef typename VectorTraits<V>::value_type value_type;
+    typedef SelfMadeMatrixTag matrix_category;
+};
+
+
+namespace create
+{
+MultiMatrix< EllSparseBlockMat<double>, thrust::host_vector<double> > fast_interpolation( const aTopology2d& t, unsigned multiplyX, unsigned multiplyY)
+{
+    unsigned n=t.n();
+    dg::Grid1d g_old( -1., 1., n, 1);
+    dg::Grid1d g_newX( -1., 1., n, multiplyX);
+    dg::Grid1d g_newY( -1., 1., n, multiplyY);
+    dg::IHMatrix interpolX = dg::create::interpolation( g_newX, g_old);
+    dg::IHMatrix interpolY = dg::create::interpolation( g_newY, g_old);
+    EllSparseBlockMat<double> iX( multiplyX*t.Nx(), t.Nx(), 1, multiplyX, t.n()); 
+    EllSparseBlockMat<double> iY( multiplyY*t.Ny(), t.Ny(), 1, multiplyY, t.n()); 
+    for( unsigned  i=0; i<n; i++)
+    for( unsigned  j=0; j<n; j++)
+    for( unsigned  k=0; k<multiplyX; k++)
+        iX.data[(k*n+i)*n+j] = interpolX.values[(k*n+i)*n+j];
+    for( unsigned  i=0; i<n; i++)
+    for( unsigned  j=0; j<n; j++)
+    for( unsigned  k=0; k<multiplyY; k++)
+        iY.data[(k*n+i)*n+j] = interpolY.values[(k*n+i)*n+j];
+
+    for( unsigned i=0; i<multiplyX*t.Nx(); i++)
+    {
+        iX.cols_idx[i] = i/multiplyX;
+        iX.data_idx[i] = 0;
+    }
+    for( unsigned i=0; i<multiplyY*t.Ny(); i++)
+    {
+        iY.cols_idx[i] = i/multiplyY;
+        iY.data_idx[i] = 0;
+    }
+    iX.left_size  = t.n()*t.Ny();
+    iY.right_size = t.n()*t.Nx()*multiplyX;
+    iX.set_default_range();
+    iY.set_default_range();
+
+    MultiMatrix < EllSparseBlockMat<double>, thrust::host_vector<double> > inter(2);
+    inter.get_matrices()[0] = iX;
+    inter.get_matrices()[1] = iY;
+    thrust::host_vector<double> vec( t.size()*multiplyX);
+    inter.get_temp()[0] = Buffer<thrust::host_vector<double > >(vec);
+    return inter;
+}
+
+MultiMatrix< EllSparseBlockMat<double>, thrust::host_vector<double> > fast_projection( const aTopology2d& t, unsigned divideX, unsigned divideY)
+{
+    unsigned n=t.n();
+    if( t.Nx()%divideX != 0) throw Error( Message(_ping_)<< "Nx and divideX don't match: Nx: " << t.Nx()<< " divideX "<< divideX);
+    if( t.Ny()%divideY != 0) throw Error( Message(_ping_)<< "Ny and divideY don't match: Nx: " << t.Ny()<< " divideY "<< divideY);
+    dg::Grid1d g_oldX( -1., 1., n, divideX);
+    dg::Grid1d g_oldY( -1., 1., n, divideY);
+    dg::Grid1d g_new(  -1., 1., n, 1);
+    dg::IHMatrix projectX = dg::create::projection( g_new, g_oldX);
+    dg::IHMatrix projectY = dg::create::projection( g_new, g_oldY);
+    EllSparseBlockMat<double> pX( t.Nx()/divideX, t.Nx(), divideX, divideY, t.n()); 
+    EllSparseBlockMat<double> pY( t.Ny()/divideY, t.Ny(), divideY, divideY, t.n()); 
+    for( unsigned i=0; i<n; i++)
+    for( unsigned j=0; j<n; j++)
+    {
+        for( unsigned k=0; k<divideX; k++)
+            pX.data[(k*n+i)*n+j] = projectX.values[i*divideX*n +k*n+j];
+        for( unsigned k=0; k<divideY; k++)
+            pY.data[(k*n+i)*n+j] = projectY.values[i*divideX*n +k*n+j];
+    }
+
+    for( unsigned i=0; i<t.Nx()/divideX; i++)
+        for( unsigned d=0; d<divideX; d++)
+        {
+            pX.cols_idx[i*divideX+d] = i*divideX+d;
+            pX.data_idx[i*divideX+d] = d;
+        }
+    for( unsigned i=0; i<t.Ny()/divideY; i++)
+        for( unsigned d=0; d<divideY; d++)
+        {
+            pY.cols_idx[i*divideY+d] = i*divideY+d;
+            pY.data_idx[i*divideY+d] = d;
+        }
+    pX.left_size  = t.n()*t.Ny();
+    pY.right_size = t.n()*t.Nx()/divideX;
+    pX.set_default_range();
+    pY.set_default_range();
+
+    MultiMatrix < EllSparseBlockMat<double>, thrust::host_vector<double> > inter(2);
+    inter.get_matrices()[0] = pX;
+    inter.get_matrices()[1] = pY;
+    thrust::host_vector<double> vec( t.size()/divideX);
+    inter.get_temp()[0] = Buffer<thrust::host_vector<double> >(vec);
+    return inter;
+}
+
+}//namespace create
+
+}//namespace dg
diff --git a/inc/dg/backend/projection_t.cu b/inc/dg/backend/projection_t.cu
index c1120917c..a0baf72a3 100644
--- a/inc/dg/backend/projection_t.cu
+++ b/inc/dg/backend/projection_t.cu
@@ -4,6 +4,7 @@
 #include "evaluation.cuh"
 #include "blas.h"
 #include "typedefs.cuh"
+#include "fast_interpolation.h"
 
 double sine( double x){ return sin(x);}
 double sine( double x, double y){return sin(x)*sin(y);}
@@ -39,13 +40,15 @@ int main()
     
     dg::Grid2d g2o (0, M_PI, 0, M_PI, n_old, N_old, N_old);
     dg::Grid2d g2n (0, M_PI, 0, M_PI, n_new, N_new, N_new);
-    cusp::coo_matrix<int, double, cusp::host_memory> proj2d = dg::create::transformation( g2n, g2o);
+    //cusp::coo_matrix<int, double, cusp::host_memory> proj2d = dg::create::transformation( g2n, g2o);
     cusp::coo_matrix<int, double, cusp::host_memory> inte2d = dg::create::interpolation( g2n, g2o);
+    dg::MultiMatrix< dg::HMatrix, thrust::host_vector<double> > proj2d = dg::create::fast_projection( g2o, N_old/N_new, N_old/N_new);
+    dg::MultiMatrix< dg::HMatrix, thrust::host_vector<double> > fast_inte2d = dg::create::fast_interpolation( g2n, N_old/N_new, N_old/N_new);
     const dg::HVec sinO = dg::evaluate( sine, g2o), 
                    sinN = dg::evaluate( sine, g2n);
     dg::HVec w2do = dg::create::weights( g2o);
     dg::HVec w2dn = dg::create::weights( g2n);
-    dg::HVec sinP( g2n.size());
+    dg::HVec sinP( g2n.size()), sinI(g2o.size());
     dg::blas2::gemv( proj2d, sinO, sinP);
     std::cout << "Original vector     "<<sqrt(dg::blas2::dot( sinO, w2do, sinO)) << "\n";
     std::cout << "Projected vector    "<<sqrt(dg::blas2::dot( sinP, w2dn, sinP)) << "\n";
@@ -56,6 +59,9 @@ int main()
     dg::blas2::gemv( inte2d, sinO, sinP);
     std::cout << "Interpolated vec    "<<sqrt(dg::blas2::dot( sinP, w2dn, sinP)) << "\n";
     std::cout << "Difference in Norms "<<sqrt(dg::blas2::dot( sinO, w2do, sinO)) - sqrt(dg::blas2::dot( sinP, w2dn, sinP)) << "\n" << std::endl;
+    dg::blas2::gemv( fast_inte2d, sinN, sinI);
+    std::cout << "Interpolated vec    "<<sqrt(dg::blas2::dot( sinI, w2do, sinI)) << "\n";
+    std::cout << "Difference in Norms "<<sqrt(dg::blas2::dot( sinI, w2do, sinI)) - sqrt(dg::blas2::dot( sinP, w2dn, sinP)) << "\n" << std::endl;
 
     return 0;
 }
diff --git a/inc/dg/backend/sparseblockmat.h b/inc/dg/backend/sparseblockmat.h
index 9cacf7b43..090fea051 100644
--- a/inc/dg/backend/sparseblockmat.h
+++ b/inc/dg/backend/sparseblockmat.h
@@ -83,11 +83,11 @@ struct EllSparseBlockMat
         right_range[1]=right_size;
     }
     
-    thrust::host_vector<value_type> data;//!< The data array is of size n*n*num_different_blocks and contains the blocks
-    IVec cols_idx; //!< is of size num_block_rows*num_blocks_per_line and contains the column indices 
-    IVec data_idx; //!< has the same size as cols_idx and contains indices into the data array
-    int num_rows; //!< number of rows, each row contains blocks
-    int num_cols; //!< number of columns
+    thrust::host_vector<value_type> data;//!< The data array is of size n*n*num_different_blocks and contains the blocks. The first block is contained in the first n*n elements, then comes the next block, etc.
+    IVec cols_idx; //!< is of size num_block_rows*num_blocks_per_line and contains the column indices % n into the vector
+    IVec data_idx; //!< has the same size as cols_idx and contains indices into the data array, i.e. the block number 
+    int num_rows; //!< number of block rows, each row contains blocks ( total number of rows is num_rows*n*left_size*right_size
+    int num_cols; //!< number of block columns (total number of columns is num_cols*n*left_size*right_size
     int blocks_per_line; //!< number of blocks in each line
     int n;  //!< each block has size n*n
     int left_size; //!< size of the left Kronecker delta
-- 
GitLab


From 57b14c6ac5ce3f36652f2b2e379575c08b5361a2 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Tue, 5 Sep 2017 23:43:39 +0200
Subject: [PATCH 257/453] added 1d and 3d fast interpolation/projection

---
 inc/dg/Makefile                     |   2 +-
 inc/dg/backend/fast_interpolation.h | 168 +++++++++++++++++-----------
 inc/dg/backend/projection_t.cu      |   7 +-
 inc/dg/blas_b.cu                    |  11 +-
 4 files changed, 112 insertions(+), 76 deletions(-)

diff --git a/inc/dg/Makefile b/inc/dg/Makefile
index 70988f2b1..6ffa22a3f 100644
--- a/inc/dg/Makefile
+++ b/inc/dg/Makefile
@@ -24,7 +24,7 @@ all: $(CPPFILES:%.cpp=%) $(CUFILES:%.cu=%)
 	$(CC) $(OPT) $(INCLUDE) -DDG_DEBUG $(CFLAGS) $< -o $@ 
 
 %_b: %_b.cu 
-	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE)  
+	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) -g -DDG_DEBUG
 
 bathRZ_t: bathRZ_t.cu 
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(GLFLAGS) $(INCLUDE) 
diff --git a/inc/dg/backend/fast_interpolation.h b/inc/dg/backend/fast_interpolation.h
index 51ef9b714..e5e8c80b7 100644
--- a/inc/dg/backend/fast_interpolation.h
+++ b/inc/dg/backend/fast_interpolation.h
@@ -19,13 +19,15 @@ struct MultiMatrix
     MultiMatrix( int dimension): inter_(dimension), temp_(dimension-1 > 0 ? dimension-1 : 0 ){}
     template<class OtherMatrix, class OtherContainer>
     MultiMatrix( const MultiMatrix<OtherMatrix, OtherContainer>& src){
-            temp_.resize( src.get_temp().size());
-            inter_.resize( src.get_matrices().size());
-            for( int i=0; i<temp_.size(); i++)
-            {
-                temp_[i].data() = src.get_temp()[i].data();
-                inter_[i] = src.get_matrices()[i];
-            }
+        unsigned dimsM = src.get_matrices().size();
+        unsigned dimsT = src.get_temp().size();
+        inter_.resize( dimsM);
+        temp_.resize(  dimsT);
+        for( unsigned i=0; i<dimsM; i++)
+            inter_[i] = src.get_matrices()[i];
+        for( unsigned i=0; i<dimsT; i++)
+            temp_[i].data() = src.get_temp()[i].data();
+
     }
 
     void symv( const container& x, container& y) const{gemv(x,y);}
@@ -67,89 +69,123 @@ struct MatrixTraits<const MultiMatrix<M, V> >
 
 namespace create
 {
-MultiMatrix< EllSparseBlockMat<double>, thrust::host_vector<double> > fast_interpolation( const aTopology2d& t, unsigned multiplyX, unsigned multiplyY)
+MultiMatrix< EllSparseBlockMat<double>, thrust::host_vector<double> > fast_interpolation( const Grid1d& t, unsigned multiply)
 {
     unsigned n=t.n();
     dg::Grid1d g_old( -1., 1., n, 1);
-    dg::Grid1d g_newX( -1., 1., n, multiplyX);
-    dg::Grid1d g_newY( -1., 1., n, multiplyY);
+    dg::Grid1d g_newX( -1., 1., n, multiply);
     dg::IHMatrix interpolX = dg::create::interpolation( g_newX, g_old);
-    dg::IHMatrix interpolY = dg::create::interpolation( g_newY, g_old);
-    EllSparseBlockMat<double> iX( multiplyX*t.Nx(), t.Nx(), 1, multiplyX, t.n()); 
-    EllSparseBlockMat<double> iY( multiplyY*t.Ny(), t.Ny(), 1, multiplyY, t.n()); 
+    EllSparseBlockMat<double> iX( multiply*t.N(), t.N(), 1, multiply, t.n()); 
     for( unsigned  i=0; i<n; i++)
     for( unsigned  j=0; j<n; j++)
-    for( unsigned  k=0; k<multiplyX; k++)
+    for( unsigned  k=0; k<multiply; k++)
         iX.data[(k*n+i)*n+j] = interpolX.values[(k*n+i)*n+j];
-    for( unsigned  i=0; i<n; i++)
-    for( unsigned  j=0; j<n; j++)
-    for( unsigned  k=0; k<multiplyY; k++)
-        iY.data[(k*n+i)*n+j] = interpolY.values[(k*n+i)*n+j];
-
-    for( unsigned i=0; i<multiplyX*t.Nx(); i++)
+    for( unsigned i=0; i<multiply*t.N(); i++)
     {
-        iX.cols_idx[i] = i/multiplyX;
+        iX.cols_idx[i] = i/multiply;
         iX.data_idx[i] = 0;
     }
-    for( unsigned i=0; i<multiplyY*t.Ny(); i++)
-    {
-        iY.cols_idx[i] = i/multiplyY;
-        iY.data_idx[i] = 0;
-    }
-    iX.left_size  = t.n()*t.Ny();
-    iY.right_size = t.n()*t.Nx()*multiplyX;
-    iX.set_default_range();
-    iY.set_default_range();
-
-    MultiMatrix < EllSparseBlockMat<double>, thrust::host_vector<double> > inter(2);
+    MultiMatrix < EllSparseBlockMat<double>, thrust::host_vector<double> > inter(1);
     inter.get_matrices()[0] = iX;
-    inter.get_matrices()[1] = iY;
-    thrust::host_vector<double> vec( t.size()*multiplyX);
-    inter.get_temp()[0] = Buffer<thrust::host_vector<double > >(vec);
     return inter;
 }
 
-MultiMatrix< EllSparseBlockMat<double>, thrust::host_vector<double> > fast_projection( const aTopology2d& t, unsigned divideX, unsigned divideY)
+MultiMatrix< EllSparseBlockMat<double>, thrust::host_vector<double> > fast_projection( const Grid1d& t, unsigned divide)
 {
     unsigned n=t.n();
-    if( t.Nx()%divideX != 0) throw Error( Message(_ping_)<< "Nx and divideX don't match: Nx: " << t.Nx()<< " divideX "<< divideX);
-    if( t.Ny()%divideY != 0) throw Error( Message(_ping_)<< "Ny and divideY don't match: Nx: " << t.Ny()<< " divideY "<< divideY);
-    dg::Grid1d g_oldX( -1., 1., n, divideX);
-    dg::Grid1d g_oldY( -1., 1., n, divideY);
+    if( t.N()%divide != 0) throw Error( Message(_ping_)<< "Nx and divide don't match: Nx: " << t.N()<< " divide "<< divide);
+    dg::Grid1d g_oldX( -1., 1., n, divide);
     dg::Grid1d g_new(  -1., 1., n, 1);
     dg::IHMatrix projectX = dg::create::projection( g_new, g_oldX);
-    dg::IHMatrix projectY = dg::create::projection( g_new, g_oldY);
-    EllSparseBlockMat<double> pX( t.Nx()/divideX, t.Nx(), divideX, divideY, t.n()); 
-    EllSparseBlockMat<double> pY( t.Ny()/divideY, t.Ny(), divideY, divideY, t.n()); 
+    EllSparseBlockMat<double> pX( t.N()/divide, t.N(), divide, divide, t.n()); 
     for( unsigned i=0; i<n; i++)
     for( unsigned j=0; j<n; j++)
-    {
-        for( unsigned k=0; k<divideX; k++)
-            pX.data[(k*n+i)*n+j] = projectX.values[i*divideX*n +k*n+j];
-        for( unsigned k=0; k<divideY; k++)
-            pY.data[(k*n+i)*n+j] = projectY.values[i*divideX*n +k*n+j];
-    }
-
-    for( unsigned i=0; i<t.Nx()/divideX; i++)
-        for( unsigned d=0; d<divideX; d++)
-        {
-            pX.cols_idx[i*divideX+d] = i*divideX+d;
-            pX.data_idx[i*divideX+d] = d;
-        }
-    for( unsigned i=0; i<t.Ny()/divideY; i++)
-        for( unsigned d=0; d<divideY; d++)
+        for( unsigned k=0; k<divide; k++)
+            pX.data[(k*n+i)*n+j] = projectX.values[i*divide*n +k*n+j];
+    for( unsigned i=0; i<t.N()/divide; i++)
+        for( unsigned d=0; d<divide; d++)
         {
-            pY.cols_idx[i*divideY+d] = i*divideY+d;
-            pY.data_idx[i*divideY+d] = d;
+            pX.cols_idx[i*divide+d] = i*divide+d;
+            pX.data_idx[i*divide+d] = d;
         }
-    pX.left_size  = t.n()*t.Ny();
-    pY.right_size = t.n()*t.Nx()/divideX;
-    pX.set_default_range();
-    pY.set_default_range();
+    MultiMatrix < EllSparseBlockMat<double>, thrust::host_vector<double> > inter(1);
+    inter.get_matrices()[0] = pX;
+    return inter;
+}
+
+MultiMatrix< EllSparseBlockMat<double>, thrust::host_vector<double> > fast_interpolation( const aTopology2d& t, unsigned multiplyX, unsigned multiplyY)
+{
+    dg::Grid1d gx(t.x0(), t.x1(), t.n(), t.Nx());
+    dg::Grid1d gy(t.y0(), t.y1(), t.n(), t.Ny());
+    MultiMatrix < EllSparseBlockMat<double>, thrust::host_vector<double> > interX = dg::create::fast_interpolation( gx, multiplyX);
+    interX.get_matrices()[0].left_size = t.n()*t.Ny();
+    interX.get_matrices()[0].set_default_range();
+    MultiMatrix < EllSparseBlockMat<double>, thrust::host_vector<double> > interY = dg::create::fast_interpolation( gy, multiplyY);
+    interY.get_matrices()[0].right_size = t.n()*t.Nx()*multiplyX;
+    interY.get_matrices()[0].set_default_range();
 
     MultiMatrix < EllSparseBlockMat<double>, thrust::host_vector<double> > inter(2);
-    inter.get_matrices()[0] = pX;
-    inter.get_matrices()[1] = pY;
+    inter.get_matrices()[0] = interX.get_matrices()[0];
+    inter.get_matrices()[1] = interY.get_matrices()[0];
+    thrust::host_vector<double> vec( t.size()*multiplyX);
+    inter.get_temp()[0] = Buffer<thrust::host_vector<double > >(vec);
+    return inter;
+}
+
+MultiMatrix< EllSparseBlockMat<double>, thrust::host_vector<double> > fast_projection( const aTopology2d& t, unsigned divideX, unsigned divideY)
+{
+    dg::Grid1d gx(t.x0(), t.x1(), t.n(), t.Nx());
+    dg::Grid1d gy(t.y0(), t.y1(), t.n(), t.Ny());
+    MultiMatrix < EllSparseBlockMat<double>, thrust::host_vector<double> > interX = dg::create::fast_projection( gx, divideX);
+    interX.get_matrices()[0].left_size = t.n()*t.Ny();
+    interX.get_matrices()[0].set_default_range();
+    MultiMatrix < EllSparseBlockMat<double>, thrust::host_vector<double> > interY = dg::create::fast_projection( gy, divideY);
+    interY.get_matrices()[0].right_size = t.n()*t.Nx()/divideX;
+    interY.get_matrices()[0].set_default_range();
+
+    MultiMatrix < EllSparseBlockMat<double>, thrust::host_vector<double> > inter(2);
+    inter.get_matrices()[0] = interX.get_matrices()[0];
+    inter.get_matrices()[1] = interY.get_matrices()[0];
+    thrust::host_vector<double> vec( t.size()/divideX);
+    inter.get_temp()[0] = Buffer<thrust::host_vector<double> >(vec);
+    return inter;
+}
+
+MultiMatrix< EllSparseBlockMat<double>, thrust::host_vector<double> > fast_interpolation( const aTopology3d& t, unsigned multiplyX, unsigned multiplyY)
+{
+    dg::Grid1d gx(t.x0(), t.x1(), t.n(), t.Nx());
+    dg::Grid1d gy(t.y0(), t.y1(), t.n(), t.Ny());
+    MultiMatrix < EllSparseBlockMat<double>, thrust::host_vector<double> > interX = dg::create::fast_interpolation( gx, multiplyX);
+    interX.get_matrices()[0].left_size = t.n()*t.Ny()*t.Nz();
+    interX.get_matrices()[0].set_default_range();
+    MultiMatrix < EllSparseBlockMat<double>, thrust::host_vector<double> > interY = dg::create::fast_interpolation( gy, multiplyY);
+    interY.get_matrices()[0].right_size = t.n()*t.Nx()*multiplyX;
+    interY.get_matrices()[0].left_size = t.Nz();
+    interY.get_matrices()[0].set_default_range();
+
+    MultiMatrix < EllSparseBlockMat<double>, thrust::host_vector<double> > inter(2);
+    inter.get_matrices()[0] = interX.get_matrices()[0];
+    inter.get_matrices()[1] = interY.get_matrices()[0];
+    thrust::host_vector<double> vec( t.size()*multiplyX);
+    inter.get_temp()[0] = Buffer<thrust::host_vector<double > >(vec);
+    return inter;
+}
+
+MultiMatrix< EllSparseBlockMat<double>, thrust::host_vector<double> > fast_projection( const aTopology3d& t, unsigned divideX, unsigned divideY)
+{
+    dg::Grid1d gx(t.x0(), t.x1(), t.n(), t.Nx());
+    dg::Grid1d gy(t.y0(), t.y1(), t.n(), t.Ny());
+    MultiMatrix < EllSparseBlockMat<double>, thrust::host_vector<double> > interX = dg::create::fast_projection( gx, divideX);
+    interX.get_matrices()[0].left_size = t.n()*t.Ny()*t.Nz();
+    interX.get_matrices()[0].set_default_range();
+    MultiMatrix < EllSparseBlockMat<double>, thrust::host_vector<double> > interY = dg::create::fast_projection( gy, divideY);
+    interY.get_matrices()[0].right_size = t.n()*t.Nx()/divideX;
+    interY.get_matrices()[0].left_size = t.Nz();
+    interY.get_matrices()[0].set_default_range();
+
+    MultiMatrix < EllSparseBlockMat<double>, thrust::host_vector<double> > inter(2);
+    inter.get_matrices()[0] = interX.get_matrices()[0];
+    inter.get_matrices()[1] = interY.get_matrices()[0];
     thrust::host_vector<double> vec( t.size()/divideX);
     inter.get_temp()[0] = Buffer<thrust::host_vector<double> >(vec);
     return inter;
diff --git a/inc/dg/backend/projection_t.cu b/inc/dg/backend/projection_t.cu
index a0baf72a3..b075f4df4 100644
--- a/inc/dg/backend/projection_t.cu
+++ b/inc/dg/backend/projection_t.cu
@@ -8,6 +8,7 @@
 
 double sine( double x){ return sin(x);}
 double sine( double x, double y){return sin(x)*sin(y);}
+double sine( double x, double y, double z){return sin(x)*sin(y);}
 
 int main()
 {
@@ -36,10 +37,10 @@ int main()
     std::cout << "Interpolated vec "<<dg::blas2::dot( onen, w1dn, w) << "\n";
     std::cout << "Difference       "<<dg::blas2::dot( oneo, w1do, v) - dg::blas2::dot( onen, w1dn, w) << "\n"<<std::endl;
 
-    std::cout << "TEST 2D\n";
+    std::cout << "TEST 2D and 3D\n";
     
-    dg::Grid2d g2o (0, M_PI, 0, M_PI, n_old, N_old, N_old);
-    dg::Grid2d g2n (0, M_PI, 0, M_PI, n_new, N_new, N_new);
+    dg::Grid3d g2o (0, M_PI, 0, M_PI, 0,1, n_old, N_old, N_old, 4);
+    dg::Grid3d g2n (0, M_PI, 0, M_PI, 0,1, n_new, N_new, N_new, 4);
     //cusp::coo_matrix<int, double, cusp::host_memory> proj2d = dg::create::transformation( g2n, g2o);
     cusp::coo_matrix<int, double, cusp::host_memory> inte2d = dg::create::interpolation( g2n, g2o);
     dg::MultiMatrix< dg::HMatrix, thrust::host_vector<double> > proj2d = dg::create::fast_projection( g2o, N_old/N_new, N_old/N_new);
diff --git a/inc/dg/blas_b.cu b/inc/dg/blas_b.cu
index 23c86de95..636bfc05d 100644
--- a/inc/dg/blas_b.cu
+++ b/inc/dg/blas_b.cu
@@ -8,7 +8,7 @@
 #include "blas.h"
 #include "backend/derivatives.h"
 #include "backend/evaluation.cuh"
-#include "backend/projection.cuh"
+#include "backend/fast_interpolation.h"
 
 const double lx = 2.*M_PI;
 const double ly = 2.*M_PI;
@@ -23,7 +23,6 @@ typedef dg::DVec Vector;
 //typedef thrust::device_vector<double> Vector;
 typedef dg::DMatrix Matrix;
 //typedef cusp::array1d<double, cusp::device_memory> Vector;
-typedef dg::IDMatrix IMatrix;
 
 int main()
 {
@@ -32,7 +31,7 @@ int main()
     std::cout << "Type n, Nx, Ny and Nz ( Nx and Ny shall be multiples of 2)\n";
     std::cin >> n >> Nx >> Ny >> Nz;
     dg::Grid3d grid(      0., lx, 0, ly, 0, ly, n, Nx, Ny, Nz);
-    dg::Grid3d grid_half( 0., lx, 0, ly, 0, ly, n, Nx/2, Ny/2, Nz);
+    dg::Grid3d grid_half = grid; grid_half.multiplyCellNumbers(0.5, 0.5);
     Vector w2d;
     dg::blas1::transfer( dg::create::weights(grid), w2d);
 
@@ -46,9 +45,9 @@ int main()
     value_type gbytes=(value_type)x.size()*sizeof(value_type)/1e9;
     std::cout << "Sizeof vectors is "<<gbytes<<" GB\n";
     std::cout << "Generate interpolation and projection\n";
-    IMatrix inter, project; 
-    dg::blas2::transfer(dg::create::interpolation( grid, grid_half), inter);
-    dg::blas2::transfer(dg::create::projection( grid_half, grid), project);
+    dg::MultiMatrix<Matrix, Vector> inter, project; 
+    dg::blas2::transfer(dg::create::fast_interpolation( grid_half, 2,2), inter);
+    dg::blas2::transfer(dg::create::fast_projection( grid, 2,2), project);
     std::cout << "Done...\n";
     int multi=200;
     t.tic();
-- 
GitLab


From b2d56af8982c686300d50d68c0ab3fc6b8a297ef Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <Matthias.Wiesenberger@uibk.ac.at>
Date: Wed, 6 Sep 2017 01:55:57 +0200
Subject: [PATCH 258/453] optimize fast projection and interpolation

---
 inc/dg/Makefile                             |   2 +-
 inc/dg/backend/sparseblockmat_omp_kernels.h | 249 ++++++++++++++------
 2 files changed, 179 insertions(+), 72 deletions(-)

diff --git a/inc/dg/Makefile b/inc/dg/Makefile
index 6ffa22a3f..63c041708 100644
--- a/inc/dg/Makefile
+++ b/inc/dg/Makefile
@@ -24,7 +24,7 @@ all: $(CPPFILES:%.cpp=%) $(CUFILES:%.cu=%)
 	$(CC) $(OPT) $(INCLUDE) -DDG_DEBUG $(CFLAGS) $< -o $@ 
 
 %_b: %_b.cu 
-	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) -g -DDG_DEBUG
+	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) 
 
 bathRZ_t: bathRZ_t.cu 
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(GLFLAGS) $(INCLUDE) 
diff --git a/inc/dg/backend/sparseblockmat_omp_kernels.h b/inc/dg/backend/sparseblockmat_omp_kernels.h
index 5f04add8a..1e9b6bf11 100644
--- a/inc/dg/backend/sparseblockmat_omp_kernels.h
+++ b/inc/dg/backend/sparseblockmat_omp_kernels.h
@@ -14,21 +14,112 @@ void ell_multiply_kernel( value_type alpha, value_type beta,
          )
 {
     //simplest implementation
-#pragma omp parallel for collapse(4)
+#pragma omp parallel for collapse(2)
     for( int s=0; s<left_size; s++)
     for( int i=0; i<num_rows; i++)
     for( int k=0; k<n; k++)
-    for( int j=right_range[0]; j<right_range[1]; j++)
     {
-        int I = ((s*num_rows + i)*n+k)*right_size+j;
-        y[I] *=beta;
+        int B[blocks_per_line], J[blocks_per_line];
+        for( int d=0; d<blocks_per_line; d++)
+        {
+            B[d] = (data_idx[i*blocks_per_line+d]*n+k)*n;
+            J[d] = (s*num_cols+cols_idx[i*blocks_per_line+d])*n;
+        }
+        for( int j=right_range[0]; j<right_range[1]; j++)
+        {
+            value_type temp = 0;
+            for( int d=0; d<blocks_per_line; d++)
+            {
+                for( int q=0; q<n; q++) //multiplication-loop
+                    temp += data[ B[d]+q]*
+                        x[(J[d]+q)*right_size+j];
+            }
+            int I = ((s*num_rows + i)*n+k)*right_size+j;
+            y[I] = alpha*temp + beta*y[I];
+        }
+    }
+}
+
+template<class value_type>
+void ell_multiply_kernel3( value_type alpha, value_type beta,
+         const value_type * RESTRICT data, const int * RESTRICT cols_idx, const int * RESTRICT data_idx, 
+         const int num_rows, const int num_cols, const int blocks_per_line,
+         const int left_size, const int right_size, 
+         const int * RESTRICT right_range,
+         const value_type * RESTRICT x, value_type * RESTRICT y
+         )
+{
+    bool trivial = true;
+    for( int i=1; i<num_rows; i++)
         for( int d=0; d<blocks_per_line; d++)
         {
-            int B = (data_idx[i*blocks_per_line+d]*n+k)*n;
-            int J = (s*num_cols+cols_idx[i*blocks_per_line+d])*n;
-            for( int q=0; q<n; q++) //multiplication-loop
-                y[I] += alpha*data[ B+q]*
-                    x[(J+q)*right_size+j];
+            if( data_idx[i*blocks_per_line+d] != data_idx[d]) trivial = false;
+        }
+    if (trivial && blocks_per_line == 1)
+    {
+        if( right_size==1)
+        {
+            #pragma omp parallel for 
+            for( int s=0; s<left_size; s++)
+            for( int i=0; i<num_rows; i++)
+            for( int k=0; k<3; k++)
+            {
+                value_type temp = 0;
+                int B = (data_idx[i]*3+k)*3;
+                int J = (s*num_cols+cols_idx[i])*3;
+
+                temp +=data[ B+0]* x[(J+0)*right_size];
+                temp +=data[ B+1]* x[(J+1)*right_size];
+                temp +=data[ B+2]* x[(J+2)*right_size];
+                int I = ((s*num_rows + i)*3+k)*right_size;
+                y[I] = alpha*temp + beta*y[I];
+            }
+        }
+        else
+        {
+            #pragma omp parallel for collapse(2)
+            for( int s=0; s<left_size; s++)
+            for( int i=0; i<num_rows; i++)
+            for( int k=0; k<3; k++)
+            for( int j=right_range[0]; j<right_range[1]; j++)
+            {
+                value_type temp = 0;
+                int B = (data_idx[i]*3+k)*3;
+                int J = (s*num_cols+cols_idx[i])*3;
+
+                temp +=data[ B+0]* x[(J+0)*right_size+j];
+                temp +=data[ B+1]* x[(J+1)*right_size+j];
+                temp +=data[ B+2]* x[(J+2)*right_size+j];
+                int I = ((s*num_rows + i)*3+k)*right_size+j;
+                y[I] = alpha*temp + beta*y[I];
+            }
+        }
+    }
+    else
+    {
+#pragma omp parallel for collapse(2)
+        for( int s=0; s<left_size; s++)
+        for( int i=0; i<num_rows; i++)
+        for( int k=0; k<3; k++)
+        {
+            int B[blocks_per_line], J[blocks_per_line];
+            for( int d=0; d<blocks_per_line; d++)
+            {
+                B[d] = (data_idx[i*blocks_per_line+d]*3+k)*3;
+                J[d] = (s*num_cols+cols_idx[i*blocks_per_line+d])*3;
+            }
+            for( int j=right_range[0]; j<right_range[1]; j++)
+            {
+                value_type temp = 0;
+                for( int d=0; d<blocks_per_line; d++)
+                {
+                    temp +=data[ B[d]+0]* x[(J[d]+0)*right_size+j];
+                    temp +=data[ B[d]+1]* x[(J[d]+1)*right_size+j];
+                    temp +=data[ B[d]+2]* x[(J[d]+2)*right_size+j];
+                }
+                int I = ((s*num_rows + i)*3+k)*right_size+j;
+                y[I] = alpha*temp + beta*y[I];
+            }
         }
     }
 }
@@ -52,7 +143,7 @@ void ell_multiply_kernel33( value_type alpha, value_type beta,
         for( int d=0; d<3; d++)
         {
             if( data_idx[i*3+d] != d) trivial = false;
-            if( cols_idx[i*3+d] != i+d-1) trivial = false;
+            //if( cols_idx[i*3+d] != i+d-1) trivial = false;
         }
     if( trivial)
     {
@@ -61,14 +152,17 @@ void ell_multiply_kernel33( value_type alpha, value_type beta,
 	{
 		for( int i=0; i<num_rows; i++)
 		{
-		        int J0 = (i==0 || i==num_rows-1)?(s*num_cols+cols_idx[i*3+0])*3:(s*num_cols+i+0-1)*3;
-		        int J1 = (i==0 || i==num_rows-1)?(s*num_cols+cols_idx[i*3+1])*3:(s*num_cols+i+1-1)*3;
-		        int J2 = (i==0 || i==num_rows-1)?(s*num_cols+cols_idx[i*3+2])*3:(s*num_cols+i+2-1)*3;
+            //int J0 = (i==0 || i==num_rows-1)?(s*num_cols+cols_idx[i*3+0])*3:(s*num_cols+i+0-1)*3;
+            //int J1 = (i==0 || i==num_rows-1)?(s*num_cols+cols_idx[i*3+1])*3:(s*num_cols+i+1-1)*3;
+            //int J2 = (i==0 || i==num_rows-1)?(s*num_cols+cols_idx[i*3+2])*3:(s*num_cols+i+2-1)*3;
+            int J0 = (i==0 || i==num_rows-1)?(s*num_cols+cols_idx[i*3+0])*3:(s*num_cols+cols_idx[i*3+0])*3;
+            int J1 = (i==0 || i==num_rows-1)?(s*num_cols+cols_idx[i*3+1])*3:(s*num_cols+cols_idx[i*3+1])*3;
+            int J2 = (i==0 || i==num_rows-1)?(s*num_cols+cols_idx[i*3+2])*3:(s*num_cols+cols_idx[i*3+2])*3;
 			for( int k=0; k<3; k++)
 			{
 				int B0 = (i==0 || i==num_rows-1)?(data_idx[i*3+0]*3+k)*3:(0*3+k)*3;
-			        int B1 = (i==0 || i==num_rows-1)?(data_idx[i*3+1]*3+k)*3:(1*3+k)*3;
-			        int B2 = (i==0 || i==num_rows-1)?(data_idx[i*3+2]*3+k)*3:(2*3+k)*3;
+                int B1 = (i==0 || i==num_rows-1)?(data_idx[i*3+1]*3+k)*3:(1*3+k)*3;
+                int B2 = (i==0 || i==num_rows-1)?(data_idx[i*3+2]*3+k)*3:(2*3+k)*3;
 				#pragma vector nontemporal(y)
 				for( int j=right_range[0]; j<right_range[1]; j++)
 				{
@@ -92,7 +186,7 @@ void ell_multiply_kernel33( value_type alpha, value_type beta,
 	}
     }
     else 
-        ell_multiply_kernel(alpha,beta, data, cols_idx, data_idx, num_rows, num_cols, 3, 3, left_size, right_size, right_range,  x, y);
+        ell_multiply_kernel3(alpha,beta, data, cols_idx, data_idx, num_rows, num_cols, 3, left_size, right_size, right_range,  x, y);
 }
 
 // multiply kernel, n=3, 2 blocks per line
@@ -110,8 +204,8 @@ void ell_multiply_kernel32( value_type alpha, value_type beta,
         for( int d=0; d<2; d++)
         {
             if( data_idx[i*2+d] != d) forward = backward = false;
-            if( cols_idx[i*2+d] != i+d-1) backward = false;
-            if( cols_idx[i*2+d] != i+d) forward = false;
+            //if( cols_idx[i*2+d] != i+d-1) backward = false;
+            //if( cols_idx[i*2+d] != i+d) forward = false;
         }
     int diff = -1;
     if(forward ) diff = 0;
@@ -141,21 +235,25 @@ void ell_multiply_kernel32( value_type alpha, value_type beta,
     for( int s=0; s<left_size; s++)
     for( int i=1; i<num_rows-1; i++)
     for( int k=0; k<3; k++)
-    for( int j=right_range[0]; j<right_range[1]; j++)
     {
-        value_type temp = 0;
         int B0 = (0*3+k)*3;
         int B1 = (1*3+k)*3;
-        int J0 = (s*num_cols+i+0+diff)*3;
-        int J1 = (s*num_cols+i+1+diff)*3;
-        temp +=data[ B0+0]* x[(J0+0)*right_size+j];
-        temp +=data[ B0+1]* x[(J0+1)*right_size+j];
-        temp +=data[ B0+2]* x[(J0+2)*right_size+j];
-        temp +=data[ B1+0]* x[(J1+0)*right_size+j];
-        temp +=data[ B1+1]* x[(J1+1)*right_size+j];
-        temp +=data[ B1+2]* x[(J1+2)*right_size+j];
-        int I = ((s*num_rows + i)*3+k)*right_size+j;
-        y[I]=alpha*temp+beta*y[I];
+        //int J0 = (s*num_cols+i+0+diff)*3;
+        //int J1 = (s*num_cols+i+1+diff)*3;
+        int J0 = (s*num_cols+cols_idx[i*2+0])*3;
+        int J1 = (s*num_cols+cols_idx[i*2+1])*3;
+        for( int j=right_range[0]; j<right_range[1]; j++)
+        {
+            value_type temp = 0;
+            temp +=data[ B0+0]* x[(J0+0)*right_size+j];
+            temp +=data[ B0+1]* x[(J0+1)*right_size+j];
+            temp +=data[ B0+2]* x[(J0+2)*right_size+j];
+            temp +=data[ B1+0]* x[(J1+0)*right_size+j];
+            temp +=data[ B1+1]* x[(J1+1)*right_size+j];
+            temp +=data[ B1+2]* x[(J1+2)*right_size+j];
+            int I = ((s*num_rows + i)*3+k)*right_size+j;
+            y[I]=alpha*temp+beta*y[I];
+        }
     }
 #pragma omp parallel for 
     for( int s=0; s<left_size; s++)
@@ -179,7 +277,7 @@ void ell_multiply_kernel32( value_type alpha, value_type beta,
     }
     }
     else
-        ell_multiply_kernel(alpha,beta, data, cols_idx, data_idx, num_rows, num_cols, 2, 3, left_size, right_size, right_range,  x, y);
+        ell_multiply_kernel3(alpha,beta, data, cols_idx, data_idx, num_rows, num_cols, 2, left_size, right_size, right_range,  x, y);
 
 }
 // multiply kernel, n=3, 3 blocks per line, right_size = 1
@@ -208,19 +306,19 @@ void ell_multiply_kernel33x( value_type alpha, value_type beta,
     {
         value_type temp = 0;
         int B0 = (data_idx[i*3+0]*3+k)*3;
+        int B1 = (data_idx[i*3+1]*3+k)*3;
+        int B2 = (data_idx[i*3+2]*3+k)*3;
         int J0 = (s*num_cols+cols_idx[i*3+0])*3;
+        int J1 = (s*num_cols+cols_idx[i*3+1])*3;
+        int J2 = (s*num_cols+cols_idx[i*3+2])*3;
         temp +=data[ B0+0]* x[(J0+0)];
         temp +=data[ B0+1]* x[(J0+1)];
         temp +=data[ B0+2]* x[(J0+2)];
 
-        int B1 = (data_idx[i*3+1]*3+k)*3;
-        int J1 = (s*num_cols+cols_idx[i*3+1])*3;
         temp +=data[ B1+0]* x[(J1+0)];
         temp +=data[ B1+1]* x[(J1+1)];
         temp +=data[ B1+2]* x[(J1+2)];
 
-        int B2 = (data_idx[i*3+2]*3+k)*3;
-        int J2 = (s*num_cols+cols_idx[i*3+2])*3;
         temp +=data[ B2+0]* x[(J2+0)];
         temp +=data[ B2+1]* x[(J2+1)];
         temp +=data[ B2+2]* x[(J2+2)];
@@ -228,27 +326,32 @@ void ell_multiply_kernel33x( value_type alpha, value_type beta,
         y[I]=alpha*temp+beta*y[I];
     }
     for( int i=1; i<num_rows-1; i++)
-    for( int k=0; k<3; k++)
     {
-        value_type temp = 0;
-        int B0 = (0*3+k)*3;
+        //int J0 = (s*num_cols+cols_idx[i*3+0])*3;
+        //int J1 = (s*num_cols+cols_idx[i*3+1])*3;
+        //int J2 = (s*num_cols+cols_idx[i*3+2])*3;
         int J0 = (s*num_cols+i+0-1)*3;
-        temp +=data[ B0+0]* x[(J0+0)];
-        temp +=data[ B0+1]* x[(J0+1)];
-        temp +=data[ B0+2]* x[(J0+2)];
-        int B1 = (1*3+k)*3;
         int J1 = (s*num_cols+i+1-1)*3;
-        temp +=data[ B1+0]* x[(J1+0)];
-        temp +=data[ B1+1]* x[(J1+1)];
-        temp +=data[ B1+2]* x[(J1+2)];
-        int B2 = (2*3+k)*3;
         int J2 = (s*num_cols+i+2-1)*3;
-        temp +=data[ B2+0]* x[(J2+0)];
-        temp +=data[ B2+1]* x[(J2+1)];
-        temp +=data[ B2+2]* x[(J2+2)];
+        for( int k=0; k<3; k++)
+        {
+            value_type temp = 0;
+            int B0 = (0*3+k)*3;
+            temp +=data[ B0+0]* x[(J0+0)];
+            temp +=data[ B0+1]* x[(J0+1)];
+            temp +=data[ B0+2]* x[(J0+2)];
+            int B1 = (1*3+k)*3;
+            temp +=data[ B1+0]* x[(J1+0)];
+            temp +=data[ B1+1]* x[(J1+1)];
+            temp +=data[ B1+2]* x[(J1+2)];
+            int B2 = (2*3+k)*3;
+            temp +=data[ B2+0]* x[(J2+0)];
+            temp +=data[ B2+1]* x[(J2+1)];
+            temp +=data[ B2+2]* x[(J2+2)];
 
-        int I = ((s*num_rows + i)*3+k);
-        y[I]=alpha*temp+beta*y[I];
+            int I = ((s*num_rows + i)*3+k);
+            y[I]=alpha*temp+beta*y[I];
+        }
     }
     for( int i=num_rows-1; i<num_rows; i++)
     for( int k=0; k<3; k++)
@@ -280,7 +383,7 @@ void ell_multiply_kernel33x( value_type alpha, value_type beta,
     else 
     {
         int right_range[2] = {0,1};
-        ell_multiply_kernel(alpha,beta, data, cols_idx, data_idx, num_rows, num_cols, 3, 3, left_size, 1, right_range,  x, y);
+        ell_multiply_kernel3(alpha,beta, data, cols_idx, data_idx, num_rows, num_cols, 3, left_size, 1, right_range,  x, y);
     }
 }
 
@@ -298,8 +401,8 @@ void ell_multiply_kernel32x( value_type alpha, value_type beta,
         for( int d=0; d<2; d++)
         {
             if( data_idx[i*2+d] != d) forward = backward = false;
-            if( cols_idx[i*2+d] != i+d-1) backward = false;
-            if( cols_idx[i*2+d] != i+d) forward = false;
+            //if( cols_idx[i*2+d] != i+d-1) backward = false;
+            //if( cols_idx[i*2+d] != i+d) forward = false;
         }
     int diff = -1;
     if(forward ) {diff = 0; }
@@ -327,21 +430,25 @@ void ell_multiply_kernel32x( value_type alpha, value_type beta,
 #pragma omp parallel for
     for( int s=0; s<left_size; s++)
     for( int i=1; i<num_rows-1; i++)
-    for( int k=0; k<3; k++)
     {
-        value_type temp = 0;
-        int B0 = (0*3+k)*3;
-        int B1 = (1*3+k)*3;
-        int J0 = (s*num_cols+i+0+diff)*3;
-        int J1 = (s*num_cols+i+1+diff)*3;
-        temp +=data[ B0+0]* x[(J0+0)];
-        temp +=data[ B0+1]* x[(J0+1)];
-        temp +=data[ B0+2]* x[(J0+2)];
-        temp +=data[ B1+0]* x[(J1+0)];
-        temp +=data[ B1+1]* x[(J1+1)];
-        temp +=data[ B1+2]* x[(J1+2)];
-        int I = ((s*num_rows + i)*3+k);
-        y[I]=alpha*temp+beta*y[I];
+        //int J0 = (s*num_cols+i+0+diff)*3;
+        //int J1 = (s*num_cols+i+1+diff)*3;
+        int J0 = (s*num_cols+cols_idx[i*2+0])*3;
+        int J1 = (s*num_cols+cols_idx[i*2+0])*3;
+        for( int k=0; k<3; k++)
+        {
+            value_type temp = 0;
+            int B0 = (0*3+k)*3;
+            int B1 = (1*3+k)*3;
+            temp +=data[ B0+0]* x[(J0+0)];
+            temp +=data[ B0+1]* x[(J0+1)];
+            temp +=data[ B0+2]* x[(J0+2)];
+            temp +=data[ B1+0]* x[(J1+0)];
+            temp +=data[ B1+1]* x[(J1+1)];
+            temp +=data[ B1+2]* x[(J1+2)];
+            int I = ((s*num_rows + i)*3+k);
+            y[I]=alpha*temp+beta*y[I];
+        }
     }
 #pragma omp parallel for
     for( int s=0; s<left_size; s++)
@@ -366,7 +473,7 @@ void ell_multiply_kernel32x( value_type alpha, value_type beta,
     else
     {
         int right_range[2] = {0,1};
-        ell_multiply_kernel(alpha,beta, data, cols_idx, data_idx, num_rows, num_cols, 2, 3, left_size, 1, right_range, x, y);
+        ell_multiply_kernel3(alpha,beta, data, cols_idx, data_idx, num_rows, num_cols, 2, left_size, 1, right_range, x, y);
     }
 
 
@@ -402,8 +509,8 @@ void EllSparseBlockMatDevice<value_type>::launch_multiply_kernel( value_type alp
                 ell_multiply_kernel32<value_type> ( alpha, beta, data_ptr, cols_ptr, block_ptr, num_rows, num_cols, left_size, right_size, right_range_ptr,  x_ptr,y_ptr);
         }
         else
-            ell_multiply_kernel<value_type>(alpha, beta,  
-                data_ptr, cols_ptr, block_ptr, num_rows, num_cols, blocks_per_line, 3, left_size, right_size, right_range_ptr,  x_ptr,y_ptr);
+            ell_multiply_kernel3<value_type>(alpha, beta,  
+                data_ptr, cols_ptr, block_ptr, num_rows, num_cols, blocks_per_line, left_size, right_size, right_range_ptr,  x_ptr,y_ptr);
     }
     else
         ell_multiply_kernel<value_type>  (alpha, beta,  
-- 
GitLab


From ef5bd064606c3e37484e8867a99e09aa7460a94c Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Wed, 6 Sep 2017 22:47:10 +0200
Subject: [PATCH 259/453] added multigrid.h

---
 inc/dg/backend/fast_interpolation.h | 13 +++++++-
 inc/dg/cg.h                         | 37 +++++++++++----------
 inc/dg/elliptic.h                   | 51 +++++++++++++++--------------
 inc/dg/multigrid.h                  | 44 +++++++++++++++++++++++++
 4 files changed, 102 insertions(+), 43 deletions(-)
 create mode 100644 inc/dg/multigrid.h

diff --git a/inc/dg/backend/fast_interpolation.h b/inc/dg/backend/fast_interpolation.h
index e5e8c80b7..ec63990ef 100644
--- a/inc/dg/backend/fast_interpolation.h
+++ b/inc/dg/backend/fast_interpolation.h
@@ -11,11 +11,19 @@
 namespace dg
 {
 
+/**
+ * @brief Struct that applies given matrices one after the other
+ * @copydoc hide_matrix_container
+ * @ingroup misc
+ */
 template <class Matrix, class container>
 struct MultiMatrix
 {
     MultiMatrix(){}
-
+    /**
+    * @brief reserve space for dimension matrices  and dimension-1 containers
+    * @param dimension # of matrices to store 
+    */
     MultiMatrix( int dimension): inter_(dimension), temp_(dimension-1 > 0 ? dimension-1 : 0 ){}
     template<class OtherMatrix, class OtherContainer>
     MultiMatrix( const MultiMatrix<OtherMatrix, OtherContainer>& src){
@@ -31,6 +39,9 @@ struct MultiMatrix
     }
 
     void symv( const container& x, container& y) const{gemv(x,y);}
+    /**
+    * @brief Applies all stored matrices one after the other
+    */
     void gemv( const container& x, container& y) const
     {
         int dims = inter_.size();
diff --git a/inc/dg/cg.h b/inc/dg/cg.h
index ca58faa80..8ee712520 100644
--- a/inc/dg/cg.h
+++ b/inc/dg/cg.h
@@ -399,6 +399,7 @@ struct Invert
      * @param phi solution (write only)
      * @param rho right-hand-side
      * @note computes inverse weights from the weights 
+     * @note If the Macro DG_BENCHMARK is defined this function will write timings to std::cout
      *
      * @return number of iterations used 
      */
@@ -424,6 +425,7 @@ struct Invert
      * @param w The weights that made the operator symmetric
      * @param p The preconditioner  
      * @note computes inverse weights from the weights 
+     * @note If the Macro DG_BENCHMARK is defined this function will write timings to std::cout
      *
      * @return number of iterations used 
      */
@@ -450,26 +452,23 @@ struct Invert
      * @param p The preconditioner  
      * @param inv_weights The inverse weights used to compute the scalar product in the CG solver
      * @note Calls the most general form of the CG solver with SquareNorm being the container class
+     * @note If the Macro DG_BENCHMARK is defined this function will write timings to std::cout
      *
      * @return number of iterations used 
      */
     template< class SymmetricOp, class Preconditioner >
     unsigned operator()( SymmetricOp& op, container& phi, const container& rho, const container& w, Preconditioner& p, const container& inv_weights)
     {
-        assert( phi0.size() != 0);
+        assert( phi.size() != 0);
         assert( &rho != &phi);
-        blas1::axpby( alpha[0], phi0, alpha[1], phi1, phi); // 1. phi0 + 0.*phi1 = phi
-        blas1::axpby( alpha[2], phi2, 1., phi); // 0. phi2 + 1. phi0 + 0.*phi1 = phi
-
-        unsigned number;
 #ifdef DG_BENCHMARK
-#ifdef MPI_VERSION
-        int rank;
-        MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-#endif //MPI
         Timer t;
         t.tic();
 #endif //DG_BENCHMARK
+        blas1::axpbygz( alpha[0], phi0, alpha[1], phi1, alpha[2], phi2); 
+        phi.swap(phi2);
+
+        unsigned number;
         if( multiplyWeights_ ) 
         {
             dg::blas2::symv( w, rho, phi2);
@@ -477,21 +476,23 @@ struct Invert
         }
         else
             number = cg( op, phi, rho, p, inv_weights, eps_, nrmb_correction_);
+
+        phi1.swap( phi2);
+        phi0.swap( phi1);
+        
+        blas1::copy( phi, phi0);
 #ifdef DG_BENCHMARK
-#ifdef MPI_VERSION
-        if(rank==0)
-#endif //MPI
-        std::cout << "# of cg iterations \t"<< number << "\t";
         t.toc();
 #ifdef MPI_VERSION
+        int rank;
+        MPI_Comm_rank(MPI_COMM_WORLD, &rank);
         if(rank==0)
 #endif //MPI
-        std::cout<< "took \t"<<t.diff()<<"s\n";
+        {
+            std::cout << "# of cg iterations \t"<< number << "\t";
+            std::cout<< "took \t"<<t.diff()<<"s\n";
+        }
 #endif //DG_BENCHMARK
-        phi1.swap( phi2);
-        phi0.swap( phi1);
-        
-        blas1::axpby( 1., phi, 0, phi0);
         return number;
     }
 
diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index 9b614fdab..197e8c8a9 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -54,6 +54,8 @@ template <class Geometry, class Matrix, class container>
 class Elliptic
 {
     public:
+    ///@brief empty object ( no memory allocation)
+    Elliptic(){}
     /**
      * @brief Construct from Grid
      *
@@ -65,9 +67,14 @@ class Elliptic
      * @note chi is assumed 1 per default
      */
     Elliptic( const Geometry& g, norm no = not_normed, direction dir = forward, double jfactor=1.): 
-        no_(no), jfactor_(jfactor)
     { 
-        construct( g, g.bcx(), g.bcy(), dir);
+        construct( g, g.bcx(), g.bcy(), no, dir, forward, jfactor);
+    }
+
+    ///@copydoc Elliptic::construct()
+    Elliptic( const Geometry& g, bc bcx, bc bcy, norm no = not_normed, direction dir = forward, double jfactor=1.): 
+    { 
+        construct( g, bcx, bcy, no, dir, jfactor);
     }
 
     /**
@@ -77,13 +84,25 @@ class Elliptic
      * @param bcy boundary contition in y
      * @param no Not normed for elliptic equations, normed else
      * @param dir Direction of the right first derivative (i.e. forward, backward or centered)
-
      * @param jfactor scale jump terms (1 is a good value but in some cases 0.1 or 0.01 might be better)
      */
-    Elliptic( const Geometry& g, bc bcx, bc bcy, norm no = not_normed, direction dir = forward, double jfactor=1.): 
-        no_(no), jfactor_(jfactor)
-    { 
-        construct( g, bcx, bcy, dir);
+    void construct( const Geometry& g, bc bcx, bc bcy, norm no = not_normed, direction dir = forward, double jfactor = 1.)
+    {
+        no_=no, jfactor_=jfactor;
+        dg::blas2::transfer( dg::create::dx( g, inverse( bcx), inverse(dir)), leftx);
+        dg::blas2::transfer( dg::create::dy( g, inverse( bcy), inverse(dir)), lefty);
+        dg::blas2::transfer( dg::create::dx( g, bcx, dir), rightx);
+        dg::blas2::transfer( dg::create::dy( g, bcy, dir), righty);
+        dg::blas2::transfer( dg::create::jumpX( g, bcx),   jumpX);
+        dg::blas2::transfer( dg::create::jumpY( g, bcy),   jumpY);
+
+        dg::blas1::transfer( dg::create::volume(g),        weights_);
+        dg::blas1::transfer( dg::create::inv_weights(g),   precond_); //weights are better preconditioners than volume
+        tempx = tempy = gradx = weights_;
+        chi_=g.metric();
+        vol_=dg::tensor::volume(chi_);
+        dg::tensor::scal( chi_, vol_);
+        dg::blas1::transfer( dg::create::weights(g), weights_wo_vol);
     }
 
     /**
@@ -162,24 +181,8 @@ class Elliptic
             dg::blas2::symv( weights_wo_vol, y, y);
 
     }
-    private:
-    void construct( const Geometry& g, bc bcx, bc bcy, direction dir)
-    {
-        dg::blas2::transfer( dg::create::dx( g, inverse( bcx), inverse(dir)), leftx);
-        dg::blas2::transfer( dg::create::dy( g, inverse( bcy), inverse(dir)), lefty);
-        dg::blas2::transfer( dg::create::dx( g, bcx, dir), rightx);
-        dg::blas2::transfer( dg::create::dy( g, bcy, dir), righty);
-        dg::blas2::transfer( dg::create::jumpX( g, bcx),   jumpX);
-        dg::blas2::transfer( dg::create::jumpY( g, bcy),   jumpY);
 
-        dg::blas1::transfer( dg::create::volume(g),        weights_);
-        dg::blas1::transfer( dg::create::inv_weights(g),   precond_); //weights are better preconditioners than volume
-        tempx = tempy = gradx = weights_;
-        chi_=g.metric();
-        vol_=dg::tensor::volume(chi_);
-        dg::tensor::scal( chi_, vol_);
-        dg::blas1::transfer( dg::create::weights(g), weights_wo_vol);
-    }
+    private:
     bc inverse( bc bound)
     {
         if( bound == DIR) return NEU;
diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
new file mode 100644
index 000000000..a1b179ce1
--- /dev/null
+++ b/inc/dg/multigrid.h
@@ -0,0 +1,44 @@
+#pragma once
+
+#include "backend/fast_interpolation.h"
+#include "backend/exceptions.h"
+#include "backend/memory.h"
+#include "blas.h"
+
+namespace dg
+{
+
+
+template< class Geometry, class Matrix, class container> 
+struct Multigrid2d
+{
+    Multigrid2d( const Geometry& grid, unsigned stages ) {
+        if( stages < 2 ) throw Error( Message(_ping_)<<" There must be minimum 2 stages in a multigrid solver! You gave " << stages);
+        grids_.resize( stages );
+        grids_[0].reset( grid);
+        for( unsigned u=1; u<stages; u++)
+        {
+            grids_[u] = grids_[u-1]; //deep copy
+            grids_[u].get().multiplyCellNumbers(0.5,0.5);
+        }
+        inter_.resize(stages-1)
+        project_.resize( stages-1);
+        for( unsigned u=0; u<stages-1; u++)
+        {
+            project_[u] = dg::create::fast_projection( grids_[u], 2,2);
+            inter_[u] = dg::create::fast_interpolation(grids_[u+1],2,2);
+        }
+
+        cg_.resize( stages);
+    }
+
+
+
+    private:
+    std::vector< Handle< Geometry> > grids_;
+    std::vector< MultiMatrix<Matrix, container>  >  inter_;
+    std::vector< MultiMatrix<Matrix, container>  >  project_;
+    std::vector< CG<container> > cg_;
+};
+
+}//namespace dg
-- 
GitLab


From 330ec087eaf53676ca9e3503e64e3de0b2997d3c Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 7 Sep 2017 20:37:44 +0200
Subject: [PATCH 260/453] replace weights() function with inv_weights

---
 inc/dg/cg.h            | 35 +++--------------------------------
 inc/dg/elliptic.h      | 38 ++++++++++++++++++++------------------
 inc/dg/elliptic2d_b.cu |  4 ++--
 inc/dg/helmholtz.h     |  8 ++++----
 4 files changed, 29 insertions(+), 56 deletions(-)

diff --git a/inc/dg/cg.h b/inc/dg/cg.h
index 8ee712520..75bccd100 100644
--- a/inc/dg/cg.h
+++ b/inc/dg/cg.h
@@ -406,9 +406,7 @@ struct Invert
     template< class SymmetricOp >
     unsigned operator()( SymmetricOp& op, container& phi, const container& rho)
     {
-        container inv_weights( op.weights());
-        dg::blas1::transform( inv_weights, inv_weights, dg::INVERT<double>());
-        return this->operator()(op, phi, rho, op.weights(), op.precond(), inv_weights);
+        return this->operator()(op, phi, rho, op.inv_weights(), op.precond());
     }
 
     /**
@@ -430,34 +428,7 @@ struct Invert
      * @return number of iterations used 
      */
     template< class SymmetricOp, class Preconditioner >
-    unsigned operator()( SymmetricOp& op, container& phi, const container& rho, const container& w, Preconditioner& p)
-    {
-        container inv_weights( w);
-        dg::blas1::transform( inv_weights, inv_weights, dg::INVERT<double>());
-        return this->operator()(op, phi, rho, w, p, inv_weights);
-    }
-
-    /**
-     * @brief Solve linear problem
-     *
-     * Solves the Equation \f[ \hat O \phi = W\rho \f] using a preconditioned 
-     * conjugate gradient method. The initial guess comes from an extrapolation 
-     * of the last solutions.
-     * @copydoc hide_symmetric_op
-     * @tparam Preconditioner A type for which the blas2::symv(Matrix&, Vector1&, Vector2&) function is callable. 
-     * @param op symmetric Matrix operator
-     * @param phi solution (write only)
-     * @param rho right-hand-side
-     * @param w The weights that made the operator symmetric
-     * @param p The preconditioner  
-     * @param inv_weights The inverse weights used to compute the scalar product in the CG solver
-     * @note Calls the most general form of the CG solver with SquareNorm being the container class
-     * @note If the Macro DG_BENCHMARK is defined this function will write timings to std::cout
-     *
-     * @return number of iterations used 
-     */
-    template< class SymmetricOp, class Preconditioner >
-    unsigned operator()( SymmetricOp& op, container& phi, const container& rho, const container& w, Preconditioner& p, const container& inv_weights)
+    unsigned operator()( SymmetricOp& op, container& phi, const container& rho, const container& inv_weights, Preconditioner& p)
     {
         assert( phi.size() != 0);
         assert( &rho != &phi);
@@ -471,7 +442,7 @@ struct Invert
         unsigned number;
         if( multiplyWeights_ ) 
         {
-            dg::blas2::symv( w, rho, phi2);
+            dg::blas1::pointwiseDivide( rho, inv_weights, phi2);
             number = cg( op, phi, phi2, p, inv_weights, eps_, nrmb_correction_);
         }
         else
diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index 197e8c8a9..e3748a2f6 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -96,9 +96,9 @@ class Elliptic
         dg::blas2::transfer( dg::create::jumpX( g, bcx),   jumpX);
         dg::blas2::transfer( dg::create::jumpY( g, bcy),   jumpY);
 
-        dg::blas1::transfer( dg::create::volume(g),        weights_);
-        dg::blas1::transfer( dg::create::inv_weights(g),   precond_); //weights are better preconditioners than volume
-        tempx = tempy = gradx = weights_;
+        dg::blas1::transfer( dg::create::inv_volume(g),    inv_weights_);
+        dg::blas1::transfer( dg::create::inv_weights(g),   precond_); 
+        tempx = tempy = gradx = inv_weights_;
         chi_=g.metric();
         vol_=dg::tensor::volume(chi_);
         dg::tensor::scal( chi_, vol_);
@@ -116,10 +116,12 @@ class Elliptic
         if( !chi_old_.isSet()) 
         {
             dg::tensor::scal( chi_, chi);
+            dg::blas1::pointwiseDivide( precond_, chi, precond_);
             chi_old_.value() = chi;
             return;
         }
-        dg::blas1::pointwiseDivide(chi, chi_old_.value(), tempx);
+        dg::blas1::pointwiseDivide( chi, chi_old_.value(), tempx);
+        dg::blas1::pointwiseDivide( precond_, tempx, precond_);
         dg::tensor::scal( chi_, tempx);
         chi_old_.value()=chi;
     }
@@ -130,7 +132,7 @@ class Elliptic
      * i.e. the volume form 
      * @return weights (volume form including dG weights)
      */
-    const container& weights()const {return weights_;}
+    const container& inv_weights()const {return inv_weights_;}
     /**
      * @brief Returns the default preconditioner to use in conjugate gradient
      *
@@ -198,7 +200,7 @@ class Elliptic
         return centered;
     }
     Matrix leftx, lefty, rightx, righty, jumpX, jumpY;
-    container weights_, precond_, weights_wo_vol; 
+    container inv_weights_, precond_, weights_wo_vol; 
     container tempx, tempy, gradx;
     norm no_;
     SparseTensor<container> chi_;
@@ -248,7 +250,7 @@ struct GeneralElliptic
         rightz( dg::create::dz( g, g.bcz(), dir)),
         jumpX ( dg::create::jumpX( g, g.bcx())),
         jumpY ( dg::create::jumpY( g, g.bcy())),
-        weights_(dg::create::volume(g)), precond_(dg::create::inv_weights(g)), 
+        inv_weights_(dg::create::inv_volume(g)), precond_(dg::create::inv_weights(g)), 
         xchi( dg::evaluate( one, g) ), ychi( xchi), zchi( xchi), 
         xx(xchi), yy(xx), zz(xx), temp0( xx), temp1(temp0),
         no_(no)
@@ -276,7 +278,7 @@ struct GeneralElliptic
         rightz( dg::create::dz( g, bcz, dir)),
         jumpX ( dg::create::jumpX( g, bcx)),
         jumpY ( dg::create::jumpY( g, bcy)),
-        weights_(dg::create::volume(g)), precond_(dg::create::inv_weights(g)), 
+        inv_weights_(dg::create::inv_volume(g)), precond_(dg::create::inv_weights(g)), 
         xchi( dg::evaluate( one, g) ), ychi( xchi), zchi( xchi), 
         xx(xchi), yy(xx), zz(xx), temp0( xx), temp1(temp0),
         no_(no)
@@ -331,7 +333,7 @@ struct GeneralElliptic
      * in this case the volume element
      * @return weights (the volume element including dG weights)
      */
-    const container& weights()const {return weights_;}
+    const container& inv_weights()const {return inv_weights_;}
     /**
      * @brief Returns the preconditioner to use in conjugate gradient
      *
@@ -383,7 +385,7 @@ struct GeneralElliptic
         if( no_==not_normed)//multiply weights w/o volume
         {
             dg::tensor::pointwiseDivide( y, vol_, y);
-            dg::blas2::symv( weights_, y, y);
+            dg::blas1::pointwiseDivide( y, inv_weights_, y);
         }
     }
     private:
@@ -402,7 +404,7 @@ struct GeneralElliptic
         return centered;
     }
     Matrix leftx, lefty, leftz, rightx, righty, rightz, jumpX, jumpY;
-    container weights_, precond_; //contain coeffs for chi multiplication
+    container inv_weights_, precond_; //contain coeffs for chi multiplication
     container xchi, ychi, zchi, xx, yy, zz, temp0, temp1;
     norm no_;
     SparseElement<container> vol_;
@@ -443,7 +445,7 @@ struct GeneralEllipticSym
      */
     GeneralEllipticSym( const Geometry& g, norm no = not_normed, direction dir = forward): 
         ellipticForward_( g, no, dir), ellipticBackward_(g,no,inverse(dir)),
-        weights_(dg::create::volume(g)), precond_(dg::create::inv_weights(g)), 
+        inv_weights_(dg::create::inv_volume(g)), precond_(dg::create::inv_weights(g)), 
         temp_( dg::evaluate( one, g) )
     { }
 
@@ -459,7 +461,7 @@ struct GeneralEllipticSym
      */
     GeneralEllipticSym( const Geometry& g, bc bcx, bc bcy,bc bcz, norm no = not_normed, direction dir = forward): 
         ellipticForward_( g, bcx, bcy, no, dir), ellipticBackward_(g,bcx,bcy,no,inverse(dir)),
-        weights_(dg::create::volume(g)), precond_(dg::create::inv_weights(g)), 
+        inv_weights_(dg::create::inv_volume(g)), precond_(dg::create::inv_weights(g)), 
         temp_( dg::evaluate( one, g) )
     { 
     }
@@ -510,7 +512,7 @@ struct GeneralEllipticSym
      *
      * @return weights
      */
-    const container& weights()const {return weights_;}
+    const container& inv_weights()const {return inv_weights_;}
     /**
      * @brief Returns the preconditioner to use in conjugate gradient
      *
@@ -539,7 +541,7 @@ struct GeneralEllipticSym
         return centered;
     }
     dg::GeneralElliptic<Geometry, Matrix, container> ellipticForward_, ellipticBackward_;
-    container weights_, precond_; //contain coeffs for chi multiplication
+    container inv_weights_, precond_; //contain coeffs for chi multiplication
     container temp_;
 };
 
@@ -628,7 +630,7 @@ struct TensorElliptic
      *
      * @return weights
      */
-    const container& weights()const {return weights_;}
+    const container& inv_weights()const {return inv_weights_;}
     /**
      * @brief Returns the preconditioner to use in conjugate gradient
      *
@@ -679,7 +681,7 @@ struct TensorElliptic
         dg::blas2::transfer( dg::create::dy( g, bcy, dir), righty);
         dg::blas2::transfer( dg::create::jumpX( g, bcx),   jumpX);
         dg::blas2::transfer( dg::create::jumpY( g, bcy),   jumpY);
-        dg::blas1::transfer( dg::create::volume(g),        weights_);
+        dg::blas1::transfer( dg::create::inv_volume(g),    inv_weights_);
         dg::blas1::transfer( dg::create::inv_weights(g),   precond_); //weights are better preconditioners than volume
         dg::blas1::transfer( dg::evaluate( dg::one, g),    chixx_);
         dg::blas1::transfer( dg::evaluate( dg::zero,g),    chixy_);
@@ -707,7 +709,7 @@ struct TensorElliptic
         return centered;
     }
     Matrix leftx, lefty, rightx, righty, jumpX, jumpY;
-    container weights_, weights_wo_vol, precond_; //contain coeffs for chi multiplication
+    container inv_weights_, weights_wo_vol, precond_; //contain coeffs for chi multiplication
     container chixx_, chixy_, chiyy_, tempx_, tempy_, gradx_;
     SparseElement<container> vol_;
     norm no_;
diff --git a/inc/dg/elliptic2d_b.cu b/inc/dg/elliptic2d_b.cu
index 3b5aa1815..1af1eb322 100644
--- a/inc/dg/elliptic2d_b.cu
+++ b/inc/dg/elliptic2d_b.cu
@@ -140,7 +140,7 @@ int main()
     pol_forward.set_chi( chi);
     x = temp;
     dg::Invert<dg::DVec > invert_fw( x, n*n*Nx*Ny, eps);
-    std::cout << " "<< invert_fw( pol_forward, x, b, w2d, chi_inv, v2d);
+    std::cout << " "<< invert_fw( pol_forward, x, b, v2d, chi_inv);
     dg::blas1::axpby( 1.,x,-1., solution, error);
     err = dg::blas2::dot( w2d, error);
     std::cout << " "<<sqrt( err/norm);
@@ -151,7 +151,7 @@ int main()
     pol_backward.set_chi( chi);
     x = temp;
     dg::Invert<dg::DVec > invert_bw( x, n*n*Nx*Ny, eps);
-    std::cout << " "<< invert_bw( pol_backward, x, b, w2d, chi_inv, v2d);
+    std::cout << " "<< invert_bw( pol_backward, x, b, v2d, chi_inv);
     dg::blas1::axpby( 1.,x,-1., solution, error);
     err = dg::blas2::dot( w2d, error);
     std::cout << " "<<sqrt( err/norm)<<std::endl;
diff --git a/inc/dg/helmholtz.h b/inc/dg/helmholtz.h
index faa6d41d9..7e546a89f 100644
--- a/inc/dg/helmholtz.h
+++ b/inc/dg/helmholtz.h
@@ -74,7 +74,7 @@ struct Helmholtz
             blas2::symv( laplaceM_, x, y);
 
         blas1::axpby( 1., temp_, -alpha_, y);
-        blas1::pointwiseDot( laplaceM_.weights(), y, y);
+        blas1::pointwiseDivide(y, laplaceM_.inv_weights(), y);
 
     }
     /**
@@ -82,7 +82,7 @@ struct Helmholtz
      *
      * @return weights
      */
-    const container& weights()const {return laplaceM_.weights();}
+    const container& inv_weights()const {return laplaceM_.inv_weights();}
     /**
      * @brief container to use in conjugate gradient solvers
      *
@@ -194,14 +194,14 @@ struct Helmholtz2
         tensor::pointwiseDot( chi_, x, y); //y = chi*x
         blas1::axpby( 1., y, -2.*alpha_, temp1_, y); 
         blas1::axpby( alpha_*alpha_, temp2_, 1., y, y);
-        blas1::pointwiseDot( laplaceM_.weights(), y, y);//Helmholtz is never normed
+        blas1::pointwiseDivide( y, laplaceM_.inv_weights(), y);//Helmholtz is never normed
     }
     /**
      * @brief These are the weights that made the operator symmetric
      *
      * @return weights
      */
-    const container& weights()const {return laplaceM_.weights();}
+    const container& inv_weights()const {return laplaceM_.inv_weights();}
     /**
      * @brief container to use in conjugate gradient solvers
      *
-- 
GitLab


From a1f4fd7491ed6ab64da86ea9b9e409688a2b605b Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 7 Sep 2017 22:53:24 +0200
Subject: [PATCH 261/453] debugging stuff

---
 inc/dg/Makefile                               |  2 +-
 inc/dg/backend/sparseblockmat.cuh             | 12 +++
 inc/dg/backend/sparseblockmat_gpu_kernels.cuh |  4 -
 inc/dg/backend/sparseblockmat_omp_kernels.h   | 21 ++---
 inc/dg/cg2d_b.cu                              |  2 +-
 inc/dg/elliptic.h                             | 84 ++++++------------
 inc/dg/elliptic2d_b.cu                        | 66 +++-----------
 inc/dg/helmholtz.h                            | 11 +--
 inc/dg/helmholtz_t.cu                         |  8 +-
 inc/dg/multigrid.h                            | 87 +++++++++++++++++--
 inc/dg/multistep.h                            |  2 +-
 11 files changed, 154 insertions(+), 145 deletions(-)

diff --git a/inc/dg/Makefile b/inc/dg/Makefile
index 63c041708..87b57e5bb 100644
--- a/inc/dg/Makefile
+++ b/inc/dg/Makefile
@@ -24,7 +24,7 @@ all: $(CPPFILES:%.cpp=%) $(CUFILES:%.cu=%)
 	$(CC) $(OPT) $(INCLUDE) -DDG_DEBUG $(CFLAGS) $< -o $@ 
 
 %_b: %_b.cu 
-	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) 
+	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) -g
 
 bathRZ_t: bathRZ_t.cu 
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(GLFLAGS) $(INCLUDE) 
diff --git a/inc/dg/backend/sparseblockmat.cuh b/inc/dg/backend/sparseblockmat.cuh
index 0bba470c2..bf5c79bf2 100644
--- a/inc/dg/backend/sparseblockmat.cuh
+++ b/inc/dg/backend/sparseblockmat.cuh
@@ -183,12 +183,24 @@ template<class value_type>
 template<class DeviceContainer>
 inline void EllSparseBlockMatDevice<value_type>::symv( value_type alpha, const DeviceContainer& x, value_type beta, DeviceContainer& y) const
 {
+    if( y.size() != (unsigned)num_rows*n*left_size*right_size) {
+        throw Error( Message(_ping_)<<"y has the wrong size "<<y.size()<<" and not "<<(unsigned)num_rows*n*left_size*right_size);
+    }
+    if( x.size() != (unsigned)num_cols*n*left_size*right_size) {
+        throw Error( Message(_ping_)<<"x has the wrong size "<<x.size()<<" and not "<<(unsigned)num_cols*n*left_size*right_size);
+    }
     launch_multiply_kernel( alpha, x, beta, y);
 }
 template<class value_type>
 template<class DeviceContainer>
 inline void CooSparseBlockMatDevice<value_type>::symv( value_type alpha, const DeviceContainer& x, value_type beta, DeviceContainer& y) const
 {
+    if( y.size() != (unsigned)num_rows*n*left_size*right_size) {
+        throw Error( Message(_ping_)<<"y has the wrong size "<<y.size()<<" and not "<<(unsigned)num_rows*n*left_size*right_size);
+    }
+    if( x.size() != (unsigned)num_cols*n*left_size*right_size) {
+        throw Error( Message(_ping_)<<"x has the wrong size "<<x.size()<<" and not "<<(unsigned)num_cols*n*left_size*right_size);
+    }
     launch_multiply_kernel(alpha, x, beta, y);
 }
 
diff --git a/inc/dg/backend/sparseblockmat_gpu_kernels.cuh b/inc/dg/backend/sparseblockmat_gpu_kernels.cuh
index 7bbd23853..56a81d525 100644
--- a/inc/dg/backend/sparseblockmat_gpu_kernels.cuh
+++ b/inc/dg/backend/sparseblockmat_gpu_kernels.cuh
@@ -234,8 +234,6 @@ template<class value_type>
 template<class DeviceContainer>
 void EllSparseBlockMatDevice<value_type>::launch_multiply_kernel(value_type alpha, const DeviceContainer& x, value_type beta, DeviceContainer& y) const
 {
-    assert( y.size() == (unsigned)num_rows*n*left_size*right_size);
-    assert( x.size() == (unsigned)num_cols*n*left_size*right_size);
     //set up kernel parameters
     const size_t BLOCK_SIZE = 256; 
     const size_t size = (left_size)*(right_range[1]-right_range[0])*num_rows*n; //number of lines
@@ -276,8 +274,6 @@ template<class value_type>
 template<class DeviceContainer>
 void CooSparseBlockMatDevice<value_type>::launch_multiply_kernel( value_type alpha, const DeviceContainer& x, value_type beta, DeviceContainer& y) const
 {
-    assert( y.size() == (unsigned)num_rows*n*left_size*right_size);
-    assert( x.size() == (unsigned)num_cols*n*left_size*right_size);
     //set up kernel parameters
     const size_t BLOCK_SIZE = 256; 
     const size_t size = left_size*right_size*n;
diff --git a/inc/dg/backend/sparseblockmat_omp_kernels.h b/inc/dg/backend/sparseblockmat_omp_kernels.h
index 1e9b6bf11..fb1f46e2b 100644
--- a/inc/dg/backend/sparseblockmat_omp_kernels.h
+++ b/inc/dg/backend/sparseblockmat_omp_kernels.h
@@ -207,8 +207,8 @@ void ell_multiply_kernel32( value_type alpha, value_type beta,
             //if( cols_idx[i*2+d] != i+d-1) backward = false;
             //if( cols_idx[i*2+d] != i+d) forward = false;
         }
-    int diff = -1;
-    if(forward ) diff = 0;
+    //int diff = -1;
+    //if(forward ) diff = 0;
     if( forward || backward )
     {
 #pragma omp parallel for 
@@ -404,8 +404,8 @@ void ell_multiply_kernel32x( value_type alpha, value_type beta,
             //if( cols_idx[i*2+d] != i+d-1) backward = false;
             //if( cols_idx[i*2+d] != i+d) forward = false;
         }
-    int diff = -1;
-    if(forward ) {diff = 0; }
+    //int diff = -1;
+    //if(forward ) {diff = 0; }
     if( forward || backward )
     {
 #pragma omp parallel for
@@ -434,7 +434,7 @@ void ell_multiply_kernel32x( value_type alpha, value_type beta,
         //int J0 = (s*num_cols+i+0+diff)*3;
         //int J1 = (s*num_cols+i+1+diff)*3;
         int J0 = (s*num_cols+cols_idx[i*2+0])*3;
-        int J1 = (s*num_cols+cols_idx[i*2+0])*3;
+        int J1 = (s*num_cols+cols_idx[i*2+1])*3;
         for( int k=0; k<3; k++)
         {
             value_type temp = 0;
@@ -483,9 +483,6 @@ template<class value_type>
 template<class DeviceContainer>
 void EllSparseBlockMatDevice<value_type>::launch_multiply_kernel( value_type alpha, const DeviceContainer& x, value_type beta, DeviceContainer& y) const
 {
-    assert( y.size() == (unsigned)num_rows*n*left_size*right_size);
-    assert( x.size() == (unsigned)num_cols*n*left_size*right_size);
-
     const value_type* data_ptr = thrust::raw_pointer_cast( &data[0]);
     const int* cols_ptr = thrust::raw_pointer_cast( &cols_idx[0]);
     const int* block_ptr = thrust::raw_pointer_cast( &data_idx[0]);
@@ -521,8 +518,12 @@ template<class value_type>
 template<class DeviceContainer>
 void CooSparseBlockMatDevice<value_type>::launch_multiply_kernel( value_type alpha, const DeviceContainer& x, value_type beta, DeviceContainer& y) const
 {
-    assert( y.size() == (unsigned)num_rows*n*left_size*right_size);
-    assert( x.size() == (unsigned)num_cols*n*left_size*right_size);
+    if( y.size() != (unsigned)num_rows*n*left_size*right_size) {
+        throw Error( Message(_ping_)<<"y has the wrong size "<<y.size()<<" and not "<<(unsigned)num_rows*n*left_size*right_size);
+    }
+    if( x.size() != (unsigned)num_cols*n*left_size*right_size) {
+        throw Error( Message(_ping_)<<"x has the wrong size "<<x.size()<<" and not "<<(unsigned)num_cols*n*left_size*right_size);
+    }
 
     for( int i=0; i<num_entries; i++)
 #pragma omp parallel for collapse(3)
diff --git a/inc/dg/cg2d_b.cu b/inc/dg/cg2d_b.cu
index 3420b0ded..029aae777 100644
--- a/inc/dg/cg2d_b.cu
+++ b/inc/dg/cg2d_b.cu
@@ -54,7 +54,7 @@ int main()
     const dg::DVec deriv = dg::evaluate( derivative, grid);
     dg::DVec b = dg::evaluate ( laplace_fct, grid);
     //compute S b
-    dg::blas2::symv( w2d, b, b);
+    dg::blas1::pointwiseDivide( b, lap.inv_weights(), b);
     //////////////////////////////////////////////////////////////////////
     std::cout << "Computing on the Grid " <<n<<" x "<<Nx<<" x "<<Ny <<std::endl;
 
diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index e3748a2f6..b24cc6a35 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -66,13 +66,13 @@ class Elliptic
      * @param jfactor (\f$ = \alpha \f$ ) scale jump terms (1 is a good value but in some cases 0.1 or 0.01 might be better)
      * @note chi is assumed 1 per default
      */
-    Elliptic( const Geometry& g, norm no = not_normed, direction dir = forward, double jfactor=1.): 
+    Elliptic( const Geometry& g, norm no = not_normed, direction dir = forward, double jfactor=1.)
     { 
-        construct( g, g.bcx(), g.bcy(), no, dir, forward, jfactor);
+        construct( g, g.bcx(), g.bcy(), no, dir, jfactor);
     }
 
     ///@copydoc Elliptic::construct()
-    Elliptic( const Geometry& g, bc bcx, bc bcy, norm no = not_normed, direction dir = forward, double jfactor=1.): 
+    Elliptic( const Geometry& g, bc bcx, bc bcy, norm no = not_normed, direction dir = forward, double jfactor=1.)
     { 
         construct( g, bcx, bcy, no, dir, jfactor);
     }
@@ -105,6 +105,11 @@ class Elliptic
         dg::blas1::transfer( dg::create::weights(g), weights_wo_vol);
     }
 
+    ///@copydoc  Elliptic::Elliptic(const Geometry&,norm,direction,double)
+    void construct( const Geometry& g, norm no = not_normed, direction dir = forward, double jfactor = 1.){
+        construct( g, g.bcx(), g.bcy(), no, dir, jfactor);
+    }
+
     /**
      * @brief Change Chi 
      *
@@ -129,17 +134,16 @@ class Elliptic
     /**
      * @brief Returns the weights used to make the matrix symmetric 
      *
-     * i.e. the volume form 
-     * @return weights (volume form including dG weights)
+     * i.e. the inverse volume form 
+     * @return inverse volume form including weights 
      */
     const container& inv_weights()const {return inv_weights_;}
     /**
      * @brief Returns the default preconditioner to use in conjugate gradient
      *
-     * Currently returns the inverse of the weights without volume elment
-     * @return inverse weights (without volume form)
-     * @note a better preconditioner might be the inverse of \f$\chi\f$ especially 
-     * when \f$ \chi\f$ exhibits large amplitudes or variations
+     * Currently returns the inverse of the weights without volume elment multiplied by the inverse of \f$ \chi\f$. 
+     * This is especially good when \f$ \chi\f$ exhibits large amplitudes or variations
+     * @return the inverse of \f$\chi\f$.       
      */
     const container& precond()const {return precond_;}
     /**
@@ -288,7 +292,7 @@ struct GeneralElliptic
         dg::tensor::sqrt(vol_); //now we have volume element
     }
     /**
-     * @brief Set x-component of \f$ chi\f$
+     * @brief Set x-component of \f$ \chi\f$
      *
      * @param chi new x-component
      */
@@ -297,7 +301,7 @@ struct GeneralElliptic
         xchi = chi;
     }
     /**
-     * @brief Set y-component of \f$ chi\f$
+     * @brief Set y-component of \f$ \chi\f$
      *
      * @param chi new y-component
      */
@@ -306,7 +310,7 @@ struct GeneralElliptic
         ychi = chi;
     }
     /**
-     * @brief Set z-component of \f$ chi\f$
+     * @brief Set z-component of \f$ \chi\f$
      *
      * @param chi new z-component
      */
@@ -316,7 +320,7 @@ struct GeneralElliptic
     }
 
     /**
-     * @brief Set new components for \f$ chi\f$
+     * @brief Set new components for \f$ \chi\f$
      *
      * @param chi chi[0] is new x-component, chi[1] the new y-component, chi[2] z-component
      */
@@ -327,12 +331,7 @@ struct GeneralElliptic
         zchi = chi[2];
     }
 
-    /**
-     * @brief Returns the weights used to make the matrix symmetric 
-     *
-     * in this case the volume element
-     * @return weights (the volume element including dG weights)
-     */
+    ///@copydoc Elliptic::inv_weights()
     const container& inv_weights()const {return inv_weights_;}
     /**
      * @brief Returns the preconditioner to use in conjugate gradient
@@ -342,12 +341,7 @@ struct GeneralElliptic
      */
     const container& precond()const {return precond_;}
 
-    /**
-     * @brief Computes the polarisation term
-     *
-     * @param x left-hand-side
-     * @param y result
-     */
+    ///@copydoc Elliptic::symv()
     void symv( container& x, container& y) 
     {
         dg::blas2::gemv( rightx, x, temp0); //R_x*x 
@@ -507,26 +501,12 @@ struct GeneralEllipticSym
         ellipticBackward_.set( chi);
     }
 
-    /**
-     * @brief Returns the weights used to make the matrix symmetric 
-     *
-     * @return weights
-     */
+    ///@copydoc Elliptic::inv_weights()
     const container& inv_weights()const {return inv_weights_;}
-    /**
-     * @brief Returns the preconditioner to use in conjugate gradient
-     *
-     * In this case inverse weights are the best choice
-     * @return inverse weights
-     */
+    ///@copydoc GeneralElliptic::precond()
     const container& precond()const {return precond_;}
 
-    /**
-     * @brief Computes the polarisation term
-     *
-     * @param x left-hand-side
-     * @param y result
-     */
+    ///@copydoc Elliptic::symv()
     void symv( container& x, container& y) 
     {
         ellipticForward_.symv( x,y);
@@ -625,26 +605,12 @@ struct TensorElliptic
         set( chixx_, chixy_, chiyy_);
     }
 
-    /**
-     * @brief Returns the weights used to make the matrix symmetric 
-     *
-     * @return weights
-     */
+    ///@copydoc Elliptic::inv_weights()
     const container& inv_weights()const {return inv_weights_;}
-    /**
-     * @brief Returns the preconditioner to use in conjugate gradient
-     *
-     * In this case inverse weights are the best choice
-     * @return inverse weights
-     */
+    ///@copydoc GeneralElliptic::precond()
     const container& precond()const {return precond_;}
 
-    /**
-     * @brief Computes the polarisation term
-     *
-     * @param x left-hand-side
-     * @param y result
-     */
+    ///@copydoc Elliptic::symv()
     void symv( container& x, container& y) 
     {
         //compute gradient
diff --git a/inc/dg/elliptic2d_b.cu b/inc/dg/elliptic2d_b.cu
index 1af1eb322..af6fe9e41 100644
--- a/inc/dg/elliptic2d_b.cu
+++ b/inc/dg/elliptic2d_b.cu
@@ -7,7 +7,7 @@
 
 
 #include "elliptic.h"
-#include "cg.h"
+#include "multigrid.h"
 #include "backend/timer.cuh"
 #include "backend/projection.cuh"
 
@@ -47,7 +47,7 @@ int main()
     std::cin >> eps >> jfactor;
     std::cout << "Computation on: "<< n <<" x "<<Nx<<" x "<<Ny<<std::endl;
     //std::cout << "# of 2d cells                 "<< Nx*Ny <<std::endl;
-    dg::Grid2d grid( 0, lx, 0, ly, n, Nx, Ny, bcx, bcy);
+    dg::CartesianGrid2d grid( 0, lx, 0, ly, n, Nx, Ny, bcx, bcy);
     dg::DVec w2d = dg::create::weights( grid);
     dg::DVec v2d = dg::create::inv_weights( grid);
     dg::DVec one = dg::evaluate( dg::one, grid);
@@ -64,63 +64,23 @@ int main()
     std::cout << "Create Polarisation object and set chi!\n";
     t.tic();
     {
-    std::vector<dg::Grid2d> g( 4, grid);
-    g[1].multiplyCellNumbers( 0.5, 0.5);
-    g[2].multiplyCellNumbers( 0.25, 0.25);
-    g[3].multiplyCellNumbers( 0.125, 0.125);
-    dg::IDMatrix project3 = dg::create::projection( g[3], g[0]);
-    dg::IDMatrix project2 = dg::create::projection( g[2], g[0]);
-    dg::IDMatrix project1 = dg::create::projection( g[1], g[0]);
-    dg::IDMatrix inter01 = dg::create::interpolation( g[0], g[1]);
-    dg::IDMatrix inter12 = dg::create::interpolation( g[1], g[2]);
-    dg::IDMatrix inter23 = dg::create::interpolation( g[2], g[3]);
-    std::vector<dg::DVec> w2d_(4, w2d), v2d_(4,v2d);
-    dg::Elliptic<dg::CartesianGrid2d, dg::DMatrix, dg::DVec>  pol0(g[0], dg::not_normed,dg::centered, jfactor);
-    dg::Elliptic<dg::CartesianGrid2d, dg::DMatrix, dg::DVec>  pol1(g[1], dg::not_normed,dg::centered, jfactor);
-    dg::Elliptic<dg::CartesianGrid2d, dg::DMatrix, dg::DVec>  pol2(g[2], dg::not_normed,dg::centered, jfactor);
-    dg::Elliptic<dg::CartesianGrid2d, dg::DMatrix, dg::DVec>  pol3(g[3], dg::not_normed,dg::centered, jfactor);
-    for( unsigned i=0; i<4; i++)
+    dg::MultigridCG2d<dg::aGeometry2d, dg::DMatrix, dg::DVec > multigrid( grid, 3);
+    std::vector<dg::DVec> chi_ = multigrid.project( chi);
+    std::vector<dg::Elliptic<dg::aGeometry2d, dg::DMatrix, dg::DVec> > multi_pol( 3);
+    for( unsigned u=0; u<3; u++)
     {
-        w2d_[i] = dg::create::weights( g[i]);
-        v2d_[i] = dg::create::inv_weights( g[i]);
+        multi_pol[u].construct( multigrid.grids()[u].get(), dg::not_normed, dg::centered, jfactor); 
+        multi_pol[u].set_chi( chi_[u]);
     }
-    std::vector<dg::DVec> chi_(w2d_), chi_inv_(chi_), b_(chi_), x_(chi_);
-    dg::blas2::symv( project3, chi, chi_[3]);
-    dg::blas2::symv( project2, chi, chi_[2]);
-    dg::blas2::symv( project1, chi, chi_[1]);
-    dg::blas2::symv( project3, chi_inv, chi_inv_[3]);
-    dg::blas2::symv( project2, chi_inv, chi_inv_[2]);
-    dg::blas2::symv( project1, chi_inv, chi_inv_[1]);
-    dg::blas2::symv( project3, b, b_[3]);
-    dg::blas2::symv( project2, b, b_[2]);
-    dg::blas2::symv( project1, b, b_[1]);
-    dg::blas2::symv( project3, x, x_[3]);
-    dg::blas2::symv( project2, x, x_[2]);
-    dg::blas2::symv( project1, x, x_[1]);
-    pol0.set_chi( chi);
-    pol1.set_chi( chi_[1]);
-    pol2.set_chi( chi_[2]);
-    pol3.set_chi( chi_[3]);
+
     t.toc();
     std::cout << "Creation of polarisation object took: "<<t.diff()<<"s\n";
-
-    std::vector<dg::CG<dg::DVec > > cg(4);
-    cg[0].construct( x, g[0].size());
-    for( unsigned i=1; i<4; i++)
-        cg[i].construct(x_[i], g[i].size());
     std::cout << eps<<" \n";
     t.tic();
-    //dg::blas2::symv( w2d_[3], b_[3], b_[3]); 
-    //std::cout << " # iterations grid "<< cg[3]( pol3, x_[3], b_[3], chi_inv_[3], v2d_[3], eps/10) << " \n";
-    //dg::blas2::symv( inter23, x_[3], x_[2]);
-    dg::blas2::symv( w2d_[2], b_[2], b_[2]); 
-    std::cout << " # iterations grid "<< cg[2]( pol2, x_[2], b_[2], chi_inv_[2], v2d_[2], eps/10) << " \n";
-    dg::blas2::symv( inter12, x_[2], x_[1]);
-    dg::blas2::symv( w2d_[1], b_[1], b_[1]); 
-    std::cout << " # iterations grid "<< cg[1]( pol1, x_[1], b_[1], chi_inv_[1], v2d_[1], eps/10) << " \n";
-    dg::blas2::symv( inter01, x_[1], x);
-    dg::blas2::symv( w2d, b, b_[0]); 
-    std::cout << " # iterations fine grid "<< cg[0]( pol0, x, b_[0], chi_inv, v2d, eps)<<std::endl;
+    std::vector<unsigned> number = multigrid.direct_solve( multi_pol, x, b, eps);
+    std::cout << " # iterations grid 2 "<< number[2] << " \n";
+    std::cout << " # iterations grid 1 "<< number[1] << " \n";
+    std::cout << " # iterations grid 0 "<< number[0] << " \n";
     t.toc();
     //std::cout << "Took "<<t.diff()<<"s\n";
     }
diff --git a/inc/dg/helmholtz.h b/inc/dg/helmholtz.h
index 7e546a89f..f23628e18 100644
--- a/inc/dg/helmholtz.h
+++ b/inc/dg/helmholtz.h
@@ -33,7 +33,7 @@ struct Helmholtz
      * @param alpha Scalar in the above formula
      * @param dir Direction of the Laplace operator
      * @param jfactor The jfactor used in the Laplace operator (probably 1 is always the best factor but one never knows...)
-     * @note The default value of \f$\chi\f$ is one
+     * @note The default value of \f$\chi\f$ is one. Helmholtz is never normed
      */
     Helmholtz( const Geometry& g, double alpha = 1., direction dir = dg::forward, double jfactor=1.):
         laplaceM_(g, normed, dir, jfactor), 
@@ -67,13 +67,14 @@ struct Helmholtz
      * @param y rhs contains solution
      * @note Takes care of sign in laplaceM and thus multiplies by -alpha
      */
-    void symv( container& x, container& y) 
+    void symv( const container& x, container& y) 
     {
-        tensor::pointwiseDot( chi_, x, temp_);
         if( alpha_ != 0)
             blas2::symv( laplaceM_, x, y);
-
-        blas1::axpby( 1., temp_, -alpha_, y);
+        if( chi_.isSet())
+            dg::blas1::pointwiseDot( 1., chi_.value(), x, -alpha_, y);
+        else
+            blas1::axpby( 1., x, -alpha_, y);
         blas1::pointwiseDivide(y, laplaceM_.inv_weights(), y);
 
     }
diff --git a/inc/dg/helmholtz_t.cu b/inc/dg/helmholtz_t.cu
index 2d8a41263..bd9b4da12 100644
--- a/inc/dg/helmholtz_t.cu
+++ b/inc/dg/helmholtz_t.cu
@@ -101,11 +101,13 @@ int main()
     dg::Helmholtz< dg::CylindricalGrid3d, dg::DMatrix, dg::DVec > helmholtz( g3d, alpha);
     dg::blas2::symv( laplaceM, fct_, temp_);
     dg::blas1::axpby( 1., laplace_fct_, -1., temp_);
-    std::cout << "error Laplace " << sqrt( dg::blas2::dot( laplaceM.weights(), temp_))<<" (Note the supraconvergence!)"<<std::endl;
+    dg::DVec w3d =  laplaceM.inv_weights();
+    dg::blas1::transform(w3d, w3d,dg::INVERT<double>());
+    std::cout << "error Laplace " << sqrt( dg::blas2::dot( w3d, temp_))<<" (Note the supraconvergence!)"<<std::endl;
     dg::blas2::symv( helmholtz, fct_, temp_);
-    dg::blas2::symv( helmholtz.precond(), temp_, temp_);
+    dg::blas1::pointwiseDot( helmholtz.inv_weights(), temp_, temp_);
     dg::blas1::axpby( 1., helmholtz_fct_, -1, temp_);
-    std::cout << "error " << sqrt( dg::blas2::dot( helmholtz.weights(), temp_))<<" (Note the supraconvergence!)"<<std::endl;
+    std::cout << "error " << sqrt( dg::blas2::dot( w3d, temp_))<<" (Note the supraconvergence!)"<<std::endl;
 
 
 
diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index a1b179ce1..af0316722 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -4,41 +4,112 @@
 #include "backend/exceptions.h"
 #include "backend/memory.h"
 #include "blas.h"
+#include "cg.h"
 
 namespace dg
 {
 
 
 template< class Geometry, class Matrix, class container> 
-struct Multigrid2d
+struct MultigridCG2d
 {
-    Multigrid2d( const Geometry& grid, unsigned stages ) {
+    MultigridCG2d( const Geometry& grid, unsigned stages, int extrapolation_type = 2 ) {
         if( stages < 2 ) throw Error( Message(_ping_)<<" There must be minimum 2 stages in a multigrid solver! You gave " << stages);
-        grids_.resize( stages );
+        grids_.resize( stages);
         grids_[0].reset( grid);
         for( unsigned u=1; u<stages; u++)
         {
             grids_[u] = grids_[u-1]; //deep copy
-            grids_[u].get().multiplyCellNumbers(0.5,0.5);
+            grids_[u].get().multiplyCellNumbers(0.5, 0.5);
         }
-        inter_.resize(stages-1)
+        inter_.resize(stages-1);
         project_.resize( stages-1);
         for( unsigned u=0; u<stages-1; u++)
         {
-            project_[u] = dg::create::fast_projection( grids_[u], 2,2);
-            inter_[u] = dg::create::fast_interpolation(grids_[u+1],2,2);
+            project_[u] = dg::create::fast_projection( grids_[u].get(), 2,2);
+            inter_[u] = dg::create::fast_interpolation(grids_[u+1].get(),2,2);
         }
 
         cg_.resize( stages);
+        dg::blas1::transfer( dg::evaluate( dg::zero, grid), x0_);
+        x1_=x0_, x2_=x0_;
+        x_ =  project( x0_), r_ = x_, b_ = x_;
+        set_extrapolationType(extrapolation_type);
     }
 
+    template<class SymmetricOp>
+    std::vector<unsigned> direct_solve( std::vector<SymmetricOp>& op, container&  x, const container& b, double eps)
+    {
+        dg::blas1::axpbygz( alpha[0], x0_, alpha[1], x1_, alpha[2], x2_); 
+        x_[0].swap(x2_);
+        project( x_[0], x_);
+        dg::blas1::pointwiseDivide( b, op[0].inv_weights(), b_[0]);
+        project( b_[0], b_);
+        std::vector<unsigned> number(stages_);
+        //for( unsigned u=stages_-1; u>0; u--)
+        //{
+        //    number[u] = cg_[u]( op[u], x_[u], b_[u], op[u].precond(), op[u].inv_weights(), eps, 1.);
+        //    dg::blas2::symv( inter_[u-1], x_[u], x_[u-1]);
+        //}
+        number[0] = cg_[0]( op[0], x, b_[0], op[0].precond(), op[0].inv_weights(), eps);
+        x1_.swap( x2_);
+        x0_.swap( x1_);
+        
+        blas1::copy( x, x0_);
+        return number;
+    }
+
+    ///src may alias first element of out
+    void project( const container& src, std::vector<container>& out)
+    {
+        dg::blas1::copy( src, out[0]);
+        for( unsigned u=0; u<grids_.size()-1; u++)
+            dg::blas2::gemv( project_[u], out[u], out[u+1]);
+    }
+
+    std::vector<container> project( const container& src)
+    {
+        std::vector<container> out( grids_.size());
+        for( unsigned u=0; u<grids_.size(); u++)
+            dg::blas1::transfer( dg::evaluate( dg::zero, grids_[u].get()), out[u]);
+        project( src, out);
+        return out;
+
+    }
+    unsigned stages()const{return stages_;}
+    /**
+     * @brief Set the extrapolation Type for following inversions
+     *
+     * @param extrapolationType number of last values to use for next extrapolation of initial guess
+     */
+    void set_extrapolationType( int extrapolationType)
+    {
+        assert( extrapolationType <= 3 && extrapolationType >= 0);
+        switch(extrapolationType)
+        {
+            case(0): alpha[0] = 0, alpha[1] = 0, alpha[2] = 0;
+                     break;
+            case(1): alpha[0] = 1, alpha[1] = 0, alpha[2] = 0;
+                     break;
+            case(2): alpha[0] = 2, alpha[1] = -1, alpha[2] = 0;
+                     break;
+            case(3): alpha[0] = 3, alpha[1] = -3, alpha[2] = 1;
+                     break;
+            default: alpha[0] = 2, alpha[1] = -1, alpha[2] = 0;
+        }
+    }
 
+    const std::vector<dg::Handle< Geometry > > grids()const{return grids_;}
 
     private:
-    std::vector< Handle< Geometry> > grids_;
+    unsigned stages_;
+    std::vector< dg::Handle< Geometry> > grids_;
     std::vector< MultiMatrix<Matrix, container>  >  inter_;
     std::vector< MultiMatrix<Matrix, container>  >  project_;
     std::vector< CG<container> > cg_;
+    std::vector< container> x_, r_, b_; 
+    container x0_, x1_, x2_;
+    double alpha[3];
 };
 
 }//namespace dg
diff --git a/inc/dg/multistep.h b/inc/dg/multistep.h
index 64a74d5d0..7374adf84 100644
--- a/inc/dg/multistep.h
+++ b/inc/dg/multistep.h
@@ -329,7 +329,7 @@ void Karniadakis<container>::operator()( Functor& f, Diffusion& diff, container&
     //double alpha[2] = {1., 0.};
     blas1::axpby( alpha[0], u_[1], alpha[1],  u_[2], u_[0]); //extrapolate previous solutions
     blas2::symv( diff.weights(), u, u);
-    detail::Implicit<Diffusion, Vector> implicit( -dt_/11.*6., diff, f_[0]);
+    detail::Implicit<Diffusion, container> implicit( -dt_/11.*6., diff, f_[0]);
 #ifdef DG_BENCHMARK
 #ifdef MPI_VERSION
     int rank;
-- 
GitLab


From 61b13f536bdc3ebdab1862eb62b0914e2fce94a0 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 7 Sep 2017 23:27:25 +0200
Subject: [PATCH 262/453] multigrid computes but wrong

---
 inc/dg/Makefile        |  2 +-
 inc/dg/elliptic2d_b.cu | 15 +++++++--------
 inc/dg/multigrid.h     | 31 +++++++++++++++++++------------
 3 files changed, 27 insertions(+), 21 deletions(-)

diff --git a/inc/dg/Makefile b/inc/dg/Makefile
index 87b57e5bb..216eb230b 100644
--- a/inc/dg/Makefile
+++ b/inc/dg/Makefile
@@ -24,7 +24,7 @@ all: $(CPPFILES:%.cpp=%) $(CUFILES:%.cu=%)
 	$(CC) $(OPT) $(INCLUDE) -DDG_DEBUG $(CFLAGS) $< -o $@ 
 
 %_b: %_b.cu 
-	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) -g
+	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) -g 
 
 bathRZ_t: bathRZ_t.cu 
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(GLFLAGS) $(INCLUDE) 
diff --git a/inc/dg/elliptic2d_b.cu b/inc/dg/elliptic2d_b.cu
index af6fe9e41..d9e19f13a 100644
--- a/inc/dg/elliptic2d_b.cu
+++ b/inc/dg/elliptic2d_b.cu
@@ -11,8 +11,6 @@
 #include "backend/timer.cuh"
 #include "backend/projection.cuh"
 
-//NOTE: IF DEVICE=CPU THEN THE POLARISATION ASSEMBLY IS NOT PARALLEL AS IT IS NOW 
-
 //global relative error in L2 norm is O(h^P)
 //as a rule of thumb with n=4 the true error is err = 1e-3 * eps as long as eps > 1e3*err
 
@@ -64,10 +62,11 @@ int main()
     std::cout << "Create Polarisation object and set chi!\n";
     t.tic();
     {
-    dg::MultigridCG2d<dg::aGeometry2d, dg::DMatrix, dg::DVec > multigrid( grid, 3);
+    unsigned stages=2;
+    dg::MultigridCG2d<dg::aGeometry2d, dg::DMatrix, dg::DVec > multigrid( grid, stages);
     std::vector<dg::DVec> chi_ = multigrid.project( chi);
-    std::vector<dg::Elliptic<dg::aGeometry2d, dg::DMatrix, dg::DVec> > multi_pol( 3);
-    for( unsigned u=0; u<3; u++)
+    std::vector<dg::Elliptic<dg::aGeometry2d, dg::DMatrix, dg::DVec> > multi_pol( stages);
+    for( unsigned u=0; u<stages; u++)
     {
         multi_pol[u].construct( multigrid.grids()[u].get(), dg::not_normed, dg::centered, jfactor); 
         multi_pol[u].set_chi( chi_[u]);
@@ -78,9 +77,9 @@ int main()
     std::cout << eps<<" \n";
     t.tic();
     std::vector<unsigned> number = multigrid.direct_solve( multi_pol, x, b, eps);
-    std::cout << " # iterations grid 2 "<< number[2] << " \n";
-    std::cout << " # iterations grid 1 "<< number[1] << " \n";
-    std::cout << " # iterations grid 0 "<< number[0] << " \n";
+    for( unsigned u=0; u<number.size(); u++)
+        std::cout << " # iterations grid "<<number.size()-1-u<<" "<<number[number.size()-1-u] << " \n";
+
     t.toc();
     //std::cout << "Took "<<t.diff()<<"s\n";
     }
diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index af0316722..8931033e1 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -14,6 +14,7 @@ template< class Geometry, class Matrix, class container>
 struct MultigridCG2d
 {
     MultigridCG2d( const Geometry& grid, unsigned stages, int extrapolation_type = 2 ) {
+        stages_=stages;
         if( stages < 2 ) throw Error( Message(_ping_)<<" There must be minimum 2 stages in a multigrid solver! You gave " << stages);
         grids_.resize( stages);
         grids_[0].reset( grid);
@@ -26,15 +27,18 @@ struct MultigridCG2d
         project_.resize( stages-1);
         for( unsigned u=0; u<stages-1; u++)
         {
-            project_[u] = dg::create::fast_projection( grids_[u].get(), 2,2);
+            project_[u] = dg::create::fast_projection( grids_[0].get(), pow(2,u+1),pow(2, u+1));
             inter_[u] = dg::create::fast_interpolation(grids_[u+1].get(),2,2);
         }
 
-        cg_.resize( stages);
         dg::blas1::transfer( dg::evaluate( dg::zero, grid), x0_);
         x1_=x0_, x2_=x0_;
-        x_ =  project( x0_), r_ = x_, b_ = x_;
+        x_ =  project( x0_); 
+        r_ = x_, b_ = x_;
         set_extrapolationType(extrapolation_type);
+        cg_.resize( stages);
+        for( unsigned u=0; u<stages; u++)
+            cg_[u].construct( x_[u], x_[u].size());
     }
 
     template<class SymmetricOp>
@@ -43,15 +47,17 @@ struct MultigridCG2d
         dg::blas1::axpbygz( alpha[0], x0_, alpha[1], x1_, alpha[2], x2_); 
         x_[0].swap(x2_);
         project( x_[0], x_);
-        dg::blas1::pointwiseDivide( b, op[0].inv_weights(), b_[0]);
-        project( b_[0], b_);
+        project( b, b_);
         std::vector<unsigned> number(stages_);
-        //for( unsigned u=stages_-1; u>0; u--)
-        //{
-        //    number[u] = cg_[u]( op[u], x_[u], b_[u], op[u].precond(), op[u].inv_weights(), eps, 1.);
-        //    dg::blas2::symv( inter_[u-1], x_[u], x_[u-1]);
-        //}
-        number[0] = cg_[0]( op[0], x, b_[0], op[0].precond(), op[0].inv_weights(), eps);
+        for( unsigned u=stages_-1; u>0; u--)
+        {
+            dg::blas1::pointwiseDivide( b_[u], op[u].inv_weights(), b_[u]);
+            number[u] = cg_[u]( op[u], x_[u], b_[u], op[u].precond(), op[u].inv_weights(), eps, 1.);
+            std::cout << " # iterations grid "<<u<<" "<<number[u] << " \n";
+            dg::blas2::symv( inter_[u-1], x_[u], x_[u-1]);
+        }
+        number[0] = cg_[0]( op[0], x_[0], b_[0], op[0].precond(), op[0].inv_weights(), eps);
+        x_[0].swap( x);
         x1_.swap( x2_);
         x0_.swap( x1_);
         
@@ -64,7 +70,8 @@ struct MultigridCG2d
     {
         dg::blas1::copy( src, out[0]);
         for( unsigned u=0; u<grids_.size()-1; u++)
-            dg::blas2::gemv( project_[u], out[u], out[u+1]);
+            dg::blas2::gemv( project_[u], src, out[u+1]);
+            //dg::blas2::gemv( project_[u], out[u], out[u+1]);
     }
 
     std::vector<container> project( const container& src)
-- 
GitLab


From 1fbe1394a1469371b8f08ae30b20c4c9d690999c Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Fri, 8 Sep 2017 00:01:36 +0200
Subject: [PATCH 263/453] finally found error

---
 inc/dg/backend/projection.cuh | 10 +++++-----
 inc/dg/elliptic2d_b.cu        |  2 +-
 inc/dg/multigrid.h            | 17 ++++++++++++-----
 3 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/inc/dg/backend/projection.cuh b/inc/dg/backend/projection.cuh
index e7a582289..a513d01b4 100644
--- a/inc/dg/backend/projection.cuh
+++ b/inc/dg/backend/projection.cuh
@@ -106,7 +106,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const aTopology
  */
 cusp::coo_matrix< int, double, cusp::host_memory> projection( const Grid1d& g_new, const Grid1d& g_old)
 {
-    if( g_old.N() % g_new.N() != 0) std::cerr << "ATTENTION: you project between incompatible grids!!\n";
+    if( g_old.N() % g_new.N() != 0) std::cerr << "ATTENTION: you project between incompatible grids!! old N: "<<g_old.N()<<" new N: "<<g_new.N()<<"\n";
     //form the adjoint
     thrust::host_vector<double> w_f = dg::create::weights( g_old);
     thrust::host_vector<double> v_c = dg::create::inv_weights( g_new );
@@ -135,8 +135,8 @@ cusp::coo_matrix< int, double, cusp::host_memory> projection( const Grid1d& g_ne
  */
 cusp::coo_matrix< int, double, cusp::host_memory> projection( const aTopology2d& g_new, const aTopology2d& g_old)
 {
-    if( g_old.Nx() % g_new.Nx() != 0) std::cerr << "ATTENTION: you project between incompatible grids in x!!\n";
-    if( g_old.Ny() % g_new.Ny() != 0) std::cerr << "ATTENTION: you project between incompatible grids in y!!\n";
+    if( g_old.Nx() % g_new.Nx() != 0) std::cerr << "ATTENTION: you project between incompatible grids in x!! old N: "<<g_old.Nx()<<" new N: "<<g_new.Nx()<<"\n";
+    if( g_old.Ny() % g_new.Ny() != 0) std::cerr << "ATTENTION: you project between incompatible grids in y!! old N: "<<g_old.Ny()<<" new N: "<<g_new.Ny()<<"\n";
     //form the adjoint
     thrust::host_vector<double> w_f = dg::create::weights( g_old);
     thrust::host_vector<double> v_c = dg::create::inv_weights( g_new );
@@ -164,8 +164,8 @@ cusp::coo_matrix< int, double, cusp::host_memory> projection( const aTopology2d&
  */
 cusp::coo_matrix< int, double, cusp::host_memory> projection( const aTopology3d& g_new, const aTopology3d& g_old)
 {
-    if( g_old.Nx() % g_new.Nx() != 0) std::cerr << "ATTENTION: you project between incompatible grids in x!!\n";
-    if( g_old.Ny() % g_new.Ny() != 0) std::cerr << "ATTENTION: you project between incompatible grids in y!!\n";
+    if( g_old.Nx() % g_new.Nx() != 0) std::cerr << "ATTENTION: you project between incompatible grids in x!! old N: "<<g_old.Nx()<<" new N: "<<g_new.Nx()<<"\n";
+    if( g_old.Ny() % g_new.Ny() != 0) std::cerr << "ATTENTION: you project between incompatible grids in y!! old N: "<<g_old.Ny()<<" new N: "<<g_new.Ny()<<"\n";
     //form the adjoint
     thrust::host_vector<double> w_f = dg::create::weights( g_old);
     thrust::host_vector<double> v_c = dg::create::inv_weights( g_new );
diff --git a/inc/dg/elliptic2d_b.cu b/inc/dg/elliptic2d_b.cu
index d9e19f13a..15b7ee8ee 100644
--- a/inc/dg/elliptic2d_b.cu
+++ b/inc/dg/elliptic2d_b.cu
@@ -62,7 +62,7 @@ int main()
     std::cout << "Create Polarisation object and set chi!\n";
     t.tic();
     {
-    unsigned stages=2;
+    unsigned stages=3;
     dg::MultigridCG2d<dg::aGeometry2d, dg::DMatrix, dg::DVec > multigrid( grid, stages);
     std::vector<dg::DVec> chi_ = multigrid.project( chi);
     std::vector<dg::Elliptic<dg::aGeometry2d, dg::DMatrix, dg::DVec> > multi_pol( stages);
diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index 8931033e1..af87127a4 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -1,6 +1,7 @@
 #pragma once
 
 #include "backend/fast_interpolation.h"
+#include "backend/interpolation.cuh"
 #include "backend/exceptions.h"
 #include "backend/memory.h"
 #include "blas.h"
@@ -18,17 +19,21 @@ struct MultigridCG2d
         if( stages < 2 ) throw Error( Message(_ping_)<<" There must be minimum 2 stages in a multigrid solver! You gave " << stages);
         grids_.resize( stages);
         grids_[0].reset( grid);
+        grids_[0].get().display();
         for( unsigned u=1; u<stages; u++)
         {
             grids_[u] = grids_[u-1]; //deep copy
             grids_[u].get().multiplyCellNumbers(0.5, 0.5);
+            grids_[u].get().display();
         }
         inter_.resize(stages-1);
         project_.resize( stages-1);
         for( unsigned u=0; u<stages-1; u++)
         {
-            project_[u] = dg::create::fast_projection( grids_[0].get(), pow(2,u+1),pow(2, u+1));
-            inter_[u] = dg::create::fast_interpolation(grids_[u+1].get(),2,2);
+            //project_[u] = dg::create::fast_projection( grids_[0].get(), pow(2,u+1),pow(2, u+1));
+            //inter_[u] = dg::create::fast_interpolation(grids_[u+1].get(),2,2);
+            project_[u] = dg::create::projection( grids_[u+1].get(), grids_[0].get());
+            inter_[u] = dg::create::interpolation(grids_[u].get(),grids_[u+1].get());
         }
 
         dg::blas1::transfer( dg::evaluate( dg::zero, grid), x0_);
@@ -53,9 +58,9 @@ struct MultigridCG2d
         {
             dg::blas1::pointwiseDivide( b_[u], op[u].inv_weights(), b_[u]);
             number[u] = cg_[u]( op[u], x_[u], b_[u], op[u].precond(), op[u].inv_weights(), eps, 1.);
-            std::cout << " # iterations grid "<<u<<" "<<number[u] << " \n";
             dg::blas2::symv( inter_[u-1], x_[u], x_[u-1]);
         }
+        dg::blas1::pointwiseDivide( b_[0], op[0].inv_weights(), b_[0]);
         number[0] = cg_[0]( op[0], x_[0], b_[0], op[0].precond(), op[0].inv_weights(), eps);
         x_[0].swap( x);
         x1_.swap( x2_);
@@ -111,8 +116,10 @@ struct MultigridCG2d
     private:
     unsigned stages_;
     std::vector< dg::Handle< Geometry> > grids_;
-    std::vector< MultiMatrix<Matrix, container>  >  inter_;
-    std::vector< MultiMatrix<Matrix, container>  >  project_;
+    //std::vector< MultiMatrix<Matrix, container>  >  inter_;
+    //std::vector< MultiMatrix<Matrix, container>  >  project_;
+    std::vector< dg::IDMatrix  >  inter_;
+    std::vector< dg::IDMatrix  >  project_;
     std::vector< CG<container> > cg_;
     std::vector< container> x_, r_, b_; 
     container x0_, x1_, x2_;
-- 
GitLab


From 62aaa345307b1f329eb9b21f80fe74cdc6c1106f Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Fri, 8 Sep 2017 00:50:07 +0200
Subject: [PATCH 264/453] found error in fast_interpolation

---
 inc/dg/backend/derivatives_b.cu             |  2 +-
 inc/dg/backend/fast_interpolation.h         |  8 ++---
 inc/dg/backend/projection_t.cu              |  2 +-
 inc/dg/backend/sparseblockmat_omp_kernels.h | 38 ++++++++++++---------
 inc/dg/elliptic2d_b.cu                      | 38 ++++++++++-----------
 inc/dg/multigrid.h                          | 17 ++++-----
 6 files changed, 55 insertions(+), 50 deletions(-)

diff --git a/inc/dg/backend/derivatives_b.cu b/inc/dg/backend/derivatives_b.cu
index 75fadddcc..b23076d62 100644
--- a/inc/dg/backend/derivatives_b.cu
+++ b/inc/dg/backend/derivatives_b.cu
@@ -55,7 +55,7 @@ int main()
     const Vector func = dg::evaluate( siny, g);
     const Vector deri = dg::evaluate( cosy, g);
 
-    Matrix dy = dg::create::dy( g); 
+    Matrix dy = dg::create::dy( g, dg::forward); 
     Vector temp( func);
     t.tic();
     dg::blas2::gemv( dy, func, temp);
diff --git a/inc/dg/backend/fast_interpolation.h b/inc/dg/backend/fast_interpolation.h
index ec63990ef..be71a2b57 100644
--- a/inc/dg/backend/fast_interpolation.h
+++ b/inc/dg/backend/fast_interpolation.h
@@ -84,17 +84,17 @@ MultiMatrix< EllSparseBlockMat<double>, thrust::host_vector<double> > fast_inter
 {
     unsigned n=t.n();
     dg::Grid1d g_old( -1., 1., n, 1);
-    dg::Grid1d g_newX( -1., 1., n, multiply);
-    dg::IHMatrix interpolX = dg::create::interpolation( g_newX, g_old);
+    dg::Grid1d g_new( -1., 1., n, multiply);
+    dg::IHMatrix interpolX = dg::create::interpolation( g_new, g_old);
     EllSparseBlockMat<double> iX( multiply*t.N(), t.N(), 1, multiply, t.n()); 
+    for( unsigned  k=0; k<multiply; k++)
     for( unsigned  i=0; i<n; i++)
     for( unsigned  j=0; j<n; j++)
-    for( unsigned  k=0; k<multiply; k++)
         iX.data[(k*n+i)*n+j] = interpolX.values[(k*n+i)*n+j];
     for( unsigned i=0; i<multiply*t.N(); i++)
     {
         iX.cols_idx[i] = i/multiply;
-        iX.data_idx[i] = 0;
+        iX.data_idx[i] = i%multiply;
     }
     MultiMatrix < EllSparseBlockMat<double>, thrust::host_vector<double> > inter(1);
     inter.get_matrices()[0] = iX;
diff --git a/inc/dg/backend/projection_t.cu b/inc/dg/backend/projection_t.cu
index b075f4df4..033025273 100644
--- a/inc/dg/backend/projection_t.cu
+++ b/inc/dg/backend/projection_t.cu
@@ -62,7 +62,7 @@ int main()
     std::cout << "Difference in Norms "<<sqrt(dg::blas2::dot( sinO, w2do, sinO)) - sqrt(dg::blas2::dot( sinP, w2dn, sinP)) << "\n" << std::endl;
     dg::blas2::gemv( fast_inte2d, sinN, sinI);
     std::cout << "Interpolated vec    "<<sqrt(dg::blas2::dot( sinI, w2do, sinI)) << "\n";
-    std::cout << "Difference in Norms "<<sqrt(dg::blas2::dot( sinI, w2do, sinI)) - sqrt(dg::blas2::dot( sinP, w2dn, sinP)) << "\n" << std::endl;
+    std::cout << "Difference in Norms "<<sqrt(dg::blas2::dot( sinI, w2do, sinI)) - sqrt(dg::blas2::dot( sinN, w2dn, sinN)) << "\n" << std::endl;
 
     return 0;
 }
diff --git a/inc/dg/backend/sparseblockmat_omp_kernels.h b/inc/dg/backend/sparseblockmat_omp_kernels.h
index fb1f46e2b..4fda62af4 100644
--- a/inc/dg/backend/sparseblockmat_omp_kernels.h
+++ b/inc/dg/backend/sparseblockmat_omp_kernels.h
@@ -13,6 +13,7 @@ void ell_multiply_kernel( value_type alpha, value_type beta,
          const value_type * RESTRICT x, value_type * RESTRICT y
          )
 {
+    std::cout << " standard\n";
     //simplest implementation
 #pragma omp parallel for collapse(2)
     for( int s=0; s<left_size; s++)
@@ -57,22 +58,24 @@ void ell_multiply_kernel3( value_type alpha, value_type beta,
         }
     if (trivial && blocks_per_line == 1)
     {
+        std::cout << " hi3";
         if( right_size==1)
         {
             #pragma omp parallel for 
             for( int s=0; s<left_size; s++)
             for( int i=0; i<num_rows; i++)
-            for( int k=0; k<3; k++)
             {
-                value_type temp = 0;
-                int B = (data_idx[i]*3+k)*3;
                 int J = (s*num_cols+cols_idx[i])*3;
-
-                temp +=data[ B+0]* x[(J+0)*right_size];
-                temp +=data[ B+1]* x[(J+1)*right_size];
-                temp +=data[ B+2]* x[(J+2)*right_size];
-                int I = ((s*num_rows + i)*3+k)*right_size;
-                y[I] = alpha*temp + beta*y[I];
+                for( int k=0; k<3; k++)
+                {
+                    value_type temp = 0;
+                    int B = (data_idx[i]*3+k)*3;
+                    temp +=data[ B+0]* x[(J+0)];
+                    temp +=data[ B+1]* x[(J+1)];
+                    temp +=data[ B+2]* x[(J+2)];
+                    int I = (s*num_rows + i)*3+k;
+                    y[I] = alpha*temp + beta*y[I];
+                }
             }
         }
         else
@@ -81,17 +84,18 @@ void ell_multiply_kernel3( value_type alpha, value_type beta,
             for( int s=0; s<left_size; s++)
             for( int i=0; i<num_rows; i++)
             for( int k=0; k<3; k++)
-            for( int j=right_range[0]; j<right_range[1]; j++)
             {
-                value_type temp = 0;
                 int B = (data_idx[i]*3+k)*3;
                 int J = (s*num_cols+cols_idx[i])*3;
-
-                temp +=data[ B+0]* x[(J+0)*right_size+j];
-                temp +=data[ B+1]* x[(J+1)*right_size+j];
-                temp +=data[ B+2]* x[(J+2)*right_size+j];
-                int I = ((s*num_rows + i)*3+k)*right_size+j;
-                y[I] = alpha*temp + beta*y[I];
+                for( int j=right_range[0]; j<right_range[1]; j++)
+                {
+                    value_type temp = 0;
+                    temp +=data[ B+0]* x[(J+0)*right_size+j];
+                    temp +=data[ B+1]* x[(J+1)*right_size+j];
+                    temp +=data[ B+2]* x[(J+2)*right_size+j];
+                    int I = ((s*num_rows + i)*3+k)*right_size+j;
+                    y[I] = alpha*temp + beta*y[I];
+                }
             }
         }
     }
diff --git a/inc/dg/elliptic2d_b.cu b/inc/dg/elliptic2d_b.cu
index 15b7ee8ee..88710a15d 100644
--- a/inc/dg/elliptic2d_b.cu
+++ b/inc/dg/elliptic2d_b.cu
@@ -46,26 +46,26 @@ int main()
     std::cout << "Computation on: "<< n <<" x "<<Nx<<" x "<<Ny<<std::endl;
     //std::cout << "# of 2d cells                 "<< Nx*Ny <<std::endl;
     dg::CartesianGrid2d grid( 0, lx, 0, ly, n, Nx, Ny, bcx, bcy);
-    dg::DVec w2d = dg::create::weights( grid);
-    dg::DVec v2d = dg::create::inv_weights( grid);
-    dg::DVec one = dg::evaluate( dg::one, grid);
+    dg::HVec w2d = dg::create::weights( grid);
+    dg::HVec v2d = dg::create::inv_weights( grid);
+    dg::HVec one = dg::evaluate( dg::one, grid);
     //create functions A(chi) x = b
-    dg::DVec x =    dg::evaluate( initial, grid);
-    dg::DVec b =    dg::evaluate( rhs, grid);
-    dg::DVec chi =  dg::evaluate( pol, grid);
-    dg::DVec chi_inv(chi); 
+    dg::HVec x =    dg::evaluate( initial, grid);
+    dg::HVec b =    dg::evaluate( rhs, grid);
+    dg::HVec chi =  dg::evaluate( pol, grid);
+    dg::HVec chi_inv(chi); 
     dg::blas1::transform( chi, chi_inv, dg::INVERT<double>());
     dg::blas1::pointwiseDot( chi_inv, v2d, chi_inv);
-    dg::DVec temp = x;
+    dg::HVec temp = x;
 
 
     std::cout << "Create Polarisation object and set chi!\n";
     t.tic();
     {
     unsigned stages=3;
-    dg::MultigridCG2d<dg::aGeometry2d, dg::DMatrix, dg::DVec > multigrid( grid, stages);
-    std::vector<dg::DVec> chi_ = multigrid.project( chi);
-    std::vector<dg::Elliptic<dg::aGeometry2d, dg::DMatrix, dg::DVec> > multi_pol( stages);
+    dg::MultigridCG2d<dg::aGeometry2d, dg::HMatrix, dg::HVec > multigrid( grid, stages);
+    std::vector<dg::HVec> chi_ = multigrid.project( chi);
+    std::vector<dg::Elliptic<dg::aGeometry2d, dg::HMatrix, dg::HVec> > multi_pol( stages);
     for( unsigned u=0; u<stages; u++)
     {
         multi_pol[u].construct( multigrid.grids()[u].get(), dg::not_normed, dg::centered, jfactor); 
@@ -85,9 +85,9 @@ int main()
     }
 
     //compute error
-    const dg::DVec solution = dg::evaluate( sol, grid);
-    const dg::DVec derivati = dg::evaluate( der, grid);
-    dg::DVec error( solution);
+    const dg::HVec solution = dg::evaluate( sol, grid);
+    const dg::HVec derivati = dg::evaluate( der, grid);
+    dg::HVec error( solution);
 
     dg::blas1::axpby( 1.,x,-1., solution, error);
     double err = dg::blas2::dot( w2d, error);
@@ -95,10 +95,10 @@ int main()
     const double norm = dg::blas2::dot( w2d, solution);
     std::cout << " "<<sqrt( err/norm);
     {
-    dg::Elliptic<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> pol_forward( grid, dg::not_normed, dg::forward, jfactor);
+    dg::Elliptic<dg::CartesianGrid2d, dg::HMatrix, dg::HVec> pol_forward( grid, dg::not_normed, dg::forward, jfactor);
     pol_forward.set_chi( chi);
     x = temp;
-    dg::Invert<dg::DVec > invert_fw( x, n*n*Nx*Ny, eps);
+    dg::Invert<dg::HVec > invert_fw( x, n*n*Nx*Ny, eps);
     std::cout << " "<< invert_fw( pol_forward, x, b, v2d, chi_inv);
     dg::blas1::axpby( 1.,x,-1., solution, error);
     err = dg::blas2::dot( w2d, error);
@@ -106,10 +106,10 @@ int main()
     }
 
     {
-    dg::Elliptic<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> pol_backward( grid, dg::not_normed, dg::backward, jfactor);
+    dg::Elliptic<dg::CartesianGrid2d, dg::HMatrix, dg::HVec> pol_backward( grid, dg::not_normed, dg::backward, jfactor);
     pol_backward.set_chi( chi);
     x = temp;
-    dg::Invert<dg::DVec > invert_bw( x, n*n*Nx*Ny, eps);
+    dg::Invert<dg::HVec > invert_bw( x, n*n*Nx*Ny, eps);
     std::cout << " "<< invert_bw( pol_backward, x, b, v2d, chi_inv);
     dg::blas1::axpby( 1.,x,-1., solution, error);
     err = dg::blas2::dot( w2d, error);
@@ -117,7 +117,7 @@ int main()
     }
 
 
-    dg::DMatrix DX = dg::create::dx( grid);
+    dg::HMatrix DX = dg::create::dx( grid);
     dg::blas2::gemv( DX, x, error);
     dg::blas1::axpby( 1.,derivati,-1., error);
     err = dg::blas2::dot( w2d, error);
diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index af87127a4..eee3b56f4 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -30,10 +30,12 @@ struct MultigridCG2d
         project_.resize( stages-1);
         for( unsigned u=0; u<stages-1; u++)
         {
-            //project_[u] = dg::create::fast_projection( grids_[0].get(), pow(2,u+1),pow(2, u+1));
-            //inter_[u] = dg::create::fast_interpolation(grids_[u+1].get(),2,2);
-            project_[u] = dg::create::projection( grids_[u+1].get(), grids_[0].get());
-            inter_[u] = dg::create::interpolation(grids_[u].get(),grids_[u+1].get());
+            //project_[u] = dg::create::fast_projection( grids_[u].get(), 2,2);
+            inter_[u] = dg::create::fast_interpolation(grids_[u+1].get(),2,2);
+        //Projecting from one grid to the next is the same as 
+        //projecting from the original grid to the coarse grids
+            project_[u] = dg::create::projection( grids_[u+1].get(), grids_[u].get());
+            //inter_[u] = dg::create::interpolation(grids_[u].get(),grids_[u+1].get());
         }
 
         dg::blas1::transfer( dg::evaluate( dg::zero, grid), x0_);
@@ -75,8 +77,7 @@ struct MultigridCG2d
     {
         dg::blas1::copy( src, out[0]);
         for( unsigned u=0; u<grids_.size()-1; u++)
-            dg::blas2::gemv( project_[u], src, out[u+1]);
-            //dg::blas2::gemv( project_[u], out[u], out[u+1]);
+            dg::blas2::gemv( project_[u], out[u], out[u+1]);
     }
 
     std::vector<container> project( const container& src)
@@ -116,9 +117,9 @@ struct MultigridCG2d
     private:
     unsigned stages_;
     std::vector< dg::Handle< Geometry> > grids_;
-    //std::vector< MultiMatrix<Matrix, container>  >  inter_;
+    std::vector< MultiMatrix<Matrix, container>  >  inter_;
     //std::vector< MultiMatrix<Matrix, container>  >  project_;
-    std::vector< dg::IDMatrix  >  inter_;
+    //std::vector< dg::IDMatrix  >  inter_;
     std::vector< dg::IDMatrix  >  project_;
     std::vector< CG<container> > cg_;
     std::vector< container> x_, r_, b_; 
-- 
GitLab


From c845212eebfa97f3d2cc842fda0324e05cd31a14 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Fri, 8 Sep 2017 00:56:54 +0200
Subject: [PATCH 265/453] remake original state

---
 inc/dg/backend/sparseblockmat_omp_kernels.h | 10 +-----
 inc/dg/elliptic2d_b.cu                      | 38 ++++++++++-----------
 inc/dg/multigrid.h                          | 12 +++----
 3 files changed, 24 insertions(+), 36 deletions(-)

diff --git a/inc/dg/backend/sparseblockmat_omp_kernels.h b/inc/dg/backend/sparseblockmat_omp_kernels.h
index 4fda62af4..ba10d07a5 100644
--- a/inc/dg/backend/sparseblockmat_omp_kernels.h
+++ b/inc/dg/backend/sparseblockmat_omp_kernels.h
@@ -13,7 +13,6 @@ void ell_multiply_kernel( value_type alpha, value_type beta,
          const value_type * RESTRICT x, value_type * RESTRICT y
          )
 {
-    std::cout << " standard\n";
     //simplest implementation
 #pragma omp parallel for collapse(2)
     for( int s=0; s<left_size; s++)
@@ -50,15 +49,8 @@ void ell_multiply_kernel3( value_type alpha, value_type beta,
          const value_type * RESTRICT x, value_type * RESTRICT y
          )
 {
-    bool trivial = true;
-    for( int i=1; i<num_rows; i++)
-        for( int d=0; d<blocks_per_line; d++)
-        {
-            if( data_idx[i*blocks_per_line+d] != data_idx[d]) trivial = false;
-        }
-    if (trivial && blocks_per_line == 1)
+    if (blocks_per_line == 1)
     {
-        std::cout << " hi3";
         if( right_size==1)
         {
             #pragma omp parallel for 
diff --git a/inc/dg/elliptic2d_b.cu b/inc/dg/elliptic2d_b.cu
index 88710a15d..15b7ee8ee 100644
--- a/inc/dg/elliptic2d_b.cu
+++ b/inc/dg/elliptic2d_b.cu
@@ -46,26 +46,26 @@ int main()
     std::cout << "Computation on: "<< n <<" x "<<Nx<<" x "<<Ny<<std::endl;
     //std::cout << "# of 2d cells                 "<< Nx*Ny <<std::endl;
     dg::CartesianGrid2d grid( 0, lx, 0, ly, n, Nx, Ny, bcx, bcy);
-    dg::HVec w2d = dg::create::weights( grid);
-    dg::HVec v2d = dg::create::inv_weights( grid);
-    dg::HVec one = dg::evaluate( dg::one, grid);
+    dg::DVec w2d = dg::create::weights( grid);
+    dg::DVec v2d = dg::create::inv_weights( grid);
+    dg::DVec one = dg::evaluate( dg::one, grid);
     //create functions A(chi) x = b
-    dg::HVec x =    dg::evaluate( initial, grid);
-    dg::HVec b =    dg::evaluate( rhs, grid);
-    dg::HVec chi =  dg::evaluate( pol, grid);
-    dg::HVec chi_inv(chi); 
+    dg::DVec x =    dg::evaluate( initial, grid);
+    dg::DVec b =    dg::evaluate( rhs, grid);
+    dg::DVec chi =  dg::evaluate( pol, grid);
+    dg::DVec chi_inv(chi); 
     dg::blas1::transform( chi, chi_inv, dg::INVERT<double>());
     dg::blas1::pointwiseDot( chi_inv, v2d, chi_inv);
-    dg::HVec temp = x;
+    dg::DVec temp = x;
 
 
     std::cout << "Create Polarisation object and set chi!\n";
     t.tic();
     {
     unsigned stages=3;
-    dg::MultigridCG2d<dg::aGeometry2d, dg::HMatrix, dg::HVec > multigrid( grid, stages);
-    std::vector<dg::HVec> chi_ = multigrid.project( chi);
-    std::vector<dg::Elliptic<dg::aGeometry2d, dg::HMatrix, dg::HVec> > multi_pol( stages);
+    dg::MultigridCG2d<dg::aGeometry2d, dg::DMatrix, dg::DVec > multigrid( grid, stages);
+    std::vector<dg::DVec> chi_ = multigrid.project( chi);
+    std::vector<dg::Elliptic<dg::aGeometry2d, dg::DMatrix, dg::DVec> > multi_pol( stages);
     for( unsigned u=0; u<stages; u++)
     {
         multi_pol[u].construct( multigrid.grids()[u].get(), dg::not_normed, dg::centered, jfactor); 
@@ -85,9 +85,9 @@ int main()
     }
 
     //compute error
-    const dg::HVec solution = dg::evaluate( sol, grid);
-    const dg::HVec derivati = dg::evaluate( der, grid);
-    dg::HVec error( solution);
+    const dg::DVec solution = dg::evaluate( sol, grid);
+    const dg::DVec derivati = dg::evaluate( der, grid);
+    dg::DVec error( solution);
 
     dg::blas1::axpby( 1.,x,-1., solution, error);
     double err = dg::blas2::dot( w2d, error);
@@ -95,10 +95,10 @@ int main()
     const double norm = dg::blas2::dot( w2d, solution);
     std::cout << " "<<sqrt( err/norm);
     {
-    dg::Elliptic<dg::CartesianGrid2d, dg::HMatrix, dg::HVec> pol_forward( grid, dg::not_normed, dg::forward, jfactor);
+    dg::Elliptic<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> pol_forward( grid, dg::not_normed, dg::forward, jfactor);
     pol_forward.set_chi( chi);
     x = temp;
-    dg::Invert<dg::HVec > invert_fw( x, n*n*Nx*Ny, eps);
+    dg::Invert<dg::DVec > invert_fw( x, n*n*Nx*Ny, eps);
     std::cout << " "<< invert_fw( pol_forward, x, b, v2d, chi_inv);
     dg::blas1::axpby( 1.,x,-1., solution, error);
     err = dg::blas2::dot( w2d, error);
@@ -106,10 +106,10 @@ int main()
     }
 
     {
-    dg::Elliptic<dg::CartesianGrid2d, dg::HMatrix, dg::HVec> pol_backward( grid, dg::not_normed, dg::backward, jfactor);
+    dg::Elliptic<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> pol_backward( grid, dg::not_normed, dg::backward, jfactor);
     pol_backward.set_chi( chi);
     x = temp;
-    dg::Invert<dg::HVec > invert_bw( x, n*n*Nx*Ny, eps);
+    dg::Invert<dg::DVec > invert_bw( x, n*n*Nx*Ny, eps);
     std::cout << " "<< invert_bw( pol_backward, x, b, v2d, chi_inv);
     dg::blas1::axpby( 1.,x,-1., solution, error);
     err = dg::blas2::dot( w2d, error);
@@ -117,7 +117,7 @@ int main()
     }
 
 
-    dg::HMatrix DX = dg::create::dx( grid);
+    dg::DMatrix DX = dg::create::dx( grid);
     dg::blas2::gemv( DX, x, error);
     dg::blas1::axpby( 1.,derivati,-1., error);
     err = dg::blas2::dot( w2d, error);
diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index eee3b56f4..2d2465d75 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -30,12 +30,10 @@ struct MultigridCG2d
         project_.resize( stages-1);
         for( unsigned u=0; u<stages-1; u++)
         {
-            //project_[u] = dg::create::fast_projection( grids_[u].get(), 2,2);
+            //Projecting from one grid to the next is the same as 
+            //projecting from the original grid to the coarse grids
+            project_[u] = dg::create::fast_projection( grids_[u].get(), 2,2);
             inter_[u] = dg::create::fast_interpolation(grids_[u+1].get(),2,2);
-        //Projecting from one grid to the next is the same as 
-        //projecting from the original grid to the coarse grids
-            project_[u] = dg::create::projection( grids_[u+1].get(), grids_[u].get());
-            //inter_[u] = dg::create::interpolation(grids_[u].get(),grids_[u+1].get());
         }
 
         dg::blas1::transfer( dg::evaluate( dg::zero, grid), x0_);
@@ -118,9 +116,7 @@ struct MultigridCG2d
     unsigned stages_;
     std::vector< dg::Handle< Geometry> > grids_;
     std::vector< MultiMatrix<Matrix, container>  >  inter_;
-    //std::vector< MultiMatrix<Matrix, container>  >  project_;
-    //std::vector< dg::IDMatrix  >  inter_;
-    std::vector< dg::IDMatrix  >  project_;
+    std::vector< MultiMatrix<Matrix, container>  >  project_;
     std::vector< CG<container> > cg_;
     std::vector< container> x_, r_, b_; 
     container x0_, x1_, x2_;
-- 
GitLab


From 2e31e0fc9bc608cb2bcb64c363bc9c0a10501012 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Fri, 8 Sep 2017 13:36:47 +0200
Subject: [PATCH 266/453]  provide some documentation

---
 inc/dg/backend/fast_interpolation.h | 4 ++++
 inc/dg/multigrid.h                  | 4 ++--
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/inc/dg/backend/fast_interpolation.h b/inc/dg/backend/fast_interpolation.h
index be71a2b57..5cec2c3c0 100644
--- a/inc/dg/backend/fast_interpolation.h
+++ b/inc/dg/backend/fast_interpolation.h
@@ -13,6 +13,9 @@ namespace dg
 
 /**
  * @brief Struct that applies given matrices one after the other
+ *
+ * \f[ y = M_{N-1}(...M_1(M_0x))\f]
+ * where \f$ M_i\f$ is the i-th matrix 
  * @copydoc hide_matrix_container
  * @ingroup misc
  */
@@ -23,6 +26,7 @@ struct MultiMatrix
     /**
     * @brief reserve space for dimension matrices  and dimension-1 containers
     * @param dimension # of matrices to store 
+    * @attention it is the user's reponsibility to allocate memory for the intermediate "temp" vectors
     */
     MultiMatrix( int dimension): inter_(dimension), temp_(dimension-1 > 0 ? dimension-1 : 0 ){}
     template<class OtherMatrix, class OtherContainer>
diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index 2d2465d75..81087de08 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -19,12 +19,12 @@ struct MultigridCG2d
         if( stages < 2 ) throw Error( Message(_ping_)<<" There must be minimum 2 stages in a multigrid solver! You gave " << stages);
         grids_.resize( stages);
         grids_[0].reset( grid);
-        grids_[0].get().display();
+        //grids_[0].get().display();
         for( unsigned u=1; u<stages; u++)
         {
             grids_[u] = grids_[u-1]; //deep copy
             grids_[u].get().multiplyCellNumbers(0.5, 0.5);
-            grids_[u].get().display();
+            //grids_[u].get().display();
         }
         inter_.resize(stages-1);
         project_.resize( stages-1);
-- 
GitLab


From 002a5e9cdd24f96811cbef90d53e28656a3f40ae Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Fri, 8 Sep 2017 14:45:48 +0200
Subject: [PATCH 267/453] some documentation on elliptic.h

---
 inc/dg/elliptic.h      | 2 +-
 inc/dg/elliptic2d_b.cu | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index b24cc6a35..23d1900c9 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -132,7 +132,7 @@ class Elliptic
     }
 
     /**
-     * @brief Returns the weights used to make the matrix symmetric 
+     * @brief Returns the vector missing in the un-normed symmetric matrix 
      *
      * i.e. the inverse volume form 
      * @return inverse volume form including weights 
diff --git a/inc/dg/elliptic2d_b.cu b/inc/dg/elliptic2d_b.cu
index 15b7ee8ee..d7a3bd9e4 100644
--- a/inc/dg/elliptic2d_b.cu
+++ b/inc/dg/elliptic2d_b.cu
@@ -73,7 +73,7 @@ int main()
     }
 
     t.toc();
-    std::cout << "Creation of polarisation object took: "<<t.diff()<<"s\n";
+    //std::cout << "Creation of polarisation object took: "<<t.diff()<<"s\n";
     std::cout << eps<<" \n";
     t.tic();
     std::vector<unsigned> number = multigrid.direct_solve( multi_pol, x, b, eps);
@@ -95,7 +95,7 @@ int main()
     const double norm = dg::blas2::dot( w2d, solution);
     std::cout << " "<<sqrt( err/norm);
     {
-    dg::Elliptic<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> pol_forward( grid, dg::not_normed, dg::forward, jfactor);
+    dg::Elliptic<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> pol_forward( grid, dg::not_normed, dg::centered, jfactor);
     pol_forward.set_chi( chi);
     x = temp;
     dg::Invert<dg::DVec > invert_fw( x, n*n*Nx*Ny, eps);
-- 
GitLab


From 28dd049456470e3412f1ee473abaccb9575e8f3c Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sat, 9 Sep 2017 10:44:57 +0200
Subject: [PATCH 268/453] picked sparseblockmat_omp_kernels from multigrid
 branch

---
 inc/dg/backend/sparseblockmat.cuh           |   4 +
 inc/dg/backend/sparseblockmat_omp_kernels.h | 259 ++++++++++++++------
 inc/dg/blas_b.cu                            |  16 +-
 3 files changed, 190 insertions(+), 89 deletions(-)

diff --git a/inc/dg/backend/sparseblockmat.cuh b/inc/dg/backend/sparseblockmat.cuh
index 0bba470c2..4b32d5f2e 100644
--- a/inc/dg/backend/sparseblockmat.cuh
+++ b/inc/dg/backend/sparseblockmat.cuh
@@ -183,12 +183,16 @@ template<class value_type>
 template<class DeviceContainer>
 inline void EllSparseBlockMatDevice<value_type>::symv( value_type alpha, const DeviceContainer& x, value_type beta, DeviceContainer& y) const
 {
+    assert( y.size() == (unsigned)num_rows*n*left_size*right_size);
+    assert( x.size() == (unsigned)num_cols*n*left_size*right_size);
     launch_multiply_kernel( alpha, x, beta, y);
 }
 template<class value_type>
 template<class DeviceContainer>
 inline void CooSparseBlockMatDevice<value_type>::symv( value_type alpha, const DeviceContainer& x, value_type beta, DeviceContainer& y) const
 {
+    assert( y.size() == (unsigned)num_rows*n*left_size*right_size);
+    assert( x.size() == (unsigned)num_cols*n*left_size*right_size);
     launch_multiply_kernel(alpha, x, beta, y);
 }
 
diff --git a/inc/dg/backend/sparseblockmat_omp_kernels.h b/inc/dg/backend/sparseblockmat_omp_kernels.h
index 5f04add8a..4b892b233 100644
--- a/inc/dg/backend/sparseblockmat_omp_kernels.h
+++ b/inc/dg/backend/sparseblockmat_omp_kernels.h
@@ -14,21 +14,108 @@ void ell_multiply_kernel( value_type alpha, value_type beta,
          )
 {
     //simplest implementation
-#pragma omp parallel for collapse(4)
+#pragma omp parallel for collapse(2)
     for( int s=0; s<left_size; s++)
     for( int i=0; i<num_rows; i++)
     for( int k=0; k<n; k++)
-    for( int j=right_range[0]; j<right_range[1]; j++)
     {
-        int I = ((s*num_rows + i)*n+k)*right_size+j;
-        y[I] *=beta;
+        int B[blocks_per_line], J[blocks_per_line];
         for( int d=0; d<blocks_per_line; d++)
         {
-            int B = (data_idx[i*blocks_per_line+d]*n+k)*n;
-            int J = (s*num_cols+cols_idx[i*blocks_per_line+d])*n;
-            for( int q=0; q<n; q++) //multiplication-loop
-                y[I] += alpha*data[ B+q]*
-                    x[(J+q)*right_size+j];
+            B[d] = (data_idx[i*blocks_per_line+d]*n+k)*n;
+            J[d] = (s*num_cols+cols_idx[i*blocks_per_line+d])*n;
+        }
+        for( int j=right_range[0]; j<right_range[1]; j++)
+        {
+            value_type temp = 0;
+            for( int d=0; d<blocks_per_line; d++)
+            {
+                for( int q=0; q<n; q++) //multiplication-loop
+                    temp += data[ B[d]+q]*
+                        x[(J[d]+q)*right_size+j];
+            }
+            int I = ((s*num_rows + i)*n+k)*right_size+j;
+            y[I] = alpha*temp + beta*y[I];
+        }
+    }
+}
+
+template<class value_type>
+void ell_multiply_kernel3( value_type alpha, value_type beta,
+         const value_type * RESTRICT data, const int * RESTRICT cols_idx, const int * RESTRICT data_idx, 
+         const int num_rows, const int num_cols, const int blocks_per_line,
+         const int left_size, const int right_size, 
+         const int * RESTRICT right_range,
+         const value_type * RESTRICT x, value_type * RESTRICT y
+         )
+{
+    if (blocks_per_line == 1)
+    {
+        if( right_size==1)
+        {
+            #pragma omp parallel for 
+            for( int s=0; s<left_size; s++)
+            for( int i=0; i<num_rows; i++)
+            {
+                int J = (s*num_cols+cols_idx[i])*3;
+                for( int k=0; k<3; k++)
+                {
+                    value_type temp = 0;
+                    int B = (data_idx[i]*3+k)*3;
+                    temp +=data[ B+0]* x[(J+0)];
+                    temp +=data[ B+1]* x[(J+1)];
+                    temp +=data[ B+2]* x[(J+2)];
+                    int I = (s*num_rows + i)*3+k;
+                    y[I] = alpha*temp + beta*y[I];
+                }
+            }
+        }
+        else
+        {
+            #pragma omp parallel for collapse(2)
+            for( int s=0; s<left_size; s++)
+            for( int i=0; i<num_rows; i++)
+            for( int k=0; k<3; k++)
+            {
+                int B = (data_idx[i]*3+k)*3;
+                int J = (s*num_cols+cols_idx[i])*3;
+                for( int j=right_range[0]; j<right_range[1]; j++)
+                {
+                    value_type temp = 0;
+                    temp +=data[ B+0]* x[(J+0)*right_size+j];
+                    temp +=data[ B+1]* x[(J+1)*right_size+j];
+                    temp +=data[ B+2]* x[(J+2)*right_size+j];
+                    int I = ((s*num_rows + i)*3+k)*right_size+j;
+                    y[I] = alpha*temp + beta*y[I];
+                }
+            }
+        }
+    }
+    else
+    {
+#pragma omp parallel for collapse(2)
+        for( int s=0; s<left_size; s++)
+        for( int i=0; i<num_rows; i++)
+        for( int k=0; k<3; k++)
+        {
+            int B[blocks_per_line], J[blocks_per_line];
+            for( int d=0; d<blocks_per_line; d++)
+            {
+                B[d] = (data_idx[i*blocks_per_line+d]*3+k)*3;
+                J[d] = (s*num_cols+cols_idx[i*blocks_per_line+d])*3;
+            }
+            for( int j=right_range[0]; j<right_range[1]; j++)
+            {
+                value_type temp = 0;
+                for( int d=0; d<blocks_per_line; d++)
+                {
+                    temp +=data[ B[d]+0]* x[(J[d]+0)*right_size+j];
+                    temp +=data[ B[d]+1]* x[(J[d]+1)*right_size+j];
+                    temp +=data[ B[d]+2]* x[(J[d]+2)*right_size+j];
+                }
+                int I = ((s*num_rows + i)*3+k)*right_size+j;
+                y[I] = alpha*temp + beta*y[I];
+            }
         }
     }
 }
@@ -52,7 +139,7 @@ void ell_multiply_kernel33( value_type alpha, value_type beta,
         for( int d=0; d<3; d++)
         {
             if( data_idx[i*3+d] != d) trivial = false;
-            if( cols_idx[i*3+d] != i+d-1) trivial = false;
+            //if( cols_idx[i*3+d] != i+d-1) trivial = false;
         }
     if( trivial)
     {
@@ -61,14 +148,17 @@ void ell_multiply_kernel33( value_type alpha, value_type beta,
 	{
 		for( int i=0; i<num_rows; i++)
 		{
-		        int J0 = (i==0 || i==num_rows-1)?(s*num_cols+cols_idx[i*3+0])*3:(s*num_cols+i+0-1)*3;
-		        int J1 = (i==0 || i==num_rows-1)?(s*num_cols+cols_idx[i*3+1])*3:(s*num_cols+i+1-1)*3;
-		        int J2 = (i==0 || i==num_rows-1)?(s*num_cols+cols_idx[i*3+2])*3:(s*num_cols+i+2-1)*3;
+            //int J0 = (i==0 || i==num_rows-1)?(s*num_cols+cols_idx[i*3+0])*3:(s*num_cols+i+0-1)*3;
+            //int J1 = (i==0 || i==num_rows-1)?(s*num_cols+cols_idx[i*3+1])*3:(s*num_cols+i+1-1)*3;
+            //int J2 = (i==0 || i==num_rows-1)?(s*num_cols+cols_idx[i*3+2])*3:(s*num_cols+i+2-1)*3;
+            int J0 = (i==0 || i==num_rows-1)?(s*num_cols+cols_idx[i*3+0])*3:(s*num_cols+cols_idx[i*3+0])*3;
+            int J1 = (i==0 || i==num_rows-1)?(s*num_cols+cols_idx[i*3+1])*3:(s*num_cols+cols_idx[i*3+1])*3;
+            int J2 = (i==0 || i==num_rows-1)?(s*num_cols+cols_idx[i*3+2])*3:(s*num_cols+cols_idx[i*3+2])*3;
 			for( int k=0; k<3; k++)
 			{
 				int B0 = (i==0 || i==num_rows-1)?(data_idx[i*3+0]*3+k)*3:(0*3+k)*3;
-			        int B1 = (i==0 || i==num_rows-1)?(data_idx[i*3+1]*3+k)*3:(1*3+k)*3;
-			        int B2 = (i==0 || i==num_rows-1)?(data_idx[i*3+2]*3+k)*3:(2*3+k)*3;
+                int B1 = (i==0 || i==num_rows-1)?(data_idx[i*3+1]*3+k)*3:(1*3+k)*3;
+                int B2 = (i==0 || i==num_rows-1)?(data_idx[i*3+2]*3+k)*3:(2*3+k)*3;
 				#pragma vector nontemporal(y)
 				for( int j=right_range[0]; j<right_range[1]; j++)
 				{
@@ -92,7 +182,7 @@ void ell_multiply_kernel33( value_type alpha, value_type beta,
 	}
     }
     else 
-        ell_multiply_kernel(alpha,beta, data, cols_idx, data_idx, num_rows, num_cols, 3, 3, left_size, right_size, right_range,  x, y);
+        ell_multiply_kernel3(alpha,beta, data, cols_idx, data_idx, num_rows, num_cols, 3, left_size, right_size, right_range,  x, y);
 }
 
 // multiply kernel, n=3, 2 blocks per line
@@ -110,11 +200,11 @@ void ell_multiply_kernel32( value_type alpha, value_type beta,
         for( int d=0; d<2; d++)
         {
             if( data_idx[i*2+d] != d) forward = backward = false;
-            if( cols_idx[i*2+d] != i+d-1) backward = false;
-            if( cols_idx[i*2+d] != i+d) forward = false;
+            //if( cols_idx[i*2+d] != i+d-1) backward = false;
+            //if( cols_idx[i*2+d] != i+d) forward = false;
         }
-    int diff = -1;
-    if(forward ) diff = 0;
+    //int diff = -1;
+    //if(forward ) diff = 0;
     if( forward || backward )
     {
 #pragma omp parallel for 
@@ -141,21 +231,25 @@ void ell_multiply_kernel32( value_type alpha, value_type beta,
     for( int s=0; s<left_size; s++)
     for( int i=1; i<num_rows-1; i++)
     for( int k=0; k<3; k++)
-    for( int j=right_range[0]; j<right_range[1]; j++)
     {
-        value_type temp = 0;
         int B0 = (0*3+k)*3;
         int B1 = (1*3+k)*3;
-        int J0 = (s*num_cols+i+0+diff)*3;
-        int J1 = (s*num_cols+i+1+diff)*3;
-        temp +=data[ B0+0]* x[(J0+0)*right_size+j];
-        temp +=data[ B0+1]* x[(J0+1)*right_size+j];
-        temp +=data[ B0+2]* x[(J0+2)*right_size+j];
-        temp +=data[ B1+0]* x[(J1+0)*right_size+j];
-        temp +=data[ B1+1]* x[(J1+1)*right_size+j];
-        temp +=data[ B1+2]* x[(J1+2)*right_size+j];
-        int I = ((s*num_rows + i)*3+k)*right_size+j;
-        y[I]=alpha*temp+beta*y[I];
+        //int J0 = (s*num_cols+i+0+diff)*3;
+        //int J1 = (s*num_cols+i+1+diff)*3;
+        int J0 = (s*num_cols+cols_idx[i*2+0])*3;
+        int J1 = (s*num_cols+cols_idx[i*2+1])*3;
+        for( int j=right_range[0]; j<right_range[1]; j++)
+        {
+            value_type temp = 0;
+            temp +=data[ B0+0]* x[(J0+0)*right_size+j];
+            temp +=data[ B0+1]* x[(J0+1)*right_size+j];
+            temp +=data[ B0+2]* x[(J0+2)*right_size+j];
+            temp +=data[ B1+0]* x[(J1+0)*right_size+j];
+            temp +=data[ B1+1]* x[(J1+1)*right_size+j];
+            temp +=data[ B1+2]* x[(J1+2)*right_size+j];
+            int I = ((s*num_rows + i)*3+k)*right_size+j;
+            y[I]=alpha*temp+beta*y[I];
+        }
     }
 #pragma omp parallel for 
     for( int s=0; s<left_size; s++)
@@ -179,7 +273,7 @@ void ell_multiply_kernel32( value_type alpha, value_type beta,
     }
     }
     else
-        ell_multiply_kernel(alpha,beta, data, cols_idx, data_idx, num_rows, num_cols, 2, 3, left_size, right_size, right_range,  x, y);
+        ell_multiply_kernel3(alpha,beta, data, cols_idx, data_idx, num_rows, num_cols, 2, left_size, right_size, right_range,  x, y);
 
 }
 // multiply kernel, n=3, 3 blocks per line, right_size = 1
@@ -208,19 +302,19 @@ void ell_multiply_kernel33x( value_type alpha, value_type beta,
     {
         value_type temp = 0;
         int B0 = (data_idx[i*3+0]*3+k)*3;
+        int B1 = (data_idx[i*3+1]*3+k)*3;
+        int B2 = (data_idx[i*3+2]*3+k)*3;
         int J0 = (s*num_cols+cols_idx[i*3+0])*3;
+        int J1 = (s*num_cols+cols_idx[i*3+1])*3;
+        int J2 = (s*num_cols+cols_idx[i*3+2])*3;
         temp +=data[ B0+0]* x[(J0+0)];
         temp +=data[ B0+1]* x[(J0+1)];
         temp +=data[ B0+2]* x[(J0+2)];
 
-        int B1 = (data_idx[i*3+1]*3+k)*3;
-        int J1 = (s*num_cols+cols_idx[i*3+1])*3;
         temp +=data[ B1+0]* x[(J1+0)];
         temp +=data[ B1+1]* x[(J1+1)];
         temp +=data[ B1+2]* x[(J1+2)];
 
-        int B2 = (data_idx[i*3+2]*3+k)*3;
-        int J2 = (s*num_cols+cols_idx[i*3+2])*3;
         temp +=data[ B2+0]* x[(J2+0)];
         temp +=data[ B2+1]* x[(J2+1)];
         temp +=data[ B2+2]* x[(J2+2)];
@@ -228,27 +322,32 @@ void ell_multiply_kernel33x( value_type alpha, value_type beta,
         y[I]=alpha*temp+beta*y[I];
     }
     for( int i=1; i<num_rows-1; i++)
-    for( int k=0; k<3; k++)
     {
-        value_type temp = 0;
-        int B0 = (0*3+k)*3;
+        //int J0 = (s*num_cols+cols_idx[i*3+0])*3;
+        //int J1 = (s*num_cols+cols_idx[i*3+1])*3;
+        //int J2 = (s*num_cols+cols_idx[i*3+2])*3;
         int J0 = (s*num_cols+i+0-1)*3;
-        temp +=data[ B0+0]* x[(J0+0)];
-        temp +=data[ B0+1]* x[(J0+1)];
-        temp +=data[ B0+2]* x[(J0+2)];
-        int B1 = (1*3+k)*3;
         int J1 = (s*num_cols+i+1-1)*3;
-        temp +=data[ B1+0]* x[(J1+0)];
-        temp +=data[ B1+1]* x[(J1+1)];
-        temp +=data[ B1+2]* x[(J1+2)];
-        int B2 = (2*3+k)*3;
         int J2 = (s*num_cols+i+2-1)*3;
-        temp +=data[ B2+0]* x[(J2+0)];
-        temp +=data[ B2+1]* x[(J2+1)];
-        temp +=data[ B2+2]* x[(J2+2)];
+        for( int k=0; k<3; k++)
+        {
+            value_type temp = 0;
+            int B0 = (0*3+k)*3;
+            temp +=data[ B0+0]* x[(J0+0)];
+            temp +=data[ B0+1]* x[(J0+1)];
+            temp +=data[ B0+2]* x[(J0+2)];
+            int B1 = (1*3+k)*3;
+            temp +=data[ B1+0]* x[(J1+0)];
+            temp +=data[ B1+1]* x[(J1+1)];
+            temp +=data[ B1+2]* x[(J1+2)];
+            int B2 = (2*3+k)*3;
+            temp +=data[ B2+0]* x[(J2+0)];
+            temp +=data[ B2+1]* x[(J2+1)];
+            temp +=data[ B2+2]* x[(J2+2)];
 
-        int I = ((s*num_rows + i)*3+k);
-        y[I]=alpha*temp+beta*y[I];
+            int I = ((s*num_rows + i)*3+k);
+            y[I]=alpha*temp+beta*y[I];
+        }
     }
     for( int i=num_rows-1; i<num_rows; i++)
     for( int k=0; k<3; k++)
@@ -280,7 +379,7 @@ void ell_multiply_kernel33x( value_type alpha, value_type beta,
     else 
     {
         int right_range[2] = {0,1};
-        ell_multiply_kernel(alpha,beta, data, cols_idx, data_idx, num_rows, num_cols, 3, 3, left_size, 1, right_range,  x, y);
+        ell_multiply_kernel3(alpha,beta, data, cols_idx, data_idx, num_rows, num_cols, 3, left_size, 1, right_range,  x, y);
     }
 }
 
@@ -298,11 +397,11 @@ void ell_multiply_kernel32x( value_type alpha, value_type beta,
         for( int d=0; d<2; d++)
         {
             if( data_idx[i*2+d] != d) forward = backward = false;
-            if( cols_idx[i*2+d] != i+d-1) backward = false;
-            if( cols_idx[i*2+d] != i+d) forward = false;
+            //if( cols_idx[i*2+d] != i+d-1) backward = false;
+            //if( cols_idx[i*2+d] != i+d) forward = false;
         }
-    int diff = -1;
-    if(forward ) {diff = 0; }
+    //int diff = -1;
+    //if(forward ) {diff = 0; }
     if( forward || backward )
     {
 #pragma omp parallel for
@@ -327,21 +426,25 @@ void ell_multiply_kernel32x( value_type alpha, value_type beta,
 #pragma omp parallel for
     for( int s=0; s<left_size; s++)
     for( int i=1; i<num_rows-1; i++)
-    for( int k=0; k<3; k++)
     {
-        value_type temp = 0;
-        int B0 = (0*3+k)*3;
-        int B1 = (1*3+k)*3;
-        int J0 = (s*num_cols+i+0+diff)*3;
-        int J1 = (s*num_cols+i+1+diff)*3;
-        temp +=data[ B0+0]* x[(J0+0)];
-        temp +=data[ B0+1]* x[(J0+1)];
-        temp +=data[ B0+2]* x[(J0+2)];
-        temp +=data[ B1+0]* x[(J1+0)];
-        temp +=data[ B1+1]* x[(J1+1)];
-        temp +=data[ B1+2]* x[(J1+2)];
-        int I = ((s*num_rows + i)*3+k);
-        y[I]=alpha*temp+beta*y[I];
+        //int J0 = (s*num_cols+i+0+diff)*3;
+        //int J1 = (s*num_cols+i+1+diff)*3;
+        int J0 = (s*num_cols+cols_idx[i*2+0])*3;
+        int J1 = (s*num_cols+cols_idx[i*2+1])*3;
+        for( int k=0; k<3; k++)
+        {
+            value_type temp = 0;
+            int B0 = (0*3+k)*3;
+            int B1 = (1*3+k)*3;
+            temp +=data[ B0+0]* x[(J0+0)];
+            temp +=data[ B0+1]* x[(J0+1)];
+            temp +=data[ B0+2]* x[(J0+2)];
+            temp +=data[ B1+0]* x[(J1+0)];
+            temp +=data[ B1+1]* x[(J1+1)];
+            temp +=data[ B1+2]* x[(J1+2)];
+            int I = ((s*num_rows + i)*3+k);
+            y[I]=alpha*temp+beta*y[I];
+        }
     }
 #pragma omp parallel for
     for( int s=0; s<left_size; s++)
@@ -366,7 +469,7 @@ void ell_multiply_kernel32x( value_type alpha, value_type beta,
     else
     {
         int right_range[2] = {0,1};
-        ell_multiply_kernel(alpha,beta, data, cols_idx, data_idx, num_rows, num_cols, 2, 3, left_size, 1, right_range, x, y);
+        ell_multiply_kernel3(alpha,beta, data, cols_idx, data_idx, num_rows, num_cols, 2, left_size, 1, right_range, x, y);
     }
 
 
@@ -376,9 +479,6 @@ template<class value_type>
 template<class DeviceContainer>
 void EllSparseBlockMatDevice<value_type>::launch_multiply_kernel( value_type alpha, const DeviceContainer& x, value_type beta, DeviceContainer& y) const
 {
-    assert( y.size() == (unsigned)num_rows*n*left_size*right_size);
-    assert( x.size() == (unsigned)num_cols*n*left_size*right_size);
-
     const value_type* data_ptr = thrust::raw_pointer_cast( &data[0]);
     const int* cols_ptr = thrust::raw_pointer_cast( &cols_idx[0]);
     const int* block_ptr = thrust::raw_pointer_cast( &data_idx[0]);
@@ -402,8 +502,8 @@ void EllSparseBlockMatDevice<value_type>::launch_multiply_kernel( value_type alp
                 ell_multiply_kernel32<value_type> ( alpha, beta, data_ptr, cols_ptr, block_ptr, num_rows, num_cols, left_size, right_size, right_range_ptr,  x_ptr,y_ptr);
         }
         else
-            ell_multiply_kernel<value_type>(alpha, beta,  
-                data_ptr, cols_ptr, block_ptr, num_rows, num_cols, blocks_per_line, 3, left_size, right_size, right_range_ptr,  x_ptr,y_ptr);
+            ell_multiply_kernel3<value_type>(alpha, beta,  
+                data_ptr, cols_ptr, block_ptr, num_rows, num_cols, blocks_per_line, left_size, right_size, right_range_ptr,  x_ptr,y_ptr);
     }
     else
         ell_multiply_kernel<value_type>  (alpha, beta,  
@@ -414,9 +514,6 @@ template<class value_type>
 template<class DeviceContainer>
 void CooSparseBlockMatDevice<value_type>::launch_multiply_kernel( value_type alpha, const DeviceContainer& x, value_type beta, DeviceContainer& y) const
 {
-    assert( y.size() == (unsigned)num_rows*n*left_size*right_size);
-    assert( x.size() == (unsigned)num_cols*n*left_size*right_size);
-
     for( int i=0; i<num_entries; i++)
 #pragma omp parallel for collapse(3)
     for( int s=0; s<left_size; s++)
diff --git a/inc/dg/blas_b.cu b/inc/dg/blas_b.cu
index 580832e0a..3aaeca20d 100644
--- a/inc/dg/blas_b.cu
+++ b/inc/dg/blas_b.cu
@@ -88,14 +88,14 @@ int main()
         dg::blas2::symv( M, x, y);
     t.toc();
     std::cout<<"jump X took                      "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
-    IMatrix inter; 
-    dg::blas2::transfer(dg::create::interpolation( grid, grid_half), inter);
-    Vector x_half = dg::evaluate( dg::zero, grid_half);
-    t.tic();
-    for( int i=0; i<multi; i++)
-        dg::blas2::gemv( inter, x_half, x);
-    t.toc();
-    std::cout<<"Interpolation on original grid  "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
+    //IMatrix inter; 
+    //dg::blas2::transfer(dg::create::interpolation( grid, grid_half), inter);
+    //Vector x_half = dg::evaluate( dg::zero, grid_half);
+    //t.tic();
+    //for( int i=0; i<multi; i++)
+    //    dg::blas2::gemv( inter, x_half, x);
+    //t.toc();
+    //std::cout<<"Interpolation on original grid  "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
     
 
     t.tic();
-- 
GitLab


From 4142f1a3a7ffa16e1ff26acca468eaa11f3495b3 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sat, 9 Sep 2017 10:56:05 +0200
Subject: [PATCH 269/453] test new sparse32x kernel

---
 inc/dg/backend/sparseblockmat_omp_kernels.h | 28 +++++++++++++++------
 1 file changed, 20 insertions(+), 8 deletions(-)

diff --git a/inc/dg/backend/sparseblockmat_omp_kernels.h b/inc/dg/backend/sparseblockmat_omp_kernels.h
index 4b892b233..770a80fed 100644
--- a/inc/dg/backend/sparseblockmat_omp_kernels.h
+++ b/inc/dg/backend/sparseblockmat_omp_kernels.h
@@ -404,6 +404,11 @@ void ell_multiply_kernel32x( value_type alpha, value_type beta,
     //if(forward ) {diff = 0; }
     if( forward || backward )
     {
+        value_type data_[18];
+        for( int k=0; k<2; k++)
+        for( int i=0; i<3; i++)
+        for( int j=0; j<3; j++)
+            data_[(i*2+k)*3+j] = data[(k*3+i)*3+j];
 #pragma omp parallel for
     for( int s=0; s<left_size; s++)
     for( int i=0; i<1; i++)
@@ -434,14 +439,21 @@ void ell_multiply_kernel32x( value_type alpha, value_type beta,
         for( int k=0; k<3; k++)
         {
             value_type temp = 0;
-            int B0 = (0*3+k)*3;
-            int B1 = (1*3+k)*3;
-            temp +=data[ B0+0]* x[(J0+0)];
-            temp +=data[ B0+1]* x[(J0+1)];
-            temp +=data[ B0+2]* x[(J0+2)];
-            temp +=data[ B1+0]* x[(J1+0)];
-            temp +=data[ B1+1]* x[(J1+1)];
-            temp +=data[ B1+2]* x[(J1+2)];
+            //int B0 = (0*3+k)*3;
+            //int B1 = (1*3+k)*3;
+            //temp +=data[ B0+0]* x[(J0+0)];
+            //temp +=data[ B0+1]* x[(J0+1)];
+            //temp +=data[ B0+2]* x[(J0+2)];
+            //temp +=data[ B1+0]* x[(J1+0)];
+            //temp +=data[ B1+1]* x[(J1+1)];
+            //temp +=data[ B1+2]* x[(J1+2)];
+            int B0 = k*6;
+            temp +=data_[ B0+0]* x[(J0+0)];
+            temp +=data_[ B0+1]* x[(J0+1)];
+            temp +=data_[ B0+2]* x[(J0+2)];
+            temp +=data_[ B0+3]* x[(J0+3)];
+            temp +=data_[ B0+4]* x[(J0+4)];
+            temp +=data_[ B0+5]* x[(J0+5)];
             int I = ((s*num_rows + i)*3+k);
             y[I]=alpha*temp+beta*y[I];
         }
-- 
GitLab


From 2c8a7becd43c4b6aabac07427a6fa90f63ac5708 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <Matthias.Wiesenberger@uibk.ac.at>
Date: Sat, 9 Sep 2017 11:59:24 +0200
Subject: [PATCH 270/453] optimizing smore

---
 inc/dg/backend/sparseblockmat_omp_kernels.h | 71 ++++++++-------------
 1 file changed, 28 insertions(+), 43 deletions(-)

diff --git a/inc/dg/backend/sparseblockmat_omp_kernels.h b/inc/dg/backend/sparseblockmat_omp_kernels.h
index 770a80fed..943174401 100644
--- a/inc/dg/backend/sparseblockmat_omp_kernels.h
+++ b/inc/dg/backend/sparseblockmat_omp_kernels.h
@@ -13,29 +13,28 @@ void ell_multiply_kernel( value_type alpha, value_type beta,
          const value_type * RESTRICT x, value_type * RESTRICT y
          )
 {
-    //simplest implementation
 #pragma omp parallel for collapse(2)
     for( int s=0; s<left_size; s++)
     for( int i=0; i<num_rows; i++)
-    for( int k=0; k<n; k++)
     {
-        int B[blocks_per_line], J[blocks_per_line];
+        int J[blocks_per_line];
         for( int d=0; d<blocks_per_line; d++)
-        {
-            B[d] = (data_idx[i*blocks_per_line+d]*n+k)*n;
             J[d] = (s*num_cols+cols_idx[i*blocks_per_line+d])*n;
-        }
-        for( int j=right_range[0]; j<right_range[1]; j++)
+        for( int k=0; k<n; k++)
         {
-            value_type temp = 0;
+            int B[blocks_per_line];
             for( int d=0; d<blocks_per_line; d++)
+                B[d] = (data_idx[i*blocks_per_line+d]*n+k)*n;
+            for( int j=right_range[0]; j<right_range[1]; j++)
             {
-                for( int q=0; q<n; q++) //multiplication-loop
-                    temp += data[ B[d]+q]*
-                        x[(J[d]+q)*right_size+j];
+                value_type temp = 0;
+                for( int d=0; d<blocks_per_line; d++)
+                    for( int q=0; q<n; q++) //multiplication-loop
+                        temp += data[ B[d]+q]*
+                            x[(J[d]+q)*right_size+j];
+                int I = ((s*num_rows + i)*n+k)*right_size+j;
+                y[I] = alpha*temp + beta*y[I];
             }
-            int I = ((s*num_rows + i)*n+k)*right_size+j;
-            y[I] = alpha*temp + beta*y[I];
         }
     }
 }
@@ -130,10 +129,6 @@ void ell_multiply_kernel33( value_type alpha, value_type beta,
          const value_type * RESTRICT x, value_type * RESTRICT y
          )
 {
- // left_size = 1
- // right_range[1]-right_range[0] = 2304
- // num_rows = 768
-
     bool trivial = true;
     for( int i=1; i<num_rows-1; i++)
         for( int d=0; d<3; d++)
@@ -294,6 +289,11 @@ void ell_multiply_kernel33x( value_type alpha, value_type beta,
         }
     if( trivial)
     {
+        value_type data_[27];
+        for( int k=0; k<3; k++)
+        for( int i=0; i<3; i++)
+        for( int j=0; j<3; j++)
+            data_[(i*3+k)*3+j] = data[(k*3+i)*3+j];
 #pragma omp parallel for
     for( int s=0; s<left_size; s++)
     {
@@ -324,26 +324,20 @@ void ell_multiply_kernel33x( value_type alpha, value_type beta,
     for( int i=1; i<num_rows-1; i++)
     {
         //int J0 = (s*num_cols+cols_idx[i*3+0])*3;
-        //int J1 = (s*num_cols+cols_idx[i*3+1])*3;
-        //int J2 = (s*num_cols+cols_idx[i*3+2])*3;
         int J0 = (s*num_cols+i+0-1)*3;
-        int J1 = (s*num_cols+i+1-1)*3;
-        int J2 = (s*num_cols+i+2-1)*3;
         for( int k=0; k<3; k++)
         {
             value_type temp = 0;
-            int B0 = (0*3+k)*3;
-            temp +=data[ B0+0]* x[(J0+0)];
-            temp +=data[ B0+1]* x[(J0+1)];
-            temp +=data[ B0+2]* x[(J0+2)];
-            int B1 = (1*3+k)*3;
-            temp +=data[ B1+0]* x[(J1+0)];
-            temp +=data[ B1+1]* x[(J1+1)];
-            temp +=data[ B1+2]* x[(J1+2)];
-            int B2 = (2*3+k)*3;
-            temp +=data[ B2+0]* x[(J2+0)];
-            temp +=data[ B2+1]* x[(J2+1)];
-            temp +=data[ B2+2]* x[(J2+2)];
+            int B0 = 9*k;
+            temp +=data_[ B0+0]* x[(J0+0)];
+            temp +=data_[ B0+1]* x[(J0+1)];
+            temp +=data_[ B0+2]* x[(J0+2)];
+            temp +=data_[ B0+3]* x[(J0+3)];
+            temp +=data_[ B0+4]* x[(J0+4)];
+            temp +=data_[ B0+5]* x[(J0+5)];
+            temp +=data_[ B0+6]* x[(J0+6)];
+            temp +=data_[ B0+7]* x[(J0+7)];
+            temp +=data_[ B0+8]* x[(J0+8)];
 
             int I = ((s*num_rows + i)*3+k);
             y[I]=alpha*temp+beta*y[I];
@@ -398,7 +392,7 @@ void ell_multiply_kernel32x( value_type alpha, value_type beta,
         {
             if( data_idx[i*2+d] != d) forward = backward = false;
             //if( cols_idx[i*2+d] != i+d-1) backward = false;
-            //if( cols_idx[i*2+d] != i+d) forward = false;
+            if( cols_idx[i*2+d] != cols_idx[i*2]+d) forward = false;
         }
     //int diff = -1;
     //if(forward ) {diff = 0; }
@@ -435,18 +429,9 @@ void ell_multiply_kernel32x( value_type alpha, value_type beta,
         //int J0 = (s*num_cols+i+0+diff)*3;
         //int J1 = (s*num_cols+i+1+diff)*3;
         int J0 = (s*num_cols+cols_idx[i*2+0])*3;
-        int J1 = (s*num_cols+cols_idx[i*2+1])*3;
         for( int k=0; k<3; k++)
         {
             value_type temp = 0;
-            //int B0 = (0*3+k)*3;
-            //int B1 = (1*3+k)*3;
-            //temp +=data[ B0+0]* x[(J0+0)];
-            //temp +=data[ B0+1]* x[(J0+1)];
-            //temp +=data[ B0+2]* x[(J0+2)];
-            //temp +=data[ B1+0]* x[(J1+0)];
-            //temp +=data[ B1+1]* x[(J1+1)];
-            //temp +=data[ B1+2]* x[(J1+2)];
             int B0 = k*6;
             temp +=data_[ B0+0]* x[(J0+0)];
             temp +=data_[ B0+1]* x[(J0+1)];
-- 
GitLab


From 3e92b292f3f38771e52998f2ebdd93304eb538c7 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <Matthias.Wiesenberger@uibk.ac.at>
Date: Sat, 9 Sep 2017 12:27:37 +0200
Subject: [PATCH 271/453] removed if else in thrust_vector_blas

---
 config/devices/devices.mk             |   2 +-
 config/marconi.mk                     |   2 +-
 inc/dg/backend/thrust_vector_blas.cuh | 167 +++-----------------------
 3 files changed, 17 insertions(+), 154 deletions(-)

diff --git a/config/devices/devices.mk b/config/devices/devices.mk
index 332f034c3..11f325b2c 100644
--- a/config/devices/devices.mk
+++ b/config/devices/devices.mk
@@ -28,5 +28,5 @@ CFLAGS+=-Wall -x c++
 CFLAGS+= -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_OMP
 CFLAGS+= $(OMPFLAG) #-mmic 
 MPICFLAGS+=$(CFLAGS) #includes values in CFLAGS defined later
-OPT=-O3 -xMIC-AVX512 -fma -finline-functions -align
+OPT=-O3 -xMIC-AVX512 
 endif #device=mic
diff --git a/config/marconi.mk b/config/marconi.mk
index afee829bf..c535a0086 100644
--- a/config/marconi.mk
+++ b/config/marconi.mk
@@ -5,7 +5,7 @@ INCLUDE += -I$(NETCDF_INC) -I$(HDF5_INC)
 GLFLAGS  = -lm 
 CC=icc
 MPICC=mpiicc
-OPT=-O3 -xHost -restrict # overwritten for mic in devices.mk
+OPT=-O3 -xHost # overwritten for mic in devices.mk
 #MPICFLAGS+= -DMPICH_IGNORE_CXX_SEEK
 OMPFLAG=-qopenmp
 CFLAGS=-restrict
diff --git a/inc/dg/backend/thrust_vector_blas.cuh b/inc/dg/backend/thrust_vector_blas.cuh
index e98d47a19..c3161abae 100644
--- a/inc/dg/backend/thrust_vector_blas.cuh
+++ b/inc/dg/backend/thrust_vector_blas.cuh
@@ -150,24 +150,9 @@ inline void doAxpby( typename Vector::value_type alpha,
     const typename Vector::value_type * RESTRICT x_ptr = thrust::raw_pointer_cast( &x.data()[0]);
     typename Vector::value_type * RESTRICT y_ptr = thrust::raw_pointer_cast( &y.data()[0]);
     unsigned size = x.size();
-    if( beta == 1.)
-    {
-        #pragma omp parallel for simd
-        for( unsigned i=0; i<size; i++)
-            y_ptr[i] += alpha*x_ptr[i];
-    }
-    else if (beta == 0)
-    {
-        #pragma omp parallel for simd
-        for( unsigned i=0; i<size; i++)
-            y_ptr[i] = alpha*x_ptr[i];
-    }
-    else
-    {
-        #pragma omp parallel for simd
-        for( unsigned i=0; i<size; i++)
-            y_ptr[i] = alpha*x_ptr[i] + beta*y_ptr[i];
-    }
+    #pragma omp parallel for simd
+    for( unsigned i=0; i<size; i++)
+        y_ptr[i] = alpha*x_ptr[i] + beta*y_ptr[i];
 #else
     if( beta != 0)
         thrust::transform( x.begin(), x.end(), y.begin(), y.begin(), 
@@ -221,24 +206,9 @@ inline void doAxpby( typename Vector::value_type alpha,
     const typename Vector::value_type * RESTRICT y_ptr = thrust::raw_pointer_cast( &y.data()[0]);
     typename Vector::value_type * RESTRICT z_ptr = thrust::raw_pointer_cast( &z.data()[0]);
     unsigned size = x.size();
-    if( gamma == 1.)
-    {
-        #pragma omp parallel for simd
-        for( unsigned i=0; i<size; i++)
-            z_ptr[i] += alpha*x_ptr[i]+beta*y_ptr[i];
-    }
-    else if (gamma == 0)
-    {
-        #pragma omp parallel for simd
-        for( unsigned i=0; i<size; i++)
-            z_ptr[i] = alpha*x_ptr[i]+beta*y_ptr[i];
-    }
-    else
-    {
-        #pragma omp parallel for simd
-        for( unsigned i=0; i<size; i++)
-            z_ptr[i] = alpha*x_ptr[i] + beta*y_ptr[i] + gamma*z_ptr[i];
-    }
+    #pragma omp parallel for simd
+    for( unsigned i=0; i<size; i++)
+        z_ptr[i] = alpha*x_ptr[i] + beta*y_ptr[i] + gamma*z_ptr[i];
 #else
     if( gamma==0)
     {
@@ -304,62 +274,11 @@ inline void doPointwiseDot(
     const typename Vector::value_type * x1_ptr = thrust::raw_pointer_cast( &(x1.data()[0]));
     const typename Vector::value_type * x2_ptr = thrust::raw_pointer_cast( &(x2.data()[0]));
      typename Vector::value_type * y_ptr = thrust::raw_pointer_cast( &(y.data()[0]));
-    if( y_ptr != x1_ptr && y_ptr != x2_ptr)
-    {
-        typename Vector::value_type * RESTRICT yr_ptr = thrust::raw_pointer_cast( &(y.data()[0]));
-        unsigned size = x1.size();
-        if( beta == 0)
-        {
-            #pragma omp parallel for simd
-            for( unsigned i=0; i<size; i++)
-            {
-                yr_ptr[i] = alpha*x1_ptr[i]*x2_ptr[i];
-            }
-        }
-        else if ( beta == 1)
-        {
-            #pragma omp parallel for simd
-            for( unsigned i=0; i<size; i++)
-            {
-                yr_ptr[i] += alpha*x1_ptr[i]*x2_ptr[i];
-            }
-        }
-        else
-        {
-            #pragma omp parallel for simd
-            for( unsigned i=0; i<size; i++)
-            {
-                yr_ptr[i] = alpha*x1_ptr[i]*x2_ptr[i]+beta*yr_ptr[i];
-            }
-        }
-    }
-    else
+    unsigned size = x1.size();
+    #pragma omp parallel for simd
+    for( unsigned i=0; i<size; i++)
     {
-        unsigned size = x1.size();
-        if( beta == 0)
-        {
-            #pragma omp parallel for simd
-            for( unsigned i=0; i<size; i++)
-            {
-                y_ptr[i] = alpha*x1_ptr[i]*x2_ptr[i];
-            }
-        }
-        else if ( beta == 1)
-        {
-            #pragma omp parallel for simd
-            for( unsigned i=0; i<size; i++)
-            {
-                y_ptr[i] += alpha*x1_ptr[i]*x2_ptr[i];
-            }
-        }
-        else
-        {
-            #pragma omp parallel for simd
-            for( unsigned i=0; i<size; i++)
-            {
-                y_ptr[i] = alpha*x1_ptr[i]*x2_ptr[i]+beta*y_ptr[i];
-            }
-        }
+        y_ptr[i] = alpha*x1_ptr[i]*x2_ptr[i]+beta*y_ptr[i];
     }
 #else
     thrust::transform( 
@@ -434,68 +353,12 @@ inline void doPointwiseDot(
           value_type * z_ptr = thrust::raw_pointer_cast( z.data());
     unsigned size = x1.size();
 #if THRUST_DEVICE_SYSTEM!=THRUST_DEVICE_SYSTEM_CUDA
-    if( z_ptr != x1_ptr && z_ptr != x2_ptr && z_ptr != y1_ptr && z_ptr != y2_ptr)
-    {
-        value_type * RESTRICT zr_ptr = thrust::raw_pointer_cast( &(z.data()[0]));
-        if(gamma==0)
-        {
-            #pragma omp parallel for simd
-            for( unsigned i=0; i<size; i++)
-            {
-                zr_ptr[i] = alpha*x1_ptr[i]*y1_ptr[i] 
-                            +beta*x2_ptr[i]*y2_ptr[i];
-            }
-        }
-        else if(gamma==1.)
-        {
-            #pragma omp parallel for simd
-            for( unsigned i=0; i<size; i++)
-            {
-                zr_ptr[i] += alpha*x1_ptr[i]*y1_ptr[i] 
-                            +beta*x2_ptr[i]*y2_ptr[i];
-            }
-        }
-        else
-        {
-            #pragma omp parallel for simd
-            for( unsigned i=0; i<size; i++)
-            {
-                zr_ptr[i] = alpha*x1_ptr[i]*y1_ptr[i] 
-                            +beta*x2_ptr[i]*y2_ptr[i]
-                            +gamma*zr_ptr[i];
-            }
-        }
-    }
-    else
+    #pragma omp parallel for simd
+    for( unsigned i=0; i<size; i++)
     {
-        if(gamma==0)
-        {
-            #pragma omp parallel for simd
-            for( unsigned i=0; i<size; i++)
-            {
-                z_ptr[i] = alpha*x1_ptr[i]*y1_ptr[i] 
-                            +beta*x2_ptr[i]*y2_ptr[i];
-            }
-        }
-        else if(gamma==1.)
-        {
-            #pragma omp parallel for simd
-            for( unsigned i=0; i<size; i++)
-            {
-                z_ptr[i] += alpha*x1_ptr[i]*y1_ptr[i] 
-                            +beta*x2_ptr[i]*y2_ptr[i];
-            }
-        }
-        else
-        {
-            #pragma omp parallel for simd
-            for( unsigned i=0; i<size; i++)
-            {
-                z_ptr[i] = alpha*x1_ptr[i]*y1_ptr[i] 
-                            +beta*x2_ptr[i]*y2_ptr[i]
-                            +gamma*z_ptr[i];
-            }
-        }
+        z_ptr[i] = alpha*x1_ptr[i]*y1_ptr[i] 
+                    +beta*x2_ptr[i]*y2_ptr[i]
+                    +gamma*z_ptr[i];
     }
 #else
     //set up kernel parameters
-- 
GitLab


From 6a6cb3a2b9768f113397d2b961f757dad1e5f514 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <Matthias.Wiesenberger@uibk.ac.at>
Date: Sat, 9 Sep 2017 12:56:05 +0200
Subject: [PATCH 272/453] introduced skylake device

---
 config/devices/devices.mk | 7 +++++++
 config/marconi.mk         | 2 +-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/config/devices/devices.mk b/config/devices/devices.mk
index 11f325b2c..64837b888 100644
--- a/config/devices/devices.mk
+++ b/config/devices/devices.mk
@@ -30,3 +30,10 @@ CFLAGS+= $(OMPFLAG) #-mmic
 MPICFLAGS+=$(CFLAGS) #includes values in CFLAGS defined later
 OPT=-O3 -xMIC-AVX512 
 endif #device=mic
+ifeq ($(strip $(device)),skl)
+CFLAGS+=-Wall -x c++ 
+CFLAGS+= -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_OMP
+CFLAGS+= $(OMPFLAG) #-mmic 
+MPICFLAGS+=$(CFLAGS) #includes values in CFLAGS defined later
+OPT=-xCORE-AVX512 -mtune=skylake -O3 
+endif #device=mic
diff --git a/config/marconi.mk b/config/marconi.mk
index c535a0086..5f7a50d8e 100644
--- a/config/marconi.mk
+++ b/config/marconi.mk
@@ -5,7 +5,7 @@ INCLUDE += -I$(NETCDF_INC) -I$(HDF5_INC)
 GLFLAGS  = -lm 
 CC=icc
 MPICC=mpiicc
-OPT=-O3 -xHost # overwritten for mic in devices.mk
+OPT=-O3 -xHost  # overwritten for mic in devices.mk
 #MPICFLAGS+= -DMPICH_IGNORE_CXX_SEEK
 OMPFLAG=-qopenmp
 CFLAGS=-restrict
-- 
GitLab


From cb023bf315e89d41fdf5cfd64e37f654d02a87a1 Mon Sep 17 00:00:00 2001
From:  <jagath@FYS-PPFE-820.win.dtu.dk>
Date: Tue, 12 Sep 2017 11:05:20 +0200
Subject: [PATCH 273/453] v-scheme upgrade multigrid

---
 inc/dg/elliptic2d_b.cu | 101 +++++++++++---------
 inc/dg/multigrid.h     | 210 +++++++++++++++++++++++++++++++++++------
 2 files changed, 237 insertions(+), 74 deletions(-)

diff --git a/inc/dg/elliptic2d_b.cu b/inc/dg/elliptic2d_b.cu
index d7a3bd9e4..53eb224c4 100644
--- a/inc/dg/elliptic2d_b.cu
+++ b/inc/dg/elliptic2d_b.cu
@@ -40,12 +40,19 @@ int main()
     unsigned n, Nx, Ny; 
     double eps;
     double jfactor;
-    std::cout << "Type n, Nx and Ny and epsilon and jfactor (1)! \n";
+    
+	n = 3;
+	Nx = Ny = 40;
+	eps = 1e-4;
+	jfactor = 1;
+	
+	/*std::cout << "Type n, Nx and Ny and epsilon and jfactor (1)! \n";
     std::cin >> n >> Nx >> Ny; //more N means less iterations for same error
-    std::cin >> eps >> jfactor;
-    std::cout << "Computation on: "<< n <<" x "<<Nx<<" x "<<Ny<<std::endl;
-    //std::cout << "# of 2d cells                 "<< Nx*Ny <<std::endl;
-    dg::CartesianGrid2d grid( 0, lx, 0, ly, n, Nx, Ny, bcx, bcy);
+    std::cin >> eps >> jfactor;*/
+    std::cout << "Computation on: "<< n <<" x "<< Nx <<" x "<< Ny << std::endl;
+    //std::cout << "# of 2d cells                 "<< Nx*Ny <<std::endl;		
+	
+	dg::CartesianGrid2d grid( 0, lx, 0, ly, n, Nx, Ny, bcx, bcy);
     dg::DVec w2d = dg::create::weights( grid);
     dg::DVec v2d = dg::create::inv_weights( grid);
     dg::DVec one = dg::evaluate( dg::one, grid);
@@ -59,29 +66,37 @@ int main()
     dg::DVec temp = x;
 
 
-    std::cout << "Create Polarisation object and set chi!\n";
-    t.tic();
+    //std::cout << "Create Polarisation object and set chi!\n";
     {
-    unsigned stages=3;
-    dg::MultigridCG2d<dg::aGeometry2d, dg::DMatrix, dg::DVec > multigrid( grid, stages);
-    std::vector<dg::DVec> chi_ = multigrid.project( chi);
-    std::vector<dg::Elliptic<dg::aGeometry2d, dg::DMatrix, dg::DVec> > multi_pol( stages);
-    for( unsigned u=0; u<stages; u++)
-    {
-        multi_pol[u].construct( multigrid.grids()[u].get(), dg::not_normed, dg::centered, jfactor); 
-        multi_pol[u].set_chi( chi_[u]);
-    }
-
-    t.toc();
-    //std::cout << "Creation of polarisation object took: "<<t.diff()<<"s\n";
-    std::cout << eps<<" \n";
-    t.tic();
-    std::vector<unsigned> number = multigrid.direct_solve( multi_pol, x, b, eps);
-    for( unsigned u=0; u<number.size(); u++)
-        std::cout << " # iterations grid "<<number.size()-1-u<<" "<<number[number.size()-1-u] << " \n";
-
-    t.toc();
-    //std::cout << "Took "<<t.diff()<<"s\n";
+		t.tic();
+
+		unsigned stages=4;
+
+		dg::MultigridCG2d<dg::aGeometry2d, dg::DMatrix, dg::DVec > multigrid( grid, stages, 1);
+		
+		std::vector<dg::DVec> chi_ = multigrid.project( chi);
+		std::vector<dg::Elliptic<dg::aGeometry2d, dg::DMatrix, dg::DVec> > multi_pol( stages);
+		
+		for(unsigned u=0; u<stages; u++)
+		{
+			multi_pol[u].construct( multigrid.grids()[u].get(), dg::not_normed, dg::centered, jfactor); 
+			multi_pol[u].set_chi( chi_[u]);
+		}
+
+		t.toc();
+    
+		//std::cout << "Creation of polarisation object took: "<<t.diff()<<"s\n";
+		//std::cout << eps<<" \n";
+    	
+		t.tic();
+
+		std::vector<unsigned> number = multigrid.solve(multi_pol, x, b, eps);
+		
+		//for( unsigned u=0; u<number.size(); u++)
+		//	std::cout << " # iterations stage "<< number.size()-1-u << " " << number[number.size()-1-u] << " \n";
+
+		t.toc();
+		std::cout << "Took "<< t.diff() <<"s\n";
     }
 
     //compute error
@@ -95,25 +110,25 @@ int main()
     const double norm = dg::blas2::dot( w2d, solution);
     std::cout << " "<<sqrt( err/norm);
     {
-    dg::Elliptic<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> pol_forward( grid, dg::not_normed, dg::centered, jfactor);
-    pol_forward.set_chi( chi);
-    x = temp;
-    dg::Invert<dg::DVec > invert_fw( x, n*n*Nx*Ny, eps);
-    std::cout << " "<< invert_fw( pol_forward, x, b, v2d, chi_inv);
-    dg::blas1::axpby( 1.,x,-1., solution, error);
-    err = dg::blas2::dot( w2d, error);
-    std::cout << " "<<sqrt( err/norm);
+		dg::Elliptic<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> pol_forward( grid, dg::not_normed, dg::centered, jfactor);
+		pol_forward.set_chi( chi);
+		x = temp;
+		dg::Invert<dg::DVec > invert_fw( x, n*n*Nx*Ny, eps);
+		std::cout << " "<< invert_fw( pol_forward, x, b, v2d, chi_inv);
+		dg::blas1::axpby( 1.,x,-1., solution, error);
+		err = dg::blas2::dot( w2d, error);
+		std::cout << " "<<sqrt( err/norm);
     }
 
     {
-    dg::Elliptic<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> pol_backward( grid, dg::not_normed, dg::backward, jfactor);
-    pol_backward.set_chi( chi);
-    x = temp;
-    dg::Invert<dg::DVec > invert_bw( x, n*n*Nx*Ny, eps);
-    std::cout << " "<< invert_bw( pol_backward, x, b, v2d, chi_inv);
-    dg::blas1::axpby( 1.,x,-1., solution, error);
-    err = dg::blas2::dot( w2d, error);
-    std::cout << " "<<sqrt( err/norm)<<std::endl;
+		dg::Elliptic<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> pol_backward( grid, dg::not_normed, dg::backward, jfactor);
+		pol_backward.set_chi( chi);
+		x = temp;
+		dg::Invert<dg::DVec > invert_bw( x, n*n*Nx*Ny, eps);
+		std::cout << " "<< invert_bw( pol_backward, x, b, v2d, chi_inv);
+		dg::blas1::axpby( 1.,x,-1., solution, error);
+		err = dg::blas2::dot( w2d, error);
+		std::cout << " "<<sqrt( err/norm)<<std::endl;
     }
 
 
diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index 81087de08..d3e392065 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -14,61 +14,129 @@ namespace dg
 template< class Geometry, class Matrix, class container> 
 struct MultigridCG2d
 {
-    MultigridCG2d( const Geometry& grid, unsigned stages, int extrapolation_type = 2 ) {
-        stages_=stages;
-        if( stages < 2 ) throw Error( Message(_ping_)<<" There must be minimum 2 stages in a multigrid solver! You gave " << stages);
-        grids_.resize( stages);
+    MultigridCG2d( const Geometry& grid, const unsigned stages, const int scheme_type = 0, const int extrapolation_type = 2 )
+    {
+        stages_= stages;
+        if(stages < 2 ) throw Error( Message(_ping_)<<" There must be minimum 2 stages in a multigrid solver! You gave " << stages);
+        
+		grids_.resize(stages);
+        cg_.resize(stages);
+
         grids_[0].reset( grid);
         //grids_[0].get().display();
-        for( unsigned u=1; u<stages; u++)
+        
+		for(unsigned u=1; u<stages; u++)
         {
-            grids_[u] = grids_[u-1]; //deep copy
+            grids_[u] = grids_[u-1]; // deep copy
             grids_[u].get().multiplyCellNumbers(0.5, 0.5);
             //grids_[u].get().display();
         }
-        inter_.resize(stages-1);
+        
+		inter_.resize(stages-1);
         project_.resize( stages-1);
-        for( unsigned u=0; u<stages-1; u++)
+        
+		for(unsigned u=0; u<stages-1; u++)
         {
-            //Projecting from one grid to the next is the same as 
-            //projecting from the original grid to the coarse grids
-            project_[u] = dg::create::fast_projection( grids_[u].get(), 2,2);
-            inter_[u] = dg::create::fast_interpolation(grids_[u+1].get(),2,2);
+            // Projecting from one grid to the next is the same as 
+            // projecting from the original grid to the coarse grids
+            project_[u] = dg::create::fast_projection(grids_[u].get(), 2, 2);
+            inter_[u] = dg::create::fast_interpolation(grids_[u+1].get(), 2, 2);
         }
 
-        dg::blas1::transfer( dg::evaluate( dg::zero, grid), x0_);
-        x1_=x0_, x2_=x0_;
-        x_ =  project( x0_); 
-        r_ = x_, b_ = x_;
-        set_extrapolationType(extrapolation_type);
-        cg_.resize( stages);
-        for( unsigned u=0; u<stages; u++)
-            cg_[u].construct( x_[u], x_[u].size());
+        dg::blas1::transfer(dg::evaluate(dg::zero, grid), x0_);
+        x1_ = x0_, x2_ = x0_;
+        x_ = project(x0_); 
+        r_ = x_, 
+		b_ = x_;        
+		set_extrapolationType(extrapolation_type);
+        set_scheme(scheme_type);        
     }
 
-    template<class SymmetricOp>
+	template<class SymmetricOp>
+	std::vector<unsigned> solve(/*const*/ std::vector<SymmetricOp>& op, container& x, const container& b, const double eps)
+	{
+		dg::blas1::axpbygz(alpha[0], x0_, alpha[1], x1_, alpha[2], x2_);
+		x_[0].swap(x2_);
+
+        project(x_[0], x_);
+		project(b, b_);
+
+        //
+        // define new rhs
+        for (unsigned u = 0; u < stages_; u++)
+            dg::blas1::pointwiseDivide(b_[u], op[u].inv_weights(), b_[u]);        
+        
+        unsigned int numStageSteps = m_schemeLayout.size();
+		std::vector<unsigned> number(numStageSteps);
+
+        unsigned u = m_startStage;
+
+        //for (unsigned u = stages_ - 1; u>0; u--)
+        for(unsigned i = 0; i < numStageSteps; i++)
+		{
+            /*if (m_schemeLayout[i].m_step > 0)
+            {
+                u += m_schemeLayout[i].m_step;
+                continue;
+            }*/
+
+            cg_[u].set_max(m_schemeLayout[i].m_niter);
+
+			number[i] = cg_[u](op[u], x_[u], b_[u], op[u].precond(), op[u].inv_weights(), eps);
+
+            std::cout << "pass: " << i << ", stage: " << u << ", max iter: " << m_schemeLayout[i].m_niter << ", iter: " << number[i] << std::endl;
+			
+            unsigned w = u + m_schemeLayout[i].m_step;
+            
+            if(m_schemeLayout[i].m_step < 0)
+                dg::blas2::symv(inter_[w], x_[u], x_[w]);
+            else if(m_schemeLayout[i].m_step > 0)
+                dg::blas2::symv(project_[u], x_[u], x_[w]);
+
+            //u += m_schemeLayout[i].m_step;
+            u = w;
+		}
+                
+        //dg::blas1::pointwiseDivide(b_[0], op[0].inv_weights(), b_[0]);
+		//number[0] = cg_[0](op[0], x_[0], b_[0], op[0].precond(), op[0].inv_weights(), eps);
+
+		x_[0].swap(x);
+		x1_.swap(x2_);
+		x0_.swap(x1_);
+
+		blas1::copy(x, x0_);
+		return number;
+	}
+
+    /*template<class SymmetricOp>
     std::vector<unsigned> direct_solve( std::vector<SymmetricOp>& op, container&  x, const container& b, double eps)
     {
         dg::blas1::axpbygz( alpha[0], x0_, alpha[1], x1_, alpha[2], x2_); 
-        x_[0].swap(x2_);
-        project( x_[0], x_);
+        
+		x_[0].swap(x2_);
+        
+		project( x_[0], x_);
         project( b, b_);
+
         std::vector<unsigned> number(stages_);
-        for( unsigned u=stages_-1; u>0; u--)
+        
+		for( unsigned u=stages_-1; u>0; u--)
         {
             dg::blas1::pointwiseDivide( b_[u], op[u].inv_weights(), b_[u]);
             number[u] = cg_[u]( op[u], x_[u], b_[u], op[u].precond(), op[u].inv_weights(), eps, 1.);
             dg::blas2::symv( inter_[u-1], x_[u], x_[u-1]);
         }
-        dg::blas1::pointwiseDivide( b_[0], op[0].inv_weights(), b_[0]);
+        
+		dg::blas1::pointwiseDivide( b_[0], op[0].inv_weights(), b_[0]);
         number[0] = cg_[0]( op[0], x_[0], b_[0], op[0].precond(), op[0].inv_weights(), eps);
-        x_[0].swap( x);
+        
+		x_[0].swap( x);
         x1_.swap( x2_);
         x0_.swap( x1_);
         
         blas1::copy( x, x0_);
         return number;
-    }
+    }*/
 
     ///src may alias first element of out
     void project( const container& src, std::vector<container>& out)
@@ -110,17 +178,97 @@ struct MultigridCG2d
         }
     }
 
-    const std::vector<dg::Handle< Geometry > > grids()const{return grids_;}
+    const std::vector<dg::Handle< Geometry > > grids()const { return grids_; }
+
+private:
+
+	void set_scheme(const int scheme_type)
+	{
+        assert(scheme_type <= 1 && scheme_type >= 0);
+        
+        // initialize one conjugate-gradient for each grid size
+        for (unsigned u = 0; u < stages_; u++)
+            cg_[u].construct(x_[u], 1);
+
+        switch (scheme_type)
+        {
+            // from coarse to fine
+        case(0):
+            
+            //m_mode = nestediteration;
+            m_startStage = stages_ - 1;
+            for (int u = stages_ - 1; u >= 0; u--)
+                m_schemeLayout.push_back(stepinfo(-1, x_[u].size()));
+            break;
+
+        case(1):
+
+            //m_mode = correctionscheme;
+            m_startStage = 0;
+            for (unsigned u = 0; u < stages_-1; u++)
+                m_schemeLayout.push_back(stepinfo(1, x_[u].size()));
+            
+            for (int u = stages_ - 1; u >= 0; u--)
+                m_schemeLayout.push_back(stepinfo(-1, x_[u].size()));
+
+            break;
+
+        default:
+            break;
+        }
+
+        // there is no step at the last stage so the step must be zero
+        m_schemeLayout.back().m_step = 0;
+
+        PrintScheme();
+
+        // checks:
+        // (0) the last entry should be the stage before the original grid
+        // (1) there can not be more than n-1 interpolations in succession
+        
+        unsigned u = m_startStage;
+        assert(u >= 0 && u <= stages_ - 1);
+        for (unsigned i = 0; i < m_schemeLayout.size(); i++)
+        {
+            u += m_schemeLayout[i].m_step;
+            assert(u >= 0 && u <= stages_ - 1);
+        }   
+        assert(u == 0);
+	}
+
+    void PrintScheme(void)
+    {
+        std::cout << "Scheme: " << std::endl;
+        unsigned u = m_startStage;
+        for (unsigned i = 0; i < m_schemeLayout.size(); i++)
+        {
+            std::cout << "num " << i << ", stage: " << u << ", iterations on current stage: " << m_schemeLayout[i].m_niter << ", step direction " << m_schemeLayout[i].m_step << std::endl;
+            u += m_schemeLayout[i].m_step;
+        }
+    }
 
-    private:
+private:
     unsigned stages_;
     std::vector< dg::Handle< Geometry> > grids_;
-    std::vector< MultiMatrix<Matrix, container>  >  inter_;
-    std::vector< MultiMatrix<Matrix, container>  >  project_;
+    std::vector< MultiMatrix<Matrix, container> >  inter_;
+    std::vector< MultiMatrix<Matrix, container> >  project_;
     std::vector< CG<container> > cg_;
     std::vector< container> x_, r_, b_; 
     container x0_, x1_, x2_;
     double alpha[3];
+
+    struct stepinfo
+    {
+        stepinfo(int step, const unsigned niter) : m_step(step), m_niter(niter)
+        {
+        };
+
+        int m_step; // {+1,-1}
+        unsigned int m_niter;        
+    };
+
+    unsigned m_startStage;
+    std::vector<stepinfo> m_schemeLayout;
 };
 
 }//namespace dg
-- 
GitLab


From 3e95a1e4006ae341d8e43a52af22a2371c3033ac Mon Sep 17 00:00:00 2001
From:  <jagath@FYS-PPFE-820.win.dtu.dk>
Date: Tue, 12 Sep 2017 16:05:23 +0200
Subject: [PATCH 274/453] multigrid general scheme

---
 inc/dg/elliptic2d_b.cu | 10 ++++--
 inc/dg/multigrid.h     | 70 ++++++++++++++++++++++++++----------------
 2 files changed, 51 insertions(+), 29 deletions(-)

diff --git a/inc/dg/elliptic2d_b.cu b/inc/dg/elliptic2d_b.cu
index 53eb224c4..0227bfa76 100644
--- a/inc/dg/elliptic2d_b.cu
+++ b/inc/dg/elliptic2d_b.cu
@@ -40,12 +40,16 @@ int main()
     unsigned n, Nx, Ny; 
     double eps;
     double jfactor;
-    
+    unsigned scheme;
+
 	n = 3;
 	Nx = Ny = 40;
-	eps = 1e-4;
+	eps = 1e-6;
 	jfactor = 1;
 	
+    std::cout << "scheme type? \n";
+    std::cin >> scheme;
+
 	/*std::cout << "Type n, Nx and Ny and epsilon and jfactor (1)! \n";
     std::cin >> n >> Nx >> Ny; //more N means less iterations for same error
     std::cin >> eps >> jfactor;*/
@@ -72,7 +76,7 @@ int main()
 
 		unsigned stages=4;
 
-		dg::MultigridCG2d<dg::aGeometry2d, dg::DMatrix, dg::DVec > multigrid( grid, stages, 1);
+		dg::MultigridCG2d<dg::aGeometry2d, dg::DMatrix, dg::DVec > multigrid( grid, stages, scheme);
 		
 		std::vector<dg::DVec> chi_ = multigrid.project( chi);
 		std::vector<dg::Elliptic<dg::aGeometry2d, dg::DMatrix, dg::DVec> > multi_pol( stages);
diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index d3e392065..ae040d96a 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -46,7 +46,7 @@ struct MultigridCG2d
         dg::blas1::transfer(dg::evaluate(dg::zero, grid), x0_);
         x1_ = x0_, x2_ = x0_;
         x_ = project(x0_); 
-        r_ = x_, 
+        m_r = x_,
 		b_ = x_;        
 		set_extrapolationType(extrapolation_type);
         set_scheme(scheme_type);        
@@ -64,41 +64,56 @@ struct MultigridCG2d
         //
         // define new rhs
         for (unsigned u = 0; u < stages_; u++)
-            dg::blas1::pointwiseDivide(b_[u], op[u].inv_weights(), b_[u]);        
+            dg::blas1::pointwiseDivide(b_[u], op[u].inv_weights(), b_[u]);
         
         unsigned int numStageSteps = m_schemeLayout.size();
 		std::vector<unsigned> number(numStageSteps);
 
         unsigned u = m_startStage;
+                        
+        for (unsigned i = 0; i < numStageSteps; i++)
+        {
+            unsigned w = u + m_schemeLayout[i].m_step;
 
-        //for (unsigned u = stages_ - 1; u>0; u--)
-        for(unsigned i = 0; i < numStageSteps; i++)
-		{
-            /*if (m_schemeLayout[i].m_step > 0)
+            //
+            // zero initial guess
+            if (m_schemeLayout[i].m_step > 0)
             {
-                u += m_schemeLayout[i].m_step;
-                continue;
-            }*/
+                std::cout << "zeroed " << w << ", ";
+                dg::blas1::scal(x_[w], 0.0);
+            }
 
+            //
+            // iterate the solver on the system A x = b, with x = 0 as inital guess
             cg_[u].set_max(m_schemeLayout[i].m_niter);
+            number[i] = cg_[u](op[u], x_[u], b_[u], op[u].precond(), op[u].inv_weights(), eps);
 
-			number[i] = cg_[u](op[u], x_[u], b_[u], op[u].precond(), op[u].inv_weights(), eps);
-
+            //
+            // debug print
             std::cout << "pass: " << i << ", stage: " << u << ", max iter: " << m_schemeLayout[i].m_niter << ", iter: " << number[i] << std::endl;
-			
-            unsigned w = u + m_schemeLayout[i].m_step;
-            
-            if(m_schemeLayout[i].m_step < 0)
-                dg::blas2::symv(inter_[w], x_[u], x_[w]);
-            else if(m_schemeLayout[i].m_step > 0)
-                dg::blas2::symv(project_[u], x_[u], x_[w]);
 
-            //u += m_schemeLayout[i].m_step;
+            if (m_schemeLayout[i].m_step > 0)
+            {
+                //
+                // compute residual r = b - A x
+                dg::blas2::symv(op[u], x_[u], m_r[u]);
+                dg::blas1::axpby(-1.0, m_r[u], 1.0, b_[u], m_r[u]);
+
+                //
+                // transfer residual to the rhs of the coarser grid
+                dg::blas2::symv(project_[u], m_r[u], b_[w]);                
+            }
+            else if (m_schemeLayout[i].m_step < 0)
+            {
+                //
+                // correct the solution vector of the finer grid
+                // x[w] = x[w] + P^{-1} x[u]
+                dg::blas2::symv(inter_[w], x_[u], m_r[w]);
+                dg::blas1::axpby(1.0, x_[w], 1.0, m_r[w], x_[w]);
+            }
+            
             u = w;
 		}
-                
-        //dg::blas1::pointwiseDivide(b_[0], op[0].inv_weights(), b_[0]);
-		//number[0] = cg_[0](op[0], x_[0], b_[0], op[0].precond(), op[0].inv_weights(), eps);
 
 		x_[0].swap(x);
 		x1_.swap(x2_);
@@ -206,10 +221,12 @@ private:
             //m_mode = correctionscheme;
             m_startStage = 0;
             for (unsigned u = 0; u < stages_-1; u++)
-                m_schemeLayout.push_back(stepinfo(1, x_[u].size()));
+                m_schemeLayout.push_back(stepinfo(1, 5));
             
-            for (int u = stages_ - 1; u >= 0; u--)
-                m_schemeLayout.push_back(stepinfo(-1, x_[u].size()));
+            m_schemeLayout.push_back(stepinfo(-1, 1000));
+
+            for (int u = stages_ - 2; u >= 0; u--)
+                m_schemeLayout.push_back(stepinfo(-1, 1000));
 
             break;
 
@@ -218,6 +235,7 @@ private:
         }
 
         // there is no step at the last stage so the step must be zero
+        m_schemeLayout.back().m_niter = x_[0].size();
         m_schemeLayout.back().m_step = 0;
 
         PrintScheme();
@@ -253,7 +271,7 @@ private:
     std::vector< MultiMatrix<Matrix, container> >  inter_;
     std::vector< MultiMatrix<Matrix, container> >  project_;
     std::vector< CG<container> > cg_;
-    std::vector< container> x_, r_, b_; 
+    std::vector< container> x_, m_r, b_; 
     container x0_, x1_, x2_;
     double alpha[3];
 
-- 
GitLab


From 8e16e2e2c678ac2b39a835e4e770dbf9de78964b Mon Sep 17 00:00:00 2001
From:  <jagath@FYS-PPFE-820.win.dtu.dk>
Date: Tue, 12 Sep 2017 17:01:22 +0200
Subject: [PATCH 275/453] two grid scheme example

---
 inc/dg/elliptic2d_b.cu | 4 ++--
 inc/dg/multigrid.h     | 8 ++++++--
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/inc/dg/elliptic2d_b.cu b/inc/dg/elliptic2d_b.cu
index 0227bfa76..77a611639 100644
--- a/inc/dg/elliptic2d_b.cu
+++ b/inc/dg/elliptic2d_b.cu
@@ -43,7 +43,7 @@ int main()
     unsigned scheme;
 
 	n = 3;
-	Nx = Ny = 40;
+	Nx = Ny = 64;
 	eps = 1e-6;
 	jfactor = 1;
 	
@@ -74,7 +74,7 @@ int main()
     {
 		t.tic();
 
-		unsigned stages=4;
+		unsigned stages = 2;
 
 		dg::MultigridCG2d<dg::aGeometry2d, dg::DMatrix, dg::DVec > multigrid( grid, stages, scheme);
 		
diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index ae040d96a..31186b613 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -101,7 +101,9 @@ struct MultigridCG2d
 
                 //
                 // transfer residual to the rhs of the coarser grid
-                dg::blas2::symv(project_[u], m_r[u], b_[w]);                
+                dg::blas2::symv(project_[u], m_r[u], b_[w]);
+
+                //dg::blas2::symv(project_[u], x_[u], x_[w]);
             }
             else if (m_schemeLayout[i].m_step < 0)
             {
@@ -110,6 +112,8 @@ struct MultigridCG2d
                 // x[w] = x[w] + P^{-1} x[u]
                 dg::blas2::symv(inter_[w], x_[u], m_r[w]);
                 dg::blas1::axpby(1.0, x_[w], 1.0, m_r[w], x_[w]);
+
+                //dg::blas2::symv(inter_[w], x_[u], x_[w]);
             }
             
             u = w;
@@ -221,7 +225,7 @@ private:
             //m_mode = correctionscheme;
             m_startStage = 0;
             for (unsigned u = 0; u < stages_-1; u++)
-                m_schemeLayout.push_back(stepinfo(1, 5));
+                m_schemeLayout.push_back(stepinfo(1, 50));
             
             m_schemeLayout.push_back(stepinfo(-1, 1000));
 
-- 
GitLab


From f1feb6eec4fe22fc2191140670f3af96d6a695e3 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Wed, 13 Sep 2017 11:40:02 +0200
Subject: [PATCH 276/453] try to use multigrid in toefl

---
 inc/dg/algorithm.h   |   1 +
 inc/dg/multistep.h   |  20 +++--
 src/toefl/toeflR.cu  |   4 +-
 src/toefl/toeflR.cuh | 177 ++++++++++++++++++++++++-------------------
 4 files changed, 116 insertions(+), 86 deletions(-)

diff --git a/inc/dg/algorithm.h b/inc/dg/algorithm.h
index a8b1ffac1..75d829621 100644
--- a/inc/dg/algorithm.h
+++ b/inc/dg/algorithm.h
@@ -14,3 +14,4 @@
 #include "multistep.h"
 #include "elliptic.h"
 #include "runge_kutta.h"
+#include "multigrid.h"
diff --git a/inc/dg/multistep.h b/inc/dg/multistep.h
index 7374adf84..4ee6dbb2f 100644
--- a/inc/dg/multistep.h
+++ b/inc/dg/multistep.h
@@ -144,18 +144,22 @@ struct Implicit
     Implicit( double alpha, LinearOp& f, container& reference): f_(f), alpha_(alpha), temp_(reference){}
     void symv( const container& x, container& y) 
     {
-        blas1::axpby( 1., x, 0, temp_);//f_ might destroy x
         if( alpha_ != 0)
+        {
+            blas1::copy( x, temp_);//f_ might destroy x
             f_( temp_,y);
+        }
         blas1::axpby( 1., x, alpha_, y, y);
-        blas2::symv( f_.weights(), y,  y);
+        blas1::pointwiseDivide( y, f_.inv_weights(),  y);
     }
     //compute without weights
     void operator()( const container& x, container& y) 
     {
-        blas1::axpby( 1., x, 0, temp_);
         if( alpha_ != 0)
+        {
+            blas1::copy( x, temp_);
             f_( temp_,y);
+        }
         blas1::axpby( 1., x, alpha_, y, y);
     }
     double& alpha( ){  return alpha_;}
@@ -323,12 +327,12 @@ void Karniadakis<container>::operator()( Functor& f, Diffusion& diff, container&
         u_[i-1].swap( u_[i]);
     }
     blas1::axpby( 1., f_[0], 1., u_[0]);
-    blas2::symv( diff.weights(), u_[0], u_[0]);
+    blas1::pointwiseDivide( u_[0], diff.inv_weights(), u_[0]);
     //compute implicit part
     double alpha[2] = {2., -1.};
     //double alpha[2] = {1., 0.};
     blas1::axpby( alpha[0], u_[1], alpha[1],  u_[2], u_[0]); //extrapolate previous solutions
-    blas2::symv( diff.weights(), u, u);
+    blas1::pointwiseDivide( u, diff.inv_weights(), u);
     detail::Implicit<Diffusion, container> implicit( -dt_/11.*6., diff, f_[0]);
 #ifdef DG_BENCHMARK
 #ifdef MPI_VERSION
@@ -394,7 +398,7 @@ struct SIRK
         u0_ = u0;
         g(u0_, g_);
         dg::blas1::axpby( dt, f_, dt, g_, rhs);
-        blas2::symv( g.weights(), rhs, rhs);
+        blas1::pointwiseDivide( rhs, g.inv_weights(), rhs);
         implicit.alpha() = -dt*d[0];
         pcg( implicit, k_[0], rhs, g.precond(), eps_);
 
@@ -403,7 +407,7 @@ struct SIRK
         dg::blas1::axpby( 1., u0_, c[1][0], k_[0], u1);
         g(u1, g_);
         dg::blas1::axpby( dt, f_, dt, g_, rhs);
-        blas2::symv( g.weights(), rhs, rhs);
+        blas1::pointwiseDivide( rhs, g.inv_weights(), rhs);
         implicit.alpha() = -dt*d[1];
         pcg( implicit, k_[1], rhs, g.precond(), eps_);
 
@@ -414,7 +418,7 @@ struct SIRK
         dg::blas1::axpby( c[2][1], k_[1], 1., u1);
         g(u1, g_);
         dg::blas1::axpby( dt, f_, dt, g_, rhs);
-        blas2::symv( g.weights(), rhs, rhs);
+        blas1::pointwiseDivide( rhs, g.inv_weights(), rhs);
         implicit.alpha() = -dt*d[2];
         pcg( implicit, k_[2], rhs, g.precond(), eps_);
         //sum up results
diff --git a/src/toefl/toeflR.cu b/src/toefl/toeflR.cu
index 23d7984ab..92556532e 100644
--- a/src/toefl/toeflR.cu
+++ b/src/toefl/toeflR.cu
@@ -51,8 +51,8 @@ int main( int argc, char* argv[])
 
     dg::Grid2d grid( 0, p.lx, 0, p.ly, p.n, p.Nx, p.Ny, p.bc_x, p.bc_y);
     //create RHS 
-    dg::ToeflR<dg::CartesianGrid2d, dg::DMatrix, dg::DVec > test( grid, p); 
-    dg::Diffusion<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> diffusion( grid, p.nu);
+    toefl::Explicit<dg::CartesianGrid2d, dg::DMatrix, dg::DVec > test( grid, p); 
+    toefl::Implicit<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> diffusion( grid, p.nu);
     //////////////////create initial vector///////////////////////////////////////
     dg::Gaussian g( p.posX*p.lx, p.posY*p.ly, p.sigma, p.sigma, p.amp); //gaussian width is in absolute values
     std::vector<dg::DVec> y0(2, dg::evaluate( g, grid)), y1(y0); // n_e' = gaussian
diff --git a/src/toefl/toeflR.cuh b/src/toefl/toeflR.cuh
index 3a4340b7b..38b77fd5c 100644
--- a/src/toefl/toeflR.cuh
+++ b/src/toefl/toeflR.cuh
@@ -10,15 +10,15 @@
 #endif
 
 
-namespace dg
+namespace toefl
 {
 
 template<class Geometry, class Matrix, class container>
-struct Diffusion
+struct Implicit
 {
-    Diffusion( const Geometry& g, double nu):
+    Implicit( const Geometry& g, double nu):
         nu_(nu), 
-        temp( dg::evaluate(dg::zero, g)), 
+        temp( dg::evaluate(dg::zero, g)), inv_weights_(2,temp),
         LaplacianM_perp( g, dg::normed, dg::centered){
     }
     void operator()( std::vector<container>& x, std::vector<container>& y)
@@ -36,26 +36,26 @@ struct Diffusion
         }
     }
     dg::Elliptic<Geometry, Matrix, container>& laplacianM() {return LaplacianM_perp;}
-    const container& weights(){return LaplacianM_perp.weights();}
+    const std::vector<container>& inv_weights(){return inv_weights_;}
     const container& precond(){return LaplacianM_perp.precond();}
 
   private:
     double nu_;
-    const container w2d, v2d;
     container temp;
+    std::vector<container> inv_weights_;
     dg::Elliptic<Geometry, Matrix, container> LaplacianM_perp;
 };
 
 template< class Geometry,  class Matrix, class container >
-struct ToeflR
+struct Explicit
 {
     /**
-     * @brief Construct a ToeflR solver object
+     * @brief Construct a Explicit solver object
      *
      * @param g The grid on which to operate
      * @param p The parameters
      */
-    ToeflR( const Geometry& g, const Parameters& p );
+    Explicit( const Geometry& g, const Parameters& p );
 
 
     /**
@@ -132,11 +132,14 @@ struct ToeflR
     container gamma_n;
 
     //matrices and solvers
-    Elliptic<Geometry, Matrix, container> pol, laplaceM; //contains normalized laplacian
-    Helmholtz<Geometry,  Matrix, container> gamma1;
-    ArakawaX< Geometry, Matrix, container> arakawa; 
+    dg::Elliptic<Geometry, Matrix, container> laplaceM; //contains normalized laplacian
+    std::vector<dg::Elliptic<Geometry, Matrix, container> > multi_pol;
+    dg::Helmholtz<Geometry,  Matrix, container> gamma1;
+    dg::ArakawaX< Geometry, Matrix, container> arakawa; 
 
-    dg::Invert<container> invert_pol, invert_invgamma;
+    dg::Invert<container> invert_invgamma;
+    dg::MultigridCG2d<Geometry, Matrix, container> multigrid_pol; 
+    std::vector<container> multi_chi;
 
     const container w2d, v2d, one;
     const double eps_pol, eps_gamma; 
@@ -149,25 +152,28 @@ struct ToeflR
 };
 
 template< class Geometry, class M, class container>
-ToeflR< Geometry, M, container>::ToeflR( const Geometry& grid, const Parameters& p ): 
+Explicit< Geometry, M, container>::Explicit( const Geometry& grid, const Parameters& p ): 
     chi( evaluate( dg::zero, grid)), omega(chi),
-    binv( evaluate( LinearX( p.kappa, 1.-p.kappa*p.posX*p.lx), grid)), 
+    binv( evaluate( dg::LinearX( p.kappa, 1.-p.kappa*p.posX*p.lx), grid)), 
     phi( 2, chi), dyphi( phi), ype(phi),
     dyy(2,chi), lny( dyy), lapy(dyy),
     gamma_n(chi),
-    pol(     grid, not_normed, dg::centered, p.jfactor), 
-    laplaceM( grid, normed, centered),
+    //pol(     grid, not_normed, dg::centered, p.jfactor), 
+    laplaceM( grid, dg::normed, dg::centered),
     gamma1(  grid, -0.5*p.tau, dg::centered),
     arakawa( grid), 
-    invert_pol(      omega, p.Nx*p.Ny*p.n*p.n, p.eps_pol),
+    //invert_pol(      omega, p.Nx*p.Ny*p.n*p.n, p.eps_pol),
     invert_invgamma( omega, p.Nx*p.Ny*p.n*p.n, p.eps_gamma),
-    w2d( create::volume(grid)), v2d( create::inv_volume(grid)), one( dg::evaluate(dg::one, grid)),
+    multigrid_pol( grid, 3), multi_chi( multigrid_pol.project( chi)),
+    w2d( dg::create::volume(grid)), v2d( dg::create::inv_volume(grid)), one( dg::evaluate(dg::one, grid)),
     eps_pol(p.eps_pol), eps_gamma( p.eps_gamma), kappa(p.kappa), friction(p.friction), nu(p.nu), tau( p.tau), equations( p.equations), boussinesq(p.boussinesq)
 {
+    for( unsigned u=0; u<3; u++)
+        multi_pol[u].construct( multigrid_pol.grids()[u].get(), dg::not_normed, dg::centered, p.jfactor);
 }
 
 template< class G, class M, class container>
-const container& ToeflR<G, M, container>::compute_psi( const container& potential)
+const container& Explicit<G, M, container>::compute_psi( const container& potential)
 {
     if(equations == "gravity_local") return potential;
     //in gyrofluid invert Gamma operator
@@ -201,33 +207,48 @@ const container& ToeflR<G, M, container>::compute_psi( const container& potentia
 
 //computes and modifies expy!!
 template<class G, class M, class container>
-const container& ToeflR<G, M, container>::polarisation( const std::vector<container>& y)
+const container& Explicit<G, M, container>::polarisation( const std::vector<container>& y)
 {
     //compute chi 
     if(equations == "global" )
     {
         dg::blas1::transfer( y[1], chi);
         dg::blas1::plus( chi, 1.); 
-        blas1::pointwiseDot( binv, chi, chi); //\chi = n_i
-        blas1::pointwiseDot( binv, chi, chi); //\chi *= binv^2
+        dg::blas1::pointwiseDot( binv, chi, chi); //\chi = n_i
+        dg::blas1::pointwiseDot( binv, chi, chi); //\chi *= binv^2
         if( !boussinesq) 
-            pol.set_chi( chi);
+        {
+            multigrid_pol.project( chi, multi_chi);
+            for( unsigned u=0; u<3; u++)
+                multi_pol[u].set_chi( multi_chi[u]);
+            //pol.set_chi( chi);
+        }
     }
     else if(equations == "gravity_global" )
     {
         dg::blas1::transfer( y[0], chi);
         dg::blas1::plus( chi, 1.); 
         if( !boussinesq) 
-            pol.set_chi( chi);
+        {
+            multigrid_pol.project( chi, multi_chi);
+            for( unsigned u=0; u<3; u++)
+                multi_pol[u].set_chi( multi_chi[u]);
+            //pol.set_chi( chi);
+        }
     }
     else if( equations == "drift_global" )
     {
         dg::blas1::transfer( y[0], chi);
         dg::blas1::plus( chi, 1.); 
-        blas1::pointwiseDot( binv, chi, chi); //\chi = n_e
-        blas1::pointwiseDot( binv, chi, chi); //\chi *= binv^2
+        dg::blas1::pointwiseDot( binv, chi, chi); //\chi = n_e
+        dg::blas1::pointwiseDot( binv, chi, chi); //\chi *= binv^2
         if( !boussinesq) 
-            pol.set_chi( chi);
+        {
+            multigrid_pol.project( chi, multi_chi);
+            for( unsigned u=0; u<3; u++)
+                multi_pol[u].set_chi( multi_chi[u]);
+            //pol.set_chi( chi);
+        }
     }
     //compute polarisation
     if( equations == "local" || equations == "global")
@@ -235,24 +256,28 @@ const container& ToeflR<G, M, container>::polarisation( const std::vector<contai
         unsigned number = invert_invgamma( gamma1, gamma_n, y[1]);
         if(  number == invert_invgamma.get_max())
             throw dg::Fail( eps_gamma);
-        blas1::axpby( -1., y[0], 1., gamma_n, omega); //omega = a_i\Gamma n_i - n_e
+        dg::blas1::axpby( -1., y[0], 1., gamma_n, omega); //omega = a_i\Gamma n_i - n_e
     }
     else 
-        blas1::axpby( -1. ,y[1], 0., omega);
+        dg::blas1::axpby( -1. ,y[1], 0., omega);
     if( equations == "global" || equations == "gravity_global" || equations == "drift_global")
         if( boussinesq) 
-            blas1::pointwiseDivide( omega, chi, omega);
+            dg::blas1::pointwiseDivide( omega, chi, omega);
     //invert 
     dg::blas1::transform( chi, chi, dg::INVERT<double>());
     dg::blas1::pointwiseDot( chi, v2d, chi);
-    unsigned number = invert_pol( pol, phi[0], omega, w2d, chi, v2d);
-    if(  number == invert_pol.get_max())
+
+    //unsigned number = invert_pol( pol, phi[0], omega, w2d, chi, v2d);
+    //if(  number == invert_pol.get_max())
+    //    throw dg::Fail( eps_pol);
+    std::vector<unsigned> number = multigrid_pol.direct_solve( multi_pol, phi[0], omega, eps_pol);
+    if(  number[0] == invert_invgamma.get_max())
         throw dg::Fail( eps_pol);
     return phi[0];
 }
 
 template< class G, class M, class container>
-void ToeflR<G, M, container>::operator()( std::vector<container>& y, std::vector<container>& yp)
+void Explicit<G, M, container>::operator()( std::vector<container>& y, std::vector<container>& yp)
 {
     //y[0] = N_e - 1
     //y[1] = N_i - 1 || y[1] = Omega
@@ -270,51 +295,51 @@ void ToeflR<G, M, container>::operator()( std::vector<container>& y, std::vector
     }
 
     /////////////////////////update energetics, 2% of total time///////////////
-    mass_ = blas2::dot( one, w2d, y[0] ); //take real ion density which is electron density!!
-    diff_ = nu*blas2::dot( one, w2d, lapy[0]);
+    mass_ = dg::blas2::dot( one, w2d, y[0] ); //take real ion density which is electron density!!
+    diff_ = nu*dg::blas2::dot( one, w2d, lapy[0]);
     if(equations == "global")
     {
-        double Ue = blas2::dot( lny[0], w2d, ype[0]);
-        double Ui = tau*blas2::dot( lny[1], w2d, ype[1]);
-        double Uphi = 0.5*blas2::dot( ype[1], w2d, omega); 
+        double Ue = dg::blas2::dot( lny[0], w2d, ype[0]);
+        double Ui = tau*dg::blas2::dot( lny[1], w2d, ype[1]);
+        double Uphi = 0.5*dg::blas2::dot( ype[1], w2d, omega); 
         energy_ = Ue + Ui + Uphi;
 
-        double Ge = - blas2::dot( one, w2d, lapy[0]) - blas2::dot( lapy[0], w2d, lny[0]); // minus 
-        double Gi = - tau*(blas2::dot( one, w2d, lapy[1]) + blas2::dot( lapy[1], w2d, lny[1])); // minus 
-        double Gphi = -blas2::dot( phi[0], w2d, lapy[0]);
-        double Gpsi = -blas2::dot( phi[1], w2d, lapy[1]);
+        double Ge = - dg::blas2::dot( one, w2d, lapy[0]) - dg::blas2::dot( lapy[0], w2d, lny[0]); // minus 
+        double Gi = - tau*(dg::blas2::dot( one, w2d, lapy[1]) + dg::blas2::dot( lapy[1], w2d, lny[1])); // minus 
+        double Gphi = -dg::blas2::dot( phi[0], w2d, lapy[0]);
+        double Gpsi = -dg::blas2::dot( phi[1], w2d, lapy[1]);
         //std::cout << "ge "<<Ge<<" gi "<<Gi<<" gphi "<<Gphi<<" gpsi "<<Gpsi<<"\n";
         ediff_ = nu*( Ge + Gi - Gphi + Gpsi);
     }
     else if ( equations == "drift_global") 
     {
-        double Se = blas2::dot( lny[0], w2d, ype[0]);
-        double Ephi = 0.5*blas2::dot( ype[0], w2d, omega); 
+        double Se = dg::blas2::dot( lny[0], w2d, ype[0]);
+        double Ephi = 0.5*dg::blas2::dot( ype[0], w2d, omega); 
         energy_ = Se + Ephi;
 
-        double Ge = - blas2::dot( one, w2d, lapy[0]) - blas2::dot( lapy[0], w2d, lny[0]); // minus 
-        double GeE = - blas2::dot( phi[1], w2d, lapy[0]); 
-        double Gpsi = -blas2::dot( phi[0], w2d, lapy[1]);
+        double Ge = - dg::blas2::dot( one, w2d, lapy[0]) - dg::blas2::dot( lapy[0], w2d, lny[0]); // minus 
+        double GeE = - dg::blas2::dot( phi[1], w2d, lapy[0]); 
+        double Gpsi = -dg::blas2::dot( phi[0], w2d, lapy[1]);
         //std::cout << "ge "<<Ge<<" gi "<<Gi<<" gphi "<<Gphi<<" gpsi "<<Gpsi<<"\n";
         ediff_ = nu*( Ge - GeE - Gpsi);
     }
     else if(equations == "gravity_global" || equations == "gravity_local")
     {
-        energy_ = 0.5*blas2::dot( y[0], w2d, y[0]);
-        double Ge = - blas2::dot( y[0], w2d, lapy[0]);
+        energy_ = 0.5*dg::blas2::dot( y[0], w2d, y[0]);
+        double Ge = - dg::blas2::dot( y[0], w2d, lapy[0]);
         ediff_ = nu* Ge;
     }
     else
     {
-        double Ue = 0.5*blas2::dot( y[0], w2d, y[0]);
-        double Ui = 0.5*tau*blas2::dot( y[1], w2d, y[1]);
-        double Uphi = 0.5*blas2::dot( one, w2d, omega); 
+        double Ue = 0.5*dg::blas2::dot( y[0], w2d, y[0]);
+        double Ui = 0.5*tau*dg::blas2::dot( y[1], w2d, y[1]);
+        double Uphi = 0.5*dg::blas2::dot( one, w2d, omega); 
         energy_ = Ue + Ui + Uphi;
 
-        double Ge = - blas2::dot( y[0], w2d, lapy[0]); // minus 
-        double Gi = - tau*(blas2::dot( y[1], w2d, lapy[1])); // minus 
-        double Gphi = -blas2::dot( phi[0], w2d, lapy[0]);
-        double Gpsi = -blas2::dot( phi[1], w2d, lapy[1]);
+        double Ge = - dg::blas2::dot( y[0], w2d, lapy[0]); // minus 
+        double Gi = - tau*(dg::blas2::dot( y[1], w2d, lapy[1])); // minus 
+        double Gphi = -dg::blas2::dot( phi[0], w2d, lapy[0]);
+        double Gpsi = -dg::blas2::dot( phi[1], w2d, lapy[1]);
         //std::cout << "ge "<<Ge<<" gi "<<Gi<<" gphi "<<Gphi<<" gpsi "<<Gpsi<<"\n";
         ediff_ = nu*( Ge + Gi - Gphi + Gpsi);
     }
@@ -334,7 +359,7 @@ void ToeflR<G, M, container>::operator()( std::vector<container>& y, std::vector
     {
         arakawa(y[0], phi[0], yp[0]);
         arakawa(y[1], phi[0], yp[1]);
-        blas2::gemv( arakawa.dy(), y[0], dyy[0]);
+        dg::blas2::gemv( arakawa.dy(), y[0], dyy[0]);
         dg::blas1::axpby( -friction, y[1], 1., yp[1]);
         dg::blas1::axpby( -1., dyy[0], 1., yp[1]);
         return;
@@ -344,20 +369,20 @@ void ToeflR<G, M, container>::operator()( std::vector<container>& y, std::vector
         arakawa(y[0], phi[0], yp[0]);
         arakawa(y[1], phi[0], yp[1]);
         arakawa(y[0], phi[1], omega);
-        blas1::pointwiseDot( binv, yp[0], yp[0]);
-        blas1::pointwiseDot( binv, yp[1], yp[1]);
-        blas1::pointwiseDot( binv, omega, omega);
+        dg::blas1::pointwiseDot( binv, yp[0], yp[0]);
+        dg::blas1::pointwiseDot( binv, yp[1], yp[1]);
+        dg::blas1::pointwiseDot( binv, omega, omega);
         dg::blas1::axpby( 1., omega, 1., yp[1]);
 
-        blas2::gemv( arakawa.dy(), phi[0], dyphi[0]);
-        blas2::gemv( arakawa.dy(), phi[1], dyphi[1]);
+        dg::blas2::gemv( arakawa.dy(), phi[0], dyphi[0]);
+        dg::blas2::gemv( arakawa.dy(), phi[1], dyphi[1]);
         //ExB compression
-        blas1::pointwiseDot( dyphi[0], ype[0], omega);
-        blas1::axpby( kappa, omega, 1., yp[0]); 
-        blas1::pointwiseDot( dyphi[0], y[1], omega);
-        blas1::axpby( kappa, omega, 1., yp[1]); 
-        blas1::pointwiseDot( dyphi[1], ype[0], omega);
-        blas1::axpby( kappa, omega, 1., yp[1]); 
+        dg::blas1::pointwiseDot( dyphi[0], ype[0], omega);
+        dg::blas1::axpby( kappa, omega, 1., yp[0]); 
+        dg::blas1::pointwiseDot( dyphi[0], y[1], omega);
+        dg::blas1::axpby( kappa, omega, 1., yp[1]); 
+        dg::blas1::pointwiseDot( dyphi[1], ype[0], omega);
+        dg::blas1::axpby( kappa, omega, 1., yp[1]); 
         // diamagnetic compression
         dg::blas2::gemv( arakawa.dy(), y[0], omega);
         dg::blas1::axpby( -kappa, omega, 1., yp[1]);
@@ -368,19 +393,19 @@ void ToeflR<G, M, container>::operator()( std::vector<container>& y, std::vector
         for( unsigned i=0; i<y.size(); i++)
         {
             arakawa( y[i], phi[i], yp[i]);
-            if(equations == "global") blas1::pointwiseDot( binv, yp[i], yp[i]);
+            if(equations == "global") dg::blas1::pointwiseDot( binv, yp[i], yp[i]);
         }
         //compute derivatives and exb compression
         for( unsigned i=0; i<y.size(); i++)
         {
-            blas2::gemv( arakawa.dy(), y[i], dyy[i]);
-            blas2::gemv( arakawa.dy(), phi[i], dyphi[i]);
-            if(equations == "global") blas1::pointwiseDot( dyphi[i], ype[i], dyphi[i]);
-            blas1::axpby( kappa, dyphi[i], 1., yp[i]);
+            dg::blas2::gemv( arakawa.dy(), y[i], dyy[i]);
+            dg::blas2::gemv( arakawa.dy(), phi[i], dyphi[i]);
+            if(equations == "global") dg::blas1::pointwiseDot( dyphi[i], ype[i], dyphi[i]);
+            dg::blas1::axpby( kappa, dyphi[i], 1., yp[i]);
         }
         // diamagnetic compression
-        blas1::axpby( -1.*kappa, dyy[0], 1., yp[0]);
-        blas1::axpby( tau*kappa, dyy[1], 1., yp[1]);
+        dg::blas1::axpby( -1.*kappa, dyy[0], 1., yp[0]);
+        dg::blas1::axpby( tau*kappa, dyy[1], 1., yp[1]);
     }
 
     return;
-- 
GitLab


From 6956c3b1f0b2ffcdd991e21e054244da044a706e Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Wed, 13 Sep 2017 11:57:39 +0200
Subject: [PATCH 277/453] added restart flag to CG

---
 inc/dg/cg.h | 131 +++++++++++++++++++++++++++-------------------------
 1 file changed, 68 insertions(+), 63 deletions(-)

diff --git a/inc/dg/cg.h b/inc/dg/cg.h
index 75bccd100..98ff0a109 100644
--- a/inc/dg/cg.h
+++ b/inc/dg/cg.h
@@ -94,7 +94,7 @@ class CG
      * @return Number of iterations used to achieve desired precision
      */
     template< class SymmetricOp, class Preconditioner >
-    unsigned operator()( SymmetricOp& A, container& x, const container& b, Preconditioner& P , value_type eps = 1e-12, value_type nrmb_correction = 1);
+    unsigned operator()( SymmetricOp& A, container& x, const container& b, Preconditioner& P , value_type eps = 1e-12, value_type nrmb_correction = 1, bool restart = false);
     //version of CG where Preconditioner is not trivial
     /**
      * @brief Solve the system A*x = b using a preconditioned conjugate gradient method
@@ -115,10 +115,11 @@ class CG
      * @return Number of iterations used to achieve desired precision
      */
     template< class SymmetricOp, class Preconditioner, class SquareNorm >
-    unsigned operator()( SymmetricOp& A, container& x, const container& b, Preconditioner& P, SquareNorm& S, value_type eps = 1e-12, value_type nrmb_correction = 1);
+    unsigned operator()( SymmetricOp& A, container& x, const container& b, Preconditioner& P, SquareNorm& S, value_type eps = 1e-12, value_type nrmb_correction = 1, bool restart = false);
   private:
     container r, p, ap; 
     unsigned max_iter;
+    value_type m_nrmb, m_alpha, m_nrm2r_old, m_nrm2r_new;
 };
 
 /*
@@ -140,38 +141,40 @@ class CG
 ///@cond
 template< class container>
 template< class Matrix, class Preconditioner>
-unsigned CG< container>::operator()( Matrix& A, container& x, const container& b, Preconditioner& P, value_type eps, value_type nrmb_correction)
+unsigned CG< container>::operator()( Matrix& A, container& x, const container& b, Preconditioner& P, value_type eps, value_type nrmb_correction, bool restart)
 {
-    value_type nrmb = sqrt( blas2::dot( P, b));
+    if( !restart)
+    {
+        m_nrmb = sqrt( blas2::dot( P, b));
 #ifdef DG_DEBUG
 #ifdef MPI_VERSION
-    int rank;
-    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-    if(rank==0)
+        int rank;
+        MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+        if(rank==0)
 #endif //MPI
-    {
-    std::cout << "Norm of b "<<nrmb <<"\n";
-    std::cout << "Residual errors: \n";
-    }
+        {
+        std::cout << "Norm of b "<<nrmb <<"\n";
+        std::cout << "Residual errors: \n";
+        }
 #endif //DG_DEBUG
-    if( nrmb == 0)
-    {
-        blas1::axpby( 1., b, 0., x);
-        return 0;
+        if( m_nrmb == 0)
+        {
+            blas1::axpby( 1., b, 0., x);
+            return 0;
+        }
+        blas2::symv( A,x,r);
+        blas1::axpby( 1., b, -1., r);
+        blas2::symv( P, r, p );//<-- compute p_0
+        //note that dot does automatically synchronize
+        m_nrm2r_old = blas2::dot( P,r); //and store the norm of it
+        if( sqrt( m_nrm2r_old ) < eps*(m_nrmb + nrmb_correction)) //if x happens to be the solution
+            return 0;
     }
-    blas2::symv( A,x,r);
-    blas1::axpby( 1., b, -1., r);
-    blas2::symv( P, r, p );//<-- compute p_0
-    //note that dot does automatically synchronize
-    value_type nrm2r_old = blas2::dot( P,r); //and store the norm of it
-    if( sqrt( nrm2r_old ) < eps*(nrmb + nrmb_correction)) //if x happens to be the solution
-        return 0;
-    value_type alpha, nrm2r_new;
     for( unsigned i=1; i<max_iter; i++)
     {
         blas2::symv( A, p, ap);
-        alpha = nrm2r_old /blas1::dot( p, ap);
-        blas1::axpby( alpha, p, 1.,x);
+        m_alpha = m_nrm2r_old /blas1::dot( p, ap);
+        blas1::axpby( m_alpha, p, 1.,x);
 	        //here one could add a ifstatement to remove accumulated floating point error
 //             if (i % 100==0) {
 //                   blas2::symv( A,x,r); 
@@ -180,22 +183,22 @@ unsigned CG< container>::operator()( Matrix& A, container& x, const container& b
 //             else {
 //                   blas1::axpby( -alpha, ap, 1., r);
 //             }
-        blas1::axpby( -alpha, ap, 1., r);
-        nrm2r_new = blas2::dot( P, r); 
+        blas1::axpby( -m_alpha, ap, 1., r);
+        m_nrm2r_new = blas2::dot( P, r); 
 #ifdef DG_DEBUG
 #ifdef MPI_VERSION
     if(rank==0)
 #endif //MPI
     {
-        std::cout << "Absolute "<<sqrt( nrm2r_new) <<"\t ";
-        std::cout << " < Critical "<<eps*nrmb + eps <<"\t ";
-        std::cout << "(Relative "<<sqrt( nrm2r_new)/nrmb << ")\n";
+        std::cout << "Absolute "<<sqrt( m_nrm2r_new) <<"\t ";
+        std::cout << " < Critical "<<eps*m_nrmb + eps <<"\t ";
+        std::cout << "(Relative "<<sqrt( m_nrm2r_new)/m_nrmb << ")\n";
     }
 #endif //DG_DEBUG
-        if( sqrt( nrm2r_new) < eps*(nrmb + nrmb_correction)) 
+        if( sqrt( m_nrm2r_new) < eps*(m_nrmb + nrmb_correction)) 
             return i;
-        blas2::symv(1.,P, r, nrm2r_new/nrm2r_old, p );
-        nrm2r_old=nrm2r_new;
+        blas2::symv(1.,P, r, m_nrm2r_new/m_nrm2r_old, p );
+        m_nrm2r_old=m_nrm2r_new;
 
     }
     return max_iter;
@@ -203,55 +206,57 @@ unsigned CG< container>::operator()( Matrix& A, container& x, const container& b
 
 template< class container>
 template< class Matrix, class Preconditioner, class SquareNorm>
-unsigned CG< container>::operator()( Matrix& A, container& x, const container& b, Preconditioner& P, SquareNorm& S, value_type eps, value_type nrmb_correction)
+unsigned CG< container>::operator()( Matrix& A, container& x, const container& b, Preconditioner& P, SquareNorm& S, value_type eps, value_type nrmb_correction, bool restart)
 {
-    value_type nrmb = sqrt( blas2::dot( S, b));
+    if( !restart)
+    {
+        m_nrmb = sqrt( blas2::dot( S, b));
 #ifdef DG_DEBUG
 #ifdef MPI_VERSION
-    int rank;
-    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-    if(rank==0)
+        int rank;
+        MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+        if(rank==0)
 #endif //MPI
-    {
-    std::cout << "Norm of S b "<<nrmb <<"\n";
-    std::cout << "Residual errors: \n";
-    }
+        {
+        std::cout << "Norm of S b "<<nrmb <<"\n";
+        std::cout << "Residual errors: \n";
+        }
 #endif //DG_DEBUG
-    if( nrmb == 0)
-    {
-        blas1::copy( b, x);
-        return 0;
+        if( m_nrmb == 0)
+        {
+            blas1::copy( b, x);
+            return 0;
+        }
+        blas2::symv( A,x,r);
+        blas1::axpby( 1., b, -1., r);
+        //note that dot does automatically synchronize
+        if( sqrt( blas2::dot(S,r) ) < eps*(m_nrmb + nrmb_correction)) //if x happens to be the solution
+            return 0;
+        blas2::symv( P, r, p );//<-- compute p_0
+        m_nrm2r_old = blas1::dot( p,r); //and store the scalar product
     }
-    blas2::symv( A,x,r);
-    blas1::axpby( 1., b, -1., r);
-    //note that dot does automatically synchronize
-    if( sqrt( blas2::dot(S,r) ) < eps*(nrmb + nrmb_correction)) //if x happens to be the solution
-        return 0;
-    blas2::symv( P, r, p );//<-- compute p_0
-    value_type nrmzr_old = blas1::dot( p,r); //and store the scalar product
-    value_type alpha, nrmzr_new;
     for( unsigned i=1; i<max_iter; i++)
     {
         blas2::symv( A, p, ap);
-        alpha =  nrmzr_old/blas1::dot( p, ap);
-        blas1::axpby( alpha, p, 1.,x);
-        blas1::axpby( -alpha, ap, 1., r);
+        m_alpha =  m_nrm2r_old/blas1::dot( p, ap);
+        blas1::axpby( m_alpha, p, 1.,x);
+        blas1::axpby( -m_alpha, ap, 1., r);
 #ifdef DG_DEBUG
 #ifdef MPI_VERSION
     if(rank==0)
 #endif //MPI
     {
         std::cout << "Absolute r*S*r "<<sqrt( blas2::dot(S,r)) <<"\t ";
-        std::cout << " < Critical "<<eps*nrmb + eps <<"\t ";
-        std::cout << "(Relative "<<sqrt( blas2::dot(S,r) )/nrmb << ")\n";
+        std::cout << " < Critical "<<eps*m_nrmb + eps <<"\t ";
+        std::cout << "(Relative "<<sqrt( blas2::dot(S,r) )/m_nrmb << ")\n";
     }
 #endif //DG_DEBUG
-        if( sqrt( blas2::dot(S,r)) < eps*(nrmb + nrmb_correction)) 
+        if( sqrt( blas2::dot(S,r)) < eps*(m_nrmb + nrmb_correction)) 
             return i;
         blas2::symv(P,r,ap);
-        nrmzr_new = blas1::dot( ap, r); 
-        blas1::axpby(1.,ap, nrmzr_new/nrmzr_old, p );
-        nrmzr_old=nrmzr_new;
+        m_nrm2r_new = blas1::dot( ap, r); 
+        blas1::axpby(1.,ap, m_nrm2r_new/m_nrm2r_old, p );
+        m_nrm2r_old=m_nrm2r_new;
     }
     return max_iter;
 }
-- 
GitLab


From b90811357262bfc93681c3c8bd8278a2793ceb6b Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Wed, 13 Sep 2017 17:22:06 +0200
Subject: [PATCH 278/453] debug toeflR

---
 src/toefl/toeflR.cuh | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/toefl/toeflR.cuh b/src/toefl/toeflR.cuh
index 38b77fd5c..9356cdc4d 100644
--- a/src/toefl/toeflR.cuh
+++ b/src/toefl/toeflR.cuh
@@ -164,10 +164,12 @@ Explicit< Geometry, M, container>::Explicit( const Geometry& grid, const Paramet
     arakawa( grid), 
     //invert_pol(      omega, p.Nx*p.Ny*p.n*p.n, p.eps_pol),
     invert_invgamma( omega, p.Nx*p.Ny*p.n*p.n, p.eps_gamma),
-    multigrid_pol( grid, 3), multi_chi( multigrid_pol.project( chi)),
+    multigrid_pol( grid, 3), 
     w2d( dg::create::volume(grid)), v2d( dg::create::inv_volume(grid)), one( dg::evaluate(dg::one, grid)),
     eps_pol(p.eps_pol), eps_gamma( p.eps_gamma), kappa(p.kappa), friction(p.friction), nu(p.nu), tau( p.tau), equations( p.equations), boussinesq(p.boussinesq)
-{
+{ 
+    multi_chi= multigrid_pol.project( chi);
+    multi_pol.resize(3);
     for( unsigned u=0; u<3; u++)
         multi_pol[u].construct( multigrid_pol.grids()[u].get(), dg::not_normed, dg::centered, p.jfactor);
 }
@@ -270,7 +272,7 @@ const container& Explicit<G, M, container>::polarisation( const std::vector<cont
     //unsigned number = invert_pol( pol, phi[0], omega, w2d, chi, v2d);
     //if(  number == invert_pol.get_max())
     //    throw dg::Fail( eps_pol);
-    std::vector<unsigned> number = multigrid_pol.direct_solve( multi_pol, phi[0], omega, eps_pol);
+    std::vector<unsigned> number = multigrid_pol.solve( multi_pol, phi[0], omega, eps_pol);
     if(  number[0] == invert_invgamma.get_max())
         throw dg::Fail( eps_pol);
     return phi[0];
-- 
GitLab


From 0aee429d45cae5592ef4e829cc588df407e0d1f1 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Wed, 13 Sep 2017 21:20:19 +0200
Subject: [PATCH 279/453] debuggin toefl more

---
 inc/dg/cg.h          | 131 +++++++++++++++++++++----------------------
 src/toefl/toeflR.cuh |  16 +++---
 2 files changed, 70 insertions(+), 77 deletions(-)

diff --git a/inc/dg/cg.h b/inc/dg/cg.h
index 98ff0a109..75bccd100 100644
--- a/inc/dg/cg.h
+++ b/inc/dg/cg.h
@@ -94,7 +94,7 @@ class CG
      * @return Number of iterations used to achieve desired precision
      */
     template< class SymmetricOp, class Preconditioner >
-    unsigned operator()( SymmetricOp& A, container& x, const container& b, Preconditioner& P , value_type eps = 1e-12, value_type nrmb_correction = 1, bool restart = false);
+    unsigned operator()( SymmetricOp& A, container& x, const container& b, Preconditioner& P , value_type eps = 1e-12, value_type nrmb_correction = 1);
     //version of CG where Preconditioner is not trivial
     /**
      * @brief Solve the system A*x = b using a preconditioned conjugate gradient method
@@ -115,11 +115,10 @@ class CG
      * @return Number of iterations used to achieve desired precision
      */
     template< class SymmetricOp, class Preconditioner, class SquareNorm >
-    unsigned operator()( SymmetricOp& A, container& x, const container& b, Preconditioner& P, SquareNorm& S, value_type eps = 1e-12, value_type nrmb_correction = 1, bool restart = false);
+    unsigned operator()( SymmetricOp& A, container& x, const container& b, Preconditioner& P, SquareNorm& S, value_type eps = 1e-12, value_type nrmb_correction = 1);
   private:
     container r, p, ap; 
     unsigned max_iter;
-    value_type m_nrmb, m_alpha, m_nrm2r_old, m_nrm2r_new;
 };
 
 /*
@@ -141,40 +140,38 @@ class CG
 ///@cond
 template< class container>
 template< class Matrix, class Preconditioner>
-unsigned CG< container>::operator()( Matrix& A, container& x, const container& b, Preconditioner& P, value_type eps, value_type nrmb_correction, bool restart)
+unsigned CG< container>::operator()( Matrix& A, container& x, const container& b, Preconditioner& P, value_type eps, value_type nrmb_correction)
 {
-    if( !restart)
-    {
-        m_nrmb = sqrt( blas2::dot( P, b));
+    value_type nrmb = sqrt( blas2::dot( P, b));
 #ifdef DG_DEBUG
 #ifdef MPI_VERSION
-        int rank;
-        MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-        if(rank==0)
+    int rank;
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+    if(rank==0)
 #endif //MPI
-        {
-        std::cout << "Norm of b "<<nrmb <<"\n";
-        std::cout << "Residual errors: \n";
-        }
+    {
+    std::cout << "Norm of b "<<nrmb <<"\n";
+    std::cout << "Residual errors: \n";
+    }
 #endif //DG_DEBUG
-        if( m_nrmb == 0)
-        {
-            blas1::axpby( 1., b, 0., x);
-            return 0;
-        }
-        blas2::symv( A,x,r);
-        blas1::axpby( 1., b, -1., r);
-        blas2::symv( P, r, p );//<-- compute p_0
-        //note that dot does automatically synchronize
-        m_nrm2r_old = blas2::dot( P,r); //and store the norm of it
-        if( sqrt( m_nrm2r_old ) < eps*(m_nrmb + nrmb_correction)) //if x happens to be the solution
-            return 0;
+    if( nrmb == 0)
+    {
+        blas1::axpby( 1., b, 0., x);
+        return 0;
     }
+    blas2::symv( A,x,r);
+    blas1::axpby( 1., b, -1., r);
+    blas2::symv( P, r, p );//<-- compute p_0
+    //note that dot does automatically synchronize
+    value_type nrm2r_old = blas2::dot( P,r); //and store the norm of it
+    if( sqrt( nrm2r_old ) < eps*(nrmb + nrmb_correction)) //if x happens to be the solution
+        return 0;
+    value_type alpha, nrm2r_new;
     for( unsigned i=1; i<max_iter; i++)
     {
         blas2::symv( A, p, ap);
-        m_alpha = m_nrm2r_old /blas1::dot( p, ap);
-        blas1::axpby( m_alpha, p, 1.,x);
+        alpha = nrm2r_old /blas1::dot( p, ap);
+        blas1::axpby( alpha, p, 1.,x);
 	        //here one could add a ifstatement to remove accumulated floating point error
 //             if (i % 100==0) {
 //                   blas2::symv( A,x,r); 
@@ -183,22 +180,22 @@ unsigned CG< container>::operator()( Matrix& A, container& x, const container& b
 //             else {
 //                   blas1::axpby( -alpha, ap, 1., r);
 //             }
-        blas1::axpby( -m_alpha, ap, 1., r);
-        m_nrm2r_new = blas2::dot( P, r); 
+        blas1::axpby( -alpha, ap, 1., r);
+        nrm2r_new = blas2::dot( P, r); 
 #ifdef DG_DEBUG
 #ifdef MPI_VERSION
     if(rank==0)
 #endif //MPI
     {
-        std::cout << "Absolute "<<sqrt( m_nrm2r_new) <<"\t ";
-        std::cout << " < Critical "<<eps*m_nrmb + eps <<"\t ";
-        std::cout << "(Relative "<<sqrt( m_nrm2r_new)/m_nrmb << ")\n";
+        std::cout << "Absolute "<<sqrt( nrm2r_new) <<"\t ";
+        std::cout << " < Critical "<<eps*nrmb + eps <<"\t ";
+        std::cout << "(Relative "<<sqrt( nrm2r_new)/nrmb << ")\n";
     }
 #endif //DG_DEBUG
-        if( sqrt( m_nrm2r_new) < eps*(m_nrmb + nrmb_correction)) 
+        if( sqrt( nrm2r_new) < eps*(nrmb + nrmb_correction)) 
             return i;
-        blas2::symv(1.,P, r, m_nrm2r_new/m_nrm2r_old, p );
-        m_nrm2r_old=m_nrm2r_new;
+        blas2::symv(1.,P, r, nrm2r_new/nrm2r_old, p );
+        nrm2r_old=nrm2r_new;
 
     }
     return max_iter;
@@ -206,57 +203,55 @@ unsigned CG< container>::operator()( Matrix& A, container& x, const container& b
 
 template< class container>
 template< class Matrix, class Preconditioner, class SquareNorm>
-unsigned CG< container>::operator()( Matrix& A, container& x, const container& b, Preconditioner& P, SquareNorm& S, value_type eps, value_type nrmb_correction, bool restart)
+unsigned CG< container>::operator()( Matrix& A, container& x, const container& b, Preconditioner& P, SquareNorm& S, value_type eps, value_type nrmb_correction)
 {
-    if( !restart)
-    {
-        m_nrmb = sqrt( blas2::dot( S, b));
+    value_type nrmb = sqrt( blas2::dot( S, b));
 #ifdef DG_DEBUG
 #ifdef MPI_VERSION
-        int rank;
-        MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-        if(rank==0)
+    int rank;
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+    if(rank==0)
 #endif //MPI
-        {
-        std::cout << "Norm of S b "<<nrmb <<"\n";
-        std::cout << "Residual errors: \n";
-        }
+    {
+    std::cout << "Norm of S b "<<nrmb <<"\n";
+    std::cout << "Residual errors: \n";
+    }
 #endif //DG_DEBUG
-        if( m_nrmb == 0)
-        {
-            blas1::copy( b, x);
-            return 0;
-        }
-        blas2::symv( A,x,r);
-        blas1::axpby( 1., b, -1., r);
-        //note that dot does automatically synchronize
-        if( sqrt( blas2::dot(S,r) ) < eps*(m_nrmb + nrmb_correction)) //if x happens to be the solution
-            return 0;
-        blas2::symv( P, r, p );//<-- compute p_0
-        m_nrm2r_old = blas1::dot( p,r); //and store the scalar product
+    if( nrmb == 0)
+    {
+        blas1::copy( b, x);
+        return 0;
     }
+    blas2::symv( A,x,r);
+    blas1::axpby( 1., b, -1., r);
+    //note that dot does automatically synchronize
+    if( sqrt( blas2::dot(S,r) ) < eps*(nrmb + nrmb_correction)) //if x happens to be the solution
+        return 0;
+    blas2::symv( P, r, p );//<-- compute p_0
+    value_type nrmzr_old = blas1::dot( p,r); //and store the scalar product
+    value_type alpha, nrmzr_new;
     for( unsigned i=1; i<max_iter; i++)
     {
         blas2::symv( A, p, ap);
-        m_alpha =  m_nrm2r_old/blas1::dot( p, ap);
-        blas1::axpby( m_alpha, p, 1.,x);
-        blas1::axpby( -m_alpha, ap, 1., r);
+        alpha =  nrmzr_old/blas1::dot( p, ap);
+        blas1::axpby( alpha, p, 1.,x);
+        blas1::axpby( -alpha, ap, 1., r);
 #ifdef DG_DEBUG
 #ifdef MPI_VERSION
     if(rank==0)
 #endif //MPI
     {
         std::cout << "Absolute r*S*r "<<sqrt( blas2::dot(S,r)) <<"\t ";
-        std::cout << " < Critical "<<eps*m_nrmb + eps <<"\t ";
-        std::cout << "(Relative "<<sqrt( blas2::dot(S,r) )/m_nrmb << ")\n";
+        std::cout << " < Critical "<<eps*nrmb + eps <<"\t ";
+        std::cout << "(Relative "<<sqrt( blas2::dot(S,r) )/nrmb << ")\n";
     }
 #endif //DG_DEBUG
-        if( sqrt( blas2::dot(S,r)) < eps*(m_nrmb + nrmb_correction)) 
+        if( sqrt( blas2::dot(S,r)) < eps*(nrmb + nrmb_correction)) 
             return i;
         blas2::symv(P,r,ap);
-        m_nrm2r_new = blas1::dot( ap, r); 
-        blas1::axpby(1.,ap, m_nrm2r_new/m_nrm2r_old, p );
-        m_nrm2r_old=m_nrm2r_new;
+        nrmzr_new = blas1::dot( ap, r); 
+        blas1::axpby(1.,ap, nrmzr_new/nrmzr_old, p );
+        nrmzr_old=nrmzr_new;
     }
     return max_iter;
 }
diff --git a/src/toefl/toeflR.cuh b/src/toefl/toeflR.cuh
index 9356cdc4d..67df92216 100644
--- a/src/toefl/toeflR.cuh
+++ b/src/toefl/toeflR.cuh
@@ -132,12 +132,12 @@ struct Explicit
     container gamma_n;
 
     //matrices and solvers
-    dg::Elliptic<Geometry, Matrix, container> laplaceM; //contains normalized laplacian
+    dg::Elliptic<Geometry, Matrix, container> pol, laplaceM; //contains normalized laplacian
     std::vector<dg::Elliptic<Geometry, Matrix, container> > multi_pol;
     dg::Helmholtz<Geometry,  Matrix, container> gamma1;
     dg::ArakawaX< Geometry, Matrix, container> arakawa; 
 
-    dg::Invert<container> invert_invgamma;
+    dg::Invert<container> invert_invgamma, invert_pol;
     dg::MultigridCG2d<Geometry, Matrix, container> multigrid_pol; 
     std::vector<container> multi_chi;
 
@@ -158,12 +158,12 @@ Explicit< Geometry, M, container>::Explicit( const Geometry& grid, const Paramet
     phi( 2, chi), dyphi( phi), ype(phi),
     dyy(2,chi), lny( dyy), lapy(dyy),
     gamma_n(chi),
-    //pol(     grid, not_normed, dg::centered, p.jfactor), 
+    pol(     grid, dg::not_normed, dg::centered, p.jfactor), 
     laplaceM( grid, dg::normed, dg::centered),
     gamma1(  grid, -0.5*p.tau, dg::centered),
     arakawa( grid), 
-    //invert_pol(      omega, p.Nx*p.Ny*p.n*p.n, p.eps_pol),
     invert_invgamma( omega, p.Nx*p.Ny*p.n*p.n, p.eps_gamma),
+    invert_pol(      omega, p.Nx*p.Ny*p.n*p.n, p.eps_pol),
     multigrid_pol( grid, 3), 
     w2d( dg::create::volume(grid)), v2d( dg::create::inv_volume(grid)), one( dg::evaluate(dg::one, grid)),
     eps_pol(p.eps_pol), eps_gamma( p.eps_gamma), kappa(p.kappa), friction(p.friction), nu(p.nu), tau( p.tau), equations( p.equations), boussinesq(p.boussinesq)
@@ -269,11 +269,9 @@ const container& Explicit<G, M, container>::polarisation( const std::vector<cont
     dg::blas1::transform( chi, chi, dg::INVERT<double>());
     dg::blas1::pointwiseDot( chi, v2d, chi);
 
-    //unsigned number = invert_pol( pol, phi[0], omega, w2d, chi, v2d);
-    //if(  number == invert_pol.get_max())
-    //    throw dg::Fail( eps_pol);
-    std::vector<unsigned> number = multigrid_pol.solve( multi_pol, phi[0], omega, eps_pol);
-    if(  number[0] == invert_invgamma.get_max())
+    unsigned number = invert_pol( pol, phi[0], omega, v2d, chi);
+    //std::vector<unsigned> number = multigrid_pol.solve( multi_pol, phi[0], omega, eps_pol);
+    if(  number == invert_pol.get_max())
         throw dg::Fail( eps_pol);
     return phi[0];
 }
-- 
GitLab


From d51d7d0406cd1456d36796fde2236aac0da23f7a Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Wed, 13 Sep 2017 23:22:47 +0200
Subject: [PATCH 280/453] thats amazing

---
 inc/dg/multigrid.h    | 42 ++++++++++++++++++++++++++++++++++--------
 inc/dg/multistep.h    | 14 +++++++-------
 inc/dg/multistep_t.cu | 12 +++++++-----
 src/toefl/toeflR.cuh  |  8 ++++----
 4 files changed, 52 insertions(+), 24 deletions(-)

diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index 31186b613..a9dd1c055 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -6,6 +6,7 @@
 #include "backend/memory.h"
 #include "blas.h"
 #include "cg.h"
+#include "backend/timer.cuh"
 
 namespace dg
 {
@@ -45,6 +46,7 @@ struct MultigridCG2d
 
         dg::blas1::transfer(dg::evaluate(dg::zero, grid), x0_);
         x1_ = x0_, x2_ = x0_;
+        
         x_ = project(x0_); 
         m_r = x_,
 		b_ = x_;        
@@ -98,10 +100,12 @@ struct MultigridCG2d
                 // compute residual r = b - A x
                 dg::blas2::symv(op[u], x_[u], m_r[u]);
                 dg::blas1::axpby(-1.0, m_r[u], 1.0, b_[u], m_r[u]);
+                dg::blas1::pointwiseDot( m_r[u], op[u].inv_weights(), m_r[u]);
 
                 //
                 // transfer residual to the rhs of the coarser grid
                 dg::blas2::symv(project_[u], m_r[u], b_[w]);
+                dg::blas1::pointwiseDivide( b_[w], op[w].inv_weights(), b_[w]);
 
                 //dg::blas2::symv(project_[u], x_[u], x_[w]);
             }
@@ -127,35 +131,57 @@ struct MultigridCG2d
 		return number;
 	}
 
-    /*template<class SymmetricOp>
+    template<class SymmetricOp>
     std::vector<unsigned> direct_solve( std::vector<SymmetricOp>& op, container&  x, const container& b, double eps)
     {
+        Timer t;
+        t.tic();
+        //compute initial guess
         dg::blas1::axpbygz( alpha[0], x0_, alpha[1], x1_, alpha[2], x2_); 
         
 		x_[0].swap(x2_);
-        
-		project( x_[0], x_);
-        project( b, b_);
+        dg::blas1::copy( x_[0], x);//save initial guess
 
+        //dg::blas1::scal( x_[0], 0.0);
+		//project( x_[0], x_);
+        project( b, b_);
+        //project( b, b_);
+        // compute residual r = b - A x
+        dg::blas2::symv(op[0], x_[0], m_r[0]);
+        dg::blas1::pointwiseDot( m_r[0], op[0].inv_weights(), m_r[0]);
+        dg::blas1::axpby(-1.0, m_r[0], 1.0, b_[0], b_[0]);
+        project( b_[0], b_);
+        dg::blas1::scal( x_[0], 0.0);
+		project( x_[0], x_);
         std::vector<unsigned> number(stages_);
         
+        //now solve residual equations
 		for( unsigned u=stages_-1; u>0; u--)
         {
+            cg_[u].set_max(grids_[u].get().size());
             dg::blas1::pointwiseDivide( b_[u], op[u].inv_weights(), b_[u]);
-            number[u] = cg_[u]( op[u], x_[u], b_[u], op[u].precond(), op[u].inv_weights(), eps, 1.);
+            number[u] = cg_[u]( op[u], x_[u], b_[u], op[u].precond(), op[u].inv_weights(), eps/2, 1.);
             dg::blas2::symv( inter_[u-1], x_[u], x_[u-1]);
+            std::cout << "stage: " << u << ", max iter: " << cg_[u].get_max() << ", iter: " << number[u] << std::endl;
         }
-        
+
 		dg::blas1::pointwiseDivide( b_[0], op[0].inv_weights(), b_[0]);
+        cg_[0].set_max(grids_[0].get().size());
         number[0] = cg_[0]( op[0], x_[0], b_[0], op[0].precond(), op[0].inv_weights(), eps);
+        std::cout << "stage: " << 0 << ", max iter: " << cg_[0].get_max() << ", iter: " << number[0] << std::endl;
+        //update initial guess
+        dg::blas1::axpby( 1., x_[0], 1., x);
+
         
-		x_[0].swap( x);
+		//x_[0].swap( x);
         x1_.swap( x2_);
         x0_.swap( x1_);
         
         blas1::copy( x, x0_);
+        t.toc();
+        std::cout<< "Took "<<t.diff()<<"s\n";
         return number;
-    }*/
+    }
 
     ///src may alias first element of out
     void project( const container& src, std::vector<container>& out)
diff --git a/inc/dg/multistep.h b/inc/dg/multistep.h
index 4ee6dbb2f..d832d8c87 100644
--- a/inc/dg/multistep.h
+++ b/inc/dg/multistep.h
@@ -297,17 +297,18 @@ void Karniadakis<container>::init( Functor& f, Diffusion& diff,  const container
 {
     dt_ = dt;
     container temp_(u0);
+    //implicit may write into temp
     detail::Implicit<Diffusion, container> implicit( -dt, diff, temp_);
-    blas1::axpby( 1., u0, 0, temp_); //copy u0
+    blas1::copy( u0, temp_); //copy u0
     f( temp_, f_[0]);
-    blas1::axpby( 1., u0, 0, u_[0]); 
+    blas1::copy(  u0, u_[0]); 
     blas1::axpby( 1., u_[0], -dt, f_[0], f_[1]); //Euler step
     implicit( f_[1], u_[1]); //explicit Euler step backwards, might destroy f_[1]
-    blas1::axpby( 1., u_[1], 0, temp_); 
+    blas1::copy( u_[1], temp_); 
     f( temp_, f_[1]);
     blas1::axpby( 1.,u_[1], -dt, f_[1], f_[2]);
     implicit( f_[2], u_[2]);
-    blas1::axpby( 1., u_[2], 0, temp_); 
+    blas1::copy( u_[2], temp_); 
     f( temp_, f_[2]);
 }
 
@@ -327,12 +328,11 @@ void Karniadakis<container>::operator()( Functor& f, Diffusion& diff, container&
         u_[i-1].swap( u_[i]);
     }
     blas1::axpby( 1., f_[0], 1., u_[0]);
-    blas1::pointwiseDivide( u_[0], diff.inv_weights(), u_[0]);
     //compute implicit part
     double alpha[2] = {2., -1.};
     //double alpha[2] = {1., 0.};
-    blas1::axpby( alpha[0], u_[1], alpha[1],  u_[2], u_[0]); //extrapolate previous solutions
-    blas1::pointwiseDivide( u, diff.inv_weights(), u);
+    blas1::axpby( alpha[0], u_[1], alpha[1],  u_[2], u); //extrapolate previous solutions
+    blas1::pointwiseDivide( u_[0], diff.inv_weights(), u_[0]);
     detail::Implicit<Diffusion, container> implicit( -dt_/11.*6., diff, f_[0]);
 #ifdef DG_BENCHMARK
 #ifdef MPI_VERSION
diff --git a/inc/dg/multistep_t.cu b/inc/dg/multistep_t.cu
index 819af8963..9a7d7bc06 100644
--- a/inc/dg/multistep_t.cu
+++ b/inc/dg/multistep_t.cu
@@ -22,7 +22,8 @@ template< class Matrix, class container>
 struct Diffusion
 {
     Diffusion( const dg::Grid2d& g, double nu): nu_(nu),
-        w2d(dg::create::weights( g)), v2d(dg::create::inv_weights(g)),
+        precon( dg::create::inv_weights(g)), 
+        v2d(2, dg::create::inv_weights(g)),
         LaplacianM( g, dg::normed) 
         { }
 
@@ -34,11 +35,12 @@ struct Diffusion
         }
         dg::blas1::axpby( 0.,y, -nu_, y);
     }
-    const container& weights(){return w2d;}
-    const container& precond(){return v2d;}
+    const std::vector<container>& inv_weights(){return v2d;}
+    const container& precond(){return precon;}
   private:
     double nu_;
-    const container w2d, v2d;
+    const container precon;
+    const std::vector<container> v2d;
     dg::Elliptic<dg::CartesianGrid2d, Matrix, container> LaplacianM;
 };
 
@@ -94,7 +96,7 @@ int main()
     dg::blas1::axpby( -1., y0[0], 1., error);
     std::cout << "Normalized solution is "<<  norm_sol<< std::endl;
     double norm_error = dg::blas2::dot( w2d, error);
-    std::cout << "Relative error is      "<< sqrt( norm_error/norm_sol)<<" (0.000141704)\n";
+    std::cout << "Relative error is      "<< sqrt( norm_error/norm_sol)<<" (0.0020084 Karniadakis) (0.000148647 SIRK)\n";
     //n = 1 -> p = 1 (Sprung in laplace macht n=1 eine Ordng schlechter) 
     //n = 2 -> p = 2
     //n = 3 -> p = 3
diff --git a/src/toefl/toeflR.cuh b/src/toefl/toeflR.cuh
index 67df92216..7d036f971 100644
--- a/src/toefl/toeflR.cuh
+++ b/src/toefl/toeflR.cuh
@@ -18,7 +18,7 @@ struct Implicit
 {
     Implicit( const Geometry& g, double nu):
         nu_(nu), 
-        temp( dg::evaluate(dg::zero, g)), inv_weights_(2,temp),
+        temp( dg::create::inv_weights( g) ), inv_weights_(2,temp),
         LaplacianM_perp( g, dg::normed, dg::centered){
     }
     void operator()( std::vector<container>& x, std::vector<container>& y)
@@ -269,9 +269,9 @@ const container& Explicit<G, M, container>::polarisation( const std::vector<cont
     dg::blas1::transform( chi, chi, dg::INVERT<double>());
     dg::blas1::pointwiseDot( chi, v2d, chi);
 
-    unsigned number = invert_pol( pol, phi[0], omega, v2d, chi);
-    //std::vector<unsigned> number = multigrid_pol.solve( multi_pol, phi[0], omega, eps_pol);
-    if(  number == invert_pol.get_max())
+    //unsigned number = invert_pol( pol, phi[0], omega, v2d, chi);
+    std::vector<unsigned> number = multigrid_pol.direct_solve( multi_pol, phi[0], omega, eps_pol);
+    if(  number[0] == invert_pol.get_max())
         throw dg::Fail( eps_pol);
     return phi[0];
 }
-- 
GitLab


From e370d3dc73da0cb571d1ced3f37d9efb256b0539 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Thu, 14 Sep 2017 10:52:45 +0200
Subject: [PATCH 281/453] also changed Helmholtz class to work for Multigrid
 and use it in toefl

---
 inc/dg/helmholtz.h   | 30 +++++++++++++++++++++---------
 src/toefl/toeflR.cuh | 23 ++++++++++++++---------
 2 files changed, 35 insertions(+), 18 deletions(-)

diff --git a/inc/dg/helmholtz.h b/inc/dg/helmholtz.h
index f23628e18..4f1278298 100644
--- a/inc/dg/helmholtz.h
+++ b/inc/dg/helmholtz.h
@@ -26,20 +26,20 @@ namespace dg{
 template< class Geometry, class Matrix, class container> 
 struct Helmholtz
 {
+    ///@brief empty object ( no memory allocation)
+    Helmholtz() {}
     /**
      * @brief Construct Helmholtz operator
      *
-     * @param g The grid to use
+     * @param g The grid to use (boundary conditions are taken from there)
      * @param alpha Scalar in the above formula
      * @param dir Direction of the Laplace operator
      * @param jfactor The jfactor used in the Laplace operator (probably 1 is always the best factor but one never knows...)
      * @note The default value of \f$\chi\f$ is one. Helmholtz is never normed
      */
-    Helmholtz( const Geometry& g, double alpha = 1., direction dir = dg::forward, double jfactor=1.):
-        laplaceM_(g, normed, dir, jfactor), 
-        temp_(dg::evaluate(dg::one, g)),
-        alpha_(alpha)
+    Helmholtz( const Geometry& g, double alpha = 1., direction dir = dg::forward, double jfactor=1.)
     { 
+        construct( g, alpha, dir, jfactor);
     }
     /**
      * @brief Construct Helmholtz operator
@@ -52,11 +52,23 @@ struct Helmholtz
      * @param jfactor The jfactor used in the Laplace operator (probably 1 is always the best factor but one never knows...)
      * @note The default value of \f$\chi\f$ is one
      */
-    Helmholtz( const Geometry& g, bc bcx, bc bcy, double alpha = 1., direction dir = dg::forward, double jfactor=1.):
-        laplaceM_(g, bcx,bcy,normed, dir, jfactor), 
-        temp_(dg::evaluate(dg::one, g)), 
-        alpha_(alpha)
+    Helmholtz( const Geometry& g, bc bcx, bc bcy, double alpha = 1., direction dir = dg::forward, double jfactor=1.)
     { 
+        construct( g, bcx, bcy, alpha, dir, jfactor);
+    }
+    ///@copydoc Helmholtz::Helmholtz(const Geometry&,bc,bc,double,direction,double)
+    void construct( const Geometry& g, bc bcx, bc bcy, double alpha = 1, direction dir = dg::forward, double jfactor = 1.) 
+    {
+        laplaceM_.construct( g, bcx, bcy, dg::normed, dir, jfactor);
+        dg::blas1::transfer( dg::evaluate( dg::one, g), temp_);
+        alpha_ = alpha; 
+    }
+    ///@copydoc Helmholtz::Helmholtz(const Geometry&,double,direction,double)
+    void construct( const Geometry& g, double alpha = 1, direction dir = dg::forward, double jfactor = 1.) 
+    {
+        laplaceM_.construct( g, dg::normed, dir, jfactor);
+        dg::blas1::transfer( dg::evaluate( dg::one, g), temp_);
+        alpha_ = alpha; 
     }
     /**
      * @brief apply operator
diff --git a/src/toefl/toeflR.cuh b/src/toefl/toeflR.cuh
index 7d036f971..dcd14dce1 100644
--- a/src/toefl/toeflR.cuh
+++ b/src/toefl/toeflR.cuh
@@ -78,7 +78,7 @@ struct Explicit
      *
      * @return Gamma operator
      */
-    dg::Helmholtz<Geometry, Matrix, container >&  gamma() {return gamma1;}
+    dg::Helmholtz<Geometry, Matrix, container >&  gamma() {return multi_gamma1[0];}
 
     /**
      * @brief Compute the right-hand side of the toefl equations
@@ -134,11 +134,11 @@ struct Explicit
     //matrices and solvers
     dg::Elliptic<Geometry, Matrix, container> pol, laplaceM; //contains normalized laplacian
     std::vector<dg::Elliptic<Geometry, Matrix, container> > multi_pol;
-    dg::Helmholtz<Geometry,  Matrix, container> gamma1;
+    std::vector<dg::Helmholtz<Geometry,  Matrix, container> > multi_gamma1;
     dg::ArakawaX< Geometry, Matrix, container> arakawa; 
 
     dg::Invert<container> invert_invgamma, invert_pol;
-    dg::MultigridCG2d<Geometry, Matrix, container> multigrid_pol; 
+    dg::MultigridCG2d<Geometry, Matrix, container> multigrid_pol, multigrid_invgammaPhi, multigrid_invgammaN; 
     std::vector<container> multi_chi;
 
     const container w2d, v2d, one;
@@ -160,18 +160,21 @@ Explicit< Geometry, M, container>::Explicit( const Geometry& grid, const Paramet
     gamma_n(chi),
     pol(     grid, dg::not_normed, dg::centered, p.jfactor), 
     laplaceM( grid, dg::normed, dg::centered),
-    gamma1(  grid, -0.5*p.tau, dg::centered),
     arakawa( grid), 
     invert_invgamma( omega, p.Nx*p.Ny*p.n*p.n, p.eps_gamma),
     invert_pol(      omega, p.Nx*p.Ny*p.n*p.n, p.eps_pol),
-    multigrid_pol( grid, 3), 
+    multigrid_pol( grid, 3), multigrid_invgammaPhi( grid, 3), multigrid_invgammaN(grid, 3),
     w2d( dg::create::volume(grid)), v2d( dg::create::inv_volume(grid)), one( dg::evaluate(dg::one, grid)),
     eps_pol(p.eps_pol), eps_gamma( p.eps_gamma), kappa(p.kappa), friction(p.friction), nu(p.nu), tau( p.tau), equations( p.equations), boussinesq(p.boussinesq)
 { 
     multi_chi= multigrid_pol.project( chi);
     multi_pol.resize(3);
+    multi_gamma1.resize(3);
     for( unsigned u=0; u<3; u++)
+    {
         multi_pol[u].construct( multigrid_pol.grids()[u].get(), dg::not_normed, dg::centered, p.jfactor);
+        multi_gamma1[u].construct( multigrid_pol.grids()[u].get(), -0.5*p.tau, dg::centered);
+    }
 }
 
 template< class G, class M, class container>
@@ -181,8 +184,9 @@ const container& Explicit<G, M, container>::compute_psi( const container& potent
     //in gyrofluid invert Gamma operator
     if( equations == "local" || equations == "global")
     {
-        unsigned number = invert_invgamma( gamma1, phi[1], potential);
-        if(  number == invert_invgamma.get_max())
+        std::vector<unsigned> number = multigrid_invgammaPhi.direct_solve( multi_gamma1, phi[1], potential, eps_gamma);
+        //unsigned number = invert_invgamma( gamma1, phi[1], potential);
+        if(  number[0] == invert_invgamma.get_max())
             throw dg::Fail( eps_gamma);
     }
     //compute (nabla phi)^2
@@ -255,8 +259,9 @@ const container& Explicit<G, M, container>::polarisation( const std::vector<cont
     //compute polarisation
     if( equations == "local" || equations == "global")
     {
-        unsigned number = invert_invgamma( gamma1, gamma_n, y[1]);
-        if(  number == invert_invgamma.get_max())
+        std::vector<unsigned> number = multigrid_invgammaN.direct_solve( multi_gamma1, gamma_n, y[1], eps_gamma);
+        //unsigned number = invert_invgamma( gamma1, gamma_n, y[1]);
+        if(  number[0] == invert_invgamma.get_max())
             throw dg::Fail( eps_gamma);
         dg::blas1::axpby( -1., y[0], 1., gamma_n, omega); //omega = a_i\Gamma n_i - n_e
     }
-- 
GitLab


From 09c5361dc5f2dc8c166d294c1746701a99b6c7c5 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Thu, 14 Sep 2017 11:08:44 +0200
Subject: [PATCH 282/453] added mathjax line in doc Makefile and updated README
 on dependencies

---
 README.md        |  7 ++++---
 inc/doc/Makefile | 22 ++++++++++++++--------
 2 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/README.md b/README.md
index 3b0b2a00c..ebb7619be 100644
--- a/README.md
+++ b/README.md
@@ -109,13 +109,14 @@ input/output parameters, etc. can be generated as a pdf with
 `make doc ` in the `path/to/feltor/src/toefl` directory.
 
 ## 2. Further reading
-Please check out our [wiki pages](https://github.com/feltor-dev/feltor/wiki) for some general information and user oriented documentation. 
+Please check out our [wiki pages](https://github.com/feltor-dev/feltor/wiki) for some general information, user oriented documentation and Troubleshooting. 
 Moreover, we maintain tex files in every src folder for technical documentation, 
  which can be compiled using pdflatex with 
 `make doc ` in the respective src folder.
 The [developer oriented documentation](http://feltor-dev.github.io/feltor/inc/dg/html/modules.html) of the dG library was generated with [Doxygen](http://www.doxygen.org). 
-You can generate a local version from source code
-by typing `make doc` in the folder `path/to/feltor/inc/doc` and then open `index.html` with your favorite browser.
+You can generate a local version from source code.
+On linux you need the `doxygen`, `libjs-mathjax` and `graphviz` packages.
+Then type `make doc` in the folder `path/to/feltor/inc/doc` and open `index.html` (a symbolic link to `dg/html/modules.html`) with your favorite browser. 
 
 ## 3. Contributions and Acknowledgements
 For instructions on how to contribute read the [wiki page](https://github.com/feltor-dev/feltor/wiki/Contributions).
diff --git a/inc/doc/Makefile b/inc/doc/Makefile
index 4b5ebcb2b..45055a934 100644
--- a/inc/doc/Makefile
+++ b/inc/doc/Makefile
@@ -1,6 +1,12 @@
 tagfiles="./dg.tag=../../dg/html ./file.tag=../../file/html"
 .PHONY: dg file geometries
 
+###Packages needed on linux:
+###doxygen
+###libjs-mathjax
+###graphviz
+#(shouldn't require latex according to doxygen documentation)
+
 all: doc
 
 dg:
@@ -9,8 +15,8 @@ dg:
 	echo "INPUT = ../dg/"; \
 	echo "OUTPUT_DIRECTORY = ./dg"; \
 	echo "HTML_HEADER = header.html"; \
-	echo "GENERATE_TAGFILE = ./dg.tag" ) | doxygen - ;
-	#echo "MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"; \ ## uncomment if run for official feltor page
+	echo "GENERATE_TAGFILE = ./dg.tag" ) | doxygen - ;\
+	echo "MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"; 
 
 
 file:
@@ -19,8 +25,8 @@ file:
 	echo "INPUT = ../file/"; \
 	echo "OUTPUT_DIRECTORY = ./file"; \
 	echo "HTML_HEADER = header.html"; \
-	echo "GENERATE_TAGFILE = ./file.tag" ) | doxygen - ;
-	#echo "MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"; \ ## uncomment if run for official feltor page
+	echo "GENERATE_TAGFILE = ./file.tag" ) | doxygen - ; \
+	echo "MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"; 
 
 geometries:
 	(cat ../geometries/Doxyfile; \
@@ -29,12 +35,12 @@ geometries:
 	echo "HTML_HEADER = header.html"; \
     echo "EXTERNAL_GROUPS=NO" ;\
     echo "EXTERNAL_PAGES=NO" ;\
-    echo "TAGFILES = $(tagfiles)") | doxygen - ;
-	#echo "MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"; \ ## uncomment if run for official feltor page
+    echo "TAGFILES = $(tagfiles)") | doxygen - ; \
+	echo "MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"; 
 
 doc: dg file geometries
-	ln -fs dg/html/modules.html index.html
-	echo "Open with: firefox index.html"
+	ln -sf dg/html/modules.html index.html
+	echo "Open with: firefox index.html or on Windows: firefox dg/html/modules.html"
 
 
 clean:
-- 
GitLab


From 7d708c90179db1f013ef9717beb74558da1b96ee Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Thu, 14 Sep 2017 12:03:16 +0200
Subject: [PATCH 283/453] made mpi version work

---
 inc/dg/backend/fast_interpolation.h | 76 ++++++++++++++++++++++++++---
 inc/dg/backend/mpi_matrix.h         | 68 +++++---------------------
 inc/dg/backend/mpi_vector.h         |  4 +-
 inc/dg/elliptic2d_mpib.cu           | 31 ++++++++----
 4 files changed, 105 insertions(+), 74 deletions(-)

diff --git a/inc/dg/backend/fast_interpolation.h b/inc/dg/backend/fast_interpolation.h
index 5cec2c3c0..0035f89d2 100644
--- a/inc/dg/backend/fast_interpolation.h
+++ b/inc/dg/backend/fast_interpolation.h
@@ -1,12 +1,18 @@
 #pragma once
 
 #include <thrust/host_vector.h>
+#include "../enums.h"
 #include "grid.h"
 #include "interpolation.cuh"
 #include "projection.cuh"
 #include "matrix_traits.h"
 #include "sparseblockmat.h"
 #include "memory.h"
+#ifdef MPI_VERSION
+#include "mpi_grid.h"
+#include "mpi_vector.h"
+#include "mpi_matrix.h"
+#endif //MPI_VERSION
 
 namespace dg
 {
@@ -105,13 +111,17 @@ MultiMatrix< EllSparseBlockMat<double>, thrust::host_vector<double> > fast_inter
     return inter;
 }
 
-MultiMatrix< EllSparseBlockMat<double>, thrust::host_vector<double> > fast_projection( const Grid1d& t, unsigned divide)
+MultiMatrix< EllSparseBlockMat<double>, thrust::host_vector<double> > fast_projection( const Grid1d& t, unsigned divide, enum dg::norm no = normed)
 {
     unsigned n=t.n();
     if( t.N()%divide != 0) throw Error( Message(_ping_)<< "Nx and divide don't match: Nx: " << t.N()<< " divide "<< divide);
     dg::Grid1d g_oldX( -1., 1., n, divide);
     dg::Grid1d g_new(  -1., 1., n, 1);
-    dg::IHMatrix projectX = dg::create::projection( g_new, g_oldX);
+    dg::IHMatrix projectX;
+    if(no == normed)
+        projectX = dg::create::projection( g_new, g_oldX);
+    else
+        projectX = dg::create::interpolationT( g_new, g_oldX);
     EllSparseBlockMat<double> pX( t.N()/divide, t.N(), divide, divide, t.n()); 
     for( unsigned i=0; i<n; i++)
     for( unsigned j=0; j<n; j++)
@@ -147,14 +157,14 @@ MultiMatrix< EllSparseBlockMat<double>, thrust::host_vector<double> > fast_inter
     return inter;
 }
 
-MultiMatrix< EllSparseBlockMat<double>, thrust::host_vector<double> > fast_projection( const aTopology2d& t, unsigned divideX, unsigned divideY)
+MultiMatrix< EllSparseBlockMat<double>, thrust::host_vector<double> > fast_projection( const aTopology2d& t, unsigned divideX, unsigned divideY, enum dg::norm no = normed)
 {
     dg::Grid1d gx(t.x0(), t.x1(), t.n(), t.Nx());
     dg::Grid1d gy(t.y0(), t.y1(), t.n(), t.Ny());
-    MultiMatrix < EllSparseBlockMat<double>, thrust::host_vector<double> > interX = dg::create::fast_projection( gx, divideX);
+    MultiMatrix < EllSparseBlockMat<double>, thrust::host_vector<double> > interX = dg::create::fast_projection( gx, divideX, no);
     interX.get_matrices()[0].left_size = t.n()*t.Ny();
     interX.get_matrices()[0].set_default_range();
-    MultiMatrix < EllSparseBlockMat<double>, thrust::host_vector<double> > interY = dg::create::fast_projection( gy, divideY);
+    MultiMatrix < EllSparseBlockMat<double>, thrust::host_vector<double> > interY = dg::create::fast_projection( gy, divideY, no);
     interY.get_matrices()[0].right_size = t.n()*t.Nx()/divideX;
     interY.get_matrices()[0].set_default_range();
 
@@ -186,14 +196,14 @@ MultiMatrix< EllSparseBlockMat<double>, thrust::host_vector<double> > fast_inter
     return inter;
 }
 
-MultiMatrix< EllSparseBlockMat<double>, thrust::host_vector<double> > fast_projection( const aTopology3d& t, unsigned divideX, unsigned divideY)
+MultiMatrix< EllSparseBlockMat<double>, thrust::host_vector<double> > fast_projection( const aTopology3d& t, unsigned divideX, unsigned divideY, enum dg::norm no = normed)
 {
     dg::Grid1d gx(t.x0(), t.x1(), t.n(), t.Nx());
     dg::Grid1d gy(t.y0(), t.y1(), t.n(), t.Ny());
-    MultiMatrix < EllSparseBlockMat<double>, thrust::host_vector<double> > interX = dg::create::fast_projection( gx, divideX);
+    MultiMatrix < EllSparseBlockMat<double>, thrust::host_vector<double> > interX = dg::create::fast_projection( gx, divideX, no);
     interX.get_matrices()[0].left_size = t.n()*t.Ny()*t.Nz();
     interX.get_matrices()[0].set_default_range();
-    MultiMatrix < EllSparseBlockMat<double>, thrust::host_vector<double> > interY = dg::create::fast_projection( gy, divideY);
+    MultiMatrix < EllSparseBlockMat<double>, thrust::host_vector<double> > interY = dg::create::fast_projection( gy, divideY, no);
     interY.get_matrices()[0].right_size = t.n()*t.Nx()/divideX;
     interY.get_matrices()[0].left_size = t.Nz();
     interY.get_matrices()[0].set_default_range();
@@ -206,6 +216,56 @@ MultiMatrix< EllSparseBlockMat<double>, thrust::host_vector<double> > fast_proje
     return inter;
 }
 
+#ifdef MPI_VERSION
+//very elaborate way of telling the compiler to just apply the local matrix to the local vector
+MultiMatrix< RowColDistMat<EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH >, MPI_Vector<thrust::host_vector<double> > > fast_interpolation( const aMPITopology2d& t, unsigned divideX, unsigned divideY)
+{
+    typedef RowColDistMat<EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> Matrix; 
+    typedef MPI_Vector<thrust::host_vector<double> > Vector; 
+    MultiMatrix<EllSparseBlockMat<double>, thrust::host_vector<double> > temp = dg::create::fast_interpolation( t.local(), divideX, divideY);
+    MultiMatrix< Matrix, Vector > inter(2); 
+    inter.get_matrices()[0] = Matrix( temp.get_matrices()[0], CooSparseBlockMat<double>(), NNCH());
+    inter.get_matrices()[1] = Matrix( temp.get_matrices()[1], CooSparseBlockMat<double>(), NNCH());
+    inter.get_temp()[0] = Buffer<Vector> ( Vector( temp.get_temp()[0].data(), t.communicator())  );
+    return inter;
+}
+MultiMatrix< RowColDistMat<EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH >, MPI_Vector<thrust::host_vector<double> > > fast_projection( const aMPITopology2d& t, unsigned divideX, unsigned divideY, enum dg::norm no = normed)
+{
+    typedef RowColDistMat<EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> Matrix; 
+    typedef MPI_Vector<thrust::host_vector<double> > Vector; 
+    MultiMatrix<EllSparseBlockMat<double>, thrust::host_vector<double> > temp = dg::create::fast_projection( t.local(), divideX, divideY, no);
+    MultiMatrix< Matrix, Vector > inter(2); 
+    inter.get_matrices()[0] = Matrix( temp.get_matrices()[0], CooSparseBlockMat<double>(), NNCH());
+    inter.get_matrices()[1] = Matrix( temp.get_matrices()[1], CooSparseBlockMat<double>(), NNCH());
+    inter.get_temp()[0] = Buffer<Vector> ( Vector( temp.get_temp()[0].data(), t.communicator())  );
+    return inter;
+}
+
+MultiMatrix< RowColDistMat<EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH >, MPI_Vector<thrust::host_vector<double> > > fast_interpolation( const aMPITopology3d& t, unsigned divideX, unsigned divideY)
+{
+    typedef RowColDistMat<EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> Matrix; 
+    typedef MPI_Vector<thrust::host_vector<double> > Vector; 
+    MultiMatrix<EllSparseBlockMat<double>, thrust::host_vector<double> > temp = dg::create::fast_interpolation( t.local(), divideX, divideY);
+    MultiMatrix< Matrix, Vector > inter(2); 
+    inter.get_matrices()[0] = Matrix( temp.get_matrices()[0], CooSparseBlockMat<double>(), NNCH());
+    inter.get_matrices()[1] = Matrix( temp.get_matrices()[1], CooSparseBlockMat<double>(), NNCH());
+    inter.get_temp()[0] = Buffer<Vector> ( Vector( temp.get_temp()[0].data(), t.communicator())  );
+    return inter;
+}
+
+MultiMatrix< RowColDistMat<EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH >, MPI_Vector<thrust::host_vector<double> > > fast_projection( const aMPITopology3d& t, unsigned divideX, unsigned divideY, enum dg::norm no = normed)
+{
+    typedef RowColDistMat<EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> Matrix; 
+    typedef MPI_Vector<thrust::host_vector<double> > Vector; 
+    MultiMatrix<EllSparseBlockMat<double>, thrust::host_vector<double> > temp = dg::create::fast_projection( t.local(), divideX, divideY, no);
+    MultiMatrix< Matrix, Vector > inter(2); 
+    inter.get_matrices()[0] = Matrix( temp.get_matrices()[0], CooSparseBlockMat<double>(), NNCH());
+    inter.get_matrices()[1] = Matrix( temp.get_matrices()[1], CooSparseBlockMat<double>(), NNCH());
+    inter.get_temp()[0] = Buffer<Vector> ( Vector( temp.get_temp()[0].data(), t.communicator())  );
+    return inter;
+}
+
+#endif //MPI_VERSION
 }//namespace create
 
 }//namespace dg
diff --git a/inc/dg/backend/mpi_matrix.h b/inc/dg/backend/mpi_matrix.h
index 6dbef6963..78ff66899 100644
--- a/inc/dg/backend/mpi_matrix.h
+++ b/inc/dg/backend/mpi_matrix.h
@@ -94,11 +94,6 @@ struct RowColDistMat
     template<class container> 
     void symv( double alpha, const MPI_Vector<container>& x, double beta, MPI_Vector<container>& y) const
     {
-        assert( x.communicator() == y.communicator());
-        assert( x.communicator() == c_.communicator());
-        //int rank;
-        //MPI_Comm_rank( MPI_COMM_WORLD, &rank);
-        //dg::Timer t;
         if( c_.size() == 0) //no communication needed
         {
             dg::blas2::detail::doSymv( alpha, m_i, x.data(), beta, y.data(), 
@@ -107,25 +102,18 @@ struct RowColDistMat
             return;
 
         }
-        //t.tic();
+        assert( x.communicator() == y.communicator());
+        assert( x.communicator() == c_.communicator());
         //1. compute inner points
         dg::blas2::detail::doSymv( alpha, m_i, x.data(), beta, y.data(), 
                        typename dg::MatrixTraits<LocalMatrixInner>::matrix_category(), 
                        typename dg::VectorTraits<container>::vector_category() );
-        //t.toc();
-        //if(rank==0)std::cout << "Inner points took "<<t.diff()<<"s\n";
         //2. communicate outer points
-        //t.tic();
         const container& temp = c_.global_gather( x.data());
-        //t.toc();
-        //if(rank==0)std::cout << "Collect      took "<<t.diff()<<"s\n";
         //3. compute and add outer points
-        //t.tic();
         dg::blas2::detail::doSymv(alpha, m_o, temp, 1., y.data(), 
                        typename dg::MatrixTraits<LocalMatrixOuter>::matrix_category(), 
                        typename dg::VectorTraits<container>::vector_category() );
-        //t.toc();
-        //if(rank==0)std::cout << "Outer points took "<<t.diff()<<"s\n";
     }
 
     /**
@@ -141,11 +129,6 @@ struct RowColDistMat
     template<class container> 
     void symv( const MPI_Vector<container>& x, MPI_Vector<container>& y) const
     {
-        assert( x.communicator() == y.communicator());
-        assert( x.communicator() == c_.communicator());
-        //int rank;
-        //MPI_Comm_rank( MPI_COMM_WORLD, &rank);
-        //dg::Timer t;
         if( c_.size() == 0) //no communication needed
         {
             dg::blas2::detail::doSymv( m_i, x.data(), y.data(), 
@@ -155,26 +138,19 @@ struct RowColDistMat
             return;
 
         }
-        //t.tic();
+        assert( x.communicator() == y.communicator());
+        assert( x.communicator() == c_.communicator());
         //1. compute inner points
         dg::blas2::detail::doSymv( m_i, x.data(), y.data(), 
                        typename dg::MatrixTraits<LocalMatrixInner>::matrix_category(), 
                        typename dg::VectorTraits<container>::vector_category(),
                        typename dg::VectorTraits<container>::vector_category() );
-        //t.toc();
-        //if(rank==0)std::cout << "Inner points took "<<t.diff()<<"s\n";
         //2. communicate outer points
-        //t.tic();
         const container& temp = c_.global_gather( x.data());
-        //t.toc();
-        //if(rank==0)std::cout << "Collect      took "<<t.diff()<<"s\n";
         //3. compute and add outer points
-        //t.tic();
         dg::blas2::detail::doSymv(1., m_o, temp, 1., y.data(), 
                        typename dg::MatrixTraits<LocalMatrixOuter>::matrix_category(), 
                        typename dg::VectorTraits<container>::vector_category() );
-        //t.toc();
-        //if(rank==0)std::cout << "Outer points took "<<t.diff()<<"s\n";
     }
         
     private:
@@ -244,13 +220,6 @@ struct RowDistMat
     template<class container> 
     void symv( double alpha, const MPI_Vector<container>& x, double beta, MPI_Vector<container>& y)
     {
-        assert( x.communicator() == y.communicator());
-        assert( x.communicator() == c_.communicator());
-        //int rank;
-        //MPI_Comm_rank( MPI_COMM_WORLD, &rank);
-        //dg::Timer t;
-
-        //t.tic();
         if( c_.size() == 0) //no communication needed
         {
             dg::blas2::detail::doSymv( alpha, m_, x.data(), beta, y.data(), 
@@ -259,26 +228,16 @@ struct RowDistMat
             return;
 
         }
+        assert( x.communicator() == y.communicator());
+        assert( x.communicator() == c_.communicator());
         container temp = c_.global_gather( x.data());
-        //t.toc();
-        //if(rank==0)std::cout << "collect took "<<t.diff()<<"s\n";
-        //t.tic();
         dg::blas2::detail::doSymv( alpha, m_, temp, beta, y.data(), 
                        typename dg::MatrixTraits<LocalMatrix>::matrix_category(), 
                        typename dg::VectorTraits<container>::vector_category() );
-        //t.toc();
-        //if(rank==0)std::cout << "symv    took "<<t.diff()<<"s\n";
     }
     template<class container> 
     void symv( const MPI_Vector<container>& x, MPI_Vector<container>& y)
     {
-        assert( x.communicator() == y.communicator());
-        assert( x.communicator() == c_.communicator());
-        //int rank;
-        //MPI_Comm_rank( MPI_COMM_WORLD, &rank);
-        //dg::Timer t;
-
-        //t.tic();
         if( c_.size() == 0) //no communication needed
         {
             dg::blas2::detail::doSymv( m_, x.data(), y.data(), 
@@ -288,16 +247,13 @@ struct RowDistMat
             return;
 
         }
+        assert( x.communicator() == y.communicator());
+        assert( x.communicator() == c_.communicator());
         container temp = c_.global_gather( x.data());
-        //t.toc();
-        //if(rank==0)std::cout << "global_gather took "<<t.diff()<<"s\n";
-        //t.tic();
         dg::blas2::detail::doSymv( m_, temp, y.data(), 
                        typename dg::MatrixTraits<LocalMatrix>::matrix_category(), 
                        typename dg::VectorTraits<container>::vector_category(),
                        typename dg::VectorTraits<container>::vector_category() );
-        //t.toc();
-        //if(rank==0)std::cout << "symv    took "<<t.diff()<<"s\n";
     }
 
         
@@ -357,8 +313,6 @@ struct ColDistMat
     template<class container> 
     void symv( double alpha, const MPI_Vector<container>& x, double beta, MPI_Vector<container>& y)
     {
-        assert( x.communicator() == y.communicator());
-        assert( x.communicator() == c_.communicator());
         if( c_.size() == 0) //no communication needed
         {
             dg::blas2::detail::doSymv( alpha, m_, x.data(), beta, y.data(), 
@@ -368,6 +322,8 @@ struct ColDistMat
             return;
 
         }
+        assert( x.communicator() == y.communicator());
+        assert( x.communicator() == c_.communicator());
         container temp( c_.size());
         dg::blas2::detail::doSymv( alpha, m_, x.data(), beta, temp, 
                        typename dg::MatrixTraits<LocalMatrix>::matrix_category(), 
@@ -377,8 +333,6 @@ struct ColDistMat
     template<class container> 
     void symv( const MPI_Vector<container>& x, MPI_Vector<container>& y)
     {
-        assert( x.communicator() == y.communicator());
-        assert( x.communicator() == c_.communicator());
         if( c_.size() == 0) //no communication needed
         {
             dg::blas2::detail::doSymv( m_, x.data(), y.data(), 
@@ -388,6 +342,8 @@ struct ColDistMat
             return;
 
         }
+        assert( x.communicator() == y.communicator());
+        assert( x.communicator() == c_.communicator());
         container temp( c_.size());
         dg::blas2::detail::doSymv( m_, x.data(), temp, 
                        typename dg::MatrixTraits<LocalMatrix>::matrix_category(), 
diff --git a/inc/dg/backend/mpi_vector.h b/inc/dg/backend/mpi_vector.h
index f38c3030b..a6d58cd95 100644
--- a/inc/dg/backend/mpi_vector.h
+++ b/inc/dg/backend/mpi_vector.h
@@ -182,7 +182,9 @@ struct NearestNeighborComm
     */
     template< class OtherIndex, class OtherVector>
     NearestNeighborComm( const NearestNeighborComm<OtherIndex, OtherVector>& src){
-        construct( src.n(), src.dims(), src.communicator(), src.direction());
+        if( src.size() == 0)  silent_=true;
+        else
+            construct( src.n(), src.dims(), src.communicator(), src.direction());
     }
 
     /**
diff --git a/inc/dg/elliptic2d_mpib.cu b/inc/dg/elliptic2d_mpib.cu
index aaaa94a38..f2cd7ec89 100644
--- a/inc/dg/elliptic2d_mpib.cu
+++ b/inc/dg/elliptic2d_mpib.cu
@@ -5,12 +5,12 @@
 
 #include "elliptic.h"
 #include "cg.h"
+#include "multigrid.h"
 
 #include "backend/timer.cuh"
 #include "backend/mpi_init.h"
 
-//NOTE: IF DEVICE=CPU THEN THE POLARISATION ASSEMBLY IS NOT PARALLEL AS IT IS NOW 
-
+//
 //global relative error in L2 norm is O(h^P)
 //as a rule of thumb with n=4 the true error is err = 1e-3 * eps as long as eps > 1e3*err
 
@@ -22,7 +22,7 @@ dg::bc bcy = dg::PER;
  // eps << relativer Abstand der exakten Lösung zur Diskretisierung vom Sinus
 
 double initial( double x, double y) {return 0.;}
-double amp = 0.5;
+double amp = 0.9999;
 double pol( double x, double y) {return 1. + amp*sin(x)*sin(y); } //must be strictly positive
 //double pol( double x, double y) {return 1.; }
 //double pol( double x, double y) {return 1. + sin(x)*sin(y) + x; } //must be strictly positive
@@ -50,7 +50,7 @@ int main(int argc, char* argv[] )
     MPI_Bcast(  &eps,1 , MPI_DOUBLE, 0, comm);
     //////////////////////begin program///////////////////////
     //create functions A(chi) x = b
-    dg::MPIGrid2d grid( 0., lx, 0, ly, n, Nx, Ny, bcx, bcy, comm);
+    dg::CartesianMPIGrid2d grid( 0., lx, 0, ly, n, Nx, Ny, bcx, bcy, comm);
     const dg::MDVec w2d = dg::create::weights( grid);
     const dg::MDVec v2d = dg::create::inv_weights( grid);
     dg::MDVec x =    dg::evaluate( initial, grid);
@@ -60,16 +60,29 @@ int main(int argc, char* argv[] )
 
     if(rank==0)std::cout << "Create Polarisation object and set chi!\n";
     t.tic();
-    dg::Elliptic<dg::CartesianMPIGrid2d, dg::MDMatrix, dg::MDVec> pol( grid, dg::not_normed, dg::centered);
-    pol.set_chi( chi);
+    //dg::Elliptic<dg::CartesianMPIGrid2d, dg::MDMatrix, dg::MDVec> pol( grid, dg::not_normed, dg::centered);
+    //pol.set_chi( chi);
+    unsigned stages = 2;
+
+    dg::MultigridCG2d<dg::aMPIGeometry2d, dg::MDMatrix, dg::MDVec > multigrid( grid, stages, 0);
+    
+    std::vector<dg::MDVec> chi_ = multigrid.project( chi);
+    std::vector<dg::Elliptic<dg::aMPIGeometry2d, dg::MDMatrix, dg::MDVec> > multi_pol( stages);
+    
+    for(unsigned u=0; u<stages; u++)
+    {
+        multi_pol[u].construct( multigrid.grids()[u].get(), dg::not_normed, dg::centered); 
+        multi_pol[u].set_chi( chi_[u]);
+    }
     t.toc();
     if(rank==0)std::cout << "Creation of polarisation object took: "<<t.diff()<<"s\n";
 
-    dg::Invert<dg::MDVec > invert( x, n*n*Nx*Ny, eps);
+    //dg::Invert<dg::MDVec > invert( x, n*n*Nx*Ny, eps);
     t.tic();
-    unsigned number = invert( pol, x, b);
+    //unsigned number = invert( pol, x, b);
+    std::vector<unsigned> number = multigrid.solve( multi_pol, x, b, eps);
     t.toc();
-    if(rank==0)std::cout << "Number of pcg iterations "<< number<<std::endl;
+    if(rank==0)std::cout << "Number of pcg iterations "<< number[0]<<std::endl;
     if(rank==0)std::cout << "For a precision of "<< eps<<std::endl;
     if(rank==0)std::cout << " took "<<t.diff()<<"s\n";
 
-- 
GitLab


From f8066ae14d822ce0718329123a29cdcc284a3436 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Thu, 14 Sep 2017 17:33:26 +0200
Subject: [PATCH 284/453] rename axpbypgz

---
 inc/dg/backend/fast_interpolation.h |  8 +++--
 inc/dg/blas1.h                      |  2 +-
 inc/dg/blas1_t.cu                   |  2 +-
 inc/dg/blas_b.cu                    |  8 ++---
 inc/dg/cg.h                         |  2 +-
 inc/dg/multigrid.h                  | 56 +++++++++++------------------
 inc/dg/multistep.h                  |  4 +--
 7 files changed, 34 insertions(+), 48 deletions(-)

diff --git a/inc/dg/backend/fast_interpolation.h b/inc/dg/backend/fast_interpolation.h
index 0035f89d2..65b57f417 100644
--- a/inc/dg/backend/fast_interpolation.h
+++ b/inc/dg/backend/fast_interpolation.h
@@ -49,21 +49,23 @@ struct MultiMatrix
     }
 
     void symv( const container& x, container& y) const{gemv(x,y);}
+    void symv( double alpha, const container& x, double beta, container& y) const{gemv(alpha, x,beta,y);}
+    void gemv( const container& x, container& y) const{ gemv( 1., x,0,y);}
     /**
     * @brief Applies all stored matrices one after the other
     */
-    void gemv( const container& x, container& y) const
+    void gemv(double alpha, const container& x, double beta, container& y) const
     {
         int dims = inter_.size();
         if( dims == 1) 
         {
-            dg::blas2::symv( inter_[0], x, y);
+            dg::blas2::symv( alpha, inter_[0], x, beta, y);
             return;
         }
         dg::blas2::symv( inter_[0], x,temp_[0].data());
         for( int i=1; i<dims-1; i++)
             dg::blas2::symv( inter_[i], temp_[i-1].data(), temp_[i].data());
-        dg::blas2::symv( inter_[dims-1], temp_[dims-2].data(), y);
+        dg::blas2::symv( alpha, inter_[dims-1], temp_[dims-2].data(), beta, y);
     }
     std::vector<Buffer<container> >& get_temp(){ return temp_;}
     const std::vector<Buffer<container> >& get_temp()const{ return temp_;}
diff --git a/inc/dg/blas1.h b/inc/dg/blas1.h
index 35464b738..bb004b0ed 100644
--- a/inc/dg/blas1.h
+++ b/inc/dg/blas1.h
@@ -130,7 +130,7 @@ inline void axpby( typename VectorTraits<Vector>::value_type alpha, const Vector
  * @attention If a thrust::device_vector is used then this routine is NON-BLOCKING!
  */
 template< class Vector>
-inline void axpbygz( typename VectorTraits<Vector>::value_type alpha, const Vector& x, typename VectorTraits<Vector>::value_type beta, const Vector& y, typename VectorTraits<Vector>::value_type gamma, Vector& z)
+inline void axpbypgz( typename VectorTraits<Vector>::value_type alpha, const Vector& x, typename VectorTraits<Vector>::value_type beta, const Vector& y, typename VectorTraits<Vector>::value_type gamma, Vector& z)
 {
     dg::blas1::detail::doAxpby( alpha, x, beta, y, gamma, z, typename dg::VectorTraits<Vector>::vector_category() );
     return;
diff --git a/inc/dg/blas1_t.cu b/inc/dg/blas1_t.cu
index 762d5a471..465461e42 100644
--- a/inc/dg/blas1_t.cu
+++ b/inc/dg/blas1_t.cu
@@ -31,7 +31,7 @@ int main()
     std::cout << "2*2+ 3*3 = " << v2[0] <<" (13)\n";
     dg::blas1::axpby( 2.5, v1, 0., v2);
     std::cout << "2.5*2+ 0 = " << v2[0] <<" (5)\n";
-    dg::blas1::axpbygz( 2.5, v1, 2., v2, 3., v3);
+    dg::blas1::axpbypgz( 2.5, v1, 2., v2, 3., v3);
     std::cout << "2.5*2+ 2.*5-3*12 = " << v3[0] <<" (-21)\n";
     dg::blas1::copy( v2, v1);
     std::cout << "5 = " << v1[0] <<" (5)"<< std::endl;
diff --git a/inc/dg/blas_b.cu b/inc/dg/blas_b.cu
index 5f316be06..f8d5398d2 100644
--- a/inc/dg/blas_b.cu
+++ b/inc/dg/blas_b.cu
@@ -113,14 +113,14 @@ int main()
     std::cout<<"AXPBY (1*y-1*x=x)                "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
     t.tic();
     for( int i=0; i<multi; i++)
-        dg::blas1::axpbygz( 1., x, -1., y, 2., z);
+        dg::blas1::axpbypgz( 1., x, -1., y, 2., z);
     t.toc();
-    std::cout<<"AXPBYGZ (1*x-1*y+2*z=z)          "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
+    std::cout<<"AXPBYPGZ (1*x-1*y+2*z=z)         "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
     t.tic();
     for( int i=0; i<multi; i++)
-        dg::blas1::axpbygz( 1., x, -1., y, 3., x);
+        dg::blas1::axpbypgz( 1., x, -1., y, 3., x);
     t.toc();
-    std::cout<<"AXPBYGZ (1*x-1.*y+3*x=x)         "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
+    std::cout<<"AXPBYPGZ (1*x-1.*y+3*x=x)        "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
 
     t.tic();
     for( int i=0; i<multi; i++)
diff --git a/inc/dg/cg.h b/inc/dg/cg.h
index 75bccd100..7af15976e 100644
--- a/inc/dg/cg.h
+++ b/inc/dg/cg.h
@@ -436,7 +436,7 @@ struct Invert
         Timer t;
         t.tic();
 #endif //DG_BENCHMARK
-        blas1::axpbygz( alpha[0], phi0, alpha[1], phi1, alpha[2], phi2); 
+        blas1::axpbypgz( alpha[0], phi0, alpha[1], phi1, alpha[2], phi2); 
         phi.swap(phi2);
 
         unsigned number;
diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index a9dd1c055..7d5652b22 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -40,7 +40,7 @@ struct MultigridCG2d
         {
             // Projecting from one grid to the next is the same as 
             // projecting from the original grid to the coarse grids
-            project_[u] = dg::create::fast_projection(grids_[u].get(), 2, 2);
+            project_[u] = dg::create::fast_projection(grids_[u].get(), 2, 2, dg::not_normed);
             inter_[u] = dg::create::fast_interpolation(grids_[u+1].get(), 2, 2);
         }
 
@@ -57,16 +57,13 @@ struct MultigridCG2d
 	template<class SymmetricOp>
 	std::vector<unsigned> solve(/*const*/ std::vector<SymmetricOp>& op, container& x, const container& b, const double eps)
 	{
-		dg::blas1::axpbygz(alpha[0], x0_, alpha[1], x1_, alpha[2], x2_);
+		dg::blas1::axpbypgz(alpha[0], x0_, alpha[1], x1_, alpha[2], x2_);
 		x_[0].swap(x2_);
 
         project(x_[0], x_);
-		project(b, b_);
+        dg::blas1::pointwiseDivide(b, op[0].inv_weights(), b_[0]);
+		project(b_[0], b_);
 
-        //
-        // define new rhs
-        for (unsigned u = 0; u < stages_; u++)
-            dg::blas1::pointwiseDivide(b_[u], op[u].inv_weights(), b_[u]);
         
         unsigned int numStageSteps = m_schemeLayout.size();
 		std::vector<unsigned> number(numStageSteps);
@@ -77,13 +74,6 @@ struct MultigridCG2d
         {
             unsigned w = u + m_schemeLayout[i].m_step;
 
-            //
-            // zero initial guess
-            if (m_schemeLayout[i].m_step > 0)
-            {
-                std::cout << "zeroed " << w << ", ";
-                dg::blas1::scal(x_[w], 0.0);
-            }
 
             //
             // iterate the solver on the system A x = b, with x = 0 as inital guess
@@ -97,26 +87,22 @@ struct MultigridCG2d
             if (m_schemeLayout[i].m_step > 0)
             {
                 //
-                // compute residual r = b - A x
+                // compute residual r = Wb - A x
                 dg::blas2::symv(op[u], x_[u], m_r[u]);
                 dg::blas1::axpby(-1.0, m_r[u], 1.0, b_[u], m_r[u]);
-                dg::blas1::pointwiseDot( m_r[u], op[u].inv_weights(), m_r[u]);
-
                 //
                 // transfer residual to the rhs of the coarser grid
                 dg::blas2::symv(project_[u], m_r[u], b_[w]);
-                dg::blas1::pointwiseDivide( b_[w], op[w].inv_weights(), b_[w]);
-
                 //dg::blas2::symv(project_[u], x_[u], x_[w]);
+                std::cout << "zeroed " << w << ", ";
+                dg::blas1::scal(x_[w], 0.0);
             }
             else if (m_schemeLayout[i].m_step < 0)
             {
                 //
                 // correct the solution vector of the finer grid
                 // x[w] = x[w] + P^{-1} x[u]
-                dg::blas2::symv(inter_[w], x_[u], m_r[w]);
-                dg::blas1::axpby(1.0, x_[w], 1.0, m_r[w], x_[w]);
-
+                dg::blas2::symv(1., inter_[w], x_[u], 1., x_[w]);
                 //dg::blas2::symv(inter_[w], x_[u], x_[w]);
             }
             
@@ -137,41 +123,39 @@ struct MultigridCG2d
         Timer t;
         t.tic();
         //compute initial guess
-        dg::blas1::axpbygz( alpha[0], x0_, alpha[1], x1_, alpha[2], x2_); 
+        dg::blas1::axpbypgz( alpha[0], x0_, alpha[1], x1_, alpha[2], x2_); 
         
 		x_[0].swap(x2_);
         dg::blas1::copy( x_[0], x);//save initial guess
-
-        //dg::blas1::scal( x_[0], 0.0);
-		//project( x_[0], x_);
-        project( b, b_);
-        //project( b, b_);
+        dg::blas1::pointwiseDivide(b, op[0].inv_weights(), b_[0]);
         // compute residual r = b - A x
         dg::blas2::symv(op[0], x_[0], m_r[0]);
-        dg::blas1::pointwiseDot( m_r[0], op[0].inv_weights(), m_r[0]);
         dg::blas1::axpby(-1.0, m_r[0], 1.0, b_[0], b_[0]);
         project( b_[0], b_);
-        dg::blas1::scal( x_[0], 0.0);
-		project( x_[0], x_);
         std::vector<unsigned> number(stages_);
+        Timer t2;
         
         //now solve residual equations
 		for( unsigned u=stages_-1; u>0; u--)
         {
+            t2.tic();
             cg_[u].set_max(grids_[u].get().size());
-            dg::blas1::pointwiseDivide( b_[u], op[u].inv_weights(), b_[u]);
+            dg::blas1::scal( x_[u], 0.0);
             number[u] = cg_[u]( op[u], x_[u], b_[u], op[u].precond(), op[u].inv_weights(), eps/2, 1.);
             dg::blas2::symv( inter_[u-1], x_[u], x_[u-1]);
-            std::cout << "stage: " << u << ", max iter: " << cg_[u].get_max() << ", iter: " << number[u] << std::endl;
+            t2.toc();
+            std::cout << "stage: " << u << ", max iter: " << cg_[u].get_max() << ", iter: " << number[u] << ", took "<<t2.diff()<<"s\n";
+
         }
+        t2.tic();
 
-		dg::blas1::pointwiseDivide( b_[0], op[0].inv_weights(), b_[0]);
         cg_[0].set_max(grids_[0].get().size());
+        dg::blas1::scal( x_[0], 0.0);
         number[0] = cg_[0]( op[0], x_[0], b_[0], op[0].precond(), op[0].inv_weights(), eps);
-        std::cout << "stage: " << 0 << ", max iter: " << cg_[0].get_max() << ", iter: " << number[0] << std::endl;
+        t2.toc();
+        std::cout << "stage: " << 0 << ", max iter: " << cg_[0].get_max() << ", iter: " << number[0] << ", took "<<t2.diff()<<"s\n";
         //update initial guess
         dg::blas1::axpby( 1., x_[0], 1., x);
-
         
 		//x_[0].swap( x);
         x1_.swap( x2_);
diff --git a/inc/dg/multistep.h b/inc/dg/multistep.h
index d832d8c87..3bdaefe05 100644
--- a/inc/dg/multistep.h
+++ b/inc/dg/multistep.h
@@ -319,8 +319,8 @@ void Karniadakis<container>::operator()( Functor& f, Diffusion& diff, container&
 
     blas1::copy( u_[0], u); //save u_[0]
     f( u, f_[0]);
-    blas1::axpbygz( dt_*b[0], f_[0], dt_*b[1], f_[1], dt_*b[2], f_[2]);
-    blas1::axpbygz( a[0], u_[0], a[1], u_[1], a[2], u_[2]);
+    blas1::axpbypgz( dt_*b[0], f_[0], dt_*b[1], f_[1], dt_*b[2], f_[2]);
+    blas1::axpbypgz( a[0], u_[0], a[1], u_[1], a[2], u_[2]);
     //permute f_[2], u_[2]  to be the new f_[0], u_[0]
     for( unsigned i=2; i>0; i--)
     {
-- 
GitLab


From d69e58dd3675da72204f17f6c2b1e666ce7eebc9 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 14 Sep 2017 21:32:50 +0200
Subject: [PATCH 285/453] toefl working correctly fast again

---
 inc/dg/multigrid.h | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index 7d5652b22..a643e9da1 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -34,14 +34,16 @@ struct MultigridCG2d
         }
         
 		inter_.resize(stages-1);
+		interT_.resize(stages-1);
         project_.resize( stages-1);
         
 		for(unsigned u=0; u<stages-1; u++)
         {
             // Projecting from one grid to the next is the same as 
             // projecting from the original grid to the coarse grids
-            project_[u] = dg::create::fast_projection(grids_[u].get(), 2, 2, dg::not_normed);
+            project_[u] = dg::create::fast_projection(grids_[u].get(), 2, 2, dg::normed);
             inter_[u] = dg::create::fast_interpolation(grids_[u+1].get(), 2, 2);
+            interT_[u] = dg::create::fast_projection(grids_[u].get(), 2, 2, dg::not_normed);
         }
 
         dg::blas1::transfer(dg::evaluate(dg::zero, grid), x0_);
@@ -62,7 +64,9 @@ struct MultigridCG2d
 
         project(x_[0], x_);
         dg::blas1::pointwiseDivide(b, op[0].inv_weights(), b_[0]);
-		project(b_[0], b_);
+        // project b down to coarse grid
+        for( unsigned u=0; u<stages_-1; u++)
+            dg::blas2::gemv( interT_[u], b_[u], b_[u+1]);
 
         
         unsigned int numStageSteps = m_schemeLayout.size();
@@ -92,7 +96,7 @@ struct MultigridCG2d
                 dg::blas1::axpby(-1.0, m_r[u], 1.0, b_[u], m_r[u]);
                 //
                 // transfer residual to the rhs of the coarser grid
-                dg::blas2::symv(project_[u], m_r[u], b_[w]);
+                dg::blas2::symv(interT_[u], m_r[u], b_[w]);
                 //dg::blas2::symv(project_[u], x_[u], x_[w]);
                 std::cout << "zeroed " << w << ", ";
                 dg::blas1::scal(x_[w], 0.0);
@@ -128,19 +132,21 @@ struct MultigridCG2d
 		x_[0].swap(x2_);
         dg::blas1::copy( x_[0], x);//save initial guess
         dg::blas1::pointwiseDivide(b, op[0].inv_weights(), b_[0]);
-        // compute residual r = b - A x
+        // compute residual r = Wb - A x
         dg::blas2::symv(op[0], x_[0], m_r[0]);
         dg::blas1::axpby(-1.0, m_r[0], 1.0, b_[0], b_[0]);
-        project( b_[0], b_);
+        // project residual down to coarse grid
+        for( unsigned u=0; u<stages_-1; u++)
+            dg::blas2::gemv( interT_[u], b_[u], b_[u+1]);
         std::vector<unsigned> number(stages_);
         Timer t2;
         
+        dg::blas1::scal( x_[stages_-1], 0.0);
         //now solve residual equations
 		for( unsigned u=stages_-1; u>0; u--)
         {
             t2.tic();
             cg_[u].set_max(grids_[u].get().size());
-            dg::blas1::scal( x_[u], 0.0);
             number[u] = cg_[u]( op[u], x_[u], b_[u], op[u].precond(), op[u].inv_weights(), eps/2, 1.);
             dg::blas2::symv( inter_[u-1], x_[u], x_[u-1]);
             t2.toc();
@@ -150,7 +156,6 @@ struct MultigridCG2d
         t2.tic();
 
         cg_[0].set_max(grids_[0].get().size());
-        dg::blas1::scal( x_[0], 0.0);
         number[0] = cg_[0]( op[0], x_[0], b_[0], op[0].precond(), op[0].inv_weights(), eps);
         t2.toc();
         std::cout << "stage: " << 0 << ", max iter: " << cg_[0].get_max() << ", iter: " << number[0] << ", took "<<t2.diff()<<"s\n";
@@ -235,7 +240,7 @@ private:
             //m_mode = correctionscheme;
             m_startStage = 0;
             for (unsigned u = 0; u < stages_-1; u++)
-                m_schemeLayout.push_back(stepinfo(1, 50));
+                m_schemeLayout.push_back(stepinfo(1, 5));
             
             m_schemeLayout.push_back(stepinfo(-1, 1000));
 
@@ -283,6 +288,7 @@ private:
     unsigned stages_;
     std::vector< dg::Handle< Geometry> > grids_;
     std::vector< MultiMatrix<Matrix, container> >  inter_;
+    std::vector< MultiMatrix<Matrix, container> >  interT_;
     std::vector< MultiMatrix<Matrix, container> >  project_;
     std::vector< CG<container> > cg_;
     std::vector< container> x_, m_r, b_; 
-- 
GitLab


From bbe5d9f6c6d6dd8f7e50998008ef13dbbc49115d Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 14 Sep 2017 21:48:20 +0200
Subject: [PATCH 286/453] correction of solution vector is the same in
 direct_solve

---
 inc/dg/multigrid.h | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index a643e9da1..a5e07448f 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -129,15 +129,14 @@ struct MultigridCG2d
         //compute initial guess
         dg::blas1::axpbypgz( alpha[0], x0_, alpha[1], x1_, alpha[2], x2_); 
         
-		x_[0].swap(x2_);
-        dg::blas1::copy( x_[0], x);//save initial guess
+        dg::blas1::copy( x2_, x);//save initial guess
         dg::blas1::pointwiseDivide(b, op[0].inv_weights(), b_[0]);
         // compute residual r = Wb - A x
-        dg::blas2::symv(op[0], x_[0], m_r[0]);
-        dg::blas1::axpby(-1.0, m_r[0], 1.0, b_[0], b_[0]);
+        dg::blas2::symv(op[0], x, m_r[0]);
+        dg::blas1::axpby(-1.0, m_r[0], 1.0, b_[0], m_r[0]);
         // project residual down to coarse grid
         for( unsigned u=0; u<stages_-1; u++)
-            dg::blas2::gemv( interT_[u], b_[u], b_[u+1]);
+            dg::blas2::gemv( interT_[u], m_r[u], m_r[u+1]);
         std::vector<unsigned> number(stages_);
         Timer t2;
         
@@ -147,7 +146,7 @@ struct MultigridCG2d
         {
             t2.tic();
             cg_[u].set_max(grids_[u].get().size());
-            number[u] = cg_[u]( op[u], x_[u], b_[u], op[u].precond(), op[u].inv_weights(), eps/2, 1.);
+            number[u] = cg_[u]( op[u], x_[u], m_r[u], op[u].precond(), op[u].inv_weights(), eps/2, 1.);
             dg::blas2::symv( inter_[u-1], x_[u], x_[u-1]);
             t2.toc();
             std::cout << "stage: " << u << ", max iter: " << cg_[u].get_max() << ", iter: " << number[u] << ", took "<<t2.diff()<<"s\n";
@@ -155,14 +154,13 @@ struct MultigridCG2d
         }
         t2.tic();
 
+        //update initial guess
+        dg::blas1::axpby( 1., x_[0], 1., x);
         cg_[0].set_max(grids_[0].get().size());
-        number[0] = cg_[0]( op[0], x_[0], b_[0], op[0].precond(), op[0].inv_weights(), eps);
+        number[0] = cg_[0]( op[0], x, b_[0], op[0].precond(), op[0].inv_weights(), eps);
         t2.toc();
         std::cout << "stage: " << 0 << ", max iter: " << cg_[0].get_max() << ", iter: " << number[0] << ", took "<<t2.diff()<<"s\n";
-        //update initial guess
-        dg::blas1::axpby( 1., x_[0], 1., x);
         
-		//x_[0].swap( x);
         x1_.swap( x2_);
         x0_.swap( x1_);
         
-- 
GitLab


From 9b4b0b65b44960b2b7b95305bf8279a42d3b3038 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Fri, 15 Sep 2017 00:03:58 +0200
Subject: [PATCH 287/453] made Extrapolation struct and updated toeflR

---
 inc/dg/backend/fast_interpolation.h         |   9 +-
 inc/dg/backend/selfmade_blas.cuh            |  13 ++
 inc/dg/backend/sparseblockmat_omp_kernels.h |   2 +
 inc/dg/blas2.h                              |  43 ++++---
 inc/dg/cg.h                                 | 125 +++++++++++++++-----
 inc/dg/elliptic2d_b.cu                      |   2 +-
 inc/dg/elliptic2d_mpib.cu                   |   2 +-
 inc/dg/multigrid.h                          |  56 ++-------
 inc/dg/multistep.h                          |   1 -
 src/toefl/toeflR.cuh                        |  61 +++++-----
 10 files changed, 178 insertions(+), 136 deletions(-)

diff --git a/inc/dg/backend/fast_interpolation.h b/inc/dg/backend/fast_interpolation.h
index 65b57f417..df490d442 100644
--- a/inc/dg/backend/fast_interpolation.h
+++ b/inc/dg/backend/fast_interpolation.h
@@ -48,13 +48,8 @@ struct MultiMatrix
 
     }
 
-    void symv( const container& x, container& y) const{gemv(x,y);}
-    void symv( double alpha, const container& x, double beta, container& y) const{gemv(alpha, x,beta,y);}
-    void gemv( const container& x, container& y) const{ gemv( 1., x,0,y);}
-    /**
-    * @brief Applies all stored matrices one after the other
-    */
-    void gemv(double alpha, const container& x, double beta, container& y) const
+    void symv( const container& x, container& y) const{ symv( 1., x,0,y);}
+    void symv(double alpha, const container& x, double beta, container& y) const
     {
         int dims = inter_.size();
         if( dims == 1) 
diff --git a/inc/dg/backend/selfmade_blas.cuh b/inc/dg/backend/selfmade_blas.cuh
index eed348f46..5dff1a0c2 100644
--- a/inc/dg/backend/selfmade_blas.cuh
+++ b/inc/dg/backend/selfmade_blas.cuh
@@ -50,6 +50,19 @@ inline void doSymv(
     m.symv( alpha, x, beta, y);
 }
 
+template< class Matrix, class Vector>
+inline void doGemv(  
+              typename VectorTraits<Vector>::value_type alpha, 
+              const Matrix& m,
+              const Vector& x, 
+              typename VectorTraits<Vector>::value_type beta, 
+              Vector& y, 
+              SelfMadeMatrixTag,
+              AnyVectorTag)
+{
+    m.symv( alpha, x, beta, y);
+}
+
 } //namespace detail
 } //namespace blas2
 } //namespace dg
diff --git a/inc/dg/backend/sparseblockmat_omp_kernels.h b/inc/dg/backend/sparseblockmat_omp_kernels.h
index 9dad75480..e2d1693ad 100644
--- a/inc/dg/backend/sparseblockmat_omp_kernels.h
+++ b/inc/dg/backend/sparseblockmat_omp_kernels.h
@@ -154,7 +154,9 @@ void ell_multiply_kernel33( value_type alpha, value_type beta,
 				int B0 = (i==0 || i==num_rows-1)?(data_idx[i*3+0]*3+k)*3:(0*3+k)*3;
                 int B1 = (i==0 || i==num_rows-1)?(data_idx[i*3+1]*3+k)*3:(1*3+k)*3;
                 int B2 = (i==0 || i==num_rows-1)?(data_idx[i*3+2]*3+k)*3:(2*3+k)*3;
+#if defined(__INTEL_COMPILER)
 				#pragma vector nontemporal(y)
+#endif //__INTEL_COMPILER
 				for( int j=right_range[0]; j<right_range[1]; j++)
 				{
 				    value_type temp = 0;
diff --git a/inc/dg/blas2.h b/inc/dg/blas2.h
index fffec1692..c5c3ae9a6 100644
--- a/inc/dg/blas2.h
+++ b/inc/dg/blas2.h
@@ -88,8 +88,6 @@ inline typename MatrixTraits<Matrix>::value_type dot( const Matrix& m, const Vec
  *
  * This routine computes \f[ y = \alpha P x + \beta y \f]
  * where \f$ P\f$ is a symmetric Preconditioner. 
- * P should be diagonal since
- * otherwise a call to symv() followed by axpby() is faster.
  * @param alpha A Scalar
  * @param P The Preconditioner
  * @param x A Vector different from y (except in the case where m is diagonal)
@@ -103,8 +101,7 @@ inline void symv( typename MatrixTraits<Precon>::value_type alpha,
                   typename MatrixTraits<Precon>::value_type beta, 
                   Vector& y)
 {
-    if(alpha == (typename MatrixTraits<Precon>::value_type)0) 
-    {
+    if(alpha == (typename MatrixTraits<Precon>::value_type)0) {
         dg::blas1::scal( y, alpha);
         return;
     }
@@ -135,19 +132,6 @@ inline void symv( Matrix& m,
                        typename dg::VectorTraits<Vector2>::vector_category() );
     return;
 }
-///@cond
-template< class Matrix, class Vector>
-inline void mv(   Matrix& m, 
-                  const Vector& x, 
-                  Vector& y)
-{
-    dg::blas2::detail::doSymv( m, x, y, 
-                       typename dg::MatrixTraits<Matrix>::matrix_category(), 
-                       typename dg::VectorTraits<const Vector>::vector_category(),
-                       typename dg::VectorTraits<Vector>::vector_category() );
-    return;
-}
-///@endcond
 
 /**
  * @brief General Matrix-Vector product
@@ -167,6 +151,31 @@ inline void gemv( Matrix& m,
                        typename dg::VectorTraits<Vector2>::vector_category() );
     return;
 }
+/**
+ * @brief General Matrix-Vector product
+ *
+ * @param alpha A Scalar
+ * @param P The Matrix
+ * @param x A Vector different from y 
+ * @param beta A Scalar
+ * @param y contains the solution on output
+ */
+template< class Precon, class Vector>
+inline void gemv( typename MatrixTraits<Precon>::value_type alpha, 
+                  const Precon& P, 
+                  const Vector& x, 
+                  typename MatrixTraits<Precon>::value_type beta, 
+                  Vector& y)
+{
+    if(alpha == (typename MatrixTraits<Precon>::value_type)0) {
+        dg::blas1::scal( y, alpha);
+        return;
+    }
+    dg::blas2::detail::doGemv( alpha, P, x, beta, y, 
+                       typename dg::MatrixTraits<Precon>::matrix_category(), 
+                       typename dg::VectorTraits<Vector>::vector_category() );
+    return;
+}
 ///@}
 
 } //namespace blas2
diff --git a/inc/dg/cg.h b/inc/dg/cg.h
index 7af15976e..88fb3438d 100644
--- a/inc/dg/cg.h
+++ b/inc/dg/cg.h
@@ -258,6 +258,94 @@ unsigned CG< container>::operator()( Matrix& A, container& x, const container& b
 ///@endcond
 
 
+/**
+* @brief Class that manages the solutions of iterative methods and
+can be used to get initial guesses based on past solutions
+*
+* @copydoc hide_container_lvl1
+*/
+template<class container>
+struct Extrapolation
+{
+    /*! @brief Set extrapolation type without initializing values
+     * @param type number of vectors to use for extrapolation ( 0<=type<=3)
+     * @attention the update function must be used at least type times before the extrapolate function can be called
+     */
+    Extrapolation( unsigned type = 2){ set_type(type); }
+    /*! @brief Set extrapolation type and initialize values
+     * @param type number of vectors to use for extrapolation ( 0<=type<=3)
+     * @param init the vectors are initialized with this value
+     */
+    Extrapolation( unsigned type, const container& init) { 
+        set_type(type, init); 
+    }
+    ///@copydoc Extrapolation(unsigned)
+    void set_type( unsigned new_type)
+    {
+        m_type = new_type;
+        m_x.resize( new_type);
+        assert( m_type <= 3 );
+    }
+    ///@copydoc Extrapolation(unsigned,const container&)
+    void set_type( unsigned new_type, const container& init)
+    {
+        m_x.assign( new_type, init);
+        m_type = new_type;
+        assert( m_type <= 3 );
+    }
+    ///read the current extrapolation type
+    unsigned get_type( ) const{return m_type;}
+
+    /**
+    * @brief Extrapolate values 
+    *
+    * @param new_x (write only) contains extrapolated value on output ( may alias the tail)
+    */
+    void extrapolate( container& new_x) const{
+        switch(m_type)
+        {
+            case(0): 
+                     break;
+            case(1): dg::blas1::copy( m_x[0], new_x);
+                     break;
+            case(2): dg::blas1::axpby( 2., m_x[0], -1., m_x[1], new_x);
+                     break;
+            case(3): dg::blas1::axpby( 1., m_x[2], -3., m_x[1], new_x);
+                     dg::blas1::axpby( 3., m_x[0], 1., new_x);
+                     break;
+            default: dg::blas1::axpby( 2., m_x[0], -1., m_x[1], new_x);
+        }
+    }
+
+    
+    /**
+    * @brief move the all values one step back and copy the given vector as current head
+    *
+    * @param new_head the new head ( may alias the tail)
+    */
+    void update( const container& new_head){
+        if( m_type == 0) return;
+        //push out last value
+        for (unsigned u=m_type-1; u>0; u--)
+            m_x[u].swap( m_x[u-1]);
+        blas1::copy( new_head, m_x[0]);
+    }
+
+    /**
+     * @brief return the current head 
+     * @return current head (undefined if m_type==0)
+     */
+    const container& head()const{return m_x[0];}
+    ///write access to tail value ( the one that will be deleted in the next update
+    container& tail(){return m_x[m_type-1];}
+    ///read access to tail value ( the one that will be deleted in the next update
+    const container& tail()const{return m_x[m_type-1];}
+
+    private:
+    unsigned m_type;
+    std::vector<container> m_x;
+};
+
 
 /**
  * @brief Smart conjugate gradient solver. 
@@ -291,7 +379,7 @@ struct Invert
      * @brief Allocate nothing
      *
      */
-    Invert() { multiplyWeights_ = true; set_extrapolationType(2); nrmb_correction_ = 1.; }
+    Invert() { multiplyWeights_ = true; nrmb_correction_ = 1.; }
 
     /**
      * @brief Constructor
@@ -320,10 +408,10 @@ struct Invert
      */
     void construct( const container& copyable, unsigned max_iter, value_type eps, int extrapolationType = 2, bool multiplyWeights = true, value_type nrmb_correction = 1.) 
     {
+        m_ex.set_type( extrapolationType);
         set_size( copyable, max_iter);
         set_accuracy( eps, nrmb_correction);
         multiplyWeights_=multiplyWeights;
-        set_extrapolationType( extrapolationType);
     }
 
 
@@ -335,7 +423,7 @@ struct Invert
      */
     void set_size( const container& assignable, unsigned max_iterations) {
         cg.construct(assignable, max_iterations);
-        phi0 = phi1 = phi2 = assignable;
+        m_ex.set_type( m_ex.get_type(), assignable);
     }
 
     /**
@@ -356,19 +444,7 @@ struct Invert
      */
     void set_extrapolationType( int extrapolationType)
     {
-        assert( extrapolationType <= 3 && extrapolationType >= 0);
-        switch(extrapolationType)
-        {
-            case(0): alpha[0] = 0, alpha[1] = 0, alpha[2] = 0;
-                     break;
-            case(1): alpha[0] = 1, alpha[1] = 0, alpha[2] = 0;
-                     break;
-            case(2): alpha[0] = 2, alpha[1] = -1, alpha[2] = 0;
-                     break;
-            case(3): alpha[0] = 3, alpha[1] = -3, alpha[2] = 1;
-                     break;
-            default: alpha[0] = 2, alpha[1] = -1, alpha[2] = 0;
-        }
+        m_ex.set_type( extrapolationType);
     }
     /**
      * @brief Set the maximum number of iterations 
@@ -386,7 +462,7 @@ struct Invert
     /**
     * @brief Return last solution
     */
-    const container& get_last() const { return phi0;}
+    const container& get_last() const { return m_ex.head();}
 
     /**
      * @brief Solve linear problem
@@ -436,22 +512,18 @@ struct Invert
         Timer t;
         t.tic();
 #endif //DG_BENCHMARK
-        blas1::axpbypgz( alpha[0], phi0, alpha[1], phi1, alpha[2], phi2); 
-        phi.swap(phi2);
+        m_ex.extrapolate( phi);
 
         unsigned number;
         if( multiplyWeights_ ) 
         {
-            dg::blas1::pointwiseDivide( rho, inv_weights, phi2);
-            number = cg( op, phi, phi2, p, inv_weights, eps_, nrmb_correction_);
+            dg::blas1::pointwiseDivide( rho, inv_weights, m_ex.tail());
+            number = cg( op, phi, m_ex.tail(), p, inv_weights, eps_, nrmb_correction_);
         }
         else
             number = cg( op, phi, rho, p, inv_weights, eps_, nrmb_correction_);
 
-        phi1.swap( phi2);
-        phi0.swap( phi1);
-        
-        blas1::copy( phi, phi0);
+        m_ex.update(phi);
 #ifdef DG_BENCHMARK
         t.toc();
 #ifdef MPI_VERSION
@@ -469,9 +541,8 @@ struct Invert
 
   private:
     value_type eps_, nrmb_correction_;
-    container phi0, phi1, phi2;
     dg::CG< container > cg;
-    value_type alpha[3];
+    Extrapolation<container> m_ex;
     bool multiplyWeights_; 
 };
 
diff --git a/inc/dg/elliptic2d_b.cu b/inc/dg/elliptic2d_b.cu
index 77a611639..e0ebe99f1 100644
--- a/inc/dg/elliptic2d_b.cu
+++ b/inc/dg/elliptic2d_b.cu
@@ -74,7 +74,7 @@ int main()
     {
 		t.tic();
 
-		unsigned stages = 2;
+		unsigned stages = 3;
 
 		dg::MultigridCG2d<dg::aGeometry2d, dg::DMatrix, dg::DVec > multigrid( grid, stages, scheme);
 		
diff --git a/inc/dg/elliptic2d_mpib.cu b/inc/dg/elliptic2d_mpib.cu
index f2cd7ec89..948d62e79 100644
--- a/inc/dg/elliptic2d_mpib.cu
+++ b/inc/dg/elliptic2d_mpib.cu
@@ -62,7 +62,7 @@ int main(int argc, char* argv[] )
     t.tic();
     //dg::Elliptic<dg::CartesianMPIGrid2d, dg::MDMatrix, dg::MDVec> pol( grid, dg::not_normed, dg::centered);
     //pol.set_chi( chi);
-    unsigned stages = 2;
+    unsigned stages = 3;
 
     dg::MultigridCG2d<dg::aMPIGeometry2d, dg::MDMatrix, dg::MDVec > multigrid( grid, stages, 0);
     
diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index a5e07448f..9bfb2bea7 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -11,7 +11,6 @@
 namespace dg
 {
 
-
 template< class Geometry, class Matrix, class container> 
 struct MultigridCG2d
 {
@@ -46,23 +45,19 @@ struct MultigridCG2d
             interT_[u] = dg::create::fast_projection(grids_[u].get(), 2, 2, dg::not_normed);
         }
 
-        dg::blas1::transfer(dg::evaluate(dg::zero, grid), x0_);
-        x1_ = x0_, x2_ = x0_;
-        
-        x_ = project(x0_); 
+        container x0;
+        dg::blas1::transfer( dg::evaluate( dg::zero, grid), x0);
+        x_ = project(x0); 
         m_r = x_,
 		b_ = x_;        
-		set_extrapolationType(extrapolation_type);
         set_scheme(scheme_type);        
     }
 
 	template<class SymmetricOp>
 	std::vector<unsigned> solve(/*const*/ std::vector<SymmetricOp>& op, container& x, const container& b, const double eps)
 	{
-		dg::blas1::axpbypgz(alpha[0], x0_, alpha[1], x1_, alpha[2], x2_);
-		x_[0].swap(x2_);
-
-        project(x_[0], x_);
+        //project initial guess down to coarse grid
+        project(x, x_);
         dg::blas1::pointwiseDivide(b, op[0].inv_weights(), b_[0]);
         // project b down to coarse grid
         for( unsigned u=0; u<stages_-1; u++)
@@ -77,8 +72,6 @@ struct MultigridCG2d
         for (unsigned i = 0; i < numStageSteps; i++)
         {
             unsigned w = u + m_schemeLayout[i].m_step;
-
-
             //
             // iterate the solver on the system A x = b, with x = 0 as inital guess
             cg_[u].set_max(m_schemeLayout[i].m_niter);
@@ -114,10 +107,6 @@ struct MultigridCG2d
 		}
 
 		x_[0].swap(x);
-		x1_.swap(x2_);
-		x0_.swap(x1_);
-
-		blas1::copy(x, x0_);
 		return number;
 	}
 
@@ -126,10 +115,6 @@ struct MultigridCG2d
     {
         Timer t;
         t.tic();
-        //compute initial guess
-        dg::blas1::axpbypgz( alpha[0], x0_, alpha[1], x1_, alpha[2], x2_); 
-        
-        dg::blas1::copy( x2_, x);//save initial guess
         dg::blas1::pointwiseDivide(b, op[0].inv_weights(), b_[0]);
         // compute residual r = Wb - A x
         dg::blas2::symv(op[0], x, m_r[0]);
@@ -161,10 +146,6 @@ struct MultigridCG2d
         t2.toc();
         std::cout << "stage: " << 0 << ", max iter: " << cg_[0].get_max() << ", iter: " << number[0] << ", took "<<t2.diff()<<"s\n";
         
-        x1_.swap( x2_);
-        x0_.swap( x1_);
-        
-        blas1::copy( x, x0_);
         t.toc();
         std::cout<< "Took "<<t.diff()<<"s\n";
         return number;
@@ -188,27 +169,6 @@ struct MultigridCG2d
 
     }
     unsigned stages()const{return stages_;}
-    /**
-     * @brief Set the extrapolation Type for following inversions
-     *
-     * @param extrapolationType number of last values to use for next extrapolation of initial guess
-     */
-    void set_extrapolationType( int extrapolationType)
-    {
-        assert( extrapolationType <= 3 && extrapolationType >= 0);
-        switch(extrapolationType)
-        {
-            case(0): alpha[0] = 0, alpha[1] = 0, alpha[2] = 0;
-                     break;
-            case(1): alpha[0] = 1, alpha[1] = 0, alpha[2] = 0;
-                     break;
-            case(2): alpha[0] = 2, alpha[1] = -1, alpha[2] = 0;
-                     break;
-            case(3): alpha[0] = 3, alpha[1] = -3, alpha[2] = 1;
-                     break;
-            default: alpha[0] = 2, alpha[1] = -1, alpha[2] = 0;
-        }
-    }
 
     const std::vector<dg::Handle< Geometry > > grids()const { return grids_; }
 
@@ -262,11 +222,11 @@ private:
         // (1) there can not be more than n-1 interpolations in succession
         
         unsigned u = m_startStage;
-        assert(u >= 0 && u <= stages_ - 1);
+        assert( u <= stages_ - 1);
         for (unsigned i = 0; i < m_schemeLayout.size(); i++)
         {
             u += m_schemeLayout[i].m_step;
-            assert(u >= 0 && u <= stages_ - 1);
+            assert( u <= stages_ - 1);
         }   
         assert(u == 0);
 	}
@@ -290,8 +250,6 @@ private:
     std::vector< MultiMatrix<Matrix, container> >  project_;
     std::vector< CG<container> > cg_;
     std::vector< container> x_, m_r, b_; 
-    container x0_, x1_, x2_;
-    double alpha[3];
 
     struct stepinfo
     {
diff --git a/inc/dg/multistep.h b/inc/dg/multistep.h
index 3bdaefe05..0cd35685a 100644
--- a/inc/dg/multistep.h
+++ b/inc/dg/multistep.h
@@ -270,7 +270,6 @@ struct Karniadakis
 
     /**
      * @brief return the current head of the computation
-     *
      * @return current head
      */
     const container& head()const{return u_[0];}
diff --git a/src/toefl/toeflR.cuh b/src/toefl/toeflR.cuh
index dcd14dce1..84e57c2a3 100644
--- a/src/toefl/toeflR.cuh
+++ b/src/toefl/toeflR.cuh
@@ -138,7 +138,8 @@ struct Explicit
     dg::ArakawaX< Geometry, Matrix, container> arakawa; 
 
     dg::Invert<container> invert_invgamma, invert_pol;
-    dg::MultigridCG2d<Geometry, Matrix, container> multigrid_pol, multigrid_invgammaPhi, multigrid_invgammaN; 
+    dg::MultigridCG2d<Geometry, Matrix, container> multigrid;
+    dg::Extrapolation<container> old_phi, old_psi, old_gammaN;
     std::vector<container> multi_chi;
 
     const container w2d, v2d, one;
@@ -163,17 +164,18 @@ Explicit< Geometry, M, container>::Explicit( const Geometry& grid, const Paramet
     arakawa( grid), 
     invert_invgamma( omega, p.Nx*p.Ny*p.n*p.n, p.eps_gamma),
     invert_pol(      omega, p.Nx*p.Ny*p.n*p.n, p.eps_pol),
-    multigrid_pol( grid, 3), multigrid_invgammaPhi( grid, 3), multigrid_invgammaN(grid, 3),
+    multigrid( grid, 3),
+    old_phi( 2, chi), old_psi( 2, chi), old_gammaN( 2, chi), 
     w2d( dg::create::volume(grid)), v2d( dg::create::inv_volume(grid)), one( dg::evaluate(dg::one, grid)),
     eps_pol(p.eps_pol), eps_gamma( p.eps_gamma), kappa(p.kappa), friction(p.friction), nu(p.nu), tau( p.tau), equations( p.equations), boussinesq(p.boussinesq)
 { 
-    multi_chi= multigrid_pol.project( chi);
+    multi_chi= multigrid.project( chi);
     multi_pol.resize(3);
     multi_gamma1.resize(3);
     for( unsigned u=0; u<3; u++)
     {
-        multi_pol[u].construct( multigrid_pol.grids()[u].get(), dg::not_normed, dg::centered, p.jfactor);
-        multi_gamma1[u].construct( multigrid_pol.grids()[u].get(), -0.5*p.tau, dg::centered);
+        multi_pol[u].construct( multigrid.grids()[u].get(), dg::not_normed, dg::centered, p.jfactor);
+        multi_gamma1[u].construct( multigrid.grids()[u].get(), -0.5*p.tau, dg::centered);
     }
 }
 
@@ -184,7 +186,9 @@ const container& Explicit<G, M, container>::compute_psi( const container& potent
     //in gyrofluid invert Gamma operator
     if( equations == "local" || equations == "global")
     {
-        std::vector<unsigned> number = multigrid_invgammaPhi.direct_solve( multi_gamma1, phi[1], potential, eps_gamma);
+        old_psi.extrapolate( phi[1]);
+        std::vector<unsigned> number = multigrid.direct_solve( multi_gamma1, phi[1], potential, eps_gamma);
+        old_psi.update( phi[1]);
         //unsigned number = invert_invgamma( gamma1, phi[1], potential);
         if(  number[0] == invert_invgamma.get_max())
             throw dg::Fail( eps_gamma);
@@ -224,7 +228,7 @@ const container& Explicit<G, M, container>::polarisation( const std::vector<cont
         dg::blas1::pointwiseDot( binv, chi, chi); //\chi *= binv^2
         if( !boussinesq) 
         {
-            multigrid_pol.project( chi, multi_chi);
+            multigrid.project( chi, multi_chi);
             for( unsigned u=0; u<3; u++)
                 multi_pol[u].set_chi( multi_chi[u]);
             //pol.set_chi( chi);
@@ -236,7 +240,7 @@ const container& Explicit<G, M, container>::polarisation( const std::vector<cont
         dg::blas1::plus( chi, 1.); 
         if( !boussinesq) 
         {
-            multigrid_pol.project( chi, multi_chi);
+            multigrid.project( chi, multi_chi);
             for( unsigned u=0; u<3; u++)
                 multi_pol[u].set_chi( multi_chi[u]);
             //pol.set_chi( chi);
@@ -250,7 +254,7 @@ const container& Explicit<G, M, container>::polarisation( const std::vector<cont
         dg::blas1::pointwiseDot( binv, chi, chi); //\chi *= binv^2
         if( !boussinesq) 
         {
-            multigrid_pol.project( chi, multi_chi);
+            multigrid.project( chi, multi_chi);
             for( unsigned u=0; u<3; u++)
                 multi_pol[u].set_chi( multi_chi[u]);
             //pol.set_chi( chi);
@@ -259,7 +263,9 @@ const container& Explicit<G, M, container>::polarisation( const std::vector<cont
     //compute polarisation
     if( equations == "local" || equations == "global")
     {
-        std::vector<unsigned> number = multigrid_invgammaN.direct_solve( multi_gamma1, gamma_n, y[1], eps_gamma);
+        old_gammaN.extrapolate( gamma_n);
+        std::vector<unsigned> number = multigrid.direct_solve( multi_gamma1, gamma_n, y[1], eps_gamma);
+        old_gammaN.update( gamma_n);
         //unsigned number = invert_invgamma( gamma1, gamma_n, y[1]);
         if(  number[0] == invert_invgamma.get_max())
             throw dg::Fail( eps_gamma);
@@ -271,11 +277,11 @@ const container& Explicit<G, M, container>::polarisation( const std::vector<cont
         if( boussinesq) 
             dg::blas1::pointwiseDivide( omega, chi, omega);
     //invert 
-    dg::blas1::transform( chi, chi, dg::INVERT<double>());
-    dg::blas1::pointwiseDot( chi, v2d, chi);
 
     //unsigned number = invert_pol( pol, phi[0], omega, v2d, chi);
-    std::vector<unsigned> number = multigrid_pol.direct_solve( multi_pol, phi[0], omega, eps_pol);
+    old_phi.extrapolate( phi[0]);
+    std::vector<unsigned> number = multigrid.direct_solve( multi_pol, phi[0], omega, eps_pol);
+    old_phi.update( phi[0]);
     if(  number[0] == invert_pol.get_max())
         throw dg::Fail( eps_pol);
     return phi[0];
@@ -354,10 +360,8 @@ void Explicit<G, M, container>::operator()( std::vector<container>& y, std::vect
         arakawa(y[0], phi[0], yp[0]);
         arakawa(y[1], phi[0], yp[1]);
         arakawa(y[0], phi[1], omega);
-        dg::blas1::axpby( 1., omega, 1., yp[1]);
-        dg::blas1::axpby( -friction, y[1], 1., yp[1]);
-        dg::blas2::gemv( arakawa.dy(), y[0], omega);
-        dg::blas1::axpby( -1., omega, 1., yp[1]);
+        dg::blas1::axpbypgz( 1., omega, -friction, y[1], 1., yp[1]);
+        dg::blas2::gemv( 1., arakawa.dy(), y[0], 1., yp[1]);
         return;
     }
     else if( equations == "gravity_local")
@@ -365,8 +369,7 @@ void Explicit<G, M, container>::operator()( std::vector<container>& y, std::vect
         arakawa(y[0], phi[0], yp[0]);
         arakawa(y[1], phi[0], yp[1]);
         dg::blas2::gemv( arakawa.dy(), y[0], dyy[0]);
-        dg::blas1::axpby( -friction, y[1], 1., yp[1]);
-        dg::blas1::axpby( -1., dyy[0], 1., yp[1]);
+        dg::blas1::axpbypgz( -friction, y[1], -1., dyy[0], 1., yp[1]);
         return;
     }
     else if( equations == "drift_global")
@@ -376,21 +379,15 @@ void Explicit<G, M, container>::operator()( std::vector<container>& y, std::vect
         arakawa(y[0], phi[1], omega);
         dg::blas1::pointwiseDot( binv, yp[0], yp[0]);
         dg::blas1::pointwiseDot( binv, yp[1], yp[1]);
-        dg::blas1::pointwiseDot( binv, omega, omega);
-        dg::blas1::axpby( 1., omega, 1., yp[1]);
+        dg::blas1::pointwiseDot( 1., binv, omega, 1., yp[1]);
 
         dg::blas2::gemv( arakawa.dy(), phi[0], dyphi[0]);
         dg::blas2::gemv( arakawa.dy(), phi[1], dyphi[1]);
         //ExB compression
-        dg::blas1::pointwiseDot( dyphi[0], ype[0], omega);
-        dg::blas1::axpby( kappa, omega, 1., yp[0]); 
-        dg::blas1::pointwiseDot( dyphi[0], y[1], omega);
-        dg::blas1::axpby( kappa, omega, 1., yp[1]); 
-        dg::blas1::pointwiseDot( dyphi[1], ype[0], omega);
-        dg::blas1::axpby( kappa, omega, 1., yp[1]); 
+        dg::blas1::pointwiseDot( kappa, dyphi[0], ype[0], 1., yp[0]);
+        dg::blas1::pointwiseDot( kappa, dyphi[0], y[1], kappa, dyphi[1], ype[0], 1.,  yp[1]);
         // diamagnetic compression
-        dg::blas2::gemv( arakawa.dy(), y[0], omega);
-        dg::blas1::axpby( -kappa, omega, 1., yp[1]);
+        dg::blas2::gemv( -kappa, arakawa.dy(), y[0], 1., yp[1]);
         return;
     }
     else
@@ -400,17 +397,15 @@ void Explicit<G, M, container>::operator()( std::vector<container>& y, std::vect
             arakawa( y[i], phi[i], yp[i]);
             if(equations == "global") dg::blas1::pointwiseDot( binv, yp[i], yp[i]);
         }
+        double _tau[2] = {-1., tau};
         //compute derivatives and exb compression
         for( unsigned i=0; i<y.size(); i++)
         {
             dg::blas2::gemv( arakawa.dy(), y[i], dyy[i]);
             dg::blas2::gemv( arakawa.dy(), phi[i], dyphi[i]);
             if(equations == "global") dg::blas1::pointwiseDot( dyphi[i], ype[i], dyphi[i]);
-            dg::blas1::axpby( kappa, dyphi[i], 1., yp[i]);
+            dg::blas1::axpbypgz( kappa, dyphi[i], _tau[i]*kappa, dyy[i], 1., yp[i]);
         }
-        // diamagnetic compression
-        dg::blas1::axpby( -1.*kappa, dyy[0], 1., yp[0]);
-        dg::blas1::axpby( tau*kappa, dyy[1], 1., yp[1]);
     }
 
     return;
-- 
GitLab


From 9868fba519ab03007b92e98a0511cf423fea2d39 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Fri, 15 Sep 2017 13:26:45 +0200
Subject: [PATCH 288/453] tests with Jacob on CG behaviour

---
 inc/dg/Makefile        |  2 +-
 inc/dg/cg.h            | 14 +++++++-----
 inc/dg/elliptic2d_b.cu | 51 +++++++++++++++++++++---------------------
 inc/dg/multigrid.h     | 29 ++++++++++++++++++++++++
 4 files changed, 65 insertions(+), 31 deletions(-)

diff --git a/inc/dg/Makefile b/inc/dg/Makefile
index 216eb230b..6ffa22a3f 100644
--- a/inc/dg/Makefile
+++ b/inc/dg/Makefile
@@ -24,7 +24,7 @@ all: $(CPPFILES:%.cpp=%) $(CUFILES:%.cu=%)
 	$(CC) $(OPT) $(INCLUDE) -DDG_DEBUG $(CFLAGS) $< -o $@ 
 
 %_b: %_b.cu 
-	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) -g 
+	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) -g -DDG_DEBUG
 
 bathRZ_t: bathRZ_t.cu 
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(GLFLAGS) $(INCLUDE) 
diff --git a/inc/dg/cg.h b/inc/dg/cg.h
index 88fb3438d..08c980a76 100644
--- a/inc/dg/cg.h
+++ b/inc/dg/cg.h
@@ -64,6 +64,9 @@ class CG
      */
     unsigned get_max() const {return max_iter;}
 
+    container& search_direction(){return p;}
+    const container& search_direction()const {return p;}
+
     /**
      * @brief Set internal storage and maximum number of iterations
      *
@@ -115,7 +118,7 @@ class CG
      * @return Number of iterations used to achieve desired precision
      */
     template< class SymmetricOp, class Preconditioner, class SquareNorm >
-    unsigned operator()( SymmetricOp& A, container& x, const container& b, Preconditioner& P, SquareNorm& S, value_type eps = 1e-12, value_type nrmb_correction = 1);
+    unsigned operator()( SymmetricOp& A, container& x, const container& b, Preconditioner& P, SquareNorm& S, value_type eps = 1e-12, value_type nrmb_correction = 1, bool initialize = true);
   private:
     container r, p, ap; 
     unsigned max_iter;
@@ -203,7 +206,7 @@ unsigned CG< container>::operator()( Matrix& A, container& x, const container& b
 
 template< class container>
 template< class Matrix, class Preconditioner, class SquareNorm>
-unsigned CG< container>::operator()( Matrix& A, container& x, const container& b, Preconditioner& P, SquareNorm& S, value_type eps, value_type nrmb_correction)
+unsigned CG< container>::operator()( Matrix& A, container& x, const container& b, Preconditioner& P, SquareNorm& S, value_type eps, value_type nrmb_correction, bool initialize)
 {
     value_type nrmb = sqrt( blas2::dot( S, b));
 #ifdef DG_DEBUG
@@ -227,7 +230,8 @@ unsigned CG< container>::operator()( Matrix& A, container& x, const container& b
     //note that dot does automatically synchronize
     if( sqrt( blas2::dot(S,r) ) < eps*(nrmb + nrmb_correction)) //if x happens to be the solution
         return 0;
-    blas2::symv( P, r, p );//<-- compute p_0
+    if( initialize) 
+        blas2::symv( P, r, p );//<-- compute p_0
     value_type nrmzr_old = blas1::dot( p,r); //and store the scalar product
     value_type alpha, nrmzr_new;
     for( unsigned i=1; i<max_iter; i++)
@@ -241,8 +245,8 @@ unsigned CG< container>::operator()( Matrix& A, container& x, const container& b
     if(rank==0)
 #endif //MPI
     {
-        std::cout << "Absolute r*S*r "<<sqrt( blas2::dot(S,r)) <<"\t ";
-        std::cout << " < Critical "<<eps*nrmb + eps <<"\t ";
+        std::cout << "Norm: r*S*r "<<sqrt( blas2::dot(S,r)) <<"\t ";
+        std::cout << "Norm p*S*p "<<sqrt( blas1::dot( p, p)) <<"\t ";
         std::cout << "(Relative "<<sqrt( blas2::dot(S,r) )/nrmb << ")\n";
     }
 #endif //DG_DEBUG
diff --git a/inc/dg/elliptic2d_b.cu b/inc/dg/elliptic2d_b.cu
index e0ebe99f1..e9efb1553 100644
--- a/inc/dg/elliptic2d_b.cu
+++ b/inc/dg/elliptic2d_b.cu
@@ -94,10 +94,11 @@ int main()
     	
 		t.tic();
 
-		std::vector<unsigned> number = multigrid.solve(multi_pol, x, b, eps);
+		//std::vector<unsigned> number = multigrid.direct_solve(multi_pol, x, b, eps);
+		std::vector<unsigned> number = multigrid.test_solve(multi_pol, x, b, eps);
 		
-		//for( unsigned u=0; u<number.size(); u++)
-		//	std::cout << " # iterations stage "<< number.size()-1-u << " " << number[number.size()-1-u] << " \n";
+		for( unsigned u=0; u<number.size(); u++)
+			std::cout << " # iterations stage "<< number.size()-1-u << " " << number[number.size()-1-u] << " \n";
 
 		t.toc();
 		std::cout << "Took "<< t.diff() <<"s\n";
@@ -113,28 +114,28 @@ int main()
     //std::cout << "L2 Norm2 of Error is                       " << err << std::endl;
     const double norm = dg::blas2::dot( w2d, solution);
     std::cout << " "<<sqrt( err/norm);
-    {
-		dg::Elliptic<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> pol_forward( grid, dg::not_normed, dg::centered, jfactor);
-		pol_forward.set_chi( chi);
-		x = temp;
-		dg::Invert<dg::DVec > invert_fw( x, n*n*Nx*Ny, eps);
-		std::cout << " "<< invert_fw( pol_forward, x, b, v2d, chi_inv);
-		dg::blas1::axpby( 1.,x,-1., solution, error);
-		err = dg::blas2::dot( w2d, error);
-		std::cout << " "<<sqrt( err/norm);
-    }
-
-    {
-		dg::Elliptic<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> pol_backward( grid, dg::not_normed, dg::backward, jfactor);
-		pol_backward.set_chi( chi);
-		x = temp;
-		dg::Invert<dg::DVec > invert_bw( x, n*n*Nx*Ny, eps);
-		std::cout << " "<< invert_bw( pol_backward, x, b, v2d, chi_inv);
-		dg::blas1::axpby( 1.,x,-1., solution, error);
-		err = dg::blas2::dot( w2d, error);
-		std::cout << " "<<sqrt( err/norm)<<std::endl;
-    }
-
+//    {
+//		dg::Elliptic<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> pol_forward( grid, dg::not_normed, dg::centered, jfactor);
+//		pol_forward.set_chi( chi);
+//		x = temp;
+//		dg::Invert<dg::DVec > invert_fw( x, n*n*Nx*Ny, eps);
+//		std::cout << " "<< invert_fw( pol_forward, x, b, v2d, chi_inv);
+//		dg::blas1::axpby( 1.,x,-1., solution, error);
+//		err = dg::blas2::dot( w2d, error);
+//		std::cout << " "<<sqrt( err/norm);
+//    }
+//
+//    {
+//		dg::Elliptic<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> pol_backward( grid, dg::not_normed, dg::backward, jfactor);
+//		pol_backward.set_chi( chi);
+//		x = temp;
+//		dg::Invert<dg::DVec > invert_bw( x, n*n*Nx*Ny, eps);
+//		std::cout << " "<< invert_bw( pol_backward, x, b, v2d, chi_inv);
+//		dg::blas1::axpby( 1.,x,-1., solution, error);
+//		err = dg::blas2::dot( w2d, error);
+//		std::cout << " "<<sqrt( err/norm)<<std::endl;
+//    }
+//
 
     dg::DMatrix DX = dg::create::dx( grid);
     dg::blas2::gemv( DX, x, error);
diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index 9bfb2bea7..88db8b1d2 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -150,6 +150,35 @@ struct MultigridCG2d
         std::cout<< "Took "<<t.diff()<<"s\n";
         return number;
     }
+    template<class SymmetricOp>
+    std::vector<unsigned> test_solve( std::vector<SymmetricOp>& op, container&  x, const container& b, double eps)
+    {
+        Timer t;
+        t.tic();
+        dg::blas1::pointwiseDivide(b, op[0].inv_weights(), b_[0]);
+        // compute residual r = Wb - A x
+        std::vector<unsigned> number( 4);
+        dg::blas1::copy( x, x_[0]);//save initial guess
+        cg_[0].set_max(1500);
+
+        number[0] = cg_[0]( op[0], x, b_[0], op[0].precond(), op[0].inv_weights(), eps, 1.);
+        container p = cg_[0].search_direction();
+        dg::blas2::symv( project_[0], p, m_r[1]);
+        dg::blas2::symv( inter_[0], m_r[1], cg_[0].search_direction());
+        dg::blas1::axpby( 1. ,p,-1., cg_[0].search_direction(), p);
+
+
+        cg_[0].set_max(10000);
+        number[1] = cg_[0]( op[0], x, b_[0], op[0].precond(), op[0].inv_weights(), eps, 1., false);
+
+        cg_[0].set_max(1500);
+        number[2] = cg_[0]( op[0], x_[0], b_[0], op[0].precond(), op[0].inv_weights(), eps, 1.);
+        cg_[0].set_max(10000);
+        number[3] = cg_[0]( op[0], x_[0], b_[0], op[0].precond(), op[0].inv_weights(), eps, 1., false);
+        std::cout << "Norm of smoothed p - Pp"<<sqrt( dg::blas1::dot(p,p) ) <<"\n";
+        
+        return number;
+    }
 
     ///src may alias first element of out
     void project( const container& src, std::vector<container>& out)
-- 
GitLab


From af4efea316496420b44f04957f173e4da09040bc Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Fri, 15 Sep 2017 13:27:33 +0200
Subject: [PATCH 289/453] go back to commit 9b4b0b

---
 inc/dg/Makefile        |  2 +-
 inc/dg/cg.h            | 14 +++++-------
 inc/dg/elliptic2d_b.cu | 51 +++++++++++++++++++++---------------------
 inc/dg/multigrid.h     | 29 ------------------------
 4 files changed, 31 insertions(+), 65 deletions(-)

diff --git a/inc/dg/Makefile b/inc/dg/Makefile
index 6ffa22a3f..216eb230b 100644
--- a/inc/dg/Makefile
+++ b/inc/dg/Makefile
@@ -24,7 +24,7 @@ all: $(CPPFILES:%.cpp=%) $(CUFILES:%.cu=%)
 	$(CC) $(OPT) $(INCLUDE) -DDG_DEBUG $(CFLAGS) $< -o $@ 
 
 %_b: %_b.cu 
-	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) -g -DDG_DEBUG
+	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) -g 
 
 bathRZ_t: bathRZ_t.cu 
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(GLFLAGS) $(INCLUDE) 
diff --git a/inc/dg/cg.h b/inc/dg/cg.h
index 08c980a76..88fb3438d 100644
--- a/inc/dg/cg.h
+++ b/inc/dg/cg.h
@@ -64,9 +64,6 @@ class CG
      */
     unsigned get_max() const {return max_iter;}
 
-    container& search_direction(){return p;}
-    const container& search_direction()const {return p;}
-
     /**
      * @brief Set internal storage and maximum number of iterations
      *
@@ -118,7 +115,7 @@ class CG
      * @return Number of iterations used to achieve desired precision
      */
     template< class SymmetricOp, class Preconditioner, class SquareNorm >
-    unsigned operator()( SymmetricOp& A, container& x, const container& b, Preconditioner& P, SquareNorm& S, value_type eps = 1e-12, value_type nrmb_correction = 1, bool initialize = true);
+    unsigned operator()( SymmetricOp& A, container& x, const container& b, Preconditioner& P, SquareNorm& S, value_type eps = 1e-12, value_type nrmb_correction = 1);
   private:
     container r, p, ap; 
     unsigned max_iter;
@@ -206,7 +203,7 @@ unsigned CG< container>::operator()( Matrix& A, container& x, const container& b
 
 template< class container>
 template< class Matrix, class Preconditioner, class SquareNorm>
-unsigned CG< container>::operator()( Matrix& A, container& x, const container& b, Preconditioner& P, SquareNorm& S, value_type eps, value_type nrmb_correction, bool initialize)
+unsigned CG< container>::operator()( Matrix& A, container& x, const container& b, Preconditioner& P, SquareNorm& S, value_type eps, value_type nrmb_correction)
 {
     value_type nrmb = sqrt( blas2::dot( S, b));
 #ifdef DG_DEBUG
@@ -230,8 +227,7 @@ unsigned CG< container>::operator()( Matrix& A, container& x, const container& b
     //note that dot does automatically synchronize
     if( sqrt( blas2::dot(S,r) ) < eps*(nrmb + nrmb_correction)) //if x happens to be the solution
         return 0;
-    if( initialize) 
-        blas2::symv( P, r, p );//<-- compute p_0
+    blas2::symv( P, r, p );//<-- compute p_0
     value_type nrmzr_old = blas1::dot( p,r); //and store the scalar product
     value_type alpha, nrmzr_new;
     for( unsigned i=1; i<max_iter; i++)
@@ -245,8 +241,8 @@ unsigned CG< container>::operator()( Matrix& A, container& x, const container& b
     if(rank==0)
 #endif //MPI
     {
-        std::cout << "Norm: r*S*r "<<sqrt( blas2::dot(S,r)) <<"\t ";
-        std::cout << "Norm p*S*p "<<sqrt( blas1::dot( p, p)) <<"\t ";
+        std::cout << "Absolute r*S*r "<<sqrt( blas2::dot(S,r)) <<"\t ";
+        std::cout << " < Critical "<<eps*nrmb + eps <<"\t ";
         std::cout << "(Relative "<<sqrt( blas2::dot(S,r) )/nrmb << ")\n";
     }
 #endif //DG_DEBUG
diff --git a/inc/dg/elliptic2d_b.cu b/inc/dg/elliptic2d_b.cu
index e9efb1553..e0ebe99f1 100644
--- a/inc/dg/elliptic2d_b.cu
+++ b/inc/dg/elliptic2d_b.cu
@@ -94,11 +94,10 @@ int main()
     	
 		t.tic();
 
-		//std::vector<unsigned> number = multigrid.direct_solve(multi_pol, x, b, eps);
-		std::vector<unsigned> number = multigrid.test_solve(multi_pol, x, b, eps);
+		std::vector<unsigned> number = multigrid.solve(multi_pol, x, b, eps);
 		
-		for( unsigned u=0; u<number.size(); u++)
-			std::cout << " # iterations stage "<< number.size()-1-u << " " << number[number.size()-1-u] << " \n";
+		//for( unsigned u=0; u<number.size(); u++)
+		//	std::cout << " # iterations stage "<< number.size()-1-u << " " << number[number.size()-1-u] << " \n";
 
 		t.toc();
 		std::cout << "Took "<< t.diff() <<"s\n";
@@ -114,28 +113,28 @@ int main()
     //std::cout << "L2 Norm2 of Error is                       " << err << std::endl;
     const double norm = dg::blas2::dot( w2d, solution);
     std::cout << " "<<sqrt( err/norm);
-//    {
-//		dg::Elliptic<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> pol_forward( grid, dg::not_normed, dg::centered, jfactor);
-//		pol_forward.set_chi( chi);
-//		x = temp;
-//		dg::Invert<dg::DVec > invert_fw( x, n*n*Nx*Ny, eps);
-//		std::cout << " "<< invert_fw( pol_forward, x, b, v2d, chi_inv);
-//		dg::blas1::axpby( 1.,x,-1., solution, error);
-//		err = dg::blas2::dot( w2d, error);
-//		std::cout << " "<<sqrt( err/norm);
-//    }
-//
-//    {
-//		dg::Elliptic<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> pol_backward( grid, dg::not_normed, dg::backward, jfactor);
-//		pol_backward.set_chi( chi);
-//		x = temp;
-//		dg::Invert<dg::DVec > invert_bw( x, n*n*Nx*Ny, eps);
-//		std::cout << " "<< invert_bw( pol_backward, x, b, v2d, chi_inv);
-//		dg::blas1::axpby( 1.,x,-1., solution, error);
-//		err = dg::blas2::dot( w2d, error);
-//		std::cout << " "<<sqrt( err/norm)<<std::endl;
-//    }
-//
+    {
+		dg::Elliptic<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> pol_forward( grid, dg::not_normed, dg::centered, jfactor);
+		pol_forward.set_chi( chi);
+		x = temp;
+		dg::Invert<dg::DVec > invert_fw( x, n*n*Nx*Ny, eps);
+		std::cout << " "<< invert_fw( pol_forward, x, b, v2d, chi_inv);
+		dg::blas1::axpby( 1.,x,-1., solution, error);
+		err = dg::blas2::dot( w2d, error);
+		std::cout << " "<<sqrt( err/norm);
+    }
+
+    {
+		dg::Elliptic<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> pol_backward( grid, dg::not_normed, dg::backward, jfactor);
+		pol_backward.set_chi( chi);
+		x = temp;
+		dg::Invert<dg::DVec > invert_bw( x, n*n*Nx*Ny, eps);
+		std::cout << " "<< invert_bw( pol_backward, x, b, v2d, chi_inv);
+		dg::blas1::axpby( 1.,x,-1., solution, error);
+		err = dg::blas2::dot( w2d, error);
+		std::cout << " "<<sqrt( err/norm)<<std::endl;
+    }
+
 
     dg::DMatrix DX = dg::create::dx( grid);
     dg::blas2::gemv( DX, x, error);
diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index 88db8b1d2..9bfb2bea7 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -150,35 +150,6 @@ struct MultigridCG2d
         std::cout<< "Took "<<t.diff()<<"s\n";
         return number;
     }
-    template<class SymmetricOp>
-    std::vector<unsigned> test_solve( std::vector<SymmetricOp>& op, container&  x, const container& b, double eps)
-    {
-        Timer t;
-        t.tic();
-        dg::blas1::pointwiseDivide(b, op[0].inv_weights(), b_[0]);
-        // compute residual r = Wb - A x
-        std::vector<unsigned> number( 4);
-        dg::blas1::copy( x, x_[0]);//save initial guess
-        cg_[0].set_max(1500);
-
-        number[0] = cg_[0]( op[0], x, b_[0], op[0].precond(), op[0].inv_weights(), eps, 1.);
-        container p = cg_[0].search_direction();
-        dg::blas2::symv( project_[0], p, m_r[1]);
-        dg::blas2::symv( inter_[0], m_r[1], cg_[0].search_direction());
-        dg::blas1::axpby( 1. ,p,-1., cg_[0].search_direction(), p);
-
-
-        cg_[0].set_max(10000);
-        number[1] = cg_[0]( op[0], x, b_[0], op[0].precond(), op[0].inv_weights(), eps, 1., false);
-
-        cg_[0].set_max(1500);
-        number[2] = cg_[0]( op[0], x_[0], b_[0], op[0].precond(), op[0].inv_weights(), eps, 1.);
-        cg_[0].set_max(10000);
-        number[3] = cg_[0]( op[0], x_[0], b_[0], op[0].precond(), op[0].inv_weights(), eps, 1., false);
-        std::cout << "Norm of smoothed p - Pp"<<sqrt( dg::blas1::dot(p,p) ) <<"\n";
-        
-        return number;
-    }
 
     ///src may alias first element of out
     void project( const container& src, std::vector<container>& out)
-- 
GitLab


From 934de4e018c0106c5d6d72c24ce0670953e8b6ee Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Fri, 15 Sep 2017 14:36:05 +0200
Subject: [PATCH 290/453] work on blas documentation

---
 inc/dg/blas1.h  | 220 +++++++++++++++++++++++++-----------------------
 inc/dg/blas2.h  |  93 ++++++++++----------
 inc/dg/dg_doc.h |   8 +-
 3 files changed, 166 insertions(+), 155 deletions(-)

diff --git a/inc/dg/blas1.h b/inc/dg/blas1.h
index bb004b0ed..4cddd1cc0 100644
--- a/inc/dg/blas1.h
+++ b/inc/dg/blas1.h
@@ -11,23 +11,15 @@
 
 /*!@file 
  *
- * blas level 1 functions
+ * Basic linear algebra level 1 functions (functions that only involve vectors and not matrices)
  */
+
 namespace dg{
 
-//TODO Throw messages instead of assertions because origin of message can 
-// be better followed ?
-//
-// eigentlich dürften die blas routinen nicht im dg namensraum liegen, denn damit 
-// haben sie erst mal nichts zu tun. Wenn man nun als Außenstehender seine eigene 
-// Vektorklasse hat und die blas spezialisieren will?
-// Synchronize ist niemals nötig mit thrust!!
-// Vielleicht wäre function overloading die bessere design - Wahl für blas
 /*! @brief BLAS Level 1 routines 
  *
  * @ingroup blas1
  * Only those routines that are actually called need to be implemented.
- * Don't forget to specialize in the dg namespace.
  * @note successive calls to blas routines are executed sequentially 
  * @note A manual synchronization of threads or devices is never needed in an application 
  * using these functions. All functions returning a value block until the value is ready.
@@ -39,213 +31,227 @@ namespace blas1
 ///@{
 
 /**
- * @brief Generic way to copy vectors of different types (e.g. from CPU to GPU, or double to float, etc.)
+ * @brief Generic way to copy and/or convert a container type to a different container type (e.g. from CPU to GPU, or double to float, etc.)
  *
- * @tparam Vector1 First vector type
- * @tparam Vector2 Second vector type
+ * @copydoc hide_container_lvl1
+ * @tparam other_container another container class
  * @param x source
  * @param y sink
  * @note y gets resized properly
  */
-template<class Vector1, class Vector2>
-inline void transfer( const Vector1& x, Vector2& y)
+template<class container, class other_container>
+inline void transfer( const container& x, other_container& y)
 {
-    dg::blas1::detail::doTransfer( x,y, typename dg::VectorTraits<Vector1>::vector_category(), typename dg::VectorTraits<Vector2>::vector_category());
+    dg::blas1::detail::doTransfer( x,y, typename dg::VectorTraits<container>::vector_category(), typename dg::VectorTraits<other_container>::vector_category());
 }
 
 
 /**
  * @brief Invoke assignment operator
  *
- * @tparam Vector Vector class
+ * Same as y=x
+ * @tparam Assignable any assignable type
  * @param x in
  * @param y out
  */
-template<class Vector>
-inline void copy( const Vector& x, Vector& y){y=x;}
+template<class Assignable>
+inline void copy( const Assignable& x, Assignable& y){y=x;}
 
-/*! @brief Euclidean dot product between two Vectors
+/*! @brief Euclidean dot product between two containers
  *
- * This routine computes \f[ x^T y = \sum_{i=0}^{N-1} x_i y_i \f]
- * @param x Left Vector
- * @param y Right Vector may equal x
- * @return Scalar product
+ * This routine computes \f[ x^T y = \sum_{i=0}^{N-1} x_i y_i \f]  i iterates over @b all elements inside the container. Specifically for a std::vector<container_type> i includes both the inner and the outer loop. If the container sizes
+ * do not match the result is undefined.
+ * @copydoc hide_container_lvl1
+ * @param x Left container
+ * @param y Right container may alias x
+ * @return Scalar product as defined above
  * @note This routine is always executed synchronously due to the 
         implicit memcpy of the result. With mpi the result is broadcasted to all
         processes
- * @note If DG_DEBUG is defined a range check shall be performed
  */
-template< class Vector>
-inline typename VectorTraits<Vector>::value_type dot( const Vector& x, const Vector& y)
+template< class container>
+inline typename VectorTraits<container>::value_type dot( const container& x, const container& y)
 {
-    return dg::blas1::detail::doDot( x, y, typename dg::VectorTraits<Vector>::vector_category() );
+    return dg::blas1::detail::doDot( x, y, typename dg::VectorTraits<container>::vector_category() );
 }
 
 /*! @brief Modified BLAS 1 routine axpy
  *
- * This routine computes \f[ y_i =  \alpha x_i + \beta y_i \f] 
+ * This routine computes \f[ y_i =  \alpha x_i + \beta y_i \f]  i iterates over @b all elements inside the container. Specifically for a std::vector<container_type> i includes both the inner and the outer loop. If the container sizes
+ * do not match the result is undefined.
+ * @copydoc hide_container_lvl1
  * @param alpha Scalar  
- * @param x Vector x may equal y 
+ * @param x container x may alias y 
  * @param beta Scalar
- * @param y Vector y contains solution on output
- * @note Checks for alpha == 0 and beta == 1
- * @note If DG_DEBUG is defined a range check shall be performed
+ * @param y container y contains solution on output
  */
-template< class Vector>
-inline void axpby( typename VectorTraits<Vector>::value_type alpha, const Vector& x, typename VectorTraits<Vector>::value_type beta, Vector& y)
+template< class container>
+inline void axpby( typename VectorTraits<container>::value_type alpha, const container& x, typename VectorTraits<container>::value_type beta, container& y)
 {
-    dg::blas1::detail::doAxpby( alpha, x, beta, y, typename dg::VectorTraits<Vector>::vector_category() );
+    dg::blas1::detail::doAxpby( alpha, x, beta, y, typename dg::VectorTraits<container>::vector_category() );
     return;
 }
 
 /*! @brief Modified BLAS 1 routine axpy
  *
- * This routine computes \f[ z_i =  \alpha x_i + \beta y_i \f] 
+ * This routine computes \f[ z_i =  \alpha x_i + \beta y_i \f]  i iterates over @b all elements inside the container. Specifically for a std::vector<container_type> i includes both the inner and the outer loop. If the container sizes
+ * do not match the result is undefined.
+
+ * @copydoc hide_container_lvl1
  * @param alpha Scalar  
- * @param x Vector x may equal result
+ * @param x container x may alias result
  * @param beta Scalar
- * @param y Vector y may equal result
- * @param result Vector contains solution on output
- * @note Checks for alpha == 0 and beta == 1
- * @note If DG_DEBUG is defined a range check shall be performed
- * @attention If a thrust::device_vector is used then this routine is NON-BLOCKING!
+ * @param y container y may alias result
+ * @param result container contains solution on output
  */
-template< class Vector>
-inline void axpby( typename VectorTraits<Vector>::value_type alpha, const Vector& x, typename VectorTraits<Vector>::value_type beta, const Vector& y, Vector& result)
+template< class container>
+inline void axpby( typename VectorTraits<container>::value_type alpha, const container& x, typename VectorTraits<container>::value_type beta, const container& y, container& result)
 {
-    dg::blas1::detail::doAxpby( alpha, x, beta, y, result, typename dg::VectorTraits<Vector>::vector_category() );
+    dg::blas1::detail::doAxpby( alpha, x, beta, y, result, typename dg::VectorTraits<container>::vector_category() );
     return;
 }
 
 /*! @brief Modified BLAS 1 routine axpy
  *
- * This routine computes \f[ z_i =  \alpha x_i + \beta y_i + \gamma z_i \f] 
+ * This routine computes \f[ z_i =  \alpha x_i + \beta y_i + \gamma z_i \f]  i iterates over @b all elements inside the container. Specifically for a std::vector<container_type> i includes both the inner and the outer loop. If the container sizes
+ * do not match the result is undefined.
+
+ * @copydoc hide_container_lvl1
  * @param alpha Scalar  
- * @param x Vector x may equal result
+ * @param x container x may alias result
  * @param beta Scalar
- * @param y Vector y may equal result
+ * @param y container y may alias result
  * @param gamma Scalar
- * @param z Vector contains solution on output
- * @note If DG_DEBUG is defined a range check shall be performed
- * @attention If a thrust::device_vector is used then this routine is NON-BLOCKING!
+ * @param z container contains solution on output
  */
-template< class Vector>
-inline void axpbypgz( typename VectorTraits<Vector>::value_type alpha, const Vector& x, typename VectorTraits<Vector>::value_type beta, const Vector& y, typename VectorTraits<Vector>::value_type gamma, Vector& z)
+template< class container>
+inline void axpbypgz( typename VectorTraits<container>::value_type alpha, const container& x, typename VectorTraits<container>::value_type beta, const container& y, typename VectorTraits<container>::value_type gamma, container& z)
 {
-    dg::blas1::detail::doAxpby( alpha, x, beta, y, gamma, z, typename dg::VectorTraits<Vector>::vector_category() );
+    dg::blas1::detail::doAxpby( alpha, x, beta, y, gamma, z, typename dg::VectorTraits<container>::vector_category() );
     return;
 }
 
 /*! @brief "new" BLAS 1 routine transform
  *
- * This routine computes \f[ y_i = op(x_i) \f] 
+ * This routine computes \f[ y_i = op(x_i) \f]
  * This is strictly speaking not a BLAS routine since f can be a nonlinear function.
- * @param x Vector x may equal y
- * @param y Vector y contains result, may equal x
+ * @copydoc hide_container_lvl1
+ * @tparam UnaryOp Type with member function: value_type operator()(value_type)  
+ * @param x container x may alias y
+ * @param y container y contains result, may alias x
  * @param op unary Operator to use on every element
- * @note In an implementation you may want to check for alpha == 0
  */
-template< class Vector, class UnaryOp>
-inline void transform( const Vector& x, Vector& y, UnaryOp op)
+template< class container, class UnaryOp>
+inline void transform( const container& x, container& y, UnaryOp op)
 {
-    dg::blas1::detail::doTransform( x, y, op, typename dg::VectorTraits<Vector>::vector_category() );
+    dg::blas1::detail::doTransform( x, y, op, typename dg::VectorTraits<container>::vector_category() );
     return;
 }
 
 /*! @brief BLAS 1 routine scal
  *
- * This routine computes \f[ \alpha x_i \f] 
+ * This routine computes \f[ \alpha x_i \f]
+ * @copydoc hide_container_lvl1
  * @param alpha Scalar  
- * @param x Vector x 
+ * @param x container x 
  */
-template< class Vector>
-inline void scal( Vector& x, typename VectorTraits<Vector>::value_type alpha)
+template< class container>
+inline void scal( container& x, typename VectorTraits<container>::value_type alpha)
 {
-    dg::blas1::detail::doScal( x, alpha, typename dg::VectorTraits<Vector>::vector_category() );
+    dg::blas1::detail::doScal( x, alpha, typename dg::VectorTraits<container>::vector_category() );
     return;
 }
 
 /*! @brief pointwise add a scalar
  *
  * This routine computes \f[ x_i + \alpha \f] 
+ * @copydoc hide_container_lvl1
  * @param alpha Scalar  
- * @param x Vector x 
+ * @param x container x 
  */
-template< class Vector>
-inline void plus( Vector& x, typename VectorTraits<Vector>::value_type alpha)
+template< class container>
+inline void plus( container& x, typename VectorTraits<container>::value_type alpha)
 {
-    dg::blas1::detail::doPlus( x, alpha, typename dg::VectorTraits<Vector>::vector_category() );
+    dg::blas1::detail::doPlus( x, alpha, typename dg::VectorTraits<container>::vector_category() );
     return;
 }
 
 /**
 * @brief A 'new' BLAS 1 routine. 
 *
-* Multiplies two vectors element by element: \f[ y_i = x_{1i}x_{2i}\f]
-* @param x1 Vector x1  
-* @param x2 Vector x2 may equal x1
-* @param y  Vector y contains result on output ( may equal x1 or x2)
-* @note If DG_DEBUG is defined a range check shall be performed 
+* Multiplies two vectors element by element: \f[ y_i = x_{1i}x_{2i}\f]  i iterates over @b all elements inside the container. Specifically for a std::vector<container_type> i includes both the inner and the outer loop. If the container sizes
+ * do not match the result is undefined.
+
+* @copydoc hide_container_lvl1
+* @param x1 container x1  
+* @param x2 container x2 may alias x1
+* @param y  container y contains result on output ( may alias x1 or x2)
 */
-template< class Vector>
-inline void pointwiseDot( const Vector& x1, const Vector& x2, Vector& y)
+template< class container>
+inline void pointwiseDot( const container& x1, const container& x2, container& y)
 {
-    dg::blas1::detail::doPointwiseDot( x1, x2, y, typename dg::VectorTraits<Vector>::vector_category() );
+    dg::blas1::detail::doPointwiseDot( x1, x2, y, typename dg::VectorTraits<container>::vector_category() );
     return;
 }
 
 /**
 * @brief A 'new' BLAS 1 routine. 
 *
-* Multiplies two vectors element by element: \f[ y_i = \alpha x_{1i}x_{2i} + \beta y_i\f]
+* Multiplies two vectors element by element: \f[ y_i = \alpha x_{1i}x_{2i} + \beta y_i\f]  i iterates over @b all elements inside the container. Specifically for a std::vector<container_type> i includes both the inner and the outer loop. If the container sizes
+ * do not match the result is undefined.
+
+* @copydoc hide_container_lvl1
 * @param alpha scalar
-* @param x1 Vector x1  
-* @param x2 Vector x2 may equal x1
+* @param x1 container x1  
+* @param x2 container x2 may alias x1
 * @param beta scalar
-* @param y  Vector y contains result on output ( may equal x1 or x2)
-* @note If DG_DEBUG is defined a range check shall be performed 
+* @param y  container y contains result on output ( may alias x1 or x2)
 */
-template< class Vector>
-inline void pointwiseDot( typename VectorTraits<Vector>::value_type alpha, const Vector& x1, const Vector& x2, typename VectorTraits<Vector>::value_type beta, Vector& y)
+template< class container>
+inline void pointwiseDot( typename VectorTraits<container>::value_type alpha, const container& x1, const container& x2, typename VectorTraits<container>::value_type beta, container& y)
 {
-    dg::blas1::detail::doPointwiseDot( alpha, x1, x2, beta, y, typename dg::VectorTraits<Vector>::vector_category() );
+    dg::blas1::detail::doPointwiseDot( alpha, x1, x2, beta, y, typename dg::VectorTraits<container>::vector_category() );
 }
 
 /**
 * @brief A 'new' BLAS 1 routine. 
 *
-* Divides two vectors element by element: \f[ y_i = x_{1i}/x_{2i}\f]
-* @param x1 Vector x1  
-* @param x2 Vector x2 may equal x1
-* @param y  Vector y contains result on output ( ma equal x1 and/or x2)
-* @note If DG_DEBUG is defined a range check shall be performed 
+* Divides two vectors element by element: \f[ y_i = x_{1i}/x_{2i}\f]  i iterates over @b all elements inside the container. Specifically for a std::vector<container_type> i includes both the inner and the outer loop. If the container sizes
+ * do not match the result is undefined.
+
+* @copydoc hide_container_lvl1
+* @param x1 container x1  
+* @param x2 container x2 may alias x1
+* @param y  container y contains result on output ( may alias x1 and/or x2)
 */
-template< class Vector>
-inline void pointwiseDivide( const Vector& x1, const Vector& x2, Vector& y)
+template< class container>
+inline void pointwiseDivide( const container& x1, const container& x2, container& y)
 {
-    dg::blas1::detail::doPointwiseDivide( x1, x2, y, typename dg::VectorTraits<Vector>::vector_category() );
+    dg::blas1::detail::doPointwiseDivide( x1, x2, y, typename dg::VectorTraits<container>::vector_category() );
     return;
 }
 
 /**
 * @brief A 'new' fused multiply-add BLAS 1 routine. 
 *
-* Multiplies and adds vectors element by element: \f[ z_i = \alpha x_{1i}y_{1i} + \beta x_{2i}y_{2i} + \gamma z_i \f]
+* Multiplies and adds vectors element by element: \f[ z_i = \alpha x_{1i}y_{1i} + \beta x_{2i}y_{2i} + \gamma z_i \f]  i iterates over @b all elements inside the container. Specifically for a std::vector<container_type> i includes both the inner and the outer loop. If the container sizes
+ * do not match the result is undefined.
+* @copydoc hide_container_lvl1
 * @param alpha scalar
-* @param x1 Vector x1  
-* @param y1 Vector y1 
+* @param x1 container x1  
+* @param y1 container y1 
 * @param beta scalar
-* @param x2 Vector x1  
-* @param y2 Vector y1 
-* @param z  Vector z contains result on output 
-* @note aliases are allowed: we perform an alias analysis on the given references to detect possible performance optimizations
+* @param x2 container x1  
+* @param y2 container y1 
+* @param z  container z contains result on output 
+* @note aliases are allowed
 */
-template<class Vector>
-void pointwiseDot(  typename VectorTraits<Vector>::value_type alpha, const Vector& x1, const Vector& y1, 
-                    typename VectorTraits<Vector>::value_type beta,  const Vector& x2, const Vector& y2, 
-                    typename VectorTraits<Vector>::value_type gamma, Vector & z)
+template<class container>
+void pointwiseDot(  typename VectorTraits<container>::value_type alpha, const container& x1, const container& y1, 
+                    typename VectorTraits<container>::value_type beta,  const container& x2, const container& y2, 
+                    typename VectorTraits<container>::value_type gamma, container & z)
 {
-    dg::blas1::detail::doPointwiseDot( alpha, x1, y1, beta, x2, y2, gamma, z, typename dg::VectorTraits<Vector>::vector_category() );
+    dg::blas1::detail::doPointwiseDot( alpha, x1, y1, beta, x2, y2, gamma, z, typename dg::VectorTraits<container>::vector_category() );
     return;
 }
 ///@}
diff --git a/inc/dg/blas2.h b/inc/dg/blas2.h
index c5c3ae9a6..440adbf61 100644
--- a/inc/dg/blas2.h
+++ b/inc/dg/blas2.h
@@ -17,32 +17,31 @@
 
 /*!@file 
  *
- * blas level 2 functions
+ * Basic linear algebra level 2 functions (functions that involve vectors and matrices)
  */
 namespace dg{
 /*! @brief BLAS Level 2 routines 
 
  @ingroup blas2
- In an implementation Vector and Matrix should be typedefed.
- Only those routines that are actually called need to be implemented.
+ @note Only those routines that are actually called need to be implemented for a given type.
 */
 namespace blas2{
 ///@addtogroup blas2
 ///@{
 
 /**
- * @brief Generic way to copy matrices of different types (e.g. from CPU to GPU, or double to float)
+ * @brief Generic way to copy and/or convert a Matrix type to a different Matrix type (e.g. from CPU to GPU, or double to float, etc.)
  *
- * @tparam Matrix1 First vector type
- * @tparam Matrix2 Second vector type
+ * @tparam Matrix First Matrix type
+ * @tparam AnotherMatrix Second Matrix type
  * @param x source
  * @param y sink 
  * @note y gets resized properly
  */
-template<class Matrix1, class Matrix2>
-inline void transfer( const Matrix1& x, Matrix2& y)
+template<class Matrix, class AnotherMatrix>
+inline void transfer( const Matrix& x, AnotherMatrix& y)
 {
-    dg::blas2::detail::doTransfer( x,y, typename dg::MatrixTraits<Matrix1>::matrix_category(), typename dg::MatrixTraits<Matrix2>::matrix_category());
+    dg::blas2::detail::doTransfer( x,y, typename dg::MatrixTraits<Matrix>::matrix_category(), typename dg::MatrixTraits<AnotherMatrix>::matrix_category());
 }
 
 /*! @brief General dot produt
@@ -51,19 +50,21 @@ inline void transfer( const Matrix1& x, Matrix2& y)
  * matrix M \f[ x^T M y = \sum_{i,j=0}^{N-1} x_i M_{ij} y_j \f]
  * ( Note that if M is not diagonal it is generally more efficient to 
  * precalculate \f$ My\f$ and then call the blas1::dot() routine!
- * @param x Left Vector
+ * @tparam DiagonalMatrix Right now DiagonalMatrix has to be the same as container, except if the container is a std::vector<container_type>, then the DiagonalMatrix has to be the container_type
+ * @copydoc hide_container_lvl1
+ * @param x Left container
  * @param m The diagonal Matrix
- * @param y Right Vector might equal Left Vector
+ * @param y Right container might equal Left container
  * @return Generalized scalar product
  * @note This routine is always executed synchronously due to the 
     implicit memcpy of the result.
  */
-template< class Matrix, class Vector>
-inline typename MatrixTraits<Matrix>::value_type dot( const Vector& x, const Matrix& m, const Vector& y)
+template< class DiagonalMatrix, class container>
+inline typename MatrixTraits<DiagonalMatrix>::value_type dot( const container& x, const DiagonalMatrix& m, const container& y)
 {
     return dg::blas2::detail::doDot( x, m, y, 
-                       typename dg::MatrixTraits<Matrix>::matrix_category(), 
-                       typename dg::VectorTraits<Vector>::vector_category() );
+                       typename dg::MatrixTraits<DiagonalMatrix>::matrix_category(), 
+                       typename dg::VectorTraits<container>::vector_category() );
 }
 
 /*! @brief General dot produt
@@ -71,35 +72,35 @@ inline typename MatrixTraits<Matrix>::value_type dot( const Vector& x, const Mat
  * This routine is equivalent to the call blas2::dot( x, m, x):
  * \f[ x^T M x = \sum_{i,j=0}^{N-1} x_i M_{ij} x_j \f]
  * @param m The diagonal Matrix
- * @param x Right Vector
+ * @param x Right container
  * @return Generalized scalar product
  * @note This routine is always executed synchronously due to the 
     implicit memcpy of the result.
  */
-template< class Matrix, class Vector>
-inline typename MatrixTraits<Matrix>::value_type dot( const Matrix& m, const Vector& x)
+template< class Matrix, class container>
+inline typename MatrixTraits<Matrix>::value_type dot( const Matrix& m, const container& x)
 {
     return dg::blas2::detail::doDot( m, x, 
                        typename dg::MatrixTraits<Matrix>::matrix_category(), 
-                       typename dg::VectorTraits<Vector>::vector_category() );
+                       typename dg::VectorTraits<container>::vector_category() );
 }
 
-/*! @brief Symmetric Matrix Vector product
+/*! @brief Symmetric Matrix container product
  *
  * This routine computes \f[ y = \alpha P x + \beta y \f]
  * where \f$ P\f$ is a symmetric Preconditioner. 
  * @param alpha A Scalar
  * @param P The Preconditioner
- * @param x A Vector different from y (except in the case where m is diagonal)
+ * @param x A container different from y (except in the case where m is diagonal)
  * @param beta A Scalar
  * @param y contains solution on output
  */
-template< class Precon, class Vector>
+template< class Precon, class container>
 inline void symv( typename MatrixTraits<Precon>::value_type alpha, 
                   const Precon& P, 
-                  const Vector& x, 
+                  const container& x, 
                   typename MatrixTraits<Precon>::value_type beta, 
-                  Vector& y)
+                  container& y)
 {
     if(alpha == (typename MatrixTraits<Precon>::value_type)0) {
         dg::blas1::scal( y, alpha);
@@ -107,65 +108,65 @@ inline void symv( typename MatrixTraits<Precon>::value_type alpha,
     }
     dg::blas2::detail::doSymv( alpha, P, x, beta, y, 
                        typename dg::MatrixTraits<Precon>::matrix_category(), 
-                       typename dg::VectorTraits<Vector>::vector_category() );
+                       typename dg::VectorTraits<container>::vector_category() );
     return;
 }
 
-/*! @brief Symmetric Matrix Vector product
+/*! @brief Symmetric Matrix container product
  *
  * This routine computes \f[ y = M x \f]
  * where \f$ M\f$ is a symmetric matrix. 
  * @param m The Matrix
- * @param x A Vector different from y (except in the case where m is diagonal)
+ * @param x A container different from y (except in the case where m is diagonal)
  *      In most applications x is assumed to remain constant. 
  * @param y contains solution on output
- * @attention Due to the SelfMadeMatrixTag and MPI_Vectors, m and x cannot be declared const
+ * @attention Due to the SelfMadeMatrixTag and MPI_containers, m and x cannot be declared const
  */
-template< class Matrix, class Vector1, class Vector2>
+template< class Matrix, class container1, class container2>
 inline void symv( Matrix& m, 
-                  Vector1& x, 
-                  Vector2& y)
+                  container1& x, 
+                  container2& y)
 {
     dg::blas2::detail::doSymv( m, x, y, 
                        typename dg::MatrixTraits<Matrix>::matrix_category(), 
-                       typename dg::VectorTraits<Vector1>::vector_category(),
-                       typename dg::VectorTraits<Vector2>::vector_category() );
+                       typename dg::VectorTraits<container1>::vector_category(),
+                       typename dg::VectorTraits<container2>::vector_category() );
     return;
 }
 
 /**
- * @brief General Matrix-Vector product
+ * @brief General Matrix-container product
  *
  * @param m The Matrix
- * @param x A Vector different from y 
+ * @param x A container different from y 
  * @param y contains the solution on output
  */
-template< class Matrix, class Vector1, class Vector2>
+template< class Matrix, class container1, class container2>
 inline void gemv( Matrix& m, 
-                  Vector1& x, 
-                  Vector2& y)
+                  container1& x, 
+                  container2& y)
 {
     dg::blas2::detail::doGemv( m, x, y, 
                        typename dg::MatrixTraits<Matrix>::matrix_category(), 
-                       typename dg::VectorTraits<Vector1>::vector_category(),
-                       typename dg::VectorTraits<Vector2>::vector_category() );
+                       typename dg::VectorTraits<container1>::vector_category(),
+                       typename dg::VectorTraits<container2>::vector_category() );
     return;
 }
 /**
- * @brief General Matrix-Vector product
+ * @brief General Matrix-container product
  *
  * @param alpha A Scalar
  * @param P The Matrix
- * @param x A Vector different from y 
+ * @param x A container different from y 
  * @param beta A Scalar
  * @param y contains the solution on output
  */
-template< class Precon, class Vector>
+template< class Precon, class container>
 inline void gemv( typename MatrixTraits<Precon>::value_type alpha, 
                   const Precon& P, 
-                  const Vector& x, 
+                  const container& x, 
                   typename MatrixTraits<Precon>::value_type beta, 
-                  Vector& y)
+                  container& y)
 {
     if(alpha == (typename MatrixTraits<Precon>::value_type)0) {
         dg::blas1::scal( y, alpha);
@@ -173,7 +174,7 @@ inline void gemv( typename MatrixTraits<Precon>::value_type alpha,
     }
     dg::blas2::detail::doGemv( alpha, P, x, beta, y, 
                        typename dg::MatrixTraits<Precon>::matrix_category(), 
-                       typename dg::VectorTraits<Vector>::vector_category() );
+                       typename dg::VectorTraits<container>::vector_category() );
     return;
 }
 ///@}
diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index f97b51bd9..e663e9a19 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -116,8 +116,12 @@
   */
 
  /** @class hide_container_lvl1
-  * @tparam container A data container class for which the blas1 functionality is overloaded. Also we assume that the type is copyable/assignable and has a swap member function. Currently this is one of 
-  *   dg::HVec, dg::DVec, dg::MHVec or dg::MDVec
+  * @tparam container A data container class for which the blas1 functionality is overloaded.
+  * We assume that container is copyable/assignable and has a swap member function. 
+  * Currently this is one of 
+  *  - dg::HVec, dg::DVec, dg::MHVec or dg::MDVec  
+  *  - std::vector<dg::HVec>, std::vector<dg::DVec>, std::vector<dg::MHVec> or std::vector<dg::MDVec> . 
+  *
   */
  /** @class hide_matrix_container
   * @tparam Matrix A class for which the blas2 functions are callable in connection with the container class
-- 
GitLab


From ff71944f456424d895bc1d0eaccaa76d665d817d Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Fri, 15 Sep 2017 15:06:12 +0200
Subject: [PATCH 291/453] worked on documentation for blas2

---
 inc/dg/blas2.h  | 117 +++++++++++++++++++++++++++---------------------
 inc/dg/dg_doc.h |  27 +++++++----
 2 files changed, 85 insertions(+), 59 deletions(-)

diff --git a/inc/dg/blas2.h b/inc/dg/blas2.h
index 440adbf61..e51634a3a 100644
--- a/inc/dg/blas2.h
+++ b/inc/dg/blas2.h
@@ -50,7 +50,8 @@ inline void transfer( const Matrix& x, AnotherMatrix& y)
  * matrix M \f[ x^T M y = \sum_{i,j=0}^{N-1} x_i M_{ij} y_j \f]
  * ( Note that if M is not diagonal it is generally more efficient to 
  * precalculate \f$ My\f$ and then call the blas1::dot() routine!
- * @tparam DiagonalMatrix Right now DiagonalMatrix has to be the same as container, except if the container is a std::vector<container_type>, then the DiagonalMatrix has to be the container_type
+ * @tparam DiagonalMatrix Right now DiagonalMatrix has to be the same as container, except if the container is a std::vector<container_type>, then the DiagonalMatrix has to be the container_type.
+ * In the latter case the Matrix is applied to all containers in the std::vector and the sum is returned. 
  * @copydoc hide_container_lvl1
  * @param x Left container
  * @param m The diagonal Matrix
@@ -71,109 +72,123 @@ inline typename MatrixTraits<DiagonalMatrix>::value_type dot( const container& x
  *
  * This routine is equivalent to the call blas2::dot( x, m, x):
  * \f[ x^T M x = \sum_{i,j=0}^{N-1} x_i M_{ij} x_j \f]
+ * @tparam DiagonalMatrix Right now DiagonalMatrix has to be the same as container, except if the container is a std::vector<container_type>, then the DiagonalMatrix has to be the container_type. 
+ * In the latter case the Matrix is applied to all containers in the std::vector and the sum is returned. 
+ * @copydoc hide_container_lvl1
  * @param m The diagonal Matrix
  * @param x Right container
  * @return Generalized scalar product
  * @note This routine is always executed synchronously due to the 
     implicit memcpy of the result.
  */
-template< class Matrix, class container>
-inline typename MatrixTraits<Matrix>::value_type dot( const Matrix& m, const container& x)
+template< class DiagonalMatrix, class container>
+inline typename MatrixTraits<DiagonalMatrix>::value_type dot( const DiagonalMatrix& m, const container& x)
 {
     return dg::blas2::detail::doDot( m, x, 
-                       typename dg::MatrixTraits<Matrix>::matrix_category(), 
+                       typename dg::MatrixTraits<DiagonalMatrix>::matrix_category(), 
                        typename dg::VectorTraits<container>::vector_category() );
 }
 
-/*! @brief Symmetric Matrix container product
+/*! @brief Matrix Vector product
  *
- * This routine computes \f[ y = \alpha P x + \beta y \f]
- * where \f$ P\f$ is a symmetric Preconditioner. 
+ * This routine computes \f[ y = \alpha M x + \beta y \f]
+ * where \f$ M\f$ is a matrix.
+ * @copydoc hide_matrix_container
  * @param alpha A Scalar
- * @param P The Preconditioner
- * @param x A container different from y (except in the case where m is diagonal)
+ * @param M The Matrix
+ * @param x A container different from y 
  * @param beta A Scalar
- * @param y contains solution on output
+ * @param y contains the solution on output (may not alias x)
+ * @attention y may never alias x
  */
-template< class Precon, class container>
-inline void symv( typename MatrixTraits<Precon>::value_type alpha, 
-                  const Precon& P, 
+template< class Matrix, class container>
+inline void symv( typename MatrixTraits<Matrix>::value_type alpha, 
+                  const Matrix& M, 
                   const container& x, 
-                  typename MatrixTraits<Precon>::value_type beta, 
+                  typename MatrixTraits<Matrix>::value_type beta, 
                   container& y)
 {
-    if(alpha == (typename MatrixTraits<Precon>::value_type)0) {
+    if(alpha == (typename MatrixTraits<Matrix>::value_type)0) {
         dg::blas1::scal( y, alpha);
         return;
     }
-    dg::blas2::detail::doSymv( alpha, P, x, beta, y, 
-                       typename dg::MatrixTraits<Precon>::matrix_category(), 
+    dg::blas2::detail::doSymv( alpha, M, x, beta, y, 
+                       typename dg::MatrixTraits<Matrix>::matrix_category(), 
                        typename dg::VectorTraits<container>::vector_category() );
     return;
 }
 
-/*! @brief Symmetric Matrix container product
+/*! @brief Matrix Vector product
  *
  * This routine computes \f[ y = M x \f]
- * where \f$ M\f$ is a symmetric matrix. 
- * @param m The Matrix
- * @param x A container different from y (except in the case where m is diagonal)
- *      In most applications x is assumed to remain constant. 
- * @param y contains solution on output
- * @attention Due to the SelfMadeMatrixTag and MPI_containers, m and x cannot be declared const
+ * where \f$ M\f$ is a matrix. 
+ * @copydoc hide_matrix_container
+ * @tparam same_or_another_container Currently needs to be the same as container.
+ * @param M The Matrix
+ * @param x A container different from y 
+ * @param y contains the solution on output (may not alias x)
+ * @attention y may never alias x
+ * @note Due to the SelfMadeMatrixTag, M cannot be declared const
  */
-template< class Matrix, class container1, class container2>
-inline void symv( Matrix& m, 
-                  container1& x, 
-                  container2& y)
+template< class Matrix, class container, class same_or_another_container>
+inline void symv( Matrix& M, 
+                  const container& x, 
+                  same_or_another_container& y)
 {
-    dg::blas2::detail::doSymv( m, x, y, 
+    dg::blas2::detail::doSymv( M, x, y, 
                        typename dg::MatrixTraits<Matrix>::matrix_category(), 
-                       typename dg::VectorTraits<container1>::vector_category(),
-                       typename dg::VectorTraits<container2>::vector_category() );
+                       typename dg::VectorTraits<container>::vector_category(),
+                       typename dg::VectorTraits<same_or_another_container>::vector_category() );
     return;
 }
 
 /**
- * @brief General Matrix-container product
+ * @brief General Matrix-Vector product
  *
- * @param m The Matrix
+ * Does exactly the same as symv. 
+ * @copydoc hide_matrix_container
+ * @tparam same_or_another_container Currently needs to be the same as container.
+ * @param M The Matrix
  * @param x A container different from y 
- * @param y contains the solution on output
+ * @param y contains the solution on output (may not alias x)
+ * @attention y may never alias x
  */
-template< class Matrix, class container1, class container2>
-inline void gemv( Matrix& m, 
-                  container1& x, 
-                  container2& y)
+template< class Matrix, class container, class same_or_another_container>
+inline void gemv( Matrix& M, 
+                  const container& x, 
+                  same_or_another_container& y)
 {
-    dg::blas2::detail::doGemv( m, x, y, 
+    dg::blas2::detail::doGemv( M, x, y, 
                        typename dg::MatrixTraits<Matrix>::matrix_category(), 
-                       typename dg::VectorTraits<container1>::vector_category(),
-                       typename dg::VectorTraits<container2>::vector_category() );
+                       typename dg::VectorTraits<container>::vector_category(),
+                       typename dg::VectorTraits<same_or_another_container>::vector_category() );
     return;
 }
 /**
- * @brief General Matrix-container product
+ * @brief General Matrix-Vector product
  *
+ * Does exactly the same as symv. 
+ * @copydoc hide_matrix_container
  * @param alpha A Scalar
- * @param P The Matrix
+ * @param M The Matrix
  * @param x A container different from y 
  * @param beta A Scalar
- * @param y contains the solution on output
+ * @param y contains the solution on output (may not alias x)
+ * @attention y may never alias x
  */
-template< class Precon, class container>
-inline void gemv( typename MatrixTraits<Precon>::value_type alpha, 
-                  const Precon& P, 
+template< class Matrix, class container>
+inline void gemv( typename MatrixTraits<Matrix>::value_type alpha, 
+                  const Matrix& M, 
                   const container& x, 
-                  typename MatrixTraits<Precon>::value_type beta, 
+                  typename MatrixTraits<Matrix>::value_type beta, 
                   container& y)
 {
-    if(alpha == (typename MatrixTraits<Precon>::value_type)0) {
+    if(alpha == (typename MatrixTraits<Matrix>::value_type)0) {
         dg::blas1::scal( y, alpha);
         return;
     }
-    dg::blas2::detail::doGemv( alpha, P, x, beta, y, 
-                       typename dg::MatrixTraits<Precon>::matrix_category(), 
+    dg::blas2::detail::doGemv( alpha, M, x, beta, y, 
+                       typename dg::MatrixTraits<Matrix>::matrix_category(), 
                        typename dg::VectorTraits<container>::vector_category() );
     return;
 }
diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index e663e9a19..783e621ef 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -116,7 +116,8 @@
   */
 
  /** @class hide_container_lvl1
-  * @tparam container A data container class for which the blas1 functionality is overloaded.
+  * @tparam container 
+  * A data container class for which the blas1 functionality is overloaded.
   * We assume that container is copyable/assignable and has a swap member function. 
   * Currently this is one of 
   *  - dg::HVec, dg::DVec, dg::MHVec or dg::MDVec  
@@ -124,14 +125,24 @@
   *
   */
  /** @class hide_matrix_container
-  * @tparam Matrix A class for which the blas2 functions are callable in connection with the container class
-  *  - dg::HMatrix with dg::HVec
-  *  - dg::DMatrix with dg::DVec
-  *  - dg::MHMatrix with dg::MHVec
-  *  - dg::MDMatrix with dg::MDVec
+  * @tparam Matrix 
+  * A class for which the blas2 functions are callable in connection with the container class. 
+  * The Matrix type can be one of:
+  *  - container: A container acts as a  diagonal matrix. 
+  *  - dg::HMatrix and dg::IHMatrix with dg::HVec or std::vector<dg::HVec>
+  *  - dg::DMatrix and dg::IDMatrix with dg::DVec or std::vector<dg::DVec>
+  *  - dg::MHMatrix with dg::MHVec or std::vector<dg::MHVec>
+  *  - dg::MDMatrix with dg::MDVec or std::vector<dg::MDVec>
+  *  - Any type that has the SelfMadeMatrixTag. In this case only those blas2 functions 
+  *  that have a corresponding member function in the Matrix class (e.g. symv( const container&, container&); ) can be called.
+  *  If the container is a std::vector, then the Matrix is applied to each of the elements.
   *
-  * @tparam container A data container class for which the blas1 functionality is overloaded. Also we assume that the type is copyable/assignable and has a swap member function. Currently this is one of 
-  *   dg::HVec, dg::DVec, dg::MHVec or dg::MDVec
+  * @tparam container 
+  * A data container class for which the blas1 functionality is overloaded.
+  * We assume that container is copyable/assignable and has a swap member function. 
+  * Currently this is one of 
+  *  - dg::HVec, dg::DVec, dg::MHVec or dg::MDVec  
+  *  - std::vector<dg::HVec>, std::vector<dg::DVec>, std::vector<dg::MHVec> or std::vector<dg::MDVec> . 
   */
 
  /** @class hide_symmetric_op
-- 
GitLab


From b397d6d6d751b1b9d54224bbaa6613ef14315e25 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Fri, 15 Sep 2017 15:50:42 +0200
Subject: [PATCH 292/453] corrected bug in Documentation generation Makefile

---
 inc/dg/blas1.h   |  5 +++--
 inc/doc/Makefile | 12 ++++++------
 2 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/inc/dg/blas1.h b/inc/dg/blas1.h
index 4cddd1cc0..d1656ee55 100644
--- a/inc/dg/blas1.h
+++ b/inc/dg/blas1.h
@@ -241,8 +241,9 @@ inline void pointwiseDivide( const container& x1, const container& x2, container
 * @param x1 container x1  
 * @param y1 container y1 
 * @param beta scalar
-* @param x2 container x1  
-* @param y2 container y1 
+* @param x2 container x2  
+* @param y2 container y2 
+* @param gamma scalar
 * @param z  container z contains result on output 
 * @note aliases are allowed
 */
diff --git a/inc/doc/Makefile b/inc/doc/Makefile
index 45055a934..5e91fa9d5 100644
--- a/inc/doc/Makefile
+++ b/inc/doc/Makefile
@@ -15,8 +15,8 @@ dg:
 	echo "INPUT = ../dg/"; \
 	echo "OUTPUT_DIRECTORY = ./dg"; \
 	echo "HTML_HEADER = header.html"; \
-	echo "GENERATE_TAGFILE = ./dg.tag" ) | doxygen - ;\
-	echo "MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"; 
+	echo "MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML";  \
+	echo "GENERATE_TAGFILE = ./dg.tag" ) | doxygen - ;
 
 
 file:
@@ -25,8 +25,8 @@ file:
 	echo "INPUT = ../file/"; \
 	echo "OUTPUT_DIRECTORY = ./file"; \
 	echo "HTML_HEADER = header.html"; \
-	echo "GENERATE_TAGFILE = ./file.tag" ) | doxygen - ; \
-	echo "MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"; 
+	echo "MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"; \
+	echo "GENERATE_TAGFILE = ./file.tag" ) | doxygen - ; 
 
 geometries:
 	(cat ../geometries/Doxyfile; \
@@ -35,8 +35,8 @@ geometries:
 	echo "HTML_HEADER = header.html"; \
     echo "EXTERNAL_GROUPS=NO" ;\
     echo "EXTERNAL_PAGES=NO" ;\
-    echo "TAGFILES = $(tagfiles)") | doxygen - ; \
-	echo "MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"; 
+	echo "MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML";  \
+    echo "TAGFILES = $(tagfiles)") | doxygen - ; 
 
 doc: dg file geometries
 	ln -sf dg/html/modules.html index.html
-- 
GitLab


From aa088c1fe7eff97857b46138da3a106d4e8cfa03 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Fri, 15 Sep 2017 16:07:03 +0200
Subject: [PATCH 293/453] update README.md with dependencies of documentation

---
 README.md                    |  5 +++--
 inc/dg/blas1.h               | 16 ++++++++--------
 inc/doc/Makefile             |  2 +-
 inc/geometries/average.h     |  2 +-
 inc/geometries/solovev_doc.h |  2 +-
 inc/geometries/utilitiesX.h  |  2 +-
 6 files changed, 15 insertions(+), 14 deletions(-)

diff --git a/README.md b/README.md
index ebb7619be..7e2e99f95 100644
--- a/README.md
+++ b/README.md
@@ -115,8 +115,8 @@ Moreover, we maintain tex files in every src folder for technical documentation,
 `make doc ` in the respective src folder.
 The [developer oriented documentation](http://feltor-dev.github.io/feltor/inc/dg/html/modules.html) of the dG library was generated with [Doxygen](http://www.doxygen.org). 
 You can generate a local version from source code.
-On linux you need the `doxygen`, `libjs-mathjax` and `graphviz` packages.
-Then type `make doc` in the folder `path/to/feltor/inc/doc` and open `index.html` (a symbolic link to `dg/html/modules.html`) with your favorite browser. 
+This depends on the `doxygen`, `libjs-mathjax` and `graphviz` packages.
+Type `make doc` in the folder `path/to/feltor/inc/doc` and open `index.html` (a symbolic link to `dg/html/modules.html`) with your favorite browser. 
 
 ## 3. Contributions and Acknowledgements
 For instructions on how to contribute read the [wiki page](https://github.com/feltor-dev/feltor/wiki/Contributions).
@@ -124,6 +124,7 @@ We gratefully acknowledge contributions from
 - Ralph Kube
 - Eduard Reiter
 - Lukas Einkemmer
+- Jakob Gath
 
 We further acknowledge support on the Knights landing architecture from the High Level Support Team from 
 - Albert Gutiérrez
diff --git a/inc/dg/blas1.h b/inc/dg/blas1.h
index d1656ee55..90eec3c5a 100644
--- a/inc/dg/blas1.h
+++ b/inc/dg/blas1.h
@@ -60,7 +60,7 @@ inline void copy( const Assignable& x, Assignable& y){y=x;}
 /*! @brief Euclidean dot product between two containers
  *
  * This routine computes \f[ x^T y = \sum_{i=0}^{N-1} x_i y_i \f]  i iterates over @b all elements inside the container. Specifically for a std::vector<container_type> i includes both the inner and the outer loop. If the container sizes
- * do not match the result is undefined.
+ * do not match, the result is undefined.
  * @copydoc hide_container_lvl1
  * @param x Left container
  * @param y Right container may alias x
@@ -78,7 +78,7 @@ inline typename VectorTraits<container>::value_type dot( const container& x, con
 /*! @brief Modified BLAS 1 routine axpy
  *
  * This routine computes \f[ y_i =  \alpha x_i + \beta y_i \f]  i iterates over @b all elements inside the container. Specifically for a std::vector<container_type> i includes both the inner and the outer loop. If the container sizes
- * do not match the result is undefined.
+ * do not match, the result is undefined.
  * @copydoc hide_container_lvl1
  * @param alpha Scalar  
  * @param x container x may alias y 
@@ -95,7 +95,7 @@ inline void axpby( typename VectorTraits<container>::value_type alpha, const con
 /*! @brief Modified BLAS 1 routine axpy
  *
  * This routine computes \f[ z_i =  \alpha x_i + \beta y_i \f]  i iterates over @b all elements inside the container. Specifically for a std::vector<container_type> i includes both the inner and the outer loop. If the container sizes
- * do not match the result is undefined.
+ * do not match, the result is undefined.
 
  * @copydoc hide_container_lvl1
  * @param alpha Scalar  
@@ -114,7 +114,7 @@ inline void axpby( typename VectorTraits<container>::value_type alpha, const con
 /*! @brief Modified BLAS 1 routine axpy
  *
  * This routine computes \f[ z_i =  \alpha x_i + \beta y_i + \gamma z_i \f]  i iterates over @b all elements inside the container. Specifically for a std::vector<container_type> i includes both the inner and the outer loop. If the container sizes
- * do not match the result is undefined.
+ * do not match, the result is undefined.
 
  * @copydoc hide_container_lvl1
  * @param alpha Scalar  
@@ -180,7 +180,7 @@ inline void plus( container& x, typename VectorTraits<container>::value_type alp
 * @brief A 'new' BLAS 1 routine. 
 *
 * Multiplies two vectors element by element: \f[ y_i = x_{1i}x_{2i}\f]  i iterates over @b all elements inside the container. Specifically for a std::vector<container_type> i includes both the inner and the outer loop. If the container sizes
- * do not match the result is undefined.
+ * do not match, the result is undefined.
 
 * @copydoc hide_container_lvl1
 * @param x1 container x1  
@@ -198,7 +198,7 @@ inline void pointwiseDot( const container& x1, const container& x2, container& y
 * @brief A 'new' BLAS 1 routine. 
 *
 * Multiplies two vectors element by element: \f[ y_i = \alpha x_{1i}x_{2i} + \beta y_i\f]  i iterates over @b all elements inside the container. Specifically for a std::vector<container_type> i includes both the inner and the outer loop. If the container sizes
- * do not match the result is undefined.
+ * do not match, the result is undefined.
 
 * @copydoc hide_container_lvl1
 * @param alpha scalar
@@ -217,7 +217,7 @@ inline void pointwiseDot( typename VectorTraits<container>::value_type alpha, co
 * @brief A 'new' BLAS 1 routine. 
 *
 * Divides two vectors element by element: \f[ y_i = x_{1i}/x_{2i}\f]  i iterates over @b all elements inside the container. Specifically for a std::vector<container_type> i includes both the inner and the outer loop. If the container sizes
- * do not match the result is undefined.
+ * do not match, the result is undefined.
 
 * @copydoc hide_container_lvl1
 * @param x1 container x1  
@@ -235,7 +235,7 @@ inline void pointwiseDivide( const container& x1, const container& x2, container
 * @brief A 'new' fused multiply-add BLAS 1 routine. 
 *
 * Multiplies and adds vectors element by element: \f[ z_i = \alpha x_{1i}y_{1i} + \beta x_{2i}y_{2i} + \gamma z_i \f]  i iterates over @b all elements inside the container. Specifically for a std::vector<container_type> i includes both the inner and the outer loop. If the container sizes
- * do not match the result is undefined.
+ * do not match, the result is undefined.
 * @copydoc hide_container_lvl1
 * @param alpha scalar
 * @param x1 container x1  
diff --git a/inc/doc/Makefile b/inc/doc/Makefile
index 5e91fa9d5..e9a2cf000 100644
--- a/inc/doc/Makefile
+++ b/inc/doc/Makefile
@@ -40,7 +40,7 @@ geometries:
 
 doc: dg file geometries
 	ln -sf dg/html/modules.html index.html
-	echo "Open with: firefox index.html or on Windows: firefox dg/html/modules.html"
+	#Open with: firefox index.html or on Windows: firefox dg/html/modules.html
 
 
 clean:
diff --git a/inc/geometries/average.h b/inc/geometries/average.h
index 04f449302..214cbd08e 100644
--- a/inc/geometries/average.h
+++ b/inc/geometries/average.h
@@ -92,7 +92,7 @@ struct Alpha
 
  with \f$ A = \int dV \delta(\psi_p(R,Z)-\psi_0)|\nabla\psi_p|\f$
  * @copydoc hide_container
- * @ingroup misc
+ * @ingroup misc_geo
  */
 template <class container = thrust::host_vector<double> >
 struct FluxSurfaceAverage
diff --git a/inc/geometries/solovev_doc.h b/inc/geometries/solovev_doc.h
index 34385d400..c53ec2c92 100644
--- a/inc/geometries/solovev_doc.h
+++ b/inc/geometries/solovev_doc.h
@@ -15,7 +15,7 @@
       @defgroup profiles 3.3 miscellaneous functors based on flux functions
  * @}
  * @defgroup fieldaligned 4. Fieldaligned derivatives
- * @defgroup misc 5. Miscellaneous additions
+ * @defgroup misc_geo 5. Miscellaneous additions
  *
  * Objects that are used to define and integrate the magnetic field lines. 
  * All objects can be used in the evaluation() function.
diff --git a/inc/geometries/utilitiesX.h b/inc/geometries/utilitiesX.h
index 081e1b5d5..6e8bba042 100644
--- a/inc/geometries/utilitiesX.h
+++ b/inc/geometries/utilitiesX.h
@@ -13,7 +13,7 @@ namespace geo
     @param psi \f$ \psi(R,Z)\f$, where R, Z are cylindrical coordinates
  * @param R_X start value on input, X-point on output
  * @param Z_X start value on input, X-point on output
- * @ingroup misc
+ * @ingroup misc_geo
  */
 void findXpoint( const BinaryFunctorsLvl2& psi, double& R_X, double& Z_X)
 {
-- 
GitLab


From 37422e6ddcf5e427abb6d6299169634d8ced866f Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sat, 16 Sep 2017 18:03:43 +0200
Subject: [PATCH 294/453] finalize blas2 documentation and unified
 geometry_matrix_container, removed Inverse class and commented function cg

---
 inc/dg/arakawa.h   |   8 ++-
 inc/dg/blas1.h     |  26 +++----
 inc/dg/blas2.h     |  20 +++---
 inc/dg/cg.h        | 167 ++++-----------------------------------------
 inc/dg/dg_doc.h    |  37 +++++-----
 inc/dg/elliptic.h  |  14 ++--
 inc/dg/helmholtz.h |   8 +--
 inc/dg/multigrid.h |  36 ++++++----
 inc/dg/poisson.h   |   5 +-
 9 files changed, 101 insertions(+), 220 deletions(-)

diff --git a/inc/dg/arakawa.h b/inc/dg/arakawa.h
index a9100bd04..f4d16acaf 100644
--- a/inc/dg/arakawa.h
+++ b/inc/dg/arakawa.h
@@ -21,8 +21,9 @@ namespace dg
 /**
  * @brief X-space generalized version of Arakawa's scheme
  *
- * @copydoc hide_matrix_container
- * @copydoc hide_geometry
+ * Computes \f[ [f,g] := 1/\sqrt{g_{2d}}\left(\partial_x f\partial_y g - \partial_y f\partial_x g\right) \f]
+ * where \f$ g_{2d} = g/g_{zz}\f$ is the two-dimensional volume element of the plane in 2x1 product space. 
+ * @copydoc hide_geometry_matrix_container
  * @ingroup arakawa
  */
 template< class Geometry, class Matrix, class container >
@@ -90,7 +91,7 @@ struct ArakawaX
     SparseElement<container> perp_vol_inv_;
     SparseTensor<container> metric_;
 };
-
+///@cond
 template<class Geometry, class Matrix, class container>
 ArakawaX<Geometry, Matrix, container>::ArakawaX( const Geometry& g ): 
     dxlhs( dg::evaluate( one, g) ), dxrhs(dxlhs), dylhs(dxlhs), dyrhs( dxlhs), helper_( dxlhs), 
@@ -133,6 +134,7 @@ void ArakawaX< Geometry, Matrix, container>::operator()( const container& lhs, c
     blas2::symv( 1., bdyf, dxrhs, 1., result);
     tensor::pointwiseDot( perp_vol_inv_, result, result);
 }
+///@endcond
 
 }//namespace dg
 
diff --git a/inc/dg/blas1.h b/inc/dg/blas1.h
index 90eec3c5a..99a58da77 100644
--- a/inc/dg/blas1.h
+++ b/inc/dg/blas1.h
@@ -33,8 +33,8 @@ namespace blas1
 /**
  * @brief Generic way to copy and/or convert a container type to a different container type (e.g. from CPU to GPU, or double to float, etc.)
  *
- * @copydoc hide_container_lvl1
- * @tparam other_container another container class
+ * @copydoc hide_container
+ * @tparam other_container another container type
  * @param x source
  * @param y sink
  * @note y gets resized properly
@@ -61,7 +61,7 @@ inline void copy( const Assignable& x, Assignable& y){y=x;}
  *
  * This routine computes \f[ x^T y = \sum_{i=0}^{N-1} x_i y_i \f]  i iterates over @b all elements inside the container. Specifically for a std::vector<container_type> i includes both the inner and the outer loop. If the container sizes
  * do not match, the result is undefined.
- * @copydoc hide_container_lvl1
+ * @copydoc hide_container
  * @param x Left container
  * @param y Right container may alias x
  * @return Scalar product as defined above
@@ -79,7 +79,7 @@ inline typename VectorTraits<container>::value_type dot( const container& x, con
  *
  * This routine computes \f[ y_i =  \alpha x_i + \beta y_i \f]  i iterates over @b all elements inside the container. Specifically for a std::vector<container_type> i includes both the inner and the outer loop. If the container sizes
  * do not match, the result is undefined.
- * @copydoc hide_container_lvl1
+ * @copydoc hide_container
  * @param alpha Scalar  
  * @param x container x may alias y 
  * @param beta Scalar
@@ -97,7 +97,7 @@ inline void axpby( typename VectorTraits<container>::value_type alpha, const con
  * This routine computes \f[ z_i =  \alpha x_i + \beta y_i \f]  i iterates over @b all elements inside the container. Specifically for a std::vector<container_type> i includes both the inner and the outer loop. If the container sizes
  * do not match, the result is undefined.
 
- * @copydoc hide_container_lvl1
+ * @copydoc hide_container
  * @param alpha Scalar  
  * @param x container x may alias result
  * @param beta Scalar
@@ -116,7 +116,7 @@ inline void axpby( typename VectorTraits<container>::value_type alpha, const con
  * This routine computes \f[ z_i =  \alpha x_i + \beta y_i + \gamma z_i \f]  i iterates over @b all elements inside the container. Specifically for a std::vector<container_type> i includes both the inner and the outer loop. If the container sizes
  * do not match, the result is undefined.
 
- * @copydoc hide_container_lvl1
+ * @copydoc hide_container
  * @param alpha Scalar  
  * @param x container x may alias result
  * @param beta Scalar
@@ -135,7 +135,7 @@ inline void axpbypgz( typename VectorTraits<container>::value_type alpha, const
  *
  * This routine computes \f[ y_i = op(x_i) \f]
  * This is strictly speaking not a BLAS routine since f can be a nonlinear function.
- * @copydoc hide_container_lvl1
+ * @copydoc hide_container
  * @tparam UnaryOp Type with member function: value_type operator()(value_type)  
  * @param x container x may alias y
  * @param y container y contains result, may alias x
@@ -151,7 +151,7 @@ inline void transform( const container& x, container& y, UnaryOp op)
 /*! @brief BLAS 1 routine scal
  *
  * This routine computes \f[ \alpha x_i \f]
- * @copydoc hide_container_lvl1
+ * @copydoc hide_container
  * @param alpha Scalar  
  * @param x container x 
  */
@@ -165,7 +165,7 @@ inline void scal( container& x, typename VectorTraits<container>::value_type alp
 /*! @brief pointwise add a scalar
  *
  * This routine computes \f[ x_i + \alpha \f] 
- * @copydoc hide_container_lvl1
+ * @copydoc hide_container
  * @param alpha Scalar  
  * @param x container x 
  */
@@ -182,7 +182,7 @@ inline void plus( container& x, typename VectorTraits<container>::value_type alp
 * Multiplies two vectors element by element: \f[ y_i = x_{1i}x_{2i}\f]  i iterates over @b all elements inside the container. Specifically for a std::vector<container_type> i includes both the inner and the outer loop. If the container sizes
  * do not match, the result is undefined.
 
-* @copydoc hide_container_lvl1
+* @copydoc hide_container
 * @param x1 container x1  
 * @param x2 container x2 may alias x1
 * @param y  container y contains result on output ( may alias x1 or x2)
@@ -200,7 +200,7 @@ inline void pointwiseDot( const container& x1, const container& x2, container& y
 * Multiplies two vectors element by element: \f[ y_i = \alpha x_{1i}x_{2i} + \beta y_i\f]  i iterates over @b all elements inside the container. Specifically for a std::vector<container_type> i includes both the inner and the outer loop. If the container sizes
  * do not match, the result is undefined.
 
-* @copydoc hide_container_lvl1
+* @copydoc hide_container
 * @param alpha scalar
 * @param x1 container x1  
 * @param x2 container x2 may alias x1
@@ -219,7 +219,7 @@ inline void pointwiseDot( typename VectorTraits<container>::value_type alpha, co
 * Divides two vectors element by element: \f[ y_i = x_{1i}/x_{2i}\f]  i iterates over @b all elements inside the container. Specifically for a std::vector<container_type> i includes both the inner and the outer loop. If the container sizes
  * do not match, the result is undefined.
 
-* @copydoc hide_container_lvl1
+* @copydoc hide_container
 * @param x1 container x1  
 * @param x2 container x2 may alias x1
 * @param y  container y contains result on output ( may alias x1 and/or x2)
@@ -236,7 +236,7 @@ inline void pointwiseDivide( const container& x1, const container& x2, container
 *
 * Multiplies and adds vectors element by element: \f[ z_i = \alpha x_{1i}y_{1i} + \beta x_{2i}y_{2i} + \gamma z_i \f]  i iterates over @b all elements inside the container. Specifically for a std::vector<container_type> i includes both the inner and the outer loop. If the container sizes
  * do not match, the result is undefined.
-* @copydoc hide_container_lvl1
+* @copydoc hide_container
 * @param alpha scalar
 * @param x1 container x1  
 * @param y1 container y1 
diff --git a/inc/dg/blas2.h b/inc/dg/blas2.h
index e51634a3a..ab65d23de 100644
--- a/inc/dg/blas2.h
+++ b/inc/dg/blas2.h
@@ -32,8 +32,8 @@ namespace blas2{
 /**
  * @brief Generic way to copy and/or convert a Matrix type to a different Matrix type (e.g. from CPU to GPU, or double to float, etc.)
  *
- * @tparam Matrix First Matrix type
- * @tparam AnotherMatrix Second Matrix type
+ * @copydoc hide_matrix
+ * @tparam AnotherMatrix Another Matrix type
  * @param x source
  * @param y sink 
  * @note y gets resized properly
@@ -52,7 +52,7 @@ inline void transfer( const Matrix& x, AnotherMatrix& y)
  * precalculate \f$ My\f$ and then call the blas1::dot() routine!
  * @tparam DiagonalMatrix Right now DiagonalMatrix has to be the same as container, except if the container is a std::vector<container_type>, then the DiagonalMatrix has to be the container_type.
  * In the latter case the Matrix is applied to all containers in the std::vector and the sum is returned. 
- * @copydoc hide_container_lvl1
+ * @copydoc hide_container
  * @param x Left container
  * @param m The diagonal Matrix
  * @param y Right container might equal Left container
@@ -74,7 +74,7 @@ inline typename MatrixTraits<DiagonalMatrix>::value_type dot( const container& x
  * \f[ x^T M x = \sum_{i,j=0}^{N-1} x_i M_{ij} x_j \f]
  * @tparam DiagonalMatrix Right now DiagonalMatrix has to be the same as container, except if the container is a std::vector<container_type>, then the DiagonalMatrix has to be the container_type. 
  * In the latter case the Matrix is applied to all containers in the std::vector and the sum is returned. 
- * @copydoc hide_container_lvl1
+ * @copydoc hide_container
  * @param m The diagonal Matrix
  * @param x Right container
  * @return Generalized scalar product
@@ -93,7 +93,8 @@ inline typename MatrixTraits<DiagonalMatrix>::value_type dot( const DiagonalMatr
  *
  * This routine computes \f[ y = \alpha M x + \beta y \f]
  * where \f$ M\f$ is a matrix.
- * @copydoc hide_matrix_container
+ * @copydoc hide_matrix
+ * @copydoc hide_container
  * @param alpha A Scalar
  * @param M The Matrix
  * @param x A container different from y 
@@ -122,7 +123,8 @@ inline void symv( typename MatrixTraits<Matrix>::value_type alpha,
  *
  * This routine computes \f[ y = M x \f]
  * where \f$ M\f$ is a matrix. 
- * @copydoc hide_matrix_container
+ * @copydoc hide_matrix
+ * @copydoc hide_container
  * @tparam same_or_another_container Currently needs to be the same as container.
  * @param M The Matrix
  * @param x A container different from y 
@@ -146,7 +148,8 @@ inline void symv( Matrix& M,
  * @brief General Matrix-Vector product
  *
  * Does exactly the same as symv. 
- * @copydoc hide_matrix_container
+ * @copydoc hide_matrix
+ * @copydoc hide_container
  * @tparam same_or_another_container Currently needs to be the same as container.
  * @param M The Matrix
  * @param x A container different from y 
@@ -168,7 +171,8 @@ inline void gemv( Matrix& M,
  * @brief General Matrix-Vector product
  *
  * Does exactly the same as symv. 
- * @copydoc hide_matrix_container
+ * @copydoc hide_matrix
+ * @copydoc hide_container
  * @param alpha A Scalar
  * @param M The Matrix
  * @param x A container different from y 
diff --git a/inc/dg/cg.h b/inc/dg/cg.h
index 88fb3438d..2a1163c3e 100644
--- a/inc/dg/cg.h
+++ b/inc/dg/cg.h
@@ -16,21 +16,14 @@
 
 namespace dg{
 
-//// TO DO: check for better stopping criteria using condition number estimates
+//// TO DO: check for better stopping criteria using condition number estimates?
 
 /**
 * @brief Functor class for the preconditioned conjugate gradient method to solve
 * \f[ Ax=b\f]
 *
  @ingroup invert
- @copydoc hide_container_lvl1
-
- The following 3 pseudo - BLAS routines need to be callable 
- \li value_type dot = dg::blas1::dot( const container&, const container&); 
- \li dg::blas1::axpby();  with the container type
- \li dg::blas2::symv(SymmetricOp& m, container1& x, container2& y ); with the SymmetricOp type
- \li value_type dot = dg::blas2::dot( );  with the Preconditioner type
- \li dg::blas2::symv( ); with the Preconditioner type
+ @copydoc hide_container
 
  @note Conjugate gradients might become unstable for positive semidefinite
  matrices arising e.g. in the discretization of the periodic laplacian
@@ -79,10 +72,9 @@ class CG
      *
      * The iteration stops if \f$ ||Ax|| < \epsilon( ||b|| + C) \f$ where \f$C\f$ is 
      * a correction factor to the absolute error
-     * @copydoc hide_symmetric_op
+     * @copydoc hide_matrix
      * @tparam Preconditioner A class for which the blas2::symv() and 
-     blas2::dot( const Matrix&, const Vector&) functions are callable. This can for example be one of the container classes (diagonal preconditioner), 
-     which should be enough if the initial guess is extrapolated from previous timesteps.
+     blas2::dot( const Matrix&, const Vector&) functions are callable. Currently Preconditioner must be the same as container (diagonal preconditioner) except when container is std::vector<container_type> then Preconditioner can be container_type
      * @param A A symmetric positive definit matrix
      * @param x Contains an initial value on input and the solution on output.
      * @param b The right hand side vector. x and b may be the same vector.
@@ -93,15 +85,15 @@ class CG
      *
      * @return Number of iterations used to achieve desired precision
      */
-    template< class SymmetricOp, class Preconditioner >
-    unsigned operator()( SymmetricOp& A, container& x, const container& b, Preconditioner& P , value_type eps = 1e-12, value_type nrmb_correction = 1);
+    template< class Matrix, class Preconditioner >
+    unsigned operator()( Matrix& A, container& x, const container& b, Preconditioner& P , value_type eps = 1e-12, value_type nrmb_correction = 1);
     //version of CG where Preconditioner is not trivial
     /**
      * @brief Solve the system A*x = b using a preconditioned conjugate gradient method
      *
      * The iteration stops if \f$ ||Ax||_S < \epsilon( ||b||_S + C) \f$ where \f$C\f$ is 
      * a correction factor to the absolute error and \f$ S \f$ defines a square norm
-     * @copydoc hide_symmetric_op
+     * @copydoc hide_matrix
      * @tparam Preconditioner A type for which the blas2::symv(Matrix&, Vector1&, Vector2&) function is callable. 
      * @tparam SquareNorm A type for which the blas2::dot( const Matrix&, const Vector&) function is callable. This can e.g. be one of the container types.
      * @param A A symmetric positive definit matrix
@@ -114,8 +106,8 @@ class CG
      *
      * @return Number of iterations used to achieve desired precision
      */
-    template< class SymmetricOp, class Preconditioner, class SquareNorm >
-    unsigned operator()( SymmetricOp& A, container& x, const container& b, Preconditioner& P, SquareNorm& S, value_type eps = 1e-12, value_type nrmb_correction = 1);
+    template< class Matrix, class Preconditioner, class SquareNorm >
+    unsigned operator()( Matrix& A, container& x, const container& b, Preconditioner& P, SquareNorm& S, value_type eps = 1e-12, value_type nrmb_correction = 1);
   private:
     container r, p, ap; 
     unsigned max_iter;
@@ -259,10 +251,10 @@ unsigned CG< container>::operator()( Matrix& A, container& x, const container& b
 
 
 /**
-* @brief Class that manages the solutions of iterative methods and
+* @brief Class that stores a number of solutions of iterative methods and
 can be used to get initial guesses based on past solutions
 *
-* @copydoc hide_container_lvl1
+* @copydoc hide_container
 */
 template<class container>
 struct Extrapolation
@@ -359,7 +351,7 @@ struct Extrapolation
  * by appropriate weights \f$W\f$ (s. comment below). 
  * It uses solutions from the last two calls to 
  * extrapolate a solution for the current call.
- * @copydoc hide_container_lvl1
+ * @copydoc hide_container
  * @note A note on weights, inverse weights and preconditioning. 
  * A normalized DG-discretized derivative or operator is normally not symmetric. 
  * The diagonal coefficient matrix that is used to make the operator 
@@ -375,10 +367,7 @@ struct Invert
 {
     typedef typename VectorTraits<container>::value_type value_type;
 
-    /**
-     * @brief Allocate nothing
-     *
-     */
+    ///@brief Allocate nothing
     Invert() { multiplyWeights_ = true; nrmb_correction_ = 1.; }
 
     /**
@@ -442,26 +431,21 @@ struct Invert
      *
      * @param extrapolationType number of last values to use for next extrapolation of initial guess
      */
-    void set_extrapolationType( int extrapolationType)
-    {
+    void set_extrapolationType( int extrapolationType) {
         m_ex.set_type( extrapolationType);
     }
     /**
      * @brief Set the maximum number of iterations 
-     *
      * @param new_max New maximum number
      */
     void set_max( unsigned new_max) {cg.set_max( new_max);}
     /**
      * @brief Get the current maximum number of iterations
-     *
      * @return the current maximum
      */
     unsigned get_max() const {return cg.get_max();}
 
-    /**
-    * @brief Return last solution
-    */
+    /// @brief Return last solution
     const container& get_last() const { return m_ex.head();}
 
     /**
@@ -546,127 +530,6 @@ struct Invert
     bool multiplyWeights_; 
 };
 
-/**
- * @brief This struct holds a matrix and applies its inverse to vectors 
- *
- * The inverse is computed with a conjugate gradient method
- * @ingroup invert
- * @copydoc hide_symmetric_op
- * @copydoc hide_container_lvl1
- */
-template<class SymmetricOp, class container>
-struct Inverse
-{
-    typedef typename VectorTraits<container>::value_type value_type;
-    Inverse( SymmetricOp& op, container& copyable, unsigned max_iter, value_type eps, int extrapolationType=0): 
-        x_(copyable), b_(copyable), op_( op), invert_( copyable, max_iter, eps, extrapolationType, false, 1.){}
-    /**
-     * @brief Computes Op^{-1} b = x
-     *
-     * @param b
-     * @param x
-     */
-    template<class OtherContainer>
-    void symv( const OtherContainer& b, OtherContainer& x)
-    {
-        //std::cout << "Number in inverse "<<invert( op, x, b, op.weights(), op.precond())<<std::endl;
-        dg::blas1::transfer(b,b_);
-        invert_( op_, x_, b_); 
-        dg::blas1::transfer(x_,x);
-    }
-    private:
-    container x_,b_;
-    SymmetricOp op_;
-    Invert<container> invert_;
-};
-
-///@cond
-template< class M, class V>
-struct MatrixTraits< Inverse< M, V > >
-{
-    typedef typename Inverse<M, V>::value_type value_type;
-    typedef SelfMadeMatrixTag matrix_category;
-};
-
-///@endcond
-
-
-/**
- * @brief Function version of CG class
- *
- * @ingroup invert 
- * @tparam Matrix Matrix type
- * @copydoc hide_container_lvl1
- * @tparam Preconditioner Preconditioner type
- * @param A Matrix 
- * @param x contains initial guess on input and solution on output
- * @param b right hand side
- * @param P Preconditioner
- * @param eps relative error
- * @param max_iter maximum iterations allowed
- *
- * @return number of iterations
- */
-/*
-template< class Matrix, class container, class Preconditioner>
-unsigned cg( Matrix& A, container& x, const container& b, const Preconditioner& P, typename VectorTraits<container>::value_type eps, unsigned max_iter)
-{
-    typedef typename VectorTraits<container>::value_type value_type;
-    value_type nrmb = sqrt( blas2::dot( P, b)); //norm of b
-#ifdef DG_DEBUG
-#ifdef MPI_VERSION
-    int rank;
-    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-    if(rank==0)
-#endif //MPI
-    {
-    std::cout << "Norm of b "<<nrmb <<"\n";
-    std::cout << "Residual errors: \n";
-    }
-#endif //DG_DEBUG
-    if( nrmb == 0)
-    {
-        blas1::axpby( 1., b, 0., x); //x=b
-        return 0;
-    }
-    container r(x.size()), p(x.size()), ap(x.size()); //1% time at 20 iterations
-    //r = b; blas2::symv( -1., A, x, 1.,r); //compute r_0 
-    blas2::symv( A,x,r); //r=A x
-    blas1::axpby( 1., b, -1., r); //r=b-Ax
-    blas2::symv( P, r, p );//<-- compute p_0  //p=P(b-Ax)
-    //note that dot does automatically synchronize
-    value_type nrm2r_old = blas2::dot( P,r); //and store the norm of it // norm of r^2 = ||r||^2 = r^T P r 
-    if( sqrt( nrm2r_old ) < eps*nrmb + eps)
-        return 0;
-    value_type alpha, nrm2r_new;
-    for( unsigned i=1; i<max_iter; i++)
-    {
-        blas2::symv( A, p, ap); // ap = A ( P(b-Ax))
-        alpha = nrm2r_old /blas1::dot( p, ap); // alpha = ||r||^2 / ( ((b-Ax)^T P^T) A (P (b-Ax))  )
-        blas1::axpby( alpha, p, 1.,x);
-        //here one could add a ifstatement to remove accumulated floating point error
-        //(if i modulo sqrt(n)) r=b-Ax else ...
-        blas1::axpby( -alpha, ap, 1., r); // r = r-alpha*A ( P(b-Ax))
-        nrm2r_new = blas2::dot( P, r);  //||r_new||^2 =  r^T P r 
-#ifdef DG_DEBUG
-#ifdef MPI_VERSION
-        if(rank==0)
-#endif //MPI
-        {
-        std::cout << "Absolute "<<sqrt( nrm2r_new) <<"\t ";
-        std::cout << " < Critical "<<eps*nrmb + eps <<"\t ";
-        std::cout << "(Relative "<<sqrt( nrm2r_new)/nrmb << ")\n";
-        }
-#endif //DG_DEBUG
-        if( sqrt( nrm2r_new) < eps*nrmb + eps) 
-            return i;
-        blas2::symv(1.,P, r, nrm2r_new/nrm2r_old, p ); //p= 1*P*r + ||r_new||^2 /||r_old||^2 *p
-        nrm2r_old=nrm2r_new;
-    }
-    return max_iter;
-}
-*/
-
 } //namespace dg
 
 
diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index 783e621ef..196a38ab7 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -108,14 +108,8 @@
   * @tparam TernaryOp A class or function type with a member/signature equivalent to
   *  - double operator()(double, double, double) const
   */
- /** @class hide_geometry
-  * @tparam Geometry A type that is or derives from one of the abstract geometry base classes (e.g. aGeometry2d, aGeometry3d, aMPIGeometry2d, ...)
-  * The functions dg::create::dx() and dg::create::dy() must be callable and return an instance convertible to the Matrix class. 
-  * Furthermore dg::evaluate() must return an instance of the container class.
-     as do calls to dg::create::weights() and dg::create::inv_weights()
-  */
 
- /** @class hide_container_lvl1
+ /** @class hide_container
   * @tparam container 
   * A data container class for which the blas1 functionality is overloaded.
   * We assume that container is copyable/assignable and has a swap member function. 
@@ -124,7 +118,7 @@
   *  - std::vector<dg::HVec>, std::vector<dg::DVec>, std::vector<dg::MHVec> or std::vector<dg::MDVec> . 
   *
   */
- /** @class hide_matrix_container
+ /** @class hide_matrix
   * @tparam Matrix 
   * A class for which the blas2 functions are callable in connection with the container class. 
   * The Matrix type can be one of:
@@ -133,22 +127,33 @@
   *  - dg::DMatrix and dg::IDMatrix with dg::DVec or std::vector<dg::DVec>
   *  - dg::MHMatrix with dg::MHVec or std::vector<dg::MHVec>
   *  - dg::MDMatrix with dg::MDVec or std::vector<dg::MDVec>
-  *  - Any type that has the SelfMadeMatrixTag. In this case only those blas2 functions 
+  *  - Any type that has the SelfMadeMatrixTag specified in a corresponding MatrixTraits class (e.g. Elliptic). In this case only those blas2 functions 
   *  that have a corresponding member function in the Matrix class (e.g. symv( const container&, container&); ) can be called.
   *  If the container is a std::vector, then the Matrix is applied to each of the elements.
-  *
+  */
+
+  /** @class hide_geometry_matrix_container
+  * @tparam Geometry 
+  A type that is or derives from one of the abstract geometry base classes (e.g. aGeometry2d, aGeometry3d, aMPIGeometry2d, ...). Geometry determines which Matrix and container types can be used:
+  * @tparam Matrix 
+  * A class for which the blas2 functions are callable in connection with the container class and to which the return type of create::dx() can be converted. 
+  * The Matrix type can be one of:
+  *  - dg::HMatrix with dg::HVec and one of the shared memory geometries
+  *  - dg::DMatrix with dg::DVec and one of the shared memory geometries
+  *  - dg::MHMatrix with dg::MHVec and one of the MPI geometries
+  *  - dg::MDMatrix with dg::MDVec and one of the MPI geometries
   * @tparam container 
-  * A data container class for which the blas1 functionality is overloaded.
+  * A data container class for which the blas1 functionality is overloaded and to which the return type of blas1::evaluate() can be converted. 
   * We assume that container is copyable/assignable and has a swap member function. 
-  * Currently this is one of 
-  *  - dg::HVec, dg::DVec, dg::MHVec or dg::MDVec  
-  *  - std::vector<dg::HVec>, std::vector<dg::DVec>, std::vector<dg::MHVec> or std::vector<dg::MDVec> . 
+  * In connection with Geometry this is one of 
+  *  - dg::HVec, dg::DVec when Geometry is a shared memory geometry
+  *  - dg::MHVec or dg::MDVec when Geometry is one of the MPI geometries
   */
 
  /** @class hide_symmetric_op
  * @tparam SymmetricOp A class for which the blas2::symv(Matrix&, Vector1&, Vector2&) function is callable 
- with the container type as argument. Also, The functions %weights() and %precond() 
- need to be callable and return weights and the preconditioner for the conjugate 
+ with the container type as argument. Also, The functions %inv_weights() and %precond() 
+ need to be callable and return inverse weights and the preconditioner for the conjugate 
  gradient method. The Operator is assumed to be linear and symmetric!
  @note you can make your own SymmetricOp by providing the member function void symv(const container&, container&);
   and specializing MatrixTraits with the SelfMadeMatrixTag as the matrix_category
diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index 23d1900c9..bc859a826 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -35,12 +35,11 @@ namespace dg
  where \f$\alpha\f$  is a scale factor ( = jfactor). Usually the default \f$ \alpha=1 \f$ is a good choice.
  However, in some cases, e.g. when \f$ \chi \f$ exhibits very large variations
  \f$ \alpha=0.1\f$ or \f$ \alpha=0.01\f$ might be better values. 
- In a time dependent problem the value of \f$\alpha\f$ also determines the 
+ In a time dependent problem the value of \f$\alpha\f$ determines the 
  numerical diffusion, i.e. for low values numerical oscillations may appear. 
  Also note that a forward discretization has more diffusion than a centered discretization.
 
- * @copydoc hide_geometry
- * @copydoc hide_matrix_container
+ * @copydoc hide_geometry_matrix_container
  * This class has the SelfMadeMatrixTag so it can be used in blas2::symv functions 
  * and thus in a conjugate gradient solver. 
  * @note The constructors initialize \f$ \chi=1\f$ so that a negative laplacian operator
@@ -228,8 +227,7 @@ class Elliptic
  *  \end{align}
  *  \f] 
  * is discretized, with \f$ b^i\f$ being the contravariant components of \f$\mathbf b\f$ . 
- * @copydoc geometry
- * @copydoc hide_matrix_container
+ * @copydoc hide_geometry_matrix_container
  * This class has the SelfMadeMatrixTag so it can be used in blas2::symv functions 
  * and thus in a conjugate gradient solver. 
  * @note The constructors initialize \f$ b^x = b^y = b^z=1\f$ 
@@ -420,8 +418,7 @@ struct GeneralElliptic
  *  \end{align}
  *  \f] 
  * is discretized, with \f$ b^i\f$ being the contravariant components of \f$\mathbf b\f$ . 
- * @copydoc hide_geometry
- * @copydoc hide_matrix_container
+ * @copydoc hide_geometry_matrix_container
  * This class has the SelfMadeMatrixTag so it can be used in blas2::symv functions 
  * and thus in a conjugate gradient solver. 
  * @note The constructors initialize \f$ \chi_x = \chi_y = \chi_z=1\f$ 
@@ -541,8 +538,7 @@ struct GeneralEllipticSym
  * -\frac{1}{\sqrt{g}} \left(\partial_x(\sqrt{g} v^x ) + \partial_y(\sqrt{g} v^y) \right)
  *  \end{align}
  *  \f] 
- * @copydoc hide_geometry
- * @copydoc hide_matrix_container
+ * @copydoc hide_geometry_matrix_container
  * This class has the SelfMadeMatrixTag so it can be used in blas2::symv functions 
  * and thus in a conjugate gradient solver. 
  * @note The constructors initialize \f$ \chi = I\f$ 
diff --git a/inc/dg/helmholtz.h b/inc/dg/helmholtz.h
index 4f1278298..4a478bc9b 100644
--- a/inc/dg/helmholtz.h
+++ b/inc/dg/helmholtz.h
@@ -19,8 +19,7 @@ namespace dg{
  * Unnormed discretization of \f[ (\chi+\alpha\Delta) \f]
  * where \f$ \chi\f$ is a function and \f$\alpha\f$ a scalar.
  * Can be used by the Invert class
- * @copydoc hide_geometry
- * @copydoc hide_matrix_container
+ * @copydoc hide_geometry_matrix_container
  * @attention The Laplacian in this formula is positive as opposed to the negative sign in the Elliptic operator
  */
 template< class Geometry, class Matrix, class container> 
@@ -146,10 +145,9 @@ struct Helmholtz
  * \f[ \left[ \chi +2 \alpha\Delta +  \alpha^2\Delta \left(\chi^{-1}\Delta \right)\right] \f] 
  * where \f$ \chi\f$ is a function and \f$\alpha\f$ a scalar.
  * Can be used by the Invert class
- * @copydoc hide_geometry
- * @copydoc hide_matrix_container
+ * @copydoc hide_geometry_matrix_container
  * @attention The Laplacian in this formula is positive as opposed to the negative sign in the Elliptic operator
- * @attention It might be better to solve the normal Helmholtz operator twice
+ * @attention It is better to solve the normal Helmholtz operator twice
  * consecutively than solving the Helmholtz2 operator once. 
  */
 template< class Geometry, class Matrix, class container> 
diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index 9bfb2bea7..de6c45c8c 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -11,6 +11,11 @@
 namespace dg
 {
 
+/**
+* @brief Class for the solution of symmetric matrix equation discretizeable on multiple grids
+*
+* @copydoc hide_geometry_matrix_container
+*/
 template< class Geometry, class Matrix, class container> 
 struct MultigridCG2d
 {
@@ -113,8 +118,6 @@ struct MultigridCG2d
     template<class SymmetricOp>
     std::vector<unsigned> direct_solve( std::vector<SymmetricOp>& op, container&  x, const container& b, double eps)
     {
-        Timer t;
-        t.tic();
         dg::blas1::pointwiseDivide(b, op[0].inv_weights(), b_[0]);
         // compute residual r = Wb - A x
         dg::blas2::symv(op[0], x, m_r[0]);
@@ -123,35 +126,37 @@ struct MultigridCG2d
         for( unsigned u=0; u<stages_-1; u++)
             dg::blas2::gemv( interT_[u], m_r[u], m_r[u+1]);
         std::vector<unsigned> number(stages_);
-        Timer t2;
+        Timer t;
         
         dg::blas1::scal( x_[stages_-1], 0.0);
         //now solve residual equations
 		for( unsigned u=stages_-1; u>0; u--)
         {
-            t2.tic();
+            t.tic();
             cg_[u].set_max(grids_[u].get().size());
             number[u] = cg_[u]( op[u], x_[u], m_r[u], op[u].precond(), op[u].inv_weights(), eps/2, 1.);
             dg::blas2::symv( inter_[u-1], x_[u], x_[u-1]);
-            t2.toc();
-            std::cout << "stage: " << u << ", max iter: " << cg_[u].get_max() << ", iter: " << number[u] << ", took "<<t2.diff()<<"s\n";
+            t.toc();
+            std::cout << "stage: " << u << ", iter: " << number[u] << ", took "<<t.diff()<<"s\n";
 
         }
-        t2.tic();
+        t.tic();
 
         //update initial guess
         dg::blas1::axpby( 1., x_[0], 1., x);
         cg_[0].set_max(grids_[0].get().size());
         number[0] = cg_[0]( op[0], x, b_[0], op[0].precond(), op[0].inv_weights(), eps);
-        t2.toc();
-        std::cout << "stage: " << 0 << ", max iter: " << cg_[0].get_max() << ", iter: " << number[0] << ", took "<<t2.diff()<<"s\n";
-        
         t.toc();
-        std::cout<< "Took "<<t.diff()<<"s\n";
+        std::cout << "stage: " << 0 << ", iter: " << number[0] << ", took "<<t.diff()<<"s\n";
+        
         return number;
     }
 
-    ///src may alias first element of out
+    /**
+    * @brief Project vector to all involved grids
+    * @param src the input vector (may alias first element of out)
+    * @param out the input vector projected to all grids ( index 0 contains a copy of src, 1 is the projetion to the first coarse grid, 2 is the next coarser grid, ...)
+    */
     void project( const container& src, std::vector<container>& out)
     {
         dg::blas1::copy( src, out[0]);
@@ -159,6 +164,11 @@ struct MultigridCG2d
             dg::blas2::gemv( project_[u], out[u], out[u+1]);
     }
 
+    /**
+    * @brief Project vector to all involved grids (allocate memory version)
+    * @param src the input vector 
+    * @return the input vector projected to all grids ( index 0 contains a copy of src, 1 is the projetion to the first coarse grid, 2 is the next coarser grid, ...)
+    */
     std::vector<container> project( const container& src)
     {
         std::vector<container> out( grids_.size());
@@ -168,6 +178,7 @@ struct MultigridCG2d
         return out;
 
     }
+    ///@return number of stages 
     unsigned stages()const{return stages_;}
 
     const std::vector<dg::Handle< Geometry > > grids()const { return grids_; }
@@ -231,6 +242,7 @@ private:
         assert(u == 0);
 	}
 
+    ///print scheme information to std::cout
     void PrintScheme(void)
     {
         std::cout << "Scheme: " << std::endl;
diff --git a/inc/dg/poisson.h b/inc/dg/poisson.h
index 5ace405af..9c0132a4d 100644
--- a/inc/dg/poisson.h
+++ b/inc/dg/poisson.h
@@ -23,8 +23,7 @@ namespace dg
  *
  * Equal to the Arakawa class except for the possitility to use mixed boundary conditions
  * @ingroup arakawa
- * @copydoc hide_geometry
- * @copydoc hide_matrix_container
+ * @copydoc hide_geometry_matrix_container
  */
 template< class Geometry, class Matrix, class container >
 struct Poisson
@@ -113,6 +112,7 @@ struct Poisson
     SparseTensor<container> metric_;
 };
 
+///@cond
 //idea: backward transform lhs and rhs and then use bdxf and bdyf , then forward transform
 //needs less memory!! and is faster
 template< class Geometry, class Matrix, class container>
@@ -165,6 +165,7 @@ void Poisson< Geometry, Matrix, container>::operator()( const container& lhs, co
     blas1::pointwiseDot( 1., dxlhslhs_, dyrhsrhs_, -1., dylhslhs_, dxrhsrhs_, 0., result);   
     tensor::pointwiseDot( perp_vol_inv_, result, result);
 }
+///@endcond
 
 }//namespace dg
 
-- 
GitLab


From 2c8270db27d31c70ab6b1854d9ed7816f9da26be Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sat, 16 Sep 2017 21:22:03 +0200
Subject: [PATCH 295/453] in the middle of documentation efforts

---
 inc/dg/Doxyfile                           | 10 ++-
 inc/dg/arakawa.h                          |  2 +-
 inc/dg/arakawa_mpib.cu                    |  2 +-
 inc/dg/arakawa_mpit.cu                    |  2 +-
 inc/dg/backend/average.cuh                |  4 +-
 inc/dg/backend/average.h                  |  4 +-
 inc/dg/backend/creation.cuh               |  1 -
 inc/dg/backend/cusp_precon_blas.h         |  9 +--
 inc/dg/backend/derivatives.h              |  4 +-
 inc/dg/backend/derivativesX.h             |  3 +-
 inc/dg/backend/dgtensor.cuh               |  5 ++
 inc/dg/backend/dlt.h                      |  5 ++
 inc/dg/backend/dx.h                       |  3 +-
 inc/dg/backend/dxX.h                      |  3 +-
 inc/dg/backend/ell_interpolation.cuh      |  7 +-
 inc/dg/backend/evaluation.cuh             |  3 +-
 inc/dg/backend/evaluationX.cuh            |  3 +-
 inc/dg/backend/exceptions.h               |  6 +-
 inc/dg/backend/fast_interpolation.h       | 12 ++-
 inc/dg/backend/functions.h                |  2 +-
 inc/dg/backend/grid.h                     | 90 +++++++++++++++--------
 inc/dg/backend/gridX.h                    |  3 +-
 inc/dg/backend/interpolation.cuh          |  2 +-
 inc/dg/backend/interpolationX.cuh         |  2 +-
 inc/dg/backend/matrix_traits.h            | 10 ++-
 inc/dg/backend/mpi_evaluation.h           |  3 +-
 inc/dg/backend/mpi_grid.h                 | 65 +++++++++-------
 inc/dg/backend/mpi_init.h                 | 38 +++++++++-
 inc/dg/backend/mpi_matrix.h               | 25 +++----
 inc/dg/backend/mpi_precon.h               |  5 ++
 inc/dg/backend/projection.cuh             |  2 +-
 inc/dg/backend/transpose.h                |  2 +-
 inc/dg/backend/typedefs.cuh               |  9 +--
 inc/dg/backend/vector_categories.h        |  6 --
 inc/dg/backend/vector_traits.h            |  6 +-
 inc/dg/backend/weights.cuh                |  3 +-
 inc/dg/backend/weightsX.cuh               |  4 +-
 inc/dg/backend/xspacelib.cuh              |  2 +-
 inc/dg/blas1_mpib.cu                      |  2 +-
 inc/dg/blas_mpib.cu                       |  2 +-
 inc/dg/cg.h                               | 11 ++-
 inc/dg/cg2d_b.cu                          | 11 ---
 inc/dg/cg2d_mpib.cu                       |  2 +-
 inc/dg/cg2d_mpit.cu                       |  2 +-
 inc/dg/cg2d_t.cu                          |  4 +-
 inc/dg/cg3d_mpib.cu                       |  2 +-
 inc/dg/dg_doc.h                           | 23 +++++-
 inc/dg/elliptic.h                         |  2 +-
 inc/dg/elliptic2d_mpib.cu                 |  2 +-
 inc/dg/elliptic_b.cu                      |  8 --
 inc/dg/elliptic_mpib.cu                   |  2 +-
 inc/dg/enums.h                            |  3 +-
 inc/dg/geometry/base_geometry.h           | 12 ++-
 inc/dg/geometry/mpi_base.h                | 62 +++++++++-------
 inc/dg/geometry/multiply.h                | 23 +++---
 inc/dg/geometry/refined_grid.h            |  4 +-
 inc/dg/geometry/tensor.h                  |  4 +-
 inc/dg/geometry/transform.h               | 19 +++--
 inc/dg/helmholtz.h                        |  2 +-
 inc/dg/multigrid.h                        | 25 ++++++-
 inc/dg/multistep.h                        |  9 +--
 inc/dg/nullstelle.h                       |  2 +-
 inc/dg/runge_kutta.h                      | 13 ++--
 inc/geometries/ds_mpib.cu                 |  2 +-
 inc/geometries/dz_mpit.cu                 |  2 +-
 inc/geometries/geometry_advection_mpib.cu |  2 +-
 inc/geometries/geometry_elliptic_mpib.cu  |  2 +-
 inc/geometries/ribeiro_mpit.cu            |  2 +-
 68 files changed, 374 insertions(+), 254 deletions(-)

diff --git a/inc/dg/Doxyfile b/inc/dg/Doxyfile
index c585ef5dd..a254af436 100644
--- a/inc/dg/Doxyfile
+++ b/inc/dg/Doxyfile
@@ -824,7 +824,12 @@ RECURSIVE              = YES
 # Note that relative paths are relative to the directory from which doxygen is
 # run.
 
-EXCLUDE                =
+EXCLUDE                = ../dg/backend/creation.cuh \
+                         ../dg/backend/matrix_traits_thrust.h \
+                         ../dg/backend/cusp_thrust_backend.h \
+                         ../dg/backend/sparseblockmat_gpu_kernels.cuh \
+                         ../dg/backend/sparseblockmat_omp_kernels.h \
+                         ../dg/backend/transpose.h \
 
 # The EXCLUDE_SYMLINKS tag can be used to select whether or not files or
 # directories that are symbolic links (a Unix file system feature) are excluded
@@ -840,7 +845,8 @@ EXCLUDE_SYMLINKS       = NO
 # Note that the wildcards are matched against the file with absolute path, so to
 # exclude all test directories for example use the pattern */test/*
 
-EXCLUDE_PATTERNS       =
+EXCLUDE_PATTERNS       = */*_blas.cuh \
+                         */*_blas.h
 
 # The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names
 # (namespaces, classes, functions, etc.) that should be excluded from the
diff --git a/inc/dg/arakawa.h b/inc/dg/arakawa.h
index f4d16acaf..46823dec6 100644
--- a/inc/dg/arakawa.h
+++ b/inc/dg/arakawa.h
@@ -13,7 +13,7 @@
 
 /*! @file 
   
-  object for computation of Poisson bracket
+  @brief object for computation of Poisson bracket
   */
 namespace dg
 {
diff --git a/inc/dg/arakawa_mpib.cu b/inc/dg/arakawa_mpib.cu
index 9bc1e833b..8dd075573 100644
--- a/inc/dg/arakawa_mpib.cu
+++ b/inc/dg/arakawa_mpib.cu
@@ -49,7 +49,7 @@ int main(int argc, char* argv[])
     int rank;
     unsigned n, Nx, Ny; 
     MPI_Comm comm;
-    mpi_init2d( bcx, bcy, n, Nx, Ny, comm);
+    dg::mpi_init2d( bcx, bcy, n, Nx, Ny, comm);
     dg::MPIGrid2d grid( 0, lx, 0, ly, n, Nx, Ny, bcx, bcy, comm);
     MPI_Comm_rank( MPI_COMM_WORLD, &rank);
     dg::Timer t;
diff --git a/inc/dg/arakawa_mpit.cu b/inc/dg/arakawa_mpit.cu
index 12c6275c1..4b216b062 100644
--- a/inc/dg/arakawa_mpit.cu
+++ b/inc/dg/arakawa_mpit.cu
@@ -61,7 +61,7 @@ int main(int argc, char* argv[])
     int rank;
     unsigned n, Nx, Ny; 
     MPI_Comm comm;
-    mpi_init2d( bcx, bcy, n, Nx, Ny, comm);
+    dg::mpi_init2d( bcx, bcy, n, Nx, Ny, comm);
     dg::MPIGrid2d grid( 0, lx, 0, ly, n, Nx, Ny, bcx, bcy, comm);
     MPI_Comm_rank( MPI_COMM_WORLD, &rank);
 
diff --git a/inc/dg/backend/average.cuh b/inc/dg/backend/average.cuh
index 6b5dbbd87..ded066493 100644
--- a/inc/dg/backend/average.cuh
+++ b/inc/dg/backend/average.cuh
@@ -6,9 +6,7 @@
 #include "../blas1.h"
 
 /*! @file 
-
-  Contains classes for poloidal and toroidal average computations.
-
+  @brief contains classes for poloidal and toroidal average computations.
   */
 namespace dg{
 //struct printf_functor
diff --git a/inc/dg/backend/average.h b/inc/dg/backend/average.h
index 646a2c3ed..bc17c4261 100644
--- a/inc/dg/backend/average.h
+++ b/inc/dg/backend/average.h
@@ -6,9 +6,7 @@
 #include "../blas1.h"
 
 /*! @file 
-
-  Contains classes for poloidal and toroidal average computations.
-
+  @brief contains classes for poloidal and toroidal average computations.
   */
 namespace dg{
 
diff --git a/inc/dg/backend/creation.cuh b/inc/dg/backend/creation.cuh
index a867d1b7d..aabab2779 100644
--- a/inc/dg/backend/creation.cuh
+++ b/inc/dg/backend/creation.cuh
@@ -77,7 +77,6 @@ void add_line( cusp::coo_matrix<int, T, cusp::host_memory>& hm,
 } //namespace detail
 } //namespace create
 } //namespace dg 
-
 ///@endcond
 
 #endif // _DG_CREATION_CUH
diff --git a/inc/dg/backend/cusp_precon_blas.h b/inc/dg/backend/cusp_precon_blas.h
index f29071fbe..a0adeca07 100644
--- a/inc/dg/backend/cusp_precon_blas.h
+++ b/inc/dg/backend/cusp_precon_blas.h
@@ -10,8 +10,7 @@
 
 namespace dg{
 
-    
-/////@cond
+///@cond
 template<class T,class M>
 struct MatrixTraits<cusp::array1d<T,M> >
 {
@@ -24,11 +23,10 @@ struct MatrixTraits<const cusp::array1d<T,M> >
     typedef typename cusp::array1d<T,M>::value_type value_type;
     typedef CuspPreconTag matrix_category; //default is a ThrustVector
 };
-/////@endcond
+
 
 
 namespace blas2{
-    ///@cond
 namespace detail{
 
 
@@ -56,6 +54,7 @@ namespace detail{
 
 
 }//namespace detail
-    ///@endcond
 } //namespace blas2
+
+///@endcond
 } //namespace dg
diff --git a/inc/dg/backend/derivatives.h b/inc/dg/backend/derivatives.h
index 6f1454b5a..377ae0e3b 100644
--- a/inc/dg/backend/derivatives.h
+++ b/inc/dg/backend/derivatives.h
@@ -4,9 +4,9 @@
 #include "dx.h"
 
 /*! @file 
-  
-  Convenience functions to create 2D derivatives
+  @brief Convenience functions to create 2D derivatives
   */
+
 namespace dg{
 
 
diff --git a/inc/dg/backend/derivativesX.h b/inc/dg/backend/derivativesX.h
index 73b0c6b66..33adedaa0 100644
--- a/inc/dg/backend/derivativesX.h
+++ b/inc/dg/backend/derivativesX.h
@@ -4,8 +4,7 @@
 #include "dxX.h"
 
 /*! @file 
-  
-  Convenience functions to create 2D derivatives
+  @brief Convenience functions to create 2D derivatives on X-point topology
   */
 namespace dg{
 
diff --git a/inc/dg/backend/dgtensor.cuh b/inc/dg/backend/dgtensor.cuh
index 1741d502d..6ad7aa870 100644
--- a/inc/dg/backend/dgtensor.cuh
+++ b/inc/dg/backend/dgtensor.cuh
@@ -9,6 +9,11 @@
 #include <thrust/inner_product.h>
 #include <thrust/iterator/zip_iterator.h>
 
+
+
+/**@file
+* @brief a tensor product function between two cusp matrices
+*/
 namespace dg
 {
 
diff --git a/inc/dg/backend/dlt.h b/inc/dg/backend/dlt.h
index 66a1fb88d..484b78bd1 100644
--- a/inc/dg/backend/dlt.h
+++ b/inc/dg/backend/dlt.h
@@ -8,6 +8,9 @@
 #include "exceptions.h"
 
 
+/**@file
+* @brief contains the discrete legendre trafo class
+*/
 namespace dg{
 
 /**
@@ -68,6 +71,7 @@ class DLT
     std::vector<T> a_, w_, forw_, back_, backEQ_;
 };
 
+///@cond
 template<class T>
 DLT<T>::DLT( unsigned n):a_(n), w_(n), forw_(n*n), back_(n*n),backEQ_(n*n)
 {
@@ -194,5 +198,6 @@ DLT<T>::DLT( unsigned n):a_(n), w_(n), forw_(n*n), back_(n*n),backEQ_(n*n)
 }
 
 
+///@endcond
 } //namespace dg
 #endif//_DLT_CUH_
diff --git a/inc/dg/backend/dx.h b/inc/dg/backend/dx.h
index a9e17b2f6..ae49dd34f 100644
--- a/inc/dg/backend/dx.h
+++ b/inc/dg/backend/dx.h
@@ -10,8 +10,7 @@
 
 //What happens for N=1?
 /*! @file 
-  
-  Simple 1d derivatives
+  @brief Simple 1d derivatives
   */
 namespace dg
 {
diff --git a/inc/dg/backend/dxX.h b/inc/dg/backend/dxX.h
index 08717953b..3677e2d66 100644
--- a/inc/dg/backend/dxX.h
+++ b/inc/dg/backend/dxX.h
@@ -8,8 +8,7 @@
 #include "sparseblockmat.h"
 
 /*! @file 
-  
-  Simple 1d derivatives
+  @brief Simple 1d derivatives on X-point topology
   */
 namespace dg
 {
diff --git a/inc/dg/backend/ell_interpolation.cuh b/inc/dg/backend/ell_interpolation.cuh
index d8b1790e0..754d03c7d 100644
--- a/inc/dg/backend/ell_interpolation.cuh
+++ b/inc/dg/backend/ell_interpolation.cuh
@@ -11,19 +11,22 @@
 /**
 * @file 
 
-Contains creation function to create an interpolation ell_matrix on the
+@brief contains creation functions to create an interpolation ell_matrix on the
     device
 */
 
 namespace dg
 {
+///@addtogroup typedefs
+///@{
 //interpolation matrices
 typedef cusp::csr_matrix<int, double, cusp::host_memory> IHMatrix; //!< CSR host Matrix
 typedef cusp::csr_matrix<int, double, cusp::device_memory> IDMatrix; //!< CSR device Matrix
+///@}
 
 namespace create
 {
-    ///@cond
+///@cond
 namespace detail
 {
 
diff --git a/inc/dg/backend/evaluation.cuh b/inc/dg/backend/evaluation.cuh
index 2d7db0cf6..7146d223e 100644
--- a/inc/dg/backend/evaluation.cuh
+++ b/inc/dg/backend/evaluation.cuh
@@ -6,8 +6,7 @@
 #include "weights.cuh"
 
 /*! @file 
-  
-  Function discretization routines
+  @brief Function discretization routines
   */
 namespace dg
 {
diff --git a/inc/dg/backend/evaluationX.cuh b/inc/dg/backend/evaluationX.cuh
index 7983e981b..70dfbbf4a 100644
--- a/inc/dg/backend/evaluationX.cuh
+++ b/inc/dg/backend/evaluationX.cuh
@@ -5,8 +5,7 @@
 
 
 /*! @file 
-  
-  Function discretization routines
+  @brief Function discretization routines on X-point topology
   */
 namespace dg
 {
diff --git a/inc/dg/backend/exceptions.h b/inc/dg/backend/exceptions.h
index e08c20378..b835f4f4a 100644
--- a/inc/dg/backend/exceptions.h
+++ b/inc/dg/backend/exceptions.h
@@ -9,9 +9,13 @@
 #include <iostream>
 #include <sstream>
 
-/*! for the simplified construction of a Message use this Macro*/
 #define _ping_ __FILE__, __LINE__ 
 
+
+
+/**@file
+* @brief Error classes or the dg library
+*/
 namespace dg
 {
 
diff --git a/inc/dg/backend/fast_interpolation.h b/inc/dg/backend/fast_interpolation.h
index df490d442..f8a194d3d 100644
--- a/inc/dg/backend/fast_interpolation.h
+++ b/inc/dg/backend/fast_interpolation.h
@@ -14,6 +14,12 @@
 #include "mpi_matrix.h"
 #endif //MPI_VERSION
 
+
+
+/**@file
+* @brief contains a matrix type for fast interpolations/projections
+*/
+
 namespace dg
 {
 
@@ -22,7 +28,8 @@ namespace dg
  *
  * \f[ y = M_{N-1}(...M_1(M_0x))\f]
  * where \f$ M_i\f$ is the i-th matrix 
- * @copydoc hide_matrix_container
+ * @copydoc hide_matrix
+ * @copydoc hide_container
  * @ingroup misc
  */
 template <class Matrix, class container>
@@ -35,6 +42,7 @@ struct MultiMatrix
     * @attention it is the user's reponsibility to allocate memory for the intermediate "temp" vectors
     */
     MultiMatrix( int dimension): inter_(dimension), temp_(dimension-1 > 0 ? dimension-1 : 0 ){}
+
     template<class OtherMatrix, class OtherContainer>
     MultiMatrix( const MultiMatrix<OtherMatrix, OtherContainer>& src){
         unsigned dimsM = src.get_matrices().size();
@@ -71,6 +79,7 @@ struct MultiMatrix
     std::vector<Buffer<container> > temp_;
 };
 
+///@cond
 template <class M, class V>
 struct MatrixTraits<MultiMatrix<M, V> >
 {
@@ -265,4 +274,5 @@ MultiMatrix< RowColDistMat<EllSparseBlockMat<double>, CooSparseBlockMat<double>,
 #endif //MPI_VERSION
 }//namespace create
 
+///@endcond
 }//namespace dg
diff --git a/inc/dg/backend/functions.h b/inc/dg/backend/functions.h
index a2fecc50f..64c4d0013 100644
--- a/inc/dg/backend/functions.h
+++ b/inc/dg/backend/functions.h
@@ -2,7 +2,7 @@
 #pragma once
 /*! @file
  *
- * Contains some utility functions for the evaluation() routines
+ * @brief contains some utility functions for the evaluation() routines
  */
 
 namespace dg{
diff --git a/inc/dg/backend/grid.h b/inc/dg/backend/grid.h
index 9c0676dbf..048c60e8b 100644
--- a/inc/dg/backend/grid.h
+++ b/inc/dg/backend/grid.h
@@ -7,10 +7,47 @@
 #include "../enums.h"
 
 /*! @file 
-  
-  aTopology objects
+  @brief base topology classes
   */
 
+/*!@class hide_grid_parameters2d
+ * @brief Construct a 2D grid
+ *
+ * @param x0 left boundary in x
+ * @param x1 right boundary in x 
+ * @param y0 lower boundary in y
+ * @param y1 upper boundary in y 
+ * @param n  # of polynomial coefficients per dimension
+ *  (1<=n<=20, note that the library is optimized for n=3 )
+ * @param Nx # of points in x 
+ * @param Ny # of points in y
+ */
+/*!@class hide_bc_parameters2d
+ * @param bcx boundary condition in x
+ * @param bcy boundary condition in y
+ */
+
+/*!@class hide_grid_parameters3d
+ * @brief Construct a 3D topology
+ *
+ * @param x0 left boundary in x
+ * @param x1 right boundary in x 
+ * @param y0 lower boundary in y
+ * @param y1 upper boundary in y 
+ * @param z0 lower boundary in z
+ * @param z1 upper boundary in z 
+ * @param n  # of polynomial coefficients per (x-,y-) dimension
+ *   (1<=n<=20, note that the library is optimized for n=3 )
+ * @attention # of polynomial coefficients in z direction is always 1
+ * @param Nx # of points in x 
+ * @param Ny # of points in y
+ * @param Nz # of points in z
+ */
+/*!@class hide_bc_parameters3d
+ * @param bcx boundary condition in x
+ * @param bcy boundary condition in y
+ * @param bcz boundary condition in z
+ */
 
 namespace dg{
 
@@ -33,7 +70,8 @@ struct Grid1d
      * 
      * @param x0 left boundary
      * @param x1 right boundary
-     * @param n # of polynomial coefficients
+     * @param n # of polynomial coefficients 
+     *  (1<=n<=20, note that the library is optimized for n=3 )
      * @param N # of cells
      * @param bcx boundary conditions
      */
@@ -371,17 +409,8 @@ struct aTopology2d
     ///disallow destruction through base class pointer
     ~aTopology2d(){}
     /**
-     * @brief Construct a 2D grid
-     *
-     * @param x0 left boundary in x
-     * @param x1 right boundary in x 
-     * @param y0 lower boundary in y
-     * @param y1 upper boundary in y 
-     * @param n  # of polynomial coefficients per dimension
-     * @param Nx # of points in x 
-     * @param Ny # of points in y
-     * @param bcx boundary condition in x
-     * @param bcy boundary condition in y
+     *@copydoc hide_grid_parameters2d
+     *@copydoc hide_bc_parameters2d
      */
     aTopology2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx, bc bcy):
         gx_(x0,x1,n,Nx,bcx), gy_(y0,y1,n,Ny,bcy) { }
@@ -397,7 +426,11 @@ struct aTopology2d
         assert( gx.n() == gy.n() );
     }
 
+    ///explicit copy constructor (default)
+    ///@param src source 
     aTopology2d(const aTopology2d& src){gx_=src.gx_, gy_=src.gy_;}
+    ///explicit assignment operator (default)
+    ///@param src source 
     aTopology2d& operator=(const aTopology2d& src){
         gx_=src.gx_;
         gy_=src.gy_;
@@ -637,22 +670,8 @@ struct aTopology3d
     ///disallow deletion through base class pointer
     ~aTopology3d(){}
     /**
-     * @brief Construct a 3D topology
-     *
-     * @param x0 left boundary in x
-     * @param x1 right boundary in x 
-     * @param y0 lower boundary in y
-     * @param y1 upper boundary in y 
-     * @param z0 lower boundary in z
-     * @param z1 upper boundary in z 
-     * @param n  # of polynomial coefficients per (x-,y-) dimension
-     * @param Nx # of points in x 
-     * @param Ny # of points in y
-     * @param Nz # of points in z
-     * @param bcx boundary condition in x
-     * @param bcy boundary condition in y
-     * @param bcz boundary condition in z
-     * @attention # of polynomial coefficients in z direction is always 1
+    @copydoc hide_grid_parameters3d
+    @copydoc hide_bc_parameters3d
      */
     aTopology3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz): 
         gx_(x0,x1,n,Nx,bcx),
@@ -670,7 +689,11 @@ struct aTopology3d
         assert( gx.n() == gy.n());
         assert( gz.n() == 1);
     }
+    ///explicit copy constructor (default)
+    ///@param src source 
     aTopology3d(const aTopology3d& src):gx_(src.gx_),gy_(src.gy_),gz_(src.gz_){}
+    ///explicit assignment operator (default)
+    ///@param src source 
     aTopology3d& operator=(const aTopology3d& src){ //use default in C++11
         gx_=src.gx_; gy_=src.gy_; gz_=src.gz_;
         return *this;
@@ -687,7 +710,8 @@ struct aTopology3d
 struct Grid2d : public aTopology2d
 {
 
-    ///@copydoc aTopology2d::aTopology2d()
+    ///@copydoc hide_grid_parameters2d
+    ///@copydoc hide_bc_parameters2d
     Grid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx = PER, bc bcy = PER):
         aTopology2d(x0,x1,y0,y1,n,Nx,Ny,bcx,bcy) { }
     ///@copydoc aTopology2d::aTopology2d(const Grid1d&,const Grid1d&)
@@ -707,12 +731,14 @@ struct Grid2d : public aTopology2d
  */
 struct Grid3d : public aTopology3d
 {
-    ///@copydoc aTopology3d::aTopology3d()
+    ///@copydoc hide_grid_parameters3d
+    ///@copydoc hide_bc_parameters3d
     Grid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = PER, bc bcy = PER, bc bcz=PER):
         aTopology3d(x0,x1,y0,y1,z0,z1,n,Nx,Ny,Nz,bcx,bcy,bcz) { }
     ///@copydoc aTopology3d::aTopology3d(const Grid1d&,const Grid1d&,const Grid1d&)
     Grid3d( const Grid1d& gx, const Grid1d& gy, const Grid1d& gz): aTopology3d(gx,gy,gz){ }
     ///allow explicit type conversion from any other topology
+    ///@param src source
     explicit Grid3d( const aTopology3d& src): aTopology3d(src){ }
     private:
     virtual void do_set( unsigned n, unsigned Nx, unsigned Ny, unsigned Nz){ 
diff --git a/inc/dg/backend/gridX.h b/inc/dg/backend/gridX.h
index 49eba04fd..87d09e3e0 100644
--- a/inc/dg/backend/gridX.h
+++ b/inc/dg/backend/gridX.h
@@ -6,8 +6,7 @@
 #include "../enums.h"
 
 /*! @file 
-  
-  GridX objects
+  @brief base X-point topology classes
   */
 
 
diff --git a/inc/dg/backend/interpolation.cuh b/inc/dg/backend/interpolation.cuh
index 9d465c4fa..5cae73f7b 100644
--- a/inc/dg/backend/interpolation.cuh
+++ b/inc/dg/backend/interpolation.cuh
@@ -12,7 +12,7 @@
 
 /*! @file
 
-  Contains 1D, 2D and 3D matrix creation functions
+  @brief contains 1D, 2D and 3D interpolation matrix creation functions
   */
 
 namespace dg{
diff --git a/inc/dg/backend/interpolationX.cuh b/inc/dg/backend/interpolationX.cuh
index 3678353ea..fe8da39b8 100644
--- a/inc/dg/backend/interpolationX.cuh
+++ b/inc/dg/backend/interpolationX.cuh
@@ -5,7 +5,7 @@
 
 /*! @file
 
-  Contains 1D, 2D and 3D matrix creation functions
+  @brief contains 1D, 2D and 3D interpolation matrix creation functions on X-point topology
   */
 
 namespace dg{
diff --git a/inc/dg/backend/matrix_traits.h b/inc/dg/backend/matrix_traits.h
index 4fa91ab1c..ba82f8b93 100644
--- a/inc/dg/backend/matrix_traits.h
+++ b/inc/dg/backend/matrix_traits.h
@@ -4,15 +4,17 @@
 #include "matrix_categories.h"
 namespace dg{
 
-    //Note to developpers: if you have problems with the compiler choosing CuspMatrixTag even if you don't want it to and you specialized the MatrixTraits for 
-    //your matrix try to specialize for const Matrix as well
-///@cond
+/*! @brief The matrix traits 
+
+Specialize this struct if you want to enable your class for the use in blas2 functions
+@note if you have problems with the compiler choosing CuspMatrixTag even if you don't want it to and you specialized the MatrixTraits for 
+your matrix try to specialize for const Matrix as well
+*/
 template< class Matrix>
 struct MatrixTraits {
     typedef typename Matrix::value_type value_type;//!< default value type
     typedef CuspMatrixTag matrix_category; //!< default is a CuspMatrix
 };
-///@endcond
 
 
 }//namespace dg
diff --git a/inc/dg/backend/mpi_evaluation.h b/inc/dg/backend/mpi_evaluation.h
index 6e0b51811..80eb14166 100644
--- a/inc/dg/backend/mpi_evaluation.h
+++ b/inc/dg/backend/mpi_evaluation.h
@@ -5,8 +5,7 @@
 #include "evaluation.cuh"
 
 /*! @file 
-  
-  Function discretization routines for mpi vectors
+  @brief Function discretization routines for mpi vectors
   */
 namespace dg
 {
diff --git a/inc/dg/backend/mpi_grid.h b/inc/dg/backend/mpi_grid.h
index 220dd6a34..b31739f33 100644
--- a/inc/dg/backend/mpi_grid.h
+++ b/inc/dg/backend/mpi_grid.h
@@ -3,14 +3,25 @@
 #include <cmath>
 #include "../enums.h"
 #include "grid.h"
+
 /*! @file 
-  
-  MPI Grid objects
+  @brief MPI Grid objects
   */
 
 namespace dg
 {
 
+/*! @class hide_comm_parameters2d
+ * @param comm a two-dimensional Cartesian communicator
+ * @note the paramateres given in the constructor are global parameters 
+ */
+/*! @class hide_comm_parameters3d
+ * @param comm a three-dimensional Cartesian communicator
+ * @note the paramateres given in the constructor are global parameters 
+ */
+
+
+
 
 /**
  * @brief 2D MPI abstract grid class 
@@ -285,20 +296,23 @@ struct aMPITopology2d
     ~aMPITopology2d(){}
 
     /**
-     * @copydoc dg::Grid2d::Grid2d()
-     * @param comm a two-dimensional Cartesian communicator
-     * @note the paramateres given in the constructor are global parameters 
+     * @copydoc hide_grid_parameters2d
+     * @copydoc hide_bc_parameters2d
+     * @copydoc hide_comm_parameters2d
      */
     aMPITopology2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx, bc bcy, MPI_Comm comm):
         g( x0, x1, y0, y1, n, Nx, Ny, bcx, bcy), comm( comm)
     {
         check_division( Nx, Ny, bcx, bcy);
     }
+    ///copydoc aTopology2d::aTopology2d(const aTopology2d&)
     aMPITopology2d(const aMPITopology2d& src):g(src.g),comm(src.comm){}
+    ///copydoc aTopology2d::operator()(const aTopology2d&)
     aMPITopology2d& operator=(const aMPITopology2d& src){
         g = src.g; comm = src.comm;
         return *this;
     }
+    ///This function has an implementation 
     virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny)=0;
     private:
     void check_division( unsigned Nx, unsigned Ny, bc bcx, bc bcy)
@@ -597,29 +611,27 @@ struct aMPITopology3d
         else
             return false;
     }
-    /**
-     *@copydoc aMPITopology2d::local()const
-     */
+    ///@copydoc aMPITopology2d::local()const
     Grid3d local() const {return Grid3d(x0(), x1(), y0(), y1(), z0(), z1(), n(), Nx(), Ny(), Nz(), bcx(), bcy(), bcz());}
-    /**
-     *@copydoc aMPITopology2d::global()const
-     */
+     ///@copydoc aMPITopology2d::global()const
     Grid3d global() const {return g;}
     protected:
     ///disallow deletion through base class pointer
     ~aMPITopology3d(){}
 
-    /**
-     * @copydoc Grid3d::Grid3d()
-     * @param comm a three-dimensional Cartesian communicator
-     * @note the paramateres given in the constructor are global parameters 
-     */
+    ///@copydoc hide_grid_parameters3d
+    ///@copydoc hide_bc_parameters3d
+    ///@copydoc hide_comm_parameters3d
     aMPITopology3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz, MPI_Comm comm):
         g( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, bcx, bcy, bcz), comm( comm)
     {
         check_division( Nx, Ny, Nz, bcx, bcy, bcz);
     }
+    ///explicit copy constructor (default)
+    ///@param src source
     aMPITopology3d(const aMPITopology3d& src):g(src.g),comm(src.comm){}
+    ///explicit assignment operator (default)
+    ///@param src source
     aMPITopology3d& operator=(const aMPITopology3d& src){
         g = src.g; comm = src.comm;
         return *this;
@@ -701,18 +713,17 @@ void aMPITopology3d::do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, u
 struct MPIGrid2d: public aMPITopology2d
 {
     /**
-     * @copydoc dg::Grid2d::Grid2d()
-     * @param comm a two-dimensional Cartesian communicator
-     * @note the paramateres given in the constructor are global parameters 
+     * @copydoc hide_grid_parameters2d
+     * @copydoc hide_comm_parameters2d
      */
     MPIGrid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, MPI_Comm comm):
         aMPITopology2d( x0,x1,y0,y1,n,Nx,Ny,dg::PER,dg::PER,comm)
     { }
 
     /**
-     * @copydoc dg::Grid2d::Grid2d()
-     * @param comm a two-dimensional Cartesian communicator
-     * @note the paramateres given in the constructor are global parameters 
+     * @copydoc hide_grid_parameters2d
+     * @copydoc hide_bc_parameters2d
+     * @copydoc hide_comm_parameters2d
      */
     MPIGrid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx, bc bcy, MPI_Comm comm):
         aMPITopology2d( x0,x1,y0,y1,n,Nx,Ny,bcx,bcy,comm)
@@ -729,15 +740,15 @@ struct MPIGrid2d: public aMPITopology2d
  */
 struct MPIGrid3d : public aMPITopology3d
 {
+    ///@copydoc hide_grid_parameters3d
+    ///@copydoc hide_comm_parameters3d
     MPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, MPI_Comm comm):
         aMPITopology3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, dg::PER, dg::PER, dg::PER,comm )
     { }
 
-    /**
-     * @copydoc Grid3d::Grid3d()
-     * @param comm a three-dimensional Cartesian communicator
-     * @note the paramateres given in the constructor are global parameters 
-     */
+    ///@copydoc hide_grid_parameters3d
+    ///@copydoc hide_bc_parameters3d
+    ///@copydoc hide_comm_parameters3d
     MPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz, MPI_Comm comm):
         aMPITopology3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, bcx, bcy, bcz, comm)
     { }
diff --git a/inc/dg/backend/mpi_init.h b/inc/dg/backend/mpi_init.h
index a20e5aebc..29ba2b58b 100644
--- a/inc/dg/backend/mpi_init.h
+++ b/inc/dg/backend/mpi_init.h
@@ -1,6 +1,27 @@
 #pragma once
-//enums need to be included before this
 
+
+/*!@file
+@brief convenience mpi init functions
+
+enums need to be included before this
+*/
+
+namespace dg
+{
+
+/**
+* @brief Read in number of processses and grid size and create Cartesian MPI communicator
+*
+* Also sets the GPU a process should use in case THRUST_DEVICE_SYSTEM_CUDA
+* @param bcx if bcx==dg::PER then the communicator is periodic in x 
+* @param bcy if bcy==dg::PER then the communicator is periodic in y 
+* @param n read in from std::cin and broadcasted to all processes in MPI_COMM_WORLD
+* @param Nx read in from std::cin and broadcasted to all processes in MPI_COMM_WORLD
+* @param Ny read in from std::cin and broadcasted to all processes in MPI_COMM_WORLD
+* @param comm (write only) a 2d Cartesian MPI communicator
+* @ingroup misc
+*/
 void mpi_init2d( dg::bc bcx, dg::bc bcy, unsigned& n, unsigned& Nx, unsigned& Ny, MPI_Comm& comm  )
 {
     int periods[2] = {false,false};
@@ -44,6 +65,20 @@ void mpi_init2d( dg::bc bcx, dg::bc bcy, unsigned& n, unsigned& Nx, unsigned& Ny
 
 }
 
+/**
+* @brief Read in number of processses and grid size and create Cartesian MPI communicator
+*
+* Also sets the GPU a process should use in case THRUST_DEVICE_SYSTEM_CUDA
+* @param bcx if bcx==dg::PER then the communicator is periodic in x 
+* @param bcy if bcy==dg::PER then the communicator is periodic in y 
+* @param bcz if bcz==dg::PER then the communicator is periodic in z 
+* @param n read in from std::cin and broadcasted to all processes in MPI_COMM_WORLD
+* @param Nx read in from std::cin and broadcasted to all processes in MPI_COMM_WORLD
+* @param Ny read in from std::cin and broadcasted to all processes in MPI_COMM_WORLD
+* @param Nz read in from std::cin and broadcasted to all processes in MPI_COMM_WORLD
+* @param comm (write only) a 3d Cartesian MPI communicator
+* @ingroup misc
+*/
 void mpi_init3d( dg::bc bcx, dg::bc bcy, dg::bc bcz, unsigned& n, unsigned& Nx, unsigned& Ny, unsigned& Nz, MPI_Comm& comm  )
 {
     int periods[3] = {false,false, false};
@@ -87,3 +122,4 @@ void mpi_init3d( dg::bc bcx, dg::bc bcy, dg::bc bcz, unsigned& n, unsigned& Nx,
 #endif//cuda
 
 }
+} //namespace dg
diff --git a/inc/dg/backend/mpi_matrix.h b/inc/dg/backend/mpi_matrix.h
index 78ff66899..2efdf7db6 100644
--- a/inc/dg/backend/mpi_matrix.h
+++ b/inc/dg/backend/mpi_matrix.h
@@ -2,7 +2,12 @@
 
 #include "mpi_vector.h"
 
-//the corresponding blas file for the Local matrix must be included before this file
+/*!@file
+
+@brief MPI matrix classes
+
+@note the corresponding blas file for the Local matrix must be included before this file
+*/
 namespace dg
 {
 
@@ -88,7 +93,9 @@ struct RowColDistMat
     * the global_gather function of the communication object is called. 
     * Finally the outer elements are added with a call to doSymv for the outer matrix
     * @tparam container container class of the vector elements
+    * @param alpha scalar
     * @param x input
+    * @param beta scalar
     * @param y output
     */
     template<class container> 
@@ -159,6 +166,7 @@ struct RowColDistMat
     Collective c_;
 };
 
+///@cond
 /**
 * @brief Distributed memory matrix class
 *
@@ -210,13 +218,6 @@ struct RowDistMat
     */
     const Collective& collective() const{return c_;}
     
-    /**
-    * @brief Apply the matrix to an MPI_Vector
-    *
-    * @tparam container 
-    * @param x
-    * @param y
-    */
     template<class container> 
     void symv( double alpha, const MPI_Vector<container>& x, double beta, MPI_Vector<container>& y)
     {
@@ -303,13 +304,6 @@ struct ColDistMat
     */
     const Collective& collective() const{return c_;}
     
-    /**
-    * @brief Apply the matrix to an MPI_Vector
-    *
-    * @tparam container 
-    * @param x
-    * @param y
-    */
     template<class container> 
     void symv( double alpha, const MPI_Vector<container>& x, double beta, MPI_Vector<container>& y)
     {
@@ -356,7 +350,6 @@ struct ColDistMat
     Collective c_;
 };
 
-///@cond
 template<class LI, class LO, class C>
 struct MatrixTraits<RowColDistMat<LI,LO, C> >
 {
diff --git a/inc/dg/backend/mpi_precon.h b/inc/dg/backend/mpi_precon.h
index 6045675a8..a9fc44098 100644
--- a/inc/dg/backend/mpi_precon.h
+++ b/inc/dg/backend/mpi_precon.h
@@ -4,6 +4,11 @@
 #include "weights.cuh"
 #include "mpi_grid.h"
 
+
+
+/**@file
+* @brief contains MPI weights
+*/
 namespace dg
 {
  
diff --git a/inc/dg/backend/projection.cuh b/inc/dg/backend/projection.cuh
index a513d01b4..0a30740d2 100644
--- a/inc/dg/backend/projection.cuh
+++ b/inc/dg/backend/projection.cuh
@@ -7,7 +7,7 @@
 
 /*!@file 
   
-  contains creation of projection matrices
+  @brief contains creation of projection matrices
  */
 namespace dg{
 ///@addtogroup interpolation
diff --git a/inc/dg/backend/transpose.h b/inc/dg/backend/transpose.h
index 0c6db1c48..9874a7bf9 100644
--- a/inc/dg/backend/transpose.h
+++ b/inc/dg/backend/transpose.h
@@ -28,6 +28,6 @@ Matrix transpose( const Matrix& src)
 {
     out = doTranspose( src, typename MatrixTraits<Matrix>::matrix_category());
 }
-///@endcond
 
 } //namespace dg
+///@endcond
diff --git a/inc/dg/backend/typedefs.cuh b/inc/dg/backend/typedefs.cuh
index 1bdbf8b83..fc9afdf9a 100644
--- a/inc/dg/backend/typedefs.cuh
+++ b/inc/dg/backend/typedefs.cuh
@@ -1,15 +1,10 @@
 #ifndef _DG_TYPEDEFS_CUH_
 #define _DG_TYPEDEFS_CUH_
 
-
-#define FELTOR_MAJOR_VERSION 3
-#define FELTOR_MINOR_VERSION 2
-#define FELTOR_SUBMINOR_VERSION 0
-
 /*! @file
-
-  This file contains useful typedefs of commonly used types.
+  @brief This file contains useful typedefs of commonly used types.
   */
+
 namespace dg{
 
 ///@addtogroup typedefs
diff --git a/inc/dg/backend/vector_categories.h b/inc/dg/backend/vector_categories.h
index 2d7b69eab..3529a8880 100644
--- a/inc/dg/backend/vector_categories.h
+++ b/inc/dg/backend/vector_categories.h
@@ -17,12 +17,6 @@ struct AnyVectorTag{};
 
 struct StdVectorTag:public AnyVectorTag {};
 
-/**
- * @brief The Thrust Vector concept
- *
- * A Thrust vector must have the two methods begin() and end() which
- * return thrust compatible iterators and the value_type typedef
- */
 struct ThrustVectorTag: public AnyVectorTag {};
 
 struct CuspVectorTag: public ThrustVectorTag {};
diff --git a/inc/dg/backend/vector_traits.h b/inc/dg/backend/vector_traits.h
index 0b90d9a0a..c0568d06d 100644
--- a/inc/dg/backend/vector_traits.h
+++ b/inc/dg/backend/vector_traits.h
@@ -6,13 +6,17 @@
 
 namespace dg{
 
-///@cond
+/*! @brief The vector traits 
+
+Specialize this struct if you want to enable your own vector/container class for the use in blas1 functions
+*/
 template< class Vector>
 struct VectorTraits {
     typedef typename Vector::value_type value_type;
     typedef ThrustVectorTag vector_category; //default is a ThrustVector
 };
 
+///@cond
 template< class Vector>
 struct VectorTraits<std::vector<Vector> >{
     typedef typename VectorTraits<Vector>::value_type value_type;
diff --git a/inc/dg/backend/weights.cuh b/inc/dg/backend/weights.cuh
index a439de3b2..da1e4355b 100644
--- a/inc/dg/backend/weights.cuh
+++ b/inc/dg/backend/weights.cuh
@@ -5,8 +5,7 @@
 
 /*! @file
 
-  * contains creation functions for integration weights 
-  * and their inverse
+  * @brief contains creation functions for integration weights and their inverse
   */
 
 namespace dg{
diff --git a/inc/dg/backend/weightsX.cuh b/inc/dg/backend/weightsX.cuh
index d92b5c186..2c58f0199 100644
--- a/inc/dg/backend/weightsX.cuh
+++ b/inc/dg/backend/weightsX.cuh
@@ -5,8 +5,8 @@
 
 /*! @file
 
-  * contains creation functions for integration weights 
-  * and their inverse
+  * @brief contains creation functions for integration weights 
+  * and their inverse on X-point topology
   */
 
 namespace dg{
diff --git a/inc/dg/backend/xspacelib.cuh b/inc/dg/backend/xspacelib.cuh
index 57087b11a..26bc70624 100644
--- a/inc/dg/backend/xspacelib.cuh
+++ b/inc/dg/backend/xspacelib.cuh
@@ -17,7 +17,7 @@
 
 /*! @file
 
-  * provides some utility functions
+  * @brief provides some utility functions
   */
 
 namespace dg{
diff --git a/inc/dg/blas1_mpib.cu b/inc/dg/blas1_mpib.cu
index 18f57d305..df92f5679 100644
--- a/inc/dg/blas1_mpib.cu
+++ b/inc/dg/blas1_mpib.cu
@@ -25,7 +25,7 @@ int main( int argc, char* argv[])
     MPI_Init(&argc, &argv);
     unsigned n, Nx, Ny; 
     MPI_Comm comm;
-    mpi_init2d( dg::PER, dg::PER, n, Nx, Ny, comm);
+    dg::mpi_init2d( dg::PER, dg::PER, n, Nx, Ny, comm);
 
     dg::MPIGrid2d grid( 0., lx, 0, ly, n, Nx, Ny, comm);
     MPIVector w2d, v2d;
diff --git a/inc/dg/blas_mpib.cu b/inc/dg/blas_mpib.cu
index 890ac9d20..30c4323cf 100644
--- a/inc/dg/blas_mpib.cu
+++ b/inc/dg/blas_mpib.cu
@@ -22,7 +22,7 @@ int main( int argc, char* argv[])
     MPI_Init(&argc, &argv);
     unsigned n, Nx, Ny; 
     MPI_Comm comm;
-    mpi_init2d( dg::PER, dg::PER, n, Nx, Ny, comm);
+    dg::mpi_init2d( dg::PER, dg::PER, n, Nx, Ny, comm);
     int rank;
     MPI_Comm_rank( MPI_COMM_WORLD, &rank);
 
diff --git a/inc/dg/cg.h b/inc/dg/cg.h
index 2a1163c3e..cc3d8a4ca 100644
--- a/inc/dg/cg.h
+++ b/inc/dg/cg.h
@@ -70,7 +70,7 @@ class CG
     /**
      * @brief Solve the system A*x = b using a preconditioned conjugate gradient method
      *
-     * The iteration stops if \f$ ||Ax|| < \epsilon( ||b|| + C) \f$ where \f$C\f$ is 
+     * The iteration stops if \f$ ||b - Ax|| < \epsilon( ||b|| + C) \f$ where \f$C\f$ is 
      * a correction factor to the absolute error
      * @copydoc hide_matrix
      * @tparam Preconditioner A class for which the blas2::symv() and 
@@ -475,20 +475,19 @@ struct Invert
      * Solves the Equation \f[ \hat O \phi = W\rho \f] using a preconditioned 
      * conjugate gradient method. The initial guess comes from an extrapolation 
      * of the last solutions.
-     * @copydoc hide_symmetric_op
+     * @copydoc hide_matrix
      * @tparam Preconditioner A type for which the blas2::symv(Matrix&, Vector1&, Vector2&) function is callable. 
      * @param op symmetric Matrix operator class
      * @param phi solution (write only)
      * @param rho right-hand-side
-     * @param w The weights that made the operator symmetric
+     * @param inv_weights The inverse weights that normalize the symmetric operator
      * @param p The preconditioner  
-     * @note computes inverse weights from the weights 
      * @note If the Macro DG_BENCHMARK is defined this function will write timings to std::cout
      *
      * @return number of iterations used 
      */
-    template< class SymmetricOp, class Preconditioner >
-    unsigned operator()( SymmetricOp& op, container& phi, const container& rho, const container& inv_weights, Preconditioner& p)
+    template< class Matrix, class Preconditioner >
+    unsigned operator()( Matrix& op, container& phi, const container& rho, const container& inv_weights, Preconditioner& p)
     {
         assert( phi.size() != 0);
         assert( &rho != &phi);
diff --git a/inc/dg/cg2d_b.cu b/inc/dg/cg2d_b.cu
index 029aae777..9b15bac1f 100644
--- a/inc/dg/cg2d_b.cu
+++ b/inc/dg/cg2d_b.cu
@@ -58,18 +58,7 @@ int main()
     //////////////////////////////////////////////////////////////////////
     std::cout << "Computing on the Grid " <<n<<" x "<<Nx<<" x "<<Ny <<std::endl;
 
-    dg::fDVec xf;
-    dg::blas1::transfer(x,xf);
-    dg::Inverse<dg::Elliptic<dg::CartesianGrid2d, dg::fDMatrix, dg::fDVec>, dg::fDVec> inverse( flap, xf, 10, 1e-15, 0);
-
-
-    
     std::cout << "... for a precision of "<< eps<<std::endl;
-    t.tic();
-    std::cout << "Number of mixed pcg iterations "<< pcg( lap, x, b, inverse, v2d, eps)<<std::endl;
-    t.toc();
-    std::cout << "... on the device took "<< t.diff()<<"s\n";
-    //dg::blas2::symv( inverse, b, x); 
     x = dg::evaluate( initial, grid);
     t.tic();
     std::cout << "Number of pcg iterations "<< pcg( lap, x, b, v2d, eps)<<std::endl;
diff --git a/inc/dg/cg2d_mpib.cu b/inc/dg/cg2d_mpib.cu
index ec425c041..7312ed925 100644
--- a/inc/dg/cg2d_mpib.cu
+++ b/inc/dg/cg2d_mpib.cu
@@ -31,7 +31,7 @@ int main( int argc, char* argv[])
     MPI_Init(&argc, &argv);
     unsigned n, Nx, Ny; 
     MPI_Comm comm;
-    mpi_init2d( bcx, dg::PER, n, Nx, Ny, comm);
+    dg::mpi_init2d( bcx, dg::PER, n, Nx, Ny, comm);
     int rank;
     MPI_Comm_rank( MPI_COMM_WORLD, &rank);
     double eps;
diff --git a/inc/dg/cg2d_mpit.cu b/inc/dg/cg2d_mpit.cu
index 137e84b05..8eecd86a0 100644
--- a/inc/dg/cg2d_mpit.cu
+++ b/inc/dg/cg2d_mpit.cu
@@ -25,7 +25,7 @@ int main( int argc, char* argv[])
     MPI_Init(&argc, &argv);
     unsigned n, Nx, Ny; 
     MPI_Comm comm;
-    mpi_init2d( bcx, dg::PER, n, Nx, Ny, comm);
+    dg::mpi_init2d( bcx, dg::PER, n, Nx, Ny, comm);
 
     dg::MPIGrid2d grid( 0., lx, 0, ly, n, Nx, Ny, bcx, dg::PER, comm);
     const dg::MDVec w2d = dg::create::weights( grid);
diff --git a/inc/dg/cg2d_t.cu b/inc/dg/cg2d_t.cu
index ecb869977..211683f69 100644
--- a/inc/dg/cg2d_t.cu
+++ b/inc/dg/cg2d_t.cu
@@ -36,11 +36,9 @@ int main()
     const dg::HVec solution = dg::evaluate ( fct, grid);
     //////////////////////////////////////////////////////////////////////
     //compute S b
-    dg::Inverse<dg::Elliptic<dg::CartesianGrid2d, dg::HMatrix, dg::HVec>, dg::HVec> inverse( A, x, 10, 1e-15, 0);
     dg::blas2::symv( w2d, b, b);
-    //std::cout << "Number of pcg iterations "<< pcg( A, x, b, v2d, eps_)<<std::endl;
+    std::cout << "Number of pcg iterations "<< pcg( A, x, b, v2d, eps_)<<std::endl;
     //std::cout << "Number of cg iterations "<< pcg( A, x, b, dg::Identity<double>(), eps)<<std::endl;
-    std::cout << "Number of pcg iterations "<< pcg( A, x, b, inverse, v2d, eps_)<<std::endl;
     std::cout << "For a precision of "<< eps_<<std::endl;
     //compute error
     dg::HVec error( solution);
diff --git a/inc/dg/cg3d_mpib.cu b/inc/dg/cg3d_mpib.cu
index d491c0e31..f44120ce7 100644
--- a/inc/dg/cg3d_mpib.cu
+++ b/inc/dg/cg3d_mpib.cu
@@ -25,7 +25,7 @@ int main( int argc, char* argv[])
     MPI_Init(&argc, &argv);
     unsigned n, Nx, Ny, Nz; 
     MPI_Comm comm;
-    mpi_init3d( bcx, dg::PER, dg::PER, n, Nx, Ny, Nz, comm);
+    dg::mpi_init3d( bcx, dg::PER, dg::PER, n, Nx, Ny, Nz, comm);
     int rank;
     MPI_Comm_rank( MPI_COMM_WORLD, &rank);
     double eps;
diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index 196a38ab7..f174b7a1b 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -127,14 +127,30 @@
   *  - dg::DMatrix and dg::IDMatrix with dg::DVec or std::vector<dg::DVec>
   *  - dg::MHMatrix with dg::MHVec or std::vector<dg::MHVec>
   *  - dg::MDMatrix with dg::MDVec or std::vector<dg::MDVec>
-  *  - Any type that has the SelfMadeMatrixTag specified in a corresponding MatrixTraits class (e.g. Elliptic). In this case only those blas2 functions 
+  *  - Any type that has the SelfMadeMatrixTag specified in a corresponding 
+  *  MatrixTraits class (e.g. Elliptic). In this case only those blas2 functions 
   *  that have a corresponding member function in the Matrix class (e.g. symv( const container&, container&); ) can be called.
   *  If the container is a std::vector, then the Matrix is applied to each of the elements.
   */
+  /** @class hide_geometry
+  * @tparam Geometry 
+  A type that is or derives from one of the abstract geometry base classes ( aGeometry2d, aGeometry3d, aMPIGeometry2d, ...). 
+  */
+
+  /** @class hide_container_geometry
+  * @tparam container 
+  * A data container class for which the blas1 functionality is overloaded and to which the return type of blas1::evaluate() can be converted. 
+  * We assume that container is copyable/assignable and has a swap member function. 
+  * In connection with Geometry this is one of 
+  *  - dg::HVec, dg::DVec when Geometry is a shared memory geometry
+  *  - dg::MHVec or dg::MDVec when Geometry is one of the MPI geometries
+  * @tparam Geometry 
+  A type that is or derives from one of the abstract geometry base classes ( aGeometry2d, aGeometry3d, aMPIGeometry2d, ...). Geometry determines which container type can be used.
+  */
 
   /** @class hide_geometry_matrix_container
   * @tparam Geometry 
-  A type that is or derives from one of the abstract geometry base classes (e.g. aGeometry2d, aGeometry3d, aMPIGeometry2d, ...). Geometry determines which Matrix and container types can be used:
+  A type that is or derives from one of the abstract geometry base classes ( aGeometry2d, aGeometry3d, aMPIGeometry2d, ...). Geometry determines which Matrix and container types can be used:
   * @tparam Matrix 
   * A class for which the blas2 functions are callable in connection with the container class and to which the return type of create::dx() can be converted. 
   * The Matrix type can be one of:
@@ -151,7 +167,8 @@
   */
 
  /** @class hide_symmetric_op
- * @tparam SymmetricOp A class for which the blas2::symv(Matrix&, Vector1&, Vector2&) function is callable 
+ * @tparam SymmetricOp 
+ A class for which the blas2::symv(Matrix&, Vector1&, Vector2&) function is callable 
  with the container type as argument. Also, The functions %inv_weights() and %precond() 
  need to be callable and return inverse weights and the preconditioner for the conjugate 
  gradient method. The Operator is assumed to be linear and symmetric!
diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index bc859a826..fc2fd650a 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -13,7 +13,7 @@
 
 /*! @file 
 
-  Contains the general negative elliptic operator
+  @brief General negative elliptic operators
   */
 namespace dg
 {
diff --git a/inc/dg/elliptic2d_mpib.cu b/inc/dg/elliptic2d_mpib.cu
index 948d62e79..652cc8715 100644
--- a/inc/dg/elliptic2d_mpib.cu
+++ b/inc/dg/elliptic2d_mpib.cu
@@ -40,7 +40,7 @@ int main(int argc, char* argv[] )
     MPI_Init( &argc, &argv);
     unsigned n, Nx, Ny; 
     MPI_Comm comm;
-    mpi_init2d( bcx, bcy, n, Nx, Ny, comm);
+    dg::mpi_init2d( bcx, bcy, n, Nx, Ny, comm);
     int rank;
     MPI_Comm_rank( MPI_COMM_WORLD, &rank);
     dg::Timer t;
diff --git a/inc/dg/elliptic_b.cu b/inc/dg/elliptic_b.cu
index e60fe22f5..746ba0507 100644
--- a/inc/dg/elliptic_b.cu
+++ b/inc/dg/elliptic_b.cu
@@ -46,7 +46,6 @@ int main()
     std::cout << "Create Laplacian\n";
     t.tic();
     dg::Elliptic<dg::aGeometry3d, dg::DMatrix, dg::DVec> laplace(grid, dg::not_normed, dg::centered);
-    dg::Elliptic<dg::aGeometry3d, dg::fDMatrix, dg::fDVec> flaplace(grid, dg::not_normed, dg::centered);
     dg::DMatrix DX = dg::create::dx( grid);
     t.toc();
     std::cout<< "Creation took "<<t.diff()<<"s\n";
@@ -59,15 +58,8 @@ int main()
     dg::DVec b = dg::evaluate ( laplace_fct, grid);
     //compute W b
     dg::blas2::symv( w3d, b, b);
-    dg::fDVec fx;
-    dg::blas1::transfer(x,fx);
-    dg::Inverse<dg::Elliptic<dg::aGeometry3d, dg::fDMatrix, dg::fDVec>, dg::fDVec> inverse( flaplace, fx, 10, 1e-15, 0);
     
     std::cout << "For a precision of "<< eps<<" ..."<<std::endl;
-    t.tic();
-    std::cout << "Number of mixed pcg iterations "<< pcg( laplace, x, b, inverse, v3d, eps)<<std::endl;
-    t.toc();
-    std::cout << "... on the device took "<< t.diff()<<"s\n";
     x = dg::evaluate( initial, grid);
     t.tic();
     std::cout << "Number of pcg iterations "<< pcg( laplace, x, b, v3d, eps)<<std::endl;
diff --git a/inc/dg/elliptic_mpib.cu b/inc/dg/elliptic_mpib.cu
index 0a0525570..3b0f07fb9 100644
--- a/inc/dg/elliptic_mpib.cu
+++ b/inc/dg/elliptic_mpib.cu
@@ -26,7 +26,7 @@ int main( int argc, char* argv[])
     MPI_Init(&argc, &argv);
     unsigned n, Nx, Ny, Nz; 
     MPI_Comm comm;
-    mpi_init3d( bcx, dg::PER, dg::PER, n, Nx, Ny, Nz, comm);
+    dg::mpi_init3d( bcx, dg::PER, dg::PER, n, Nx, Ny, Nz, comm);
 
     dg::CylindricalMPIGrid3d grid( R_0, R_0+lx, 0, ly, 0,lz, n, Nx, Ny,Nz, bcx, dg::PER, dg::PER, comm);
     const dg::MDVec w3d = dg::create::volume( grid);
diff --git a/inc/dg/enums.h b/inc/dg/enums.h
index 8fdb0e74a..fe845b07c 100644
--- a/inc/dg/enums.h
+++ b/inc/dg/enums.h
@@ -3,8 +3,7 @@
 #include "backend/exceptions.h"
 
 /*! @file 
-  
-  enums
+  @brief enums
   */
 
 namespace dg
diff --git a/inc/dg/geometry/base_geometry.h b/inc/dg/geometry/base_geometry.h
index 8409900d1..cb587b7ab 100644
--- a/inc/dg/geometry/base_geometry.h
+++ b/inc/dg/geometry/base_geometry.h
@@ -62,7 +62,9 @@ struct aGeometry2d : public aTopology2d
      * @note the default coordinate map will be the identity 
      */
     aGeometry2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx, bc bcy):aTopology2d( x0,x1,y0,y1,n,Nx,Ny,bcx,bcy){}
+    ///@copydoc aTopology2d::aTopology2d(const aTopology2d&)
     aGeometry2d( const aGeometry2d& src):aTopology2d(src){}
+    ///@copydoc aTopology2d::operator=(const aTopology2d&)
     aGeometry2d& operator=( const aGeometry2d& src){
         aTopology2d::operator=(src);
         return *this;
@@ -193,11 +195,12 @@ struct CartesianGrid2d: public dg::aGeometry2d
 struct CartesianGrid3d: public dg::aGeometry3d
 {
     typedef CartesianGrid2d perpendicular_grid;
-    ///@copydoc Grid3d::Grid3d()
+    ///@copydoc hide_grid_parameters3d
+    ///@copydoc hide_bc_parameters3d
     CartesianGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = PER, bc bcy = PER, bc bcz = PER): dg::aGeometry3d(x0,x1,y0,y1,z0,z1,n,Nx,Ny,Nz,bcx,bcy,bcz){}
     /**
      * @brief Implicit type conversion from Grid3d
-     * @param g existing grid class
+     * @param g existing grid object
      */
     CartesianGrid3d( const dg::Grid3d& g):dg::aGeometry3d(g.x0(), g.x1(), g.y0(), g.y1(), g.z0(), g.z1(),g.n(),g.Nx(),g.Ny(),g.Nz(),g.bcx(),g.bcy(),g.bcz()){}
     virtual CartesianGrid3d* clone()const{return new CartesianGrid3d(*this);}
@@ -214,7 +217,10 @@ struct CartesianGrid3d: public dg::aGeometry3d
 struct CylindricalGrid3d: public dg::aGeometry3d
 {
     typedef CartesianGrid2d perpendicular_grid;
-    CylindricalGrid3d( double R0, double R1, double Z0, double Z1, double phi0, double phi1, unsigned n, unsigned NR, unsigned NZ, unsigned Nphi, bc bcR, bc bcZ, bc bcphi = PER): dg::aGeometry3d(R0,R1,Z0,Z1,phi0,phi1,n,NR,NZ,Nphi,bcR,bcZ,bcphi){}
+    ///@copydoc hide_grid_parameters3d
+    ///@copydoc hide_bc_parameters3d
+    ///@note x corresponds to R, y to Z and z to phi, the volume element is R
+    CylindricalGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = PER, bc bcy = PER, bc bcz = PER): dg::aGeometry3d(x0,x1,y0,y1,z0,z1,n,Nx,Ny,Nz,bcx,bcy,bcz){}
     virtual CylindricalGrid3d* clone()const{return new CylindricalGrid3d(*this);}
     CartesianGrid2d perp_grid() const{ return CartesianGrid2d(x0(),x1(),y0(),y1(),n(),Nx(),Ny(),bcx(),bcy());}
     private:
diff --git a/inc/dg/geometry/mpi_base.h b/inc/dg/geometry/mpi_base.h
index 2050d82a5..897c2a791 100644
--- a/inc/dg/geometry/mpi_base.h
+++ b/inc/dg/geometry/mpi_base.h
@@ -32,10 +32,13 @@ struct aMPIGeometry2d : public aMPITopology2d
     ///allow deletion through base class pointer
     virtual ~aMPIGeometry2d(){}
     protected:
+    ///@copydoc aMPITopology2d::aMPITopology2d()
     aMPIGeometry2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx, bc bcy, MPI_Comm comm):
         aMPITopology2d( x0, x1, y0, y1, n, Nx, Ny, bcx, bcy, comm)
     { }
+    ///@copydoc aMPITopology2d::aMPITopology2d(const aMPITopology2d&)
     aMPIGeometry2d( const aMPIGeometry2d& src):aMPITopology2d(src){}
+    ///@copydoc aMPITopology2d::operator=(const aMPITopology2d&)
     aMPIGeometry2d& operator=( const aMPIGeometry2d& src){
         aMPITopology2d::operator=(src);
         return *this;
@@ -78,9 +81,12 @@ struct aMPIGeometry3d : public aMPITopology3d
     ///allow deletion through base class pointer
     virtual ~aMPIGeometry3d(){}
     protected:
+    ///@copydoc aMPITopology3d::aMPITopology3d()
     aMPIGeometry3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz, MPI_Comm comm):
         aMPITopology3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, bcx, bcy, bcz, comm){}
+    ///@copydoc aMPITopology3d::aMPITopology3d(const aMPITopology3d&)
     aMPIGeometry3d( const aMPIGeometry3d& src):aMPITopology3d(src){}
+    ///@copydoc aMPITopology3d::operator=(const aMPITopology3d&)
     aMPIGeometry3d& operator=( const aMPIGeometry3d& src){
         aMPITopology3d::operator=(src);
         return *this;
@@ -107,23 +113,20 @@ struct aMPIGeometry3d : public aMPITopology3d
 ///@{
 
 /**
- * @brief The mpi version of a cartesian grid
+ * @brief The mpi version of CartesianGrid2d
  */
 struct CartesianMPIGrid2d : public aMPIGeometry2d
 {
-    /**
-     * @copydoc Grid2d::Grid2d()
-     * @param comm a two-dimensional Cartesian communicator
-     * @note the paramateres given in the constructor are global parameters 
-     */
+    ///@copydoc hide_grid_parameters2d
+    ///@copydoc hide_comm_parameters2d
     CartesianMPIGrid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, MPI_Comm comm): aMPIGeometry2d( x0, x1, y0, y1, n, Nx, Ny, dg::PER,dg::PER,comm){}
 
-    /**
-     * @copydoc Grid2d::Grid2d()
-     * @param comm a two-dimensional Cartesian communicator
-     * @note the paramateres given in the constructor are global parameters 
-     */
+    ///@copydoc hide_grid_parameters2d
+    ///@copydoc hide_bc_parameters2d
+    ///@copydoc hide_comm_parameters2d
     CartesianMPIGrid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx, bc bcy, MPI_Comm comm):dg::aMPIGeometry2d( x0, x1, y0, y1, n, Nx, Ny,bcx, bcy, comm){}
+    ///@brief Implicit type conversion from MPIGrid2d
+    ///@param g existing grid object
     CartesianMPIGrid2d( const dg::MPIGrid2d& g): aMPIGeometry2d( g.global().x0(),g.global().x1(),g.global().y0(),g.global().y1(),g.global().n(),g.global().Nx(),g.global().Ny(),g.global().bcx(),g.global().bcy(),g.communicator()){}
     virtual CartesianMPIGrid2d* clone()const{return new CartesianMPIGrid2d(*this);}
     private:
@@ -134,24 +137,22 @@ struct CartesianMPIGrid2d : public aMPIGeometry2d
 };
 
 /**
- * @brief The mpi version of a cartesian grid
+ * @brief The mpi version of CartesianGrid3d
  */
 struct CartesianMPIGrid3d : public aMPIGeometry3d
 {
-    /**
-     * @copydoc MPIGrid3d::MPIGrid3d()
-     * @param comm a three-dimensional Cartesian communicator
-     * @note the paramateres given in the constructor are global parameters 
-     */
+    typedef CartesianGrid2d perpendicular_grid;
+    ///@copydoc hide_grid_parameters3d
+    ///@copydoc hide_comm_parameters3d
     CartesianMPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, MPI_Comm comm): aMPIGeometry3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, dg::PER,dg::PER,dg::PER, comm){}
 
-    /**
-     * @copydoc MPIGrid3d::MPIGrid3d()
-     * @param comm a three-dimensional Cartesian communicator
-     * @note the paramaters given in the constructor are global parameters 
-     */
+    ///@copydoc hide_grid_parameters3d
+    ///@copydoc hide_bc_parameters3d
+    ///@copydoc hide_comm_parameters3d
     CartesianMPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz, MPI_Comm comm):aMPIGeometry3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, bcx, bcy, bcz, comm){}
 
+    ///@brief Implicit type conversion from MPIGrid3d
+    ///@param g existing grid object
     CartesianMPIGrid3d( const dg::MPIGrid3d& g): aMPIGeometry3d( g.global().x0(),g.global().x1(),g.global().y0(),g.global().y1(),g.global().z0(),g.global().z1(),g.global().n(),g.global().Nx(),g.global().Ny(),g.global().Nz(),g.global().bcx(),g.global().bcy(),g.global().bcz(),g.communicator()){}
     virtual CartesianMPIGrid3d* clone()const{return new CartesianMPIGrid3d(*this);}
     private:
@@ -161,13 +162,22 @@ struct CartesianMPIGrid3d : public aMPIGeometry3d
 };
 
 /**
- * @brief three-dimensional Grid with Cylindrical metric
+ * @brief the mpi version of CylindricalGrid3d
  */
 struct CylindricalMPIGrid3d: public aMPIGeometry3d
 {
-    CylindricalMPIGrid3d( double R0, double R1, double Z0, double Z1, double phi0, double phi1, unsigned n, unsigned NR, unsigned NZ, unsigned Nphi, bc bcR, bc bcZ, bc bcphi, MPI_Comm comm): dg::aMPIGeometry3d(R0,R1,Z0,Z1,phi0,phi1,n,NR,NZ,Nphi,bcR,bcZ,bcphi,comm){}
-    ///take PER for bcphi
-    CylindricalMPIGrid3d( double R0, double R1, double Z0, double Z1, double phi0, double phi1, unsigned n, unsigned NR, unsigned NZ, unsigned Nphi, bc bcR, bc bcZ, MPI_Comm comm): dg::aMPIGeometry3d(R0,R1,Z0,Z1,phi0,phi1,n,NR,NZ,Nphi,bcR,bcZ,dg::PER,comm){}
+    typedef CartesianMPIGrid2d perpendicular_grid;
+    ///@copydoc hide_grid_parameters3d
+    ///@copydoc hide_bc_parameters3d
+    ///@copydoc hide_comm_parameters3d
+    ///@note x corresponds to R, y to Z and z to phi, the volume element is R
+    CylindricalMPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz, MPI_Comm comm):aMPIGeometry3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, bcx, bcy, bcz, comm){}
+    ///@copydoc hide_grid_parameters3d
+    ///@copydoc hide_bc_parameters2d
+    ///@note bcz is dg::PER
+    ///@copydoc hide_comm_parameters3d
+    ///@note x corresponds to R, y to Z and z to phi, the volume element is R
+    CylindricalMPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, MPI_Comm comm):aMPIGeometry3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, bcx, bcy, dg::PER, comm){}
 
     virtual CylindricalMPIGrid3d* clone()const{return new CylindricalMPIGrid3d(*this);}
     private:
diff --git a/inc/dg/geometry/multiply.h b/inc/dg/geometry/multiply.h
index 22fb7996f..4e175f647 100644
--- a/inc/dg/geometry/multiply.h
+++ b/inc/dg/geometry/multiply.h
@@ -7,6 +7,7 @@
 
 namespace dg
 {
+///@brief functions used in connection with the SparseElement and SparseTensor classes
 namespace tensor
 {
 
@@ -14,7 +15,7 @@ namespace tensor
 ///@{
 /**
  * @brief calls sqrt transform function on value
- * @copydoc hide_container_lvl1
+ * @copydoc hide_container
  * @param mu if empty, stays empty, else contains sqrt of input
  */
 template<class container>
@@ -25,7 +26,7 @@ void sqrt( SparseElement<container>& mu){
 
 /**
  * @brief calls invert transform function on value
- * @copydoc hide_container_lvl1
+ * @copydoc hide_container
  * @param mu if empty, stays empty, else contains inverse of input
  */
 template<class container>
@@ -38,7 +39,7 @@ void invert(SparseElement<container>& mu){
  * @brief Scale tensor with a container
  *
  * Computes \f$ t^{ij} = \mu t^{ij}\f$ 
- * @copydoc hide_container_lvl1
+ * @copydoc hide_container
  * @param t input (contains result on output)
  * @param mu all elements in t are scaled with mu
  */
@@ -61,7 +62,7 @@ void scal( SparseTensor<container>& t, const container& mu)
  * @brief Scale tensor with a form
  *
  * Computes \f$ t^{ij} = \mu t^{ij}\f$ 
- * @copydoc hide_container_lvl1
+ * @copydoc hide_container
  * @param t input (contains result on output)
  * @param mu if mu.isEmpty() then nothing happens, else all elements in t are scaled with its value
  */
@@ -75,7 +76,7 @@ void scal( SparseTensor<container>& t, const SparseElement<container>& mu)
 /**
  * @brief Multiply container with form
  *
- * @copydoc hide_container_lvl1
+ * @copydoc hide_container
  * @param mu if mu.isEmpty() then out=in, else the input is pointwise multiplied with the value in mu
  * @param in input vector
  * @param out output vector (may alias in)
@@ -91,7 +92,7 @@ void pointwiseDot( const SparseElement<container>& mu, const container& in, cont
 /**
  * @brief Multiply container with form
  *
- * @copydoc hide_container_lvl1
+ * @copydoc hide_container
  * @param in input vector
  * @param mu if mu.isEmpty() then out=in, else the input is pointwise multiplied with the value in mu
  * @param out output vector (may alias in)
@@ -105,7 +106,7 @@ void pointwiseDot( const container& in, const SparseElement<container>& mu, cont
 /**
  * @brief Divide container with form
  *
- * @copydoc hide_container_lvl1
+ * @copydoc hide_container
  * @param in input vector
  * @param mu if mu.isEmpty() then out=in, else the input is pointwise divided with the value in mu
  * @param out output vector (may alias in)
@@ -143,7 +144,7 @@ void multiply2d_helper( const SparseTensor<container>& t, const container& in0,
  * @brief Multiply a tensor with a vector in 2d
  *
  * Compute \f$ w^i = t^{ij}v_j\f$ for \f$ i,j\in \{1,2\}\f$ in the first two dimensions (ignores the 3rd dimension in t)
- * @copydoc hide_container_lvl1
+ * @copydoc hide_container
  * @param t input Tensor
  * @param in0 (input) first component    (restricted)
  * @param in1 (input) second component   (may alias out1)
@@ -167,7 +168,7 @@ void multiply2d( const SparseTensor<container>& t, const container& in0, const c
  * @brief Multiply a tensor with a vector in 2d
  *
  * Compute \f$ w^i = t^{ij}v_j\f$ for \f$ i,j\in \{1,2,3\}\f$
- * @copydoc hide_container_lvl1
+ * @copydoc hide_container
  * @param t input Tensor
  * @param in0 (input)  first component  (restricted)
  * @param in1 (input)  second component (restricted)
@@ -193,7 +194,7 @@ void multiply3d( const SparseTensor<container>& t, const container& in0, const c
 
 /**
 * @brief Compute the determinant of a tensor
-* @copydoc hide_container_lvl1
+* @copydoc hide_container
 * @param t the input tensor 
 * @return the determinant of t as a SparseElement (unset if t is empty)
 */
@@ -232,7 +233,7 @@ SparseElement<container> determinant( const SparseTensor<container>& t)
     invert(volume);
     sqrt(volume);
     @endcode
- * @copydoc hide_container_lvl1
+ * @copydoc hide_container
  * @param t the input tensor 
  * @return the inverse square root of the determinant of t as a SparseElement (unset if t is empty)
  */
diff --git a/inc/dg/geometry/refined_grid.h b/inc/dg/geometry/refined_grid.h
index 12de67c6a..f0a356f5a 100644
--- a/inc/dg/geometry/refined_grid.h
+++ b/inc/dg/geometry/refined_grid.h
@@ -13,10 +13,10 @@
 namespace dg
 {
 
+///@cond
 namespace detail
 {
 
-///@cond
 /**
  * @brief Normalize the given weights and compute the abscissas of the grid
  *
@@ -45,9 +45,9 @@ thrust::host_vector<double> normalize_weights_and_compute_abscissas( const Grid1
     }
     return abs;
 }
-///@endcond
 
 }//namespace detail
+///@endcond
 
 ///@addtogroup generators
 ///@{
diff --git a/inc/dg/geometry/tensor.h b/inc/dg/geometry/tensor.h
index 857fe4cdf..edd7701df 100644
--- a/inc/dg/geometry/tensor.h
+++ b/inc/dg/geometry/tensor.h
@@ -276,7 +276,7 @@ namespace tensor
  /**
  * @brief Construct a tensor with all unset values filled with explicit 0 or 1
  *
- * @copydoc hide_container_lvl1
+ * @copydoc hide_container
  * @return a dense tensor
  * @note undefined if t.isEmpty() returns true
  */
@@ -318,7 +318,7 @@ SparseTensor<container> dense(const SparseTensor<container>& tensor)
  * @brief data structure to hold the LDL^T decomposition of a symmetric positive definite matrix
  *
  * LDL^T stands for a lower triangular matrix L,  a diagonal matrix D and the transpose L^T
- * @copydoc hide_container_lvl1
+ * @copydoc hide_container
  * @attention the tensor in the Elliptic classes actually only need to be positive **semi-definite**
  * and unfortunately the decomposition is unstable for semi-definite matrices.
 * @ingroup misc
diff --git a/inc/dg/geometry/transform.h b/inc/dg/geometry/transform.h
index 5c7225a8d..cadf88c26 100644
--- a/inc/dg/geometry/transform.h
+++ b/inc/dg/geometry/transform.h
@@ -103,8 +103,8 @@ MPI_Vector<thrust::host_vector<double> > pullback( const Functor& f, const aMPIG
                 v^y(x,y) = y_R (x,y) v^R(R(x,y), Z(x,y)) + y_Z v^Z(R(x,y), Z(x,y)) \f]
    where \f$ x_R = \frac{\partial x}{\partial R}\f$, ... 
  * @tparam Functor1 Binary or Ternary functor
- * @copydoc hide_container_lvl1
- * @tparam Geometry The Geometry class
+ * @tparam Functor2 Binary or Ternary functor
+ * @copydoc hide_container_geometry
  * @param vR input R-component in cylindrical coordinates
  * @param vZ input Z-component in cylindrical coordinates
  * @param vx x-component of vector (gets properly resized)
@@ -132,8 +132,9 @@ void pushForwardPerp( const Functor1& vR, const Functor2& vZ,
                 v^y(x,y) = y_R (x,y) v^R(R(x,y), Z(x,y)) + y_Z v^Z(R(x,y), Z(x,y)) \f]
    where \f$ x_R = \frac{\partial x}{\partial R}\f$, ... 
  * @tparam Functor1 Binary or Ternary functor
- * @copydoc hide_container_lvl1
- * @tparam Geometry The Geometry class
+ * @tparam Functor2 Binary or Ternary functor
+ * @tparam Functor3 Binary or Ternary functor
+ * @copydoc hide_container_geometry
  * @param vR input R-component in cartesian or cylindrical coordinates
  * @param vZ input Z-component in cartesian or cylindrical coordinates
  * @param vPhi input Z-component in cartesian or cylindrical coordinates
@@ -167,8 +168,10 @@ void pushForward( const Functor1& vR, const Functor2& vZ, const Functor3& vPhi,
  \chi^{yy}(x,y) = y_R y_R \chi^{RR} + 2y_Ry_Z \chi^{RZ} + y_Zy_Z\chi^{ZZ} \\
                \f]
    where \f$ x_R = \frac{\partial x}{\partial R}\f$, ... 
- * @copydoc hide_container_lvl1
- * @tparam Geometry The Geometry class
+ * @tparam FunctorRR Binary or Ternary functor
+ * @tparam FunctorRZ Binary or Ternary functor
+ * @tparam FunctorZZ Binary or Ternary functor
+ * @copydoc hide_container_geometry
  * @param chiRR input RR-component in cylindrical coordinates
  * @param chiRZ input RZ-component in cylindrical coordinates
  * @param chiZZ input ZZ-component in cylindrical coordinates
@@ -221,7 +224,7 @@ namespace create{
  * @brief Create the inverse volume element on the grid (including weights!!)
  *
  * This is the same as the inv_weights divided by the volume form \f$ \sqrt{g}\f$
- * @tparam Geometry any Geometry class
+ * @copydoc hide_geometry
  * @param g Geometry object
  *
  * @return  The inverse volume form
@@ -241,7 +244,7 @@ typename GeometryTraits<Geometry>::host_vector inv_volume( const Geometry& g)
  * @brief Create the volume element on the grid (including weights!!)
  *
  * This is the same as the weights multiplied by the volume form \f$ \sqrt{g}\f$
- * @tparam Geometry any Geometry class
+ * @copydoc hide_geometry
  * @param g Geometry object
  *
  * @return  The volume form
diff --git a/inc/dg/helmholtz.h b/inc/dg/helmholtz.h
index 4a478bc9b..27cd1e229 100644
--- a/inc/dg/helmholtz.h
+++ b/inc/dg/helmholtz.h
@@ -7,7 +7,7 @@
 
 /*!@file
  *
- * Contains Helmholtz and Maxwell operators
+ * @brief contains Helmholtz and Maxwell operators
  */
 namespace dg{
 
diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index de6c45c8c..f8d9aef77 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -14,12 +14,21 @@ namespace dg
 /**
 * @brief Class for the solution of symmetric matrix equation discretizeable on multiple grids
 *
+* We use conjugate gradien (CG) at each stage and refine the grids in the first two dimensions (2d / x and y) 
 * @copydoc hide_geometry_matrix_container
 */
 template< class Geometry, class Matrix, class container> 
 struct MultigridCG2d
 {
-    MultigridCG2d( const Geometry& grid, const unsigned stages, const int scheme_type = 0, const int extrapolation_type = 2 )
+    /**
+    * @brief Construct the grids and the interpolation/projection operators
+    *
+    * @param grid the original grid (Nx() and Ny() must be evenly divisable by pow(2, stages-1)
+    * @param stages number of grids in total (The second grid contains half the points of the original grids,  
+    *   The third grid contains half of the second grid ...). Must be > 1
+    * @param scheme_type scheme type in the solve function
+    */
+    MultigridCG2d( const Geometry& grid, const unsigned stages, const int scheme_type = 0 )
     {
         stages_= stages;
         if(stages < 2 ) throw Error( Message(_ping_)<<" There must be minimum 2 stages in a multigrid solver! You gave " << stages);
@@ -115,6 +124,20 @@ struct MultigridCG2d
 		return number;
 	}
 
+    /**
+    * @brief Nested iterations
+    *
+    * - Compute residual with given initial guess. 
+    * - Project residual down to the coarsest grid. 
+    * - Solve equation on the coarse grid 
+    * - interpolate solution up to next finer grid and repeat until the original grid is reached. 
+    * @copydoc hide_symmetric_op
+    * @param op Index 0 is the matrix on the original grid, 1 on the half grid, 2 on the quarter grid, ...
+    * @param x (read/write) contains initial guess on input and the solution on output
+    * @param b The right hand side (will be multiplied by weights)
+    * @param eps the accuracy: iteration stops if \f$ ||b - Ax|| < \epsilon( ||b|| + 1) \f$ 
+    * @return the number of iterations in each of the stages
+    */
     template<class SymmetricOp>
     std::vector<unsigned> direct_solve( std::vector<SymmetricOp>& op, container&  x, const container& b, double eps)
     {
diff --git a/inc/dg/multistep.h b/inc/dg/multistep.h
index 0cd35685a..a9a40cfce 100644
--- a/inc/dg/multistep.h
+++ b/inc/dg/multistep.h
@@ -4,8 +4,7 @@
 
 
 /*! @file
-
-  This file contains multistep explicit& implicit time-integrators
+  @brief contains multistep explicit& implicit time-integrators
   */
 namespace dg{
 
@@ -35,7 +34,7 @@ const double ab_coeff<5>::b[5] = {1901./720., -1387./360., 109./30., -637./360.,
 * Uses only blas1::axpby routines to integrate one step
 * and only one right-hand-side evaluation per step.
 * @tparam k Order of the method (Currently one of 1, 2, 3, 4 or 5)
-* @copydoc hide_container_lvl1
+* @copydoc hide_container
 */
 template< size_t k, class container>
 struct AB
@@ -200,7 +199,7 @@ struct MatrixTraits< detail::Implicit<M, V> >
 * and only one right-hand-side evaluation per step. 
 * Uses a conjugate gradient method for the implicit operator  
 * @ingroup time
-* @copydoc hide_container_lvl1
+* @copydoc hide_container
 */
 template<class container>
 struct Karniadakis
@@ -358,7 +357,7 @@ void Karniadakis<container>::operator()( Functor& f, Diffusion& diff, container&
  * @brief Semi implicit Runge Kutta method after Yoh and Zhong (AIAA 42, 2004)
  *
  * @ingroup time
- * @copydoc hide_container_lvl1
+ * @copydoc hide_container
  */
 template <class container>
 struct SIRK
diff --git a/inc/dg/nullstelle.h b/inc/dg/nullstelle.h
index 40c5a45cb..c7242ce4f 100644
--- a/inc/dg/nullstelle.h
+++ b/inc/dg/nullstelle.h
@@ -1,5 +1,5 @@
 /*! @file 
- * @brief contains root finding method
+ * @brief Root finding method
  * @author Matthias Wiesenberger
  * @date 12.4.2010
  */
diff --git a/inc/dg/runge_kutta.h b/inc/dg/runge_kutta.h
index a5bef9075..1bdc1f095 100644
--- a/inc/dg/runge_kutta.h
+++ b/inc/dg/runge_kutta.h
@@ -9,9 +9,8 @@
 
 
 /*! @file
-
-  This file contains runge-kutta explicit time-integrators
-  */
+ * @brief contains runge-kutta explicit time-integrators
+ */
 namespace dg{
 
 //namespace detail <--------- ??
@@ -334,7 +333,7 @@ const double rk_classic<17>::b[17] = {
  *  by ones on the left and \f$ D\f$ its
  *  diagonal part. 
 * @tparam k Order of the method (1, 2, 3 or 4)
-* @copydoc hide_container_lvl1
+* @copydoc hide_container
 */
 template< size_t k, class container>
 struct RK
@@ -409,7 +408,7 @@ void RK<k, container>::operator()( Functor& f, const container& u0, container& u
 * The coefficients are chosen in the classic form given by Runge and Kutta. 
 * Needs more calls for axpby than our RK class but we implemented higher orders
 * @tparam s Order of the method (1, 2, 3, 4, 6, 17)
-* @copydoc hide_container_lvl1
+* @copydoc hide_container
 */
 template< size_t s, class container>
 struct RK_classic
@@ -490,7 +489,7 @@ void stepperRK(RHS& rhs, const container& begin, container& end, double T_min, d
  * @brief Integrate differential equation with a stage 1 Runge-Kutta scheme and a fixed number of steps
  *
  * @tparam RHS The right-hand side class
- * @copydoc hide_container_lvl1
+ * @copydoc hide_container
  * @param rhs The right-hand-side
  * @param begin initial condition 
  * @param end (write-only) contains solution on output
@@ -549,7 +548,7 @@ void stepperRK17(RHS& rhs, const container& begin, container& end, double T_min,
  *
  * @tparam RHS The right-hand side class. There must be the function bool monitor( const container& end); available which is called after every step. Return true if everything is ok and false if the integrator certainly fails.
  * The other function is the double error( const container& end0, const container& end1); which computes the error norm in which the integrator should converge. 
- * @copydoc hide_container_lvl1
+ * @copydoc hide_container
  * @param rhs The right-hand-side
  * @param begin initial condition (size 3)
  * @param end (write-only) contains solution on output
diff --git a/inc/geometries/ds_mpib.cu b/inc/geometries/ds_mpib.cu
index 26b046399..b6d0e6f22 100644
--- a/inc/geometries/ds_mpib.cu
+++ b/inc/geometries/ds_mpib.cu
@@ -64,7 +64,7 @@ int main(int argc, char* argv[])
     int rank;
     unsigned n, Nx, Ny, Nz; 
     MPI_Comm comm;
-    mpi_init3d( dg::NEU, dg::NEU, dg::PER, n, Nx, Ny, Nz, comm);
+    dg::mpi_init3d( dg::NEU, dg::NEU, dg::PER, n, Nx, Ny, Nz, comm);
     MPI_Comm_rank( MPI_COMM_WORLD, &rank);
 
     Field field( R_0, I_0);
diff --git a/inc/geometries/dz_mpit.cu b/inc/geometries/dz_mpit.cu
index 6f77c4d71..c89515c64 100644
--- a/inc/geometries/dz_mpit.cu
+++ b/inc/geometries/dz_mpit.cu
@@ -36,7 +36,7 @@ int main(int argc, char **argv)
     MPI_Init(&argc, &argv);
     unsigned n, Nx, Ny, Nz; 
     MPI_Comm comm;
-    mpi_init3d( dg::DIR, dg::DIR, dg::NEU, n, Nx, Ny, Nz, comm);
+    dg::mpi_init3d( dg::DIR, dg::DIR, dg::NEU, n, Nx, Ny, Nz, comm);
     int rank;
     MPI_Comm_rank( MPI_COMM_WORLD, &rank);
     dg::CartesianMPIGrid3d g3d( -1, 1, -1, 1, 0.1, M_PI+0.1, n, Nx, Ny, Nz, dg::DIR, dg::DIR, dg::NEU, comm);
diff --git a/inc/geometries/geometry_advection_mpib.cu b/inc/geometries/geometry_advection_mpib.cu
index 0d4d38897..60c345a55 100644
--- a/inc/geometries/geometry_advection_mpib.cu
+++ b/inc/geometries/geometry_advection_mpib.cu
@@ -110,7 +110,7 @@ int main(int argc, char** argv)
     int rank;
     unsigned n, Nx, Ny, Nz; 
     MPI_Comm comm;
-    mpi_init3d( dg::DIR, dg::PER, dg::PER, n, Nx, Ny, Nz, comm);
+    dg::mpi_init3d( dg::DIR, dg::PER, dg::PER, n, Nx, Ny, Nz, comm);
     MPI_Comm_rank( MPI_COMM_WORLD, &rank);
     Json::Reader reader;
     Json::Value js;
diff --git a/inc/geometries/geometry_elliptic_mpib.cu b/inc/geometries/geometry_elliptic_mpib.cu
index de1a5adb4..f774b71cc 100644
--- a/inc/geometries/geometry_elliptic_mpib.cu
+++ b/inc/geometries/geometry_elliptic_mpib.cu
@@ -24,7 +24,7 @@ int main(int argc, char**argv)
     int rank;
     unsigned n, Nx, Ny, Nz; 
     MPI_Comm comm;
-    mpi_init3d( dg::DIR, dg::PER, dg::PER, n, Nx, Ny, Nz, comm);
+    dg::mpi_init3d( dg::DIR, dg::PER, dg::PER, n, Nx, Ny, Nz, comm);
     MPI_Comm_rank( MPI_COMM_WORLD, &rank);
     Json::Reader reader;
     Json::Value js;
diff --git a/inc/geometries/ribeiro_mpit.cu b/inc/geometries/ribeiro_mpit.cu
index b3bdeb25e..e14029d05 100644
--- a/inc/geometries/ribeiro_mpit.cu
+++ b/inc/geometries/ribeiro_mpit.cu
@@ -36,7 +36,7 @@ int main( int argc, char* argv[])
     int rank;
     unsigned n, Nx, Ny, Nz; 
     MPI_Comm comm;
-    mpi_init3d( dg::DIR, dg::PER, dg::PER, n, Nx, Ny, Nz, comm);
+    dg::mpi_init3d( dg::DIR, dg::PER, dg::PER, n, Nx, Ny, Nz, comm);
     MPI_Comm_rank( MPI_COMM_WORLD, &rank);
     Json::Reader reader;
     Json::Value js;
-- 
GitLab


From 24fda38e4e5e66a8724a3d38e540a2f841b0ebb4 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sat, 16 Sep 2017 21:49:15 +0200
Subject: [PATCH 296/453] compile doc without warning

---
 inc/dg/backend/gridX.h                 | 82 ++++++++++++++------------
 inc/dg/backend/sparseblockmat.cuh      |  9 ++-
 inc/dg/backend/sparseblockmat.h        |  9 ++-
 inc/dg/cg.h                            | 12 ++--
 inc/dg/geometry/base_geometry.h        |  2 +
 inc/dg/geometry/base_geometryX.h       |  4 ++
 inc/dg/refined_elliptic.h              | 15 ++---
 inc/geometries/flux.h                  |  4 +-
 inc/geometries/generator.h             |  2 +-
 inc/geometries/generatorX.h            |  2 +-
 inc/geometries/hector.h                |  4 +-
 inc/geometries/ribeiro.h               |  2 +-
 inc/geometries/ribeiroX.h              |  2 +-
 inc/geometries/separatrix_orthogonal.h |  4 +-
 inc/geometries/simple_orthogonal.h     |  2 +-
 inc/geometries/solovev_doc.h           | 26 ++++++--
 16 files changed, 111 insertions(+), 70 deletions(-)

diff --git a/inc/dg/backend/gridX.h b/inc/dg/backend/gridX.h
index 87d09e3e0..85b1ba47d 100644
--- a/inc/dg/backend/gridX.h
+++ b/inc/dg/backend/gridX.h
@@ -9,6 +9,39 @@
   @brief base X-point topology classes
   */
 
+/*!@class hide_gridX_parameters2d
+ * @brief Construct a 2D X-point grid
+ *
+ * @param x0 left boundary in x
+ * @param x1 right boundary in x 
+ * @param y0 lower boundary in y
+ * @param y1 upper boundary in y 
+ * @param fx factor for the partition in x-direction (fx*Nx will be rounded)
+ * @param fy factor for the partition in y-direction (fy*Ny will be rounded)
+ * @param n  # of polynomial coefficients per dimension
+ *   (1<=n<=20, note that the library is optimized for n=3 )
+ * @param Nx # of points in x 
+ * @param Ny # of points in y
+ */
+
+/*!@class hide_gridX_parameters3d
+ * @brief Construct a 3D X-point grid
+ * @param x0 left boundary in x
+ * @param x1 right boundary in x 
+ * @param y0 lower boundary in y
+ * @param y1 upper boundary in y 
+ * @param z0 lower boundary in z
+ * @param z1 upper boundary in z 
+ * @param fx factor for the partition in x-direction
+ * @param fy factor for the partition in y-direction
+ * @param n  # of polynomial coefficients per (x-,y-) dimension
+*   (1<=n<=20, note that the library is optimized for n=3 )
+* @attention # of polynomial coefficients in z direction is always 1
+ * @param Nx # of points in x 
+ * @param Ny # of points in y
+ * @param Nz # of points in z
+ */
+
 
 namespace dg{
 
@@ -434,21 +467,8 @@ struct aTopologyX2d
   protected:
     ///disallow destruction through base class pointer
     ~aTopologyX2d(){}
-    /**
-     * @brief Construct a 2D grid
-     *
-     * @param x0 left boundary in x
-     * @param x1 right boundary in x 
-     * @param y0 lower boundary in y
-     * @param y1 upper boundary in y 
-     * @param fx factor for the partition in x-direction (fx*Nx will be rounded)
-     * @param fy factor for the partition in y-direction (fy*Ny will be rounded)
-     * @param n  # of polynomial coefficients per dimension
-     * @param Nx # of points in x 
-     * @param Ny # of points in y
-     * @param bcx boundary condition in x
-     * @param bcy boundary condition in y
-     */
+    ///@copydoc hide_gridX_parameters2d
+    ///@copydoc hide_bc_parameters2d
     aTopologyX2d( double x0, double x1, double y0, double y1, double fx, double fy, unsigned n, unsigned Nx, unsigned Ny, bc bcx, bc bcy):
         x0_(x0), x1_(x1), y0_(y0), y1_(y1), fx_(fx), fy_(fy),
         n_(n), Nx_(Nx), Ny_(Ny), bcx_(bcx), bcy_( bcy), dlt_(n)
@@ -462,6 +482,7 @@ struct aTopologyX2d
         assert( Nx_ > 0  && Ny > 0 );
         assert( bcy != PER);
     }
+    ///@copydoc aTopology2d::aTopology2d(const aTopology2d&)
     aTopologyX2d(const aTopologyX2d& src){
         x0_=src.x0_, x1_=src.x1_;
         y0_=src.y0_, y1_=src.y1_;
@@ -469,6 +490,7 @@ struct aTopologyX2d
         n_=src.n_, Nx_=src.Nx_, Ny_=src.Ny_, bcx_=src.bcx_, bcy_=src.bcy_;
         dlt_=src.dlt_;
     }
+    ///@copydoc aTopology2d::operator=(const aTopology2d&)
     aTopologyX2d& operator=(const aTopologyX2d& src){
         x0_=src.x0_, x1_=src.x1_;
         y0_=src.y0_, y1_=src.y1_;
@@ -490,7 +512,8 @@ struct aTopologyX2d
  */
 struct GridX2d : public aTopologyX2d
 {
-    ///@copydoc aTopologyX2d::aTopologyX2d()
+    ///@copydoc hide_gridX_parameters2d
+    ///@copydoc hide_bc_parameters2d
     GridX2d( double x0, double x1, double y0, double y1, double fx, double fy, unsigned n, unsigned Nx, unsigned Ny, bc bcx=PER, bc bcy=NEU):
         aTopologyX2d(x0,x1,y0,y1,fx,fy,n,Nx,Ny,bcx,bcy) { }
     ///allow explicit type conversion from any other topology
@@ -743,26 +766,8 @@ struct aTopologyX3d
   protected:
     ///disallow destruction through base class pointer
     ~aTopologyX3d(){}
-    /**
-     * @brief Construct a 3D grid
-     *
-     * @param x0 left boundary in x
-     * @param x1 right boundary in x 
-     * @param y0 lower boundary in y
-     * @param y1 upper boundary in y 
-     * @param z0 lower boundary in z
-     * @param z1 upper boundary in z 
-     * @param fx factor for the partition in x-direction
-     * @param fy factor for the partition in y-direction
-     * @param n  # of polynomial coefficients per (x-,y-) dimension
-     * @param Nx # of points in x 
-     * @param Ny # of points in y
-     * @param Nz # of points in z
-     * @param bcx boundary condition in x
-     * @param bcy boundary condition in y
-     * @param bcz boundary condition in z
-     * @attention # of polynomial coefficients in z direction is always 1
-     */
+    ///@copydoc hide_gridX_parameters3d
+    ///@copydoc hide_bc_parameters3d
     aTopologyX3d( double x0, double x1, double y0, double y1, double z0, double z1, double fx, double fy, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz):
         x0_(x0), x1_(x1), y0_(y0), y1_(y1), z0_(z0), z1_(z1), fx_(fx), fy_(fy),
         n_(n), Nx_(Nx), Ny_(Ny), Nz_(Nz), bcx_(bcx), bcy_( bcy), bcz_( bcz), dlt_(n)
@@ -775,6 +780,7 @@ struct aTopologyX3d
         assert( x1 > x0 && y1 > y0 ); assert( z1 > z0 );         
         assert( Nx_ > 0  && Ny > 0); assert( Nz > 0);
     }
+    ///@copydoc aTopology3d::aTopology3d(const aTopology3d&)
     aTopologyX3d(const aTopologyX3d& src){
         x0_=src.x0_, x1_=src.x1_;
         y0_=src.y0_, y1_=src.y1_;
@@ -783,6 +789,7 @@ struct aTopologyX3d
         n_=src.n_, Nx_=src.Nx_, Ny_=src.Ny_, Nz_=src.Nz_,bcx_=src.bcx_, bcy_=src.bcy_, bcz_=src.bcz_;
         dlt_=src.dlt_;
     }
+    ///@copydoc aTopology3d::operator=(const aTopology3d&)
     aTopologyX3d& operator=(const aTopologyX3d& src){
         x0_=src.x0_, x1_=src.x1_;
         y0_=src.y0_, y1_=src.y1_;
@@ -806,7 +813,8 @@ struct aTopologyX3d
  */
 struct GridX3d : public aTopologyX3d
 {
-    ///@copydoc aTopologyX2d::aTopologyX2d()
+    ///@copydoc hide_gridX_parameters3d
+    ///@copydoc hide_bc_parameters3d
     GridX3d( double x0, double x1, double y0, double y1, double z0, double z1, double fx, double fy, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx=PER, bc bcy=NEU, bc bcz=PER):
         aTopologyX3d(x0,x1,y0,y1,z0,z1,fx,fy,n,Nx,Ny,Nz,bcx,bcy,bcz) { }
     ///allow explicit type conversion from any other topology
diff --git a/inc/dg/backend/sparseblockmat.cuh b/inc/dg/backend/sparseblockmat.cuh
index bf5c79bf2..8681521e3 100644
--- a/inc/dg/backend/sparseblockmat.cuh
+++ b/inc/dg/backend/sparseblockmat.cuh
@@ -38,15 +38,20 @@ struct EllSparseBlockMatDevice
     
     /**
     * @brief Apply the matrix to a vector
-    *
+    * \f[  y= \alpha M x + \beta y\f]
+    * @tparam deviceContainer one of the containers of the thrust library
+    * @param alpha multiplies input
     * @param x input
-    * @param y output may not equal input
+    * @param beta premultiplies output
+    * @param y output may not alias input
     */
     template <class deviceContainer>
     void symv(value_type alpha, const deviceContainer& x, value_type beta, deviceContainer& y) const;
     /**
     * @brief Apply the matrix to a vector
     *
+    * same as symv( 1., x,0.,y);
+    * @tparam deviceContainer one of the containers of the thrust library
     * @param x input
     * @param y output may not equal input
     */
diff --git a/inc/dg/backend/sparseblockmat.h b/inc/dg/backend/sparseblockmat.h
index 090fea051..7141f00ac 100644
--- a/inc/dg/backend/sparseblockmat.h
+++ b/inc/dg/backend/sparseblockmat.h
@@ -63,15 +63,18 @@ struct EllSparseBlockMat
     /**
     * @brief Apply the matrix to a vector
     *
+    * \f[  y= \alpha M x + \beta y\f]
+    * @param alpha multiplies input
     * @param x input
-    * @param y output may not equal input
+    * @param beta premultiplies output
+    * @param y output may not alias input
     */
     void symv(value_type alpha, const thrust::host_vector<value_type>& x, value_type beta, thrust::host_vector<value_type>& y) const;
     /**
     * @brief Apply the matrix to a vector
     *
     * @param x input
-    * @param y output may not equal input
+    * @param y output may not alias input
     */
     void symv(const thrust::host_vector<value_type>& x, thrust::host_vector<value_type>& y) const {symv( 1., x, 0., y);}
 
@@ -167,7 +170,7 @@ struct CooSparseBlockMat
     * @param alpha multiplies input
     * @param x input
     * @param beta premultiplies output
-    * @param y output may not equal input
+    * @param y output may not alias input
     */
     void symv(value_type alpha, const thrust::host_vector<value_type>& x, value_type beta, thrust::host_vector<value_type>& y) const;
     /**
diff --git a/inc/dg/cg.h b/inc/dg/cg.h
index cc3d8a4ca..e5615870f 100644
--- a/inc/dg/cg.h
+++ b/inc/dg/cg.h
@@ -272,17 +272,17 @@ struct Extrapolation
         set_type(type, init); 
     }
     ///@copydoc Extrapolation(unsigned)
-    void set_type( unsigned new_type)
+    void set_type( unsigned type)
     {
-        m_type = new_type;
-        m_x.resize( new_type);
+        m_type = type;
+        m_x.resize( type);
         assert( m_type <= 3 );
     }
     ///@copydoc Extrapolation(unsigned,const container&)
-    void set_type( unsigned new_type, const container& init)
+    void set_type( unsigned type, const container& init)
     {
-        m_x.assign( new_type, init);
-        m_type = new_type;
+        m_x.assign( type, init);
+        m_type = type;
         assert( m_type <= 3 );
     }
     ///read the current extrapolation type
diff --git a/inc/dg/geometry/base_geometry.h b/inc/dg/geometry/base_geometry.h
index cb587b7ab..dc81e5834 100644
--- a/inc/dg/geometry/base_geometry.h
+++ b/inc/dg/geometry/base_geometry.h
@@ -144,7 +144,9 @@ struct aGeometry3d : public aTopology3d
      * @note the default coordinate map will be the identity 
      */
     aGeometry3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz): aTopology3d(x0,x1,y0,y1,z0,z1,n,Nx,Ny,Nz,bcx,bcy,bcz){}
+    ///@copydoc aTopology3d::aTopology3d(const aTopology3d&)
     aGeometry3d( const aGeometry3d& src):aTopology3d(src){}
+    ///@copydoc aTopology3d::operator=(const aTopology3d&)
     aGeometry3d& operator=( const aGeometry3d& src){
         aTopology3d::operator=(src);
         return *this;
diff --git a/inc/dg/geometry/base_geometryX.h b/inc/dg/geometry/base_geometryX.h
index 573aa5ce4..8ba724d80 100644
--- a/inc/dg/geometry/base_geometryX.h
+++ b/inc/dg/geometry/base_geometryX.h
@@ -36,7 +36,9 @@ struct aGeometryX2d : public aTopologyX2d
      * @note the default coordinate map will be the identity 
      */
     aGeometryX2d( double x0, double x1, double y0, double y1, double fx, double fy, unsigned n, unsigned Nx, unsigned Ny, bc bcx, bc bcy):aTopologyX2d( x0,x1,y0,y1,fx,fy,n,Nx,Ny,bcx,bcy){}
+    ///@copydoc aTopologyX2d::aTopologyX2d(const aTopologyX2d&)
     aGeometryX2d( const aGeometryX2d& src):aTopologyX2d(src){}
+    ///@copydoc aTopologyX2d::operator=(const aTopologyX2d&)
     aGeometryX2d& operator=( const aGeometryX2d& src){
         aTopologyX2d::operator=(src);
         return *this;
@@ -85,7 +87,9 @@ struct aGeometryX3d : public aTopologyX3d
      * @note the default coordinate map will be the identity 
      */
     aGeometryX3d( double x0, double x1, double y0, double y1, double z0, double z1, double fx, double fy, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz): aTopologyX3d(x0,x1,y0,y1,z0,z1,fx,fy,n,Nx,Ny,Nz,bcx,bcy,bcz){}
+    ///@copydoc aTopologyX3d::aTopologyX3d(const aTopologyX3d&)
     aGeometryX3d( const aGeometryX3d& src):aTopologyX3d(src){}
+    ///@copydoc aTopologyX3d::operator=(const aTopologyX3d&)
     aGeometryX3d& operator=( const aGeometryX3d& src){
         aTopologyX3d::operator=(src);
         return *this;
diff --git a/inc/dg/refined_elliptic.h b/inc/dg/refined_elliptic.h
index f58e2ebec..8850e976f 100644
--- a/inc/dg/refined_elliptic.h
+++ b/inc/dg/refined_elliptic.h
@@ -7,7 +7,7 @@
 
 /*! @file 
 
-  Contains an elliptic method on a refined grid
+  @brief contains an elliptic method on a refined grid
   */
 namespace dg
 {
@@ -19,7 +19,8 @@ class RefinedElliptic
     /**
      * @brief Construct from Grid
      *
-     * @param g The Grid, boundary conditions are taken from here
+     * @param g_coarse The coarse Grid
+     * @param g_fine The fine Grid, boundary conditions are taken from here
      * @param no Not normed for elliptic equations, normed else
      * @param dir Direction of the right first derivative
      */
@@ -32,7 +33,8 @@ class RefinedElliptic
     /**
      * @brief Construct from grid and boundary conditions
      *
-     * @param g The Grid
+     * @param g_coarse The coarse Grid
+     * @param g_fine The fine Grid
      * @param bcx boundary condition in x
      * @param bcy boundary contition in y
      * @param no Not normed for elliptic equations, normed else
@@ -57,11 +59,10 @@ class RefinedElliptic
     }
 
     /**
-     * @brief Returns the weights used to make the matrix symmetric 
-     *
-     * @return weights
+     * @brief Returns the inverse weights used to make the matrix normed
+     * @return inverse weights
      */
-    const container& weights()const {return weights_;}
+    const container& inv_weights()const {return inv_weights_;}
     /**
      * @brief Returns the preconditioner to use in conjugate gradient
      *
diff --git a/inc/geometries/flux.h b/inc/geometries/flux.h
index 550cef491..2f3ba3d6a 100644
--- a/inc/geometries/flux.h
+++ b/inc/geometries/flux.h
@@ -123,7 +123,7 @@ struct Fpsi
 
 /**
  * @brief A symmetry flux generator
- * @ingroup generators
+ * @ingroup generators_geo
  */
 struct FluxGenerator : public aGenerator2d
 {
@@ -213,7 +213,7 @@ struct FluxGenerator : public aGenerator2d
 
 /**
  * @brief Same as the Ribeiro class just but uses psi as a flux label directly
- * @ingroup generators
+ * @ingroup generators_geo
  */
 struct RibeiroFluxGenerator : public aGenerator2d
 {
diff --git a/inc/geometries/generator.h b/inc/geometries/generator.h
index ba81a44d4..6f5f42c25 100644
--- a/inc/geometries/generator.h
+++ b/inc/geometries/generator.h
@@ -10,7 +10,7 @@ A generator is there to construct coordinate transformations from physical coord
 \f$ x,y\f$ to the computational domain \f$\zeta, \eta\f$, which
 is a product space. 
 @note the origin of the computational space is assumed to be (0,0)
- @ingroup generators
+ @ingroup generators_geo
 */
 struct aGenerator2d
 {
diff --git a/inc/geometries/generatorX.h b/inc/geometries/generatorX.h
index 221484a99..af483a6af 100644
--- a/inc/geometries/generatorX.h
+++ b/inc/geometries/generatorX.h
@@ -10,7 +10,7 @@ A generator is there to construct coordinate transformations from physical coord
 \f$ x,y\f$ to the computational domain \f$\zeta, \eta\f$, which
 is a product space. 
  @note the origin of the computational space is assumed to be (0,0)
- @ingroup generators
+ @ingroup generators_geo
 */
 struct aGeneratorX2d
 {
diff --git a/inc/geometries/hector.h b/inc/geometries/hector.h
index 3de7bd6cd..29ed2759e 100644
--- a/inc/geometries/hector.h
+++ b/inc/geometries/hector.h
@@ -200,9 +200,9 @@ void transform(
 /**
  * @brief The High PrEcision Conformal grid generaTOR 
  *
- * @ingroup generators
+ * @ingroup generators_geo
  * @tparam IMatrix The interpolation matrix type
- * @tparam Matrix  The matrix type in the elliptic equation
+ * @copydoc hide_matrix
  * @copydoc hide_container
  */
 template <class IMatrix = dg::IHMatrix, class Matrix = dg::HMatrix, class container = dg::HVec>
diff --git a/inc/geometries/ribeiro.h b/inc/geometries/ribeiro.h
index c43a428cc..0cb7ab7c8 100644
--- a/inc/geometries/ribeiro.h
+++ b/inc/geometries/ribeiro.h
@@ -183,7 +183,7 @@ struct FieldFinv
 
 /**
  * @brief A two-dimensional grid based on "almost-conformal" coordinates by %Ribeiro and Scott 2010 
- * @ingroup generators
+ * @ingroup generators_geo
  */
 struct Ribeiro : public aGenerator2d
 {
diff --git a/inc/geometries/ribeiroX.h b/inc/geometries/ribeiroX.h
index e54faeccd..00aff3310 100644
--- a/inc/geometries/ribeiroX.h
+++ b/inc/geometries/ribeiroX.h
@@ -237,7 +237,7 @@ struct XFieldFinv
 
 /**
  * @brief A two-dimensional grid based on "almost-conformal" coordinates by %Ribeiro and Scott 2010 
- * @ingroup generators
+ * @ingroup generators_geo
  * @tparam Psi All the template parameters must model aBinaryOperator i.e. the bracket operator() must be callable with two arguments and return a double. 
  */
 struct RibeiroX : public aGeneratorX2d
diff --git a/inc/geometries/separatrix_orthogonal.h b/inc/geometries/separatrix_orthogonal.h
index 1e9f5a28f..0ad647b3a 100644
--- a/inc/geometries/separatrix_orthogonal.h
+++ b/inc/geometries/separatrix_orthogonal.h
@@ -116,7 +116,7 @@ void computeX_rzy( const BinaryFunctorsLvl1& psi,
 /**
  * @brief Choose points on inside or outside line 
  *
- * @ingroup generators
+ * @ingroup generators_geo
  */
 struct SimpleOrthogonalX : public aGeneratorX2d
 {
@@ -181,7 +181,7 @@ struct SimpleOrthogonalX : public aGeneratorX2d
 /**
  * @brief Choose points on separatrix 
  *
- * @ingroup generators
+ * @ingroup generators_geo
  */
 struct SeparatrixOrthogonal : public aGeneratorX2d
 {
diff --git a/inc/geometries/simple_orthogonal.h b/inc/geometries/simple_orthogonal.h
index dd6ab3839..bb5e1322a 100644
--- a/inc/geometries/simple_orthogonal.h
+++ b/inc/geometries/simple_orthogonal.h
@@ -273,7 +273,7 @@ void construct_rz( Nemov nemov,
  * @brief Generate a simple orthogonal grid 
  *
  * Psi is the radial coordinate and you can choose various discretizations of the first line
- * @ingroup generators
+ * @ingroup generators_geo
  */
 struct SimpleOrthogonal : public aGenerator2d
 {
diff --git a/inc/geometries/solovev_doc.h b/inc/geometries/solovev_doc.h
index c53ec2c92..b2387e4ee 100644
--- a/inc/geometries/solovev_doc.h
+++ b/inc/geometries/solovev_doc.h
@@ -2,7 +2,7 @@
 /*! 
  * 
  * @defgroup grids 1. New Geometry classes
- * @defgroup generators 2. Grid generators
+ * @defgroup generators_geo 2. Grid generators
  *
       All the grids introduced by this extension can be constructed with 
       generator classes. 
@@ -31,8 +31,26 @@
  * - there are some miscellaneous additions like a flux surface average class
  * and one used to integrate the field lines for parallel derivatives all in the dg::geo namespace.
  */
-
  /** @class hide_container
-  * @tparam container A data container class for which the blas1 functionality is overloaded. Also we assume that the type is copyable/assignable and has a swap member function. Currently this is one of 
-  *   dg::HVec, dg::DVec, dg::MHVec or dg::MDVec
+  * @tparam container 
+  * A data container class for which the blas1 functionality is overloaded.
+  * We assume that container is copyable/assignable and has a swap member function. 
+  * Currently this is one of 
+  *  - dg::HVec, dg::DVec, dg::MHVec or dg::MDVec  
+  *  - std::vector<dg::HVec>, std::vector<dg::DVec>, std::vector<dg::MHVec> or std::vector<dg::MDVec> . 
+  *
+  */
+ /** @class hide_matrix
+  * @tparam Matrix 
+  * A class for which the blas2 functions are callable in connection with the container class. 
+  * The Matrix type can be one of:
+  *  - container: A container acts as a  diagonal matrix. 
+  *  - dg::HMatrix and dg::IHMatrix with dg::HVec or std::vector<dg::HVec>
+  *  - dg::DMatrix and dg::IDMatrix with dg::DVec or std::vector<dg::DVec>
+  *  - dg::MHMatrix with dg::MHVec or std::vector<dg::MHVec>
+  *  - dg::MDMatrix with dg::MDVec or std::vector<dg::MDVec>
+  *  - Any type that has the SelfMadeMatrixTag specified in a corresponding 
+  *  MatrixTraits class (e.g. Elliptic). In this case only those blas2 functions 
+  *  that have a corresponding member function in the Matrix class (e.g. symv( const container&, container&); ) can be called.
+  *  If the container is a std::vector, then the Matrix is applied to each of the elements.
   */
-- 
GitLab


From ae98a813d92079ed6087019ec16b08bd0afab1d5 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sat, 16 Sep 2017 22:06:35 +0200
Subject: [PATCH 297/453] documentation !!

---
 inc/dg/Doxyfile                     | 1 +
 inc/dg/average.h                    | 2 +-
 inc/dg/backend/matrix_traits.h      | 2 +-
 inc/dg/backend/topological_traits.h | 8 +++-----
 inc/dg/cg.h                         | 6 +++++-
 inc/dg/helmholtz.h                  | 4 ++--
 inc/dg/multigrid.h                  | 1 +
 7 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/inc/dg/Doxyfile b/inc/dg/Doxyfile
index a254af436..45c9eb5ee 100644
--- a/inc/dg/Doxyfile
+++ b/inc/dg/Doxyfile
@@ -830,6 +830,7 @@ EXCLUDE                = ../dg/backend/creation.cuh \
                          ../dg/backend/sparseblockmat_gpu_kernels.cuh \
                          ../dg/backend/sparseblockmat_omp_kernels.h \
                          ../dg/backend/transpose.h \
+                         ../dg/backend/average.h \
 
 # The EXCLUDE_SYMLINKS tag can be used to select whether or not files or
 # directories that are symbolic links (a Unix file system feature) are excluded
diff --git a/inc/dg/average.h b/inc/dg/average.h
index 14c281de2..5f8d771d4 100644
--- a/inc/dg/average.h
+++ b/inc/dg/average.h
@@ -8,5 +8,5 @@
 
 /*!@file 
  *
- * This file includes the appropriate headers for parallel derivatives
+ * @brief This file includes the appropriate headers for parallel derivatives
  */
diff --git a/inc/dg/backend/matrix_traits.h b/inc/dg/backend/matrix_traits.h
index ba82f8b93..16a045c85 100644
--- a/inc/dg/backend/matrix_traits.h
+++ b/inc/dg/backend/matrix_traits.h
@@ -6,7 +6,7 @@ namespace dg{
 
 /*! @brief The matrix traits 
 
-Specialize this struct if you want to enable your class for the use in blas2 functions
+Specialize this struct with the SelfMadeMatrixTag as matrix_category if you want to enable your class for the use in blas2 functions
 @note if you have problems with the compiler choosing CuspMatrixTag even if you don't want it to and you specialized the MatrixTraits for 
 your matrix try to specialize for const Matrix as well
 */
diff --git a/inc/dg/backend/topological_traits.h b/inc/dg/backend/topological_traits.h
index d71fb0180..18bf53899 100644
--- a/inc/dg/backend/topological_traits.h
+++ b/inc/dg/backend/topological_traits.h
@@ -3,16 +3,14 @@
 namespace dg
 {
 
-///@cond
 template <class Topology>
 struct TopologyTraits{
-    typedef typename Topology::memory_category memory_category; //either shared or distributed
-    typedef typename Topology::dimensionality dimensionality; //either shared or distributed
+    typedef typename Topology::memory_category memory_category; //!<either shared or distributed
+    typedef typename Topology::dimensionality dimensionality; //!< two-dimensional or three-dimensional
 };
-///@endcond
 
 //memory categories
-struct MPITag{}; //!< distributed memory  system
+struct MPITag{}; //!< distributed memory system
 struct SharedTag{}; //!<  shared memory system
 
 //dimensionality 
diff --git a/inc/dg/cg.h b/inc/dg/cg.h
index e5615870f..1524b22ba 100644
--- a/inc/dg/cg.h
+++ b/inc/dg/cg.h
@@ -251,10 +251,14 @@ unsigned CG< container>::operator()( Matrix& A, container& x, const container& b
 
 
 /**
-* @brief Class that stores a number of solutions of iterative methods and
+* @brief Class that stores up to three solutions of iterative methods and
 can be used to get initial guesses based on past solutions
+
+ \f[ x_{init} = \alpha_0 x_0 + \alpha_{-1}x_{-1} + \alpha_{-2} x_{-2}\f]
+ where the indices indicate the current (0) and past (negative) solutions.
 *
 * @copydoc hide_container
+* @ingroup misc
 */
 template<class container>
 struct Extrapolation
diff --git a/inc/dg/helmholtz.h b/inc/dg/helmholtz.h
index 27cd1e229..9cc548516 100644
--- a/inc/dg/helmholtz.h
+++ b/inc/dg/helmholtz.h
@@ -147,8 +147,8 @@ struct Helmholtz
  * Can be used by the Invert class
  * @copydoc hide_geometry_matrix_container
  * @attention The Laplacian in this formula is positive as opposed to the negative sign in the Elliptic operator
- * @attention It is better to solve the normal Helmholtz operator twice
- * consecutively than solving the Helmholtz2 operator once. 
+ * @attention It is MUCH better to solve the normal Helmholtz operator twice,
+ * consecutively, than solving the Helmholtz2 operator once. 
  */
 template< class Geometry, class Matrix, class container> 
 struct Helmholtz2
diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index f8d9aef77..d28ac804d 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -16,6 +16,7 @@ namespace dg
 *
 * We use conjugate gradien (CG) at each stage and refine the grids in the first two dimensions (2d / x and y) 
 * @copydoc hide_geometry_matrix_container
+* @ingroup numerical1
 */
 template< class Geometry, class Matrix, class container> 
 struct MultigridCG2d
-- 
GitLab


From c7906a2e62fb5f524ee84305aafac49581b2d615 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sat, 16 Sep 2017 22:35:17 +0200
Subject: [PATCH 298/453] compile everything except cluster_mpib

---
 inc/dg/backend/average.cuh           |  2 +-
 inc/dg/backend/backscatter_t.cu      |  2 +-
 inc/dg/backend/derivativesX.h        | 13 ++++++++++---
 inc/dg/backend/ell_interpolation.cuh |  2 +-
 inc/dg/backend/mpi_collective.h      |  8 ++++----
 inc/dg/bathRZ_t.cu                   |  4 ++--
 inc/dg/elliptic.h                    |  6 +++---
 inc/dg/helmholtz.h                   |  2 +-
 inc/dg/helmholtzg2_t.cu              |  2 +-
 9 files changed, 24 insertions(+), 17 deletions(-)

diff --git a/inc/dg/backend/average.cuh b/inc/dg/backend/average.cuh
index ded066493..76e1c6204 100644
--- a/inc/dg/backend/average.cuh
+++ b/inc/dg/backend/average.cuh
@@ -2,7 +2,7 @@
 
 #include "evaluation.cuh"
 #include "xspacelib.cuh"
-#include "cloneable.h"
+#include "memory.h"
 #include "../blas1.h"
 
 /*! @file 
diff --git a/inc/dg/backend/backscatter_t.cu b/inc/dg/backend/backscatter_t.cu
index 01a31d1f0..b1c82491f 100644
--- a/inc/dg/backend/backscatter_t.cu
+++ b/inc/dg/backend/backscatter_t.cu
@@ -30,7 +30,7 @@ int main()
     dg::HVec visual( grid.size());
     //transform vector to an equidistant grid
     dg::Matrix equidistant = dg::create::backscatter( grid);
-    dg::blas2::mv( equidistant, vector, visual );
+    dg::blas2::gemv( equidistant, vector, visual );
 
     //create a colormap
     draw::ColorMapRedBlueExt colors( 1.);
diff --git a/inc/dg/backend/derivativesX.h b/inc/dg/backend/derivativesX.h
index 33adedaa0..b214bd757 100644
--- a/inc/dg/backend/derivativesX.h
+++ b/inc/dg/backend/derivativesX.h
@@ -20,11 +20,18 @@ struct Composite
     Composite& operator=( const Composite<Matrix2>& src){ Composite c(src); 
         *this = c; return *this;}
     template< class container>
-    void symv( const  container& v1, container& v2)
+    void symv( const  container& v1, container& v2) const
     {
-        m1.symv( v1, v2);
+        m1.symv( v1, v2); //computes first part
         if( dual)
-            m2.symv( v1, v2);
+            m2.symv( v1, v2); //computes second part
+    }
+    template< class container>
+    void symv( double alpha, const  container& v1, double beta, container& v2) const
+    {
+        m1.symv( alpha, v1, beta, v2); //computes first part
+        if( dual)
+            m2.symv( alpha, v1, beta, v2); //computes second part
     }
     void display( std::ostream& os = std::cout) const
     {
diff --git a/inc/dg/backend/ell_interpolation.cuh b/inc/dg/backend/ell_interpolation.cuh
index 754d03c7d..743719ae2 100644
--- a/inc/dg/backend/ell_interpolation.cuh
+++ b/inc/dg/backend/ell_interpolation.cuh
@@ -201,7 +201,7 @@ __launch_bounds__(BLOCK_SIZE, 1) //cuda performance hint macro, (max_threads_per
  * @note n must be smaller than 5
  * @attention no range check is performed on the input vectors
  */
-cusp::ell_matrix<double, int, cusp::device_memory> ell_interpolation( const thrust::device_vector<double>& x, const aGrid1d& g  )
+cusp::ell_matrix<double, int, cusp::device_memory> ell_interpolation( const thrust::device_vector<double>& x, const Grid1d& g  )
 {
     assert( g.n()<=4);
     //allocate ell matrix storage
diff --git a/inc/dg/backend/mpi_collective.h b/inc/dg/backend/mpi_collective.h
index 271a2f08e..023e73699 100644
--- a/inc/dg/backend/mpi_collective.h
+++ b/inc/dg/backend/mpi_collective.h
@@ -190,7 +190,7 @@ struct BijectiveComm
         thrust::gather( idx_.begin(), idx_.end(), values.begin(), values_.data().begin());
         //senden
         Vector store( p_.store_size());
-        p_.scatter( *values_.data(), store);
+        p_.scatter( values_.data(), store);
         return store;
     }
 
@@ -205,7 +205,7 @@ struct BijectiveComm
     void global_scatter_reduce( const Vector& toScatter, Vector& values) const
     {
         //actually this is a gather but we constructed it invertedly
-        p_.gather( toScatter, *values_.data());
+        p_.gather( toScatter, values_.data());
         //nach PID geordnete Werte wieder umsortieren
         thrust::scatter( values_.data().begin(), values_.data().end(), idx_.begin(), values.begin());
     }
@@ -325,7 +325,7 @@ struct SurjectiveComm
         thrust::gather( gatherMap_.begin(), gatherMap_.end(), values.begin(), store_.data().begin());
         //now gather from store into buffer
         Vector buffer( buffer_size_);
-        bijectiveComm_.global_scatter_reduce( *store_.data(), buffer);
+        bijectiveComm_.global_scatter_reduce( store_.data(), buffer);
         return buffer;
     }
     void global_scatter_reduce( const Vector& toScatter, Vector& values)
@@ -413,7 +413,7 @@ struct GeneralComm
     }
     void global_scatter_reduce( const Vector& toScatter, Vector& values)
     {
-        surjectiveComm_.global_scatter_reduce( toScatter, *store_.data());
+        surjectiveComm_.global_scatter_reduce( toScatter, store_.data());
         thrust::scatter( store_.data().begin(), store_.data().end(), scatterMap_.begin(), values.begin());
     }
 
diff --git a/inc/dg/bathRZ_t.cu b/inc/dg/bathRZ_t.cu
index 3a29f45c8..261aa1c81 100644
--- a/inc/dg/bathRZ_t.cu
+++ b/inc/dg/bathRZ_t.cu
@@ -1,6 +1,8 @@
 #include <iostream>
 #include <vector>
 #include <sstream>
+#include "blas.h"
+#include "dg/backend/interpolation.cuh"
 #include "dg/backend/xspacelib.cuh"
 #include "dg/backend/projection.cuh"
 #include "dg/backend/typedefs.cuh"
@@ -35,9 +37,7 @@ int main()
     dg::HVec hvisual_old = dg::evaluate( bathRZ, grid_old);
     dg::HVec hvisual_new( grid_new.size());
     dg::blas2::gemv( interpolate, hvisual_old, hvisual_new);
-    dg::DifferenceNorm<dg::HVec> diff( grid_old, grid_new);
     dg::HVec w2d = dg::create::weights( grid_old);
-    std::cout << "Relative error norm between original and interpolation: "<<diff( hvisual_old, hvisual_new)/dg::blas2::dot( w2d, hvisual_old)<<std::endl;
     //allocate mem for visual
     dg::HVec visual_old( grid_old.size());
     dg::HVec visual_new( grid_new.size());
diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index fc2fd650a..e674eec1e 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -340,7 +340,7 @@ struct GeneralElliptic
     const container& precond()const {return precond_;}
 
     ///@copydoc Elliptic::symv()
-    void symv( container& x, container& y) 
+    void symv( const container& x, container& y) 
     {
         dg::blas2::gemv( rightx, x, temp0); //R_x*x 
         dg::blas1::pointwiseDot( xchi, temp0, xx); //Chi_x*R_x*x 
@@ -504,7 +504,7 @@ struct GeneralEllipticSym
     const container& precond()const {return precond_;}
 
     ///@copydoc Elliptic::symv()
-    void symv( container& x, container& y) 
+    void symv( const container& x, container& y) 
     {
         ellipticForward_.symv( x,y);
         ellipticBackward_.symv( x,temp_);
@@ -607,7 +607,7 @@ struct TensorElliptic
     const container& precond()const {return precond_;}
 
     ///@copydoc Elliptic::symv()
-    void symv( container& x, container& y) 
+    void symv( const container& x, container& y) 
     {
         //compute gradient
         dg::blas2::gemv( rightx, x, tempx_); //R_x*f 
diff --git a/inc/dg/helmholtz.h b/inc/dg/helmholtz.h
index 9cc548516..89a02f601 100644
--- a/inc/dg/helmholtz.h
+++ b/inc/dg/helmholtz.h
@@ -194,7 +194,7 @@ struct Helmholtz2
      * @param y rhs contains solution
      * @note Takes care of sign in laplaceM and thus multiplies by -alpha
      */
-    void symv( container& x, container& y) 
+    void symv(const container& x, container& y) 
     {
         if( alpha_ != 0)
         {
diff --git a/inc/dg/helmholtzg2_t.cu b/inc/dg/helmholtzg2_t.cu
index 2761b9f48..8b15ff1e7 100644
--- a/inc/dg/helmholtzg2_t.cu
+++ b/inc/dg/helmholtzg2_t.cu
@@ -4,8 +4,8 @@
 
 #include "helmholtz.h"
 #include "backend/xspacelib.cuh"
+#include "backend/exceptions.h"
 #include "multistep.h"
-#include "exceptions.h"
 #include "cg.h"
 #include "functors.h"
 
-- 
GitLab


From f242fb273863cae70216c7b0c15c3885d10e9d61 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 18 Sep 2017 11:37:34 +0200
Subject: [PATCH 299/453] reimplement ZShifter as two functions in detail
 namespace in mpi_fieldaligned.h

---
 inc/dg/backend/mpi_vector.h       |  26 +++----
 inc/geometries/mpi_fieldaligned.h | 115 ++++++++++--------------------
 2 files changed, 52 insertions(+), 89 deletions(-)

diff --git a/inc/dg/backend/mpi_vector.h b/inc/dg/backend/mpi_vector.h
index a6d58cd95..7a2f35022 100644
--- a/inc/dg/backend/mpi_vector.h
+++ b/inc/dg/backend/mpi_vector.h
@@ -340,26 +340,26 @@ template<class I, class V>
 const V& NearestNeighborComm<I,V>::global_gather( const V& input) const
 {
     if( silent_) return values.data();
-        //int rank;
-        //MPI_Comm_rank( MPI_COMM_WORLD, &rank);
-        //dg::Timer t;
-        //t.tic();
+    //int rank;
+    //MPI_Comm_rank( MPI_COMM_WORLD, &rank);
+    //dg::Timer t;
+    //t.tic();
     //gather values from input into sendbuffer
     thrust::gather( gather_map1.begin(), gather_map1.end(), input.begin(), buffer1.data().begin());
     thrust::gather( gather_map2.begin(), gather_map2.end(), input.begin(), buffer2.data().begin());
-        //t.toc();
-        //if(rank==0)std::cout << "Gather       took "<<t.diff()<<"s\n";
-        //t.tic();
+    //t.toc();
+    //if(rank==0)std::cout << "Gather       took "<<t.diff()<<"s\n";
+    //t.tic();
     //mpi sendrecv
     sendrecv( buffer1.data(), buffer2.data(), rb1.data(), rb2.data());
-        //t.toc();
-        //if(rank==0)std::cout << "MPI sendrecv took "<<t.diff()<<"s\n";
-        //t.tic();
+    //t.toc();
+    //if(rank==0)std::cout << "MPI sendrecv took "<<t.diff()<<"s\n";
+    //t.tic();
     //scatter received values into values array
     thrust::scatter( rb1.data().begin(), rb1.data().end(), scatter_map1.begin(), values.data().begin());
     thrust::scatter( rb2.data().begin(), rb2.data().end(), scatter_map2.begin(), values.data().begin());
-        //t.toc();
-        //if(rank==0)std::cout << "Scatter      took "<<t.diff()<<"s\n";
+    //t.toc();
+    //if(rank==0)std::cout << "Scatter      took "<<t.diff()<<"s\n";
     return values.data();
 }
 
@@ -371,7 +371,7 @@ void NearestNeighborComm<I,V>::sendrecv( V& sb1, V& sb2 , V& rb1, V& rb2) const
     //mpi_cart_shift may return MPI_PROC_NULL then the receive buffer is not modified 
     MPI_Cart_shift( comm_, direction_, -1, &source, &dest);
 #if THRUST_DEVICE_SYSTEM==THRUST_DEVICE_SYSTEM_CUDA
-    cudaDeviceSynchronize(); //needs to be called 
+    cudaDeviceSynchronize(); //wait until device functions are finished before sending data
 #endif //THRUST_DEVICE_SYSTEM
     MPI_Sendrecv(   thrust::raw_pointer_cast(sb1.data()), buffer_size(), MPI_DOUBLE,  //sender
                     dest, 3,  //destination
diff --git a/inc/geometries/mpi_fieldaligned.h b/inc/geometries/mpi_fieldaligned.h
index 4ba263551..b5ecaaaca 100644
--- a/inc/geometries/mpi_fieldaligned.h
+++ b/inc/geometries/mpi_fieldaligned.h
@@ -14,72 +14,43 @@
 namespace dg{
  
 ///@cond
+namespace detail{
 
-/**
- * @brief Class to shift values in the z - direction 
- */
-struct ZShifter
+///basically a copy across processes
+template<class InputIterator, class OutputIterator>
+void sendForward( InputIterator begin, InputIterator end, OutputIterator result, MPI_Comm comm) //send to next plane
 {
-    ZShifter(){}
-    /**
-     * @brief Constructor
-     *
-     * @param size number of elements to exchange between processes
-     * @param comm the communicator (cartesian)
-     */
-    ZShifter( int size, MPI_Comm comm) 
-    {
-        number_ = size;
-        comm_ = comm;
-        sb_.resize( number_), rb_.resize( number_);
-    }
-    int number() const {return number_;}
-    int size() const {return number_;}
-    MPI_Comm communicator() const {return comm_;}
-    //host and device versions
-    template<class container>
-    void sendForward( const container& sb, container& rb)
-    {
-        dg::blas1::transfer( sb, sb_);
-        sendForward_( sb_, rb_);
-        dg::blas1::transfer( rb_, rb);
-    }
-    template<class container>
-    void sendBackward( const container& sb, container& rb)
-    {
-        dg::blas1::transfer( sb, sb_);
-        sendBackward_( sb_, rb_);
-        dg::blas1::transfer( rb_, rb);
-    }
-    private:
-    void sendForward_( HVec& sb, HVec& rb)const //send to next plane
-    {
-        int source, dest;
-        MPI_Status status;
-        MPI_Cart_shift( comm_, 2, +1, &source, &dest);
-        MPI_Sendrecv(   sb.data(), number_, MPI_DOUBLE,  //sender
-                        dest, 9,  //destination
-                        rb.data(), number_, MPI_DOUBLE, //receiver
-                        source, 9, //source
-                        comm_, &status);
-    }
-    void sendBackward_( HVec& sb, HVec& rb)const //send to previous plane
-    {
-        int source, dest;
-        MPI_Status status;
-        MPI_Cart_shift( comm_, 2, -1, &source, &dest);
-        MPI_Sendrecv(   sb.data(), number_, MPI_DOUBLE,  //sender
-                        dest, 3,  //destination
-                        rb.data(), number_, MPI_DOUBLE, //receiver
-                        source, 3, //source
-                        comm_, &status);
-    }
-    typedef thrust::host_vector<double> HVec;
-    HVec sb_, rb_;
-    int number_; //deepness, dimensions
-    MPI_Comm comm_;
-
-};
+    int source, dest;
+    MPI_Status status;
+    MPI_Cart_shift( comm_, 2, +1, &source, &dest);
+#if THRUST_DEVICE_SYSTEM==THRUST_DEVICE_SYSTEM_CUDA
+    cudaDeviceSynchronize();//wait until device functions are finished before sending data
+#endif //THRUST_DEVICE_SYSTEM
+    unsigned size = thrust::distance( begin, end);
+    MPI_Sendrecv(   thrust::raw_pointer_cast(begin), size, MPI_DOUBLE,  //sender
+                    dest, 9,  //destination
+                    thrust::raw_pointer_cast(result), size, MPI_DOUBLE, //receiver
+                    source, 9, //source
+                    comm, &status);
+}
+///basically a copy across processes
+template<class InputIterator, class OutputIterator>
+void sendBackward( InputIterator begin, InputIterator end, OutputIterator result, MPI_Comm comm) //send to next plane
+{
+    int source, dest;
+    MPI_Status status;
+    MPI_Cart_shift( comm_, 2, -1, &source, &dest);
+#if THRUST_DEVICE_SYSTEM==THRUST_DEVICE_SYSTEM_CUDA
+    cudaDeviceSynchronize();//wait until device functions are finished before sending data
+#endif //THRUST_DEVICE_SYSTEM
+    unsigned size = thrust::distance( begin, end);
+    MPI_Sendrecv(   thrust::raw_pointer_cast(begin), size, MPI_DOUBLE,  //sender
+                    dest, 3,  //destination
+                    thrust::raw_pointer_cast(result), size, MPI_DOUBLE, //receiver
+                    source, 3, //source
+                    comm, &status);
+}
+}//namespace detail
 
 ///@endcond
 
@@ -271,9 +242,7 @@ struct FieldAligned< Geometry, RowDistMat<LocalMatrix, Communicator>, MPI_Vector
     LocalContainer left_, right_;
     LocalContainer limiter_;
     std::vector<LocalContainer> tempXYplus_, tempXYminus_, temp_; 
-    LocalContainer tempZ_;
     Communicator commXYplus_, commXYminus_;
-    ZShifter  commZ_;
     LocalMatrix plus, minus; //interpolation matrices
     LocalMatrix plusT, minusT; //interpolation matrices
 };
@@ -371,8 +340,6 @@ FieldAligned<MPIGeometry, RowDistMat<LocalMatrix, CommunicatorXY>, MPI_Vector<Lo
         tempXYminus_[i].resize( commXYminus_.size());
         temp_[i].resize( localsize);
     }
-    commZ_ = ZShifter( localsize, g_.communicator() );
-    tempZ_.resize( commZ_.size());
 }
 
 template<class G, class M, class C, class container>
@@ -504,8 +471,7 @@ void FieldAligned<G,RowDistMat<M,C>, MPI_Vector<container> >::einsPlus( const MP
     }
     if( sizeZ != 1)
     {
-        commZ_.sendBackward( temp_[0], tempZ_);
-        thrust::copy( tempZ_.begin(), tempZ_.end(), out.begin() + (g_.Nz()-1)*size2d);
+        detail::sendBackward( temp_[0].begin(), temp_[0].end(), out.begin() + (g_.Nz()-1)*size2d);
     }
 
     //make ghostcells in last plane
@@ -579,8 +545,7 @@ void FieldAligned<G,RowDistMat<M,C>,MPI_Vector<container> >::einsMinus( const MP
     }
     if( sizeZ != 1)
     {
-        commZ_.sendForward( temp_[g_.Nz()-1], tempZ_);
-        thrust::copy( tempZ_.begin(), tempZ_.end(), out.begin());
+        detail::sendForward( temp_[g_.Nz()-1].begin(), temp_[g_.Nz()-1].end(), out.begin());
     }
     //make ghostcells in first plane
     unsigned size = g_.n()*g_.n()*g_.Nx()*g_.Ny();
@@ -652,8 +617,7 @@ void FieldAligned<G,RowDistMat<M,C>,MPI_Vector<container> >::einsMinusT( const M
     }
     if( sizeZ != 1)
     {
-        commZ_.sendBackward( temp_[0], tempZ_);
-        thrust::copy( tempZ_.begin(), tempZ_.end(), out.begin() + (g_.Nz()-1)*size2d);
+        detail::sendBackward( temp_[0].begin(), temp_[0].end(), out.begin() + ( g_.Nz()-1)*size2d);
     }
     //make ghostcells in last plane
     unsigned size = g_.n()*g_.n()*g_.Nx()*g_.Ny();
@@ -726,8 +690,7 @@ void FieldAligned<G,RowDistMat<M,C>,MPI_Vector<container> >::einsPlusT( const MP
     }
     if( sizeZ != 1)
     {
-        commZ_.sendForward( temp_[g_.Nz()-1], tempZ_);
-        thrust::copy( tempZ_.begin(), tempZ_.end(), out.begin());
+        detail::sendForward( temp_[g_.Nz()-1].begin(), temp_[g_.Nz()-1].end(), out.begin());
     }
     //make ghostcells in first plane
     unsigned size = g_.n()*g_.n()*g_.Nx()*g_.Ny();
-- 
GitLab


From 1236abdf5d19c90d67adba319266577a918b5f40 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 18 Sep 2017 14:27:57 +0200
Subject: [PATCH 300/453] get rid of display and access member functions in
 MPI_Vector

---
 inc/dg/backend/mpi_collective.h   |  95 ++++++++++++++++++++-
 inc/dg/backend/mpi_matrix.h       | 134 +++++++++---------------------
 inc/dg/backend/mpi_vector.h       |  46 ++--------
 inc/dg/dg_doc.h                   |  91 +-------------------
 inc/geometries/mpi_fieldaligned.h |  17 ++--
 5 files changed, 148 insertions(+), 235 deletions(-)

diff --git a/inc/dg/backend/mpi_collective.h b/inc/dg/backend/mpi_collective.h
index 023e73699..9367d333f 100644
--- a/inc/dg/backend/mpi_collective.h
+++ b/inc/dg/backend/mpi_collective.h
@@ -121,6 +121,97 @@ void Collective<Index, Device>::gather( const Device& gatherFrom, Device& values
 //BijectiveComm ist der Spezialfall, dass jedes Element nur ein einziges Mal gebraucht wird. 
 ///@endcond
 //
+/**
+ * @brief Struct that performs collective scatter and gather operations across processes
+ * on distributed vectors using mpi
+ *
+ * In order to understand the issue you must first really(!) understand what 
+ gather and scatter operations are, so grab pen and paper: 
+
+ First we note that gather and scatter are most often used in the context
+ of memory buffers. The buffer needs to be filled wih values (gather) or these
+ values need to be written back into the original place (scatter).
+
+ Gather: imagine a buffer vector w and a map that gives to every element in this vector w
+ an index into a source vector v where the value of this element should be taken from
+ i.e. \f$ w[i] = v[\text{idx}[i]] \f$ 
+ Note that an index in the source vector can appear several times or not at all. 
+ This is why the source vector w can have any size and even be smaller than w. 
+
+ Scatter: imagine a buffer vector w and a map that gives to every element in the buffer w an
+ index in a target vector v where this element should go to, 
+ i.e. \f$ v[\text{idx}[i]] = w[i] \f$. This is ill-defined.
+ Note again that an index in v can appear several times or never at all. 
+ Then in our case we perform a reduction operation (we sum up all elements) beginning
+ with 0 which remedies the defintion. 
+
+Since it is a vector operation the gather and scatter operation can 
+also be represented/defined by a matrix. The gather matrix is just a 
+(permutation) matrix of 1's and 0's with exactly one "1" in each line.
+In a "coo" formatted sparse matrix format the values array would consist only of "1"s, 
+row array is just the index and column array is the gather map.
+We uniquely define the corresponding scatter matrix as the transpose of the gather matrix. 
+The scatter matrix can have zero, one or more "1"s in each line.
+\f[ w = G v \\
+    v = S w \f]
+
+The scatter matrix S is the actual inverse of G if and only if the gather map is bijective.
+In this case the buffer and the vector can swap their roles. 
+
+Finally note that when v is filled with its indices, i.e. \f$ v[i] = i \f$, then
+the gather operation will reproduce the index map in the buffer w \f$ w[i] = \text{idx}[i]\f$ .
+
+ * @ingroup mpi_structures
+ */
+struct aCommunicator
+{
+
+    /**
+     * @brief Gather data across processes
+     *
+     * @param values data to gather from
+     * @tparam LocalContainer a container on a shared memory system
+     *
+     * @return the buffer vector of size size()
+     */
+    template< class LocalContainer>
+    LocalContainer global_gather( const LocalContainer& values)const;
+    template< class LocalContainer>
+    void global_gather( const LocalContainer& values, LocalContainer& gathered)const;
+    //actually the return type in NNC is const LocalContainer& 
+
+    /**
+     * @brief Scatters data accross processes and reduces on double indices
+     *
+     * @tparam LocalContainer a container on a shared memory system
+     * @param toScatter buffer vector (has to be of size given by size())
+     * @param values contains values from other processes sent back to the origin 
+     */
+    template< class LocalContainer>
+    void global_scatter_reduce( const LocalContainer& toScatter, LocalContainer& values) const;
+    template< class LocalContainer>
+    LocalContainer global_scatter_reduce( const LocalContainer& toScatter, LocalContainer& values) const;
+
+    /**
+    * @brief The size of the local buffer = local map size
+    *
+ Consider that both the vector v and the buffer w are distributed across processes.
+ In Feltor the vector v is distributed equally among processes and the local size
+ of v is the same for all processes. However the buffer size might be different for each process. 
+    * @note may return 0 to indicate identity between v and w and that no MPI communication is needed 
+    * @note we assume that the vector size is always the local size of a dg::MPI_Vector
+    * @return buffer size
+    */
+    unsigned size() const;
+    /**
+    * @brief The internal mpi communicator used 
+    *
+    * used to assert that communicators of matrix and vector are the same
+    * @return MPI Communicator
+    */
+    MPI_Comm communicator() const;
+};
+
 /**
  * @ingroup mpi_structures
  * @brief Struct that performs bijective collective scatter and gather operations across processes
@@ -144,7 +235,7 @@ void Collective<Index, Device>::gather( const Device& gatherFrom, Device& values
  @note models aCommunicator
  */
 template< class Index, class Vector>
-struct BijectiveComm
+struct BijectiveComm 
 {
     /**
      * @brief Construct empty class
@@ -382,7 +473,7 @@ struct SurjectiveComm
  * This Communicator can perform general global gather and
  scatter operations. 
  @tparam Index an integer Vector
- @tparam Vector a Vector (the data type of Vector must be double)
+ @tparam Vector a Vector (the data type of Vector must be double, and the Vector must have size() and resize() function)
  @note models aCommunicator
  */
 template< class Index, class Vector>
diff --git a/inc/dg/backend/mpi_matrix.h b/inc/dg/backend/mpi_matrix.h
index 2efdf7db6..3e5dde024 100644
--- a/inc/dg/backend/mpi_matrix.h
+++ b/inc/dg/backend/mpi_matrix.h
@@ -166,7 +166,14 @@ struct RowColDistMat
     Collective c_;
 };
 
-///@cond
+
+///Type of distribution of MPI distributed matrices
+enum dist_type
+{
+    row_dist; //!< Row distributed
+    col_dist; //!< Column distributed
+};
+
 /**
 * @brief Distributed memory matrix class
 *
@@ -181,19 +188,20 @@ Gather all points (including the ones that the process already has) necessary fo
 product into one vector, such that the local matrix can be applied.
 int size(); 
 should give the size of the vector that global_gather returns. If size()==0 the global_gather() function won't be called and
-only the inner matrix is applied.
+only the local matrix is applied.
 */
 template<class LocalMatrix, class Collective >
-struct RowDistMat
+struct MPIDistMat
 {
-    RowDistMat( ) { }
+    MPIDistMat( ) { }
     /**
     * @brief Constructor 
     *
     * @param m The local matrix
     * @param c The communication object
+    * @param dist either row or column distributed
     */
-    RowDistMat( const LocalMatrix& m, const Collective& c):m_(m), c_(c) { }
+    MPIDistMat( const LocalMatrix& m, const Collective& c, enum dist_type dist = row_dist):m_(m), c_(c), m_dist( dist) { }
 
     /**
     * @brief Copy Constructor 
@@ -203,7 +211,7 @@ struct RowDistMat
     * @param src The other matrix
     */
     template< class OtherMatrix, class OtherCollective>
-    RowDistMat( const RowDistMat<OtherMatrix, OtherCollective>& src):m_(src.matrix()), c_(src.collective())
+    MPIDistMat( const MPIDistMat<OtherMatrix, OtherCollective>& src):m_(src.matrix()), c_(src.collective())
     { }
     /**
     * @brief Access to the local matrix
@@ -217,6 +225,10 @@ struct RowDistMat
     * @return Reference to the collective object
     */
     const Collective& collective() const{return c_;}
+
+    enum dist_type get_dist() const {return m_dist;}
+    void set_dist(enum dist_type dist){m_dist=dist;}
+
     
     template<class container> 
     void symv( double alpha, const MPI_Vector<container>& x, double beta, MPI_Vector<container>& y)
@@ -231,10 +243,19 @@ struct RowDistMat
         }
         assert( x.communicator() == y.communicator());
         assert( x.communicator() == c_.communicator());
-        container temp = c_.global_gather( x.data());
-        dg::blas2::detail::doSymv( alpha, m_, temp, beta, y.data(), 
+        if( m_dist == row_dist){
+            container temp = c_.global_gather( x.data());
+            dg::blas2::detail::doSymv( alpha, m_, temp, beta, y.data(), 
                        typename dg::MatrixTraits<LocalMatrix>::matrix_category(), 
                        typename dg::VectorTraits<container>::vector_category() );
+        }
+        if( m_dist == col_dist){
+            container temp( c_.size());
+            dg::blas2::detail::doSymv( alpha, m_, x.data(), beta, temp, 
+                           typename dg::MatrixTraits<LocalMatrix>::matrix_category(), 
+                           typename dg::VectorTraits<container>::vector_category() );
+            c_.global_scatter_reduce( temp, y.data());
+        }
     }
     template<class container> 
     void symv( const MPI_Vector<container>& x, MPI_Vector<container>& y)
@@ -250,106 +271,31 @@ struct RowDistMat
         }
         assert( x.communicator() == y.communicator());
         assert( x.communicator() == c_.communicator());
-        container temp = c_.global_gather( x.data());
-        dg::blas2::detail::doSymv( m_, temp, y.data(), 
-                       typename dg::MatrixTraits<LocalMatrix>::matrix_category(), 
-                       typename dg::VectorTraits<container>::vector_category(),
-                       typename dg::VectorTraits<container>::vector_category() );
-    }
-
-        
-    private:
-    LocalMatrix m_;
-    Collective c_;
-};
-
-/**
-* @brief Distributed memory matrix class
-*
-* The idea of this mpi matrix is to separate communication and computation in order to reuse existing optimized matrix formats for the computation. 
-* It can be expected that this works particularly well for cases in which the communication to computation ratio is low. 
-* This class assumes that the matrix and vector elements are distributed columnwise among mpi processes.
-* @tparam LocalMatrix The class of the matrix for local computations. 
- symv needs to be callable on the container class of the MPI_Vector
-* @tparam Collective models aCommunicator 
-void global_scatter_reduce( const container& input, container& output);
-Sends the results of the local computations to the processes they belong to. 
-After that the results of the same lines need to be reduced.
-int size(); 
-should give the size of the vector that global_scatter_reduce needs. If size()==0 the global_scatter_reduce() function won't be called and
-only the inner matrix is applied.
-*/
-template<class LocalMatrix, class Collective >
-struct ColDistMat
-{
-    ColDistMat( ){}
-    /**
-    * @brief Constructor 
-    *
-    * @param m The local matrix
-    * @param c The communication object
-    */
-    ColDistMat( const LocalMatrix& m, const Collective& c):m_(m), c_(c)
-    { }
-    /**
-    * @brief Access to the local matrix
-    *
-    * @return Reference to the local matrix
-    */
-    const LocalMatrix& matrix() const{return m_;}
-    /**
-    * @brief Access to the communication object
-    *
-    * @return Reference to the collective object
-    */
-    const Collective& collective() const{return c_;}
-    
-    template<class container> 
-    void symv( double alpha, const MPI_Vector<container>& x, double beta, MPI_Vector<container>& y)
-    {
-        if( c_.size() == 0) //no communication needed
-        {
-            dg::blas2::detail::doSymv( alpha, m_, x.data(), beta, y.data(), 
+        if( m_dist == row_dist){
+            container temp = c_.global_gather( x.data());
+            dg::blas2::detail::doSymv( m_, temp, y.data(), 
                        typename dg::MatrixTraits<LocalMatrix>::matrix_category(), 
                        typename dg::VectorTraits<container>::vector_category(),
                        typename dg::VectorTraits<container>::vector_category() );
-            return;
-
         }
-        assert( x.communicator() == y.communicator());
-        assert( x.communicator() == c_.communicator());
-        container temp( c_.size());
-        dg::blas2::detail::doSymv( alpha, m_, x.data(), beta, temp, 
-                       typename dg::MatrixTraits<LocalMatrix>::matrix_category(), 
-                       typename dg::VectorTraits<container>::vector_category() );
-        c_.global_scatter_reduce( temp, y.data());
-    }
-    template<class container> 
-    void symv( const MPI_Vector<container>& x, MPI_Vector<container>& y)
-    {
-        if( c_.size() == 0) //no communication needed
-        {
-            dg::blas2::detail::doSymv( m_, x.data(), y.data(), 
+        if( m_dist == col_dist){
+            container temp( c_.size());
+            dg::blas2::detail::doSymv( m_, x.data(), temp, 
                        typename dg::MatrixTraits<LocalMatrix>::matrix_category(), 
                        typename dg::VectorTraits<container>::vector_category(),
                        typename dg::VectorTraits<container>::vector_category() );
-            return;
-
+            c_.global_scatter_reduce( temp, y.data());
         }
-        assert( x.communicator() == y.communicator());
-        assert( x.communicator() == c_.communicator());
-        container temp( c_.size());
-        dg::blas2::detail::doSymv( m_, x.data(), temp, 
-                       typename dg::MatrixTraits<LocalMatrix>::matrix_category(), 
-                       typename dg::VectorTraits<container>::vector_category(),
-                       typename dg::VectorTraits<container>::vector_category() );
-        c_.global_scatter_reduce( temp, y.data());
     }
+
+        
     private:
+    enum dist_type m_dist;
     LocalMatrix m_;
     Collective c_;
 };
 
+///@cond
 template<class LI, class LO, class C>
 struct MatrixTraits<RowColDistMat<LI,LO, C> >
 {
diff --git a/inc/dg/backend/mpi_vector.h b/inc/dg/backend/mpi_vector.h
index 7a2f35022..dbb2cb68a 100644
--- a/inc/dg/backend/mpi_vector.h
+++ b/inc/dg/backend/mpi_vector.h
@@ -40,7 +40,7 @@ struct MPI_Vector
     * @brief Conversion operator
     *
     * uses conversion between compatible containers
-    * @tparam OtherContainer Another container class
+    * @tparam OtherContainer Another container class (container must be copy constructible from OtherContainer)
     * @param src the source 
     */
     template<class OtherContainer>
@@ -79,40 +79,6 @@ struct MPI_Vector
      */
     MPI_Comm communicator() const{return comm_;}
 
-    /**
-     * @brief Display local data
-     *
-     * @param os outstream
-     */
-    void display( std::ostream& os) const
-    {
-        for( unsigned j=0; j<data_.size(); j++)
-            os << data_[j] << " ";
-        os << "\n";
-    }
-    /**
-    * @brief Access single data points
-    *
-    * @param i index
-    *
-    * @return a value
-    * @attention if the container class is a device vector communication is needed to transfer the value from the device to the host
-    */
-    double operator[](int i) const{return data_[i];}
-    /**
-     * @brief Disply local data
-     *
-     * @param os outstream
-     * @param v a vector
-     *
-     * @return  the outsream
-     */
-    friend std::ostream& operator<<( std::ostream& os, const MPI_Vector& v)
-    {
-        os << "Vector of size  "<<v.size()<<"\n";
-        v.display(os);
-        return os;
-    }
     /**
      * @brief Swap data 
      *
@@ -131,12 +97,12 @@ struct MPI_Vector
 
 template<class container> 
 struct VectorTraits<MPI_Vector<container> > {
-    typedef double value_type;
+    typedef typename container::value_type value_type;
     typedef MPIVectorTag vector_category;
 };
 template<class container> 
 struct VectorTraits<const MPI_Vector<container> > {
-    typedef double value_type;
+    typedef typename container::value_type value_type;
     typedef MPIVectorTag vector_category;
 };
 
@@ -148,8 +114,9 @@ struct VectorTraits<const MPI_Vector<container> > {
 *
 * exchanges a halo of given size among next neighbors in a given direction
 * @ingroup mpi_structures
-* @tparam Index the type of index container
-* @tparam Vector the vector container type
+* @tparam Index the type of index container (must be either thrust::host_vector<int> or thrust::device_vector<int>)
+* @tparam Vector the vector container type must have a resize() function and work 
+* in the thrust library functions ( i.e. must a thrust::host_vector or thrust::device_vector)
 * @note models aCommunicator
 */
 template<class Index, class Vector>
@@ -228,7 +195,6 @@ struct NearestNeighborComm
     int direction() const {return direction_;}
     private:
     void construct( int n, const int vector_dimensions[3], MPI_Comm comm, int direction);
-    typedef thrust::host_vector<double> HVec;
 
     int n_, dim_[3]; //deepness, dimensions
     MPI_Comm comm_;
diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index f174b7a1b..3899c675b 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -84,9 +84,7 @@
  *     @defgroup arakawa Discretization of Poisson bracket
  *     @defgroup matrixoperators Elliptic and Helmholtz operators
  * @}
- * @defgroup templates Level 99: Template models
- * Documentation for template models
- * @defgroup misc Level 00: Miscellaneous additions
+ * @defgroup misc Level 99: Miscellaneous additions
  * @{
  *     @defgroup timer Timer class
  *     @defgroup functions Functions and Functors
@@ -176,90 +174,3 @@
   and specializing MatrixTraits with the SelfMadeMatrixTag as the matrix_category
   */
 
-/**
- * @brief Struct that performs collective scatter and gather operations across processes
- * on distributed vectors using mpi
- *
- * In order to understand the issue you must first really(!) understand what 
- gather and scatter operations are, so grab pen and paper: 
-
- First we note that gather and scatter are most often used in the context
- of memory buffers. The buffer needs to be filled wih values (gather) or these
- values need to be written back into the original place (scatter).
-
- Gather: imagine a buffer vector w and a map that gives to every element in this vector w
- an index into a source vector v where the value of this element should be taken from
- i.e. \f$ w[i] = v[\text{idx}[i]] \f$ 
- Note that an index in the source vector can appear several times or not at all. 
- This is why the source vector w can have any size and even be smaller than w. 
-
- Scatter: imagine a buffer vector w and a map that gives to every element in the buffer w an
- index in a target vector v where this element should go to, 
- i.e. \f$ v[\text{idx}[i]] = w[i] \f$. This is ill-defined.
- Note again that an index in v can appear several times or never at all. 
- Then in our case we perform a reduction operation (we sum up all elements) beginning
- with 0 which remedies the defintion. 
-
-Since it is a vector operation the gather and scatter operation can 
-also be represented/defined by a matrix. The gather matrix is just a 
-(permutation) matrix of 1's and 0's with exactly one "1" in each line.
-In a "coo" formatted sparse matrix format the values array would consist only of "1"s, 
-row array is just the index and column array is the gather map.
-We uniquely define the corresponding scatter matrix as the transpose of the gather matrix. 
-The scatter matrix can have zero, one or more "1"s in each line.
-\f[ w = G v \\
-    v = S w \f]
-
-The scatter matrix S is the actual inverse of G if and only if the gather map is bijective.
-In this case the buffer and the vector can swap their roles. 
-
-Finally note that when v is filled with its indices, i.e. \f$ v[i] = i \f$, then
-the gather operation will reproduce the index map in the buffer w \f$ w[i] = \text{idx}[i]\f$ .
-
- * @ingroup templates
- @attention this is not a real class it's there for documentation only
- */
-struct aCommunicator
-{
-
-    /**
-     * @brief Gather data across processes
-     *
-     * @param values data to gather from
-     * @tparam LocalContainer a container on a shared memory system
-     *
-     * @return the buffer vector of size size()
-     */
-    template< class LocalContainer>
-    LocalContainer global_gather( const LocalContainer& values)const;
-    //actually the return type in NNC is const LocalContainer& 
-
-    /**
-     * @brief Scatters data accross processes and reduces on double indices
-     *
-     * @tparam LocalContainer a container on a shared memory system
-     * @param toScatter buffer vector (has to be of size given by size())
-     * @param values contains values from other processes sent back to the origin 
-     */
-    template< class LocalContainer>
-    void global_scatter_reduce( const LocalContainer& toScatter, LocalContainer& values) const;
-
-    /**
-    * @brief The size of the local buffer = local map size
-    *
- Consider that both the vector v and the buffer w are distributed across processes.
- In Feltor the vector v is distributed equally among processes and the local size
- of v is the same for all processes. However the buffer size might be different for each process. 
-    * @note may return 0 to indicate identity between v and w and that no MPI communication is needed 
-    * @note we assume that the vector size is always the local size of a dg::MPI_Vector
-    * @return buffer size
-    */
-    unsigned size() const;
-    /**
-    * @brief The internal mpi communicator used 
-    *
-    * used to assert that communicators of matrix and vector are the same
-    * @return MPI Communicator
-    */
-    MPI_Comm communicator() const;
-};
diff --git a/inc/geometries/mpi_fieldaligned.h b/inc/geometries/mpi_fieldaligned.h
index b5ecaaaca..b5a49f537 100644
--- a/inc/geometries/mpi_fieldaligned.h
+++ b/inc/geometries/mpi_fieldaligned.h
@@ -51,7 +51,6 @@ void sendBackward( InputIterator begin, InputIterator end, OutputIterator result
                     comm, &status);
 }
 }//namespace detail
-
 ///@endcond
 
 
@@ -60,12 +59,12 @@ void sendBackward( InputIterator begin, InputIterator end, OutputIterator result
  * @brief Class for the evaluation of a parallel derivative (MPI Version)
  *
  * @ingroup fieldaligned
- * @tparam LocalMatrix The matrix class of the interpolation matrix
+ * @tparam LocalIMatrix The matrix class of the interpolation matrix
  * @tparam Communicator The communicator used to exchange data in the RZ planes
  * @tparam LocalContainer The container-class to on which the interpolation matrix operates on (does not need to be dg::HVec)
  */
-template <class Geometry, class LocalMatrix, class Communicator, class LocalContainer>
-struct FieldAligned< Geometry, RowDistMat<LocalMatrix, Communicator>, MPI_Vector<LocalContainer> > 
+template <class Geometry, class LocalIMatrix, class CommunicatorXY, class LocalContainer>
+struct FieldAligned< Geometry, RowDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vector<LocalContainer> > 
 {
     /**
     * @brief Construct from a field and a grid
@@ -242,15 +241,15 @@ struct FieldAligned< Geometry, RowDistMat<LocalMatrix, Communicator>, MPI_Vector
     LocalContainer left_, right_;
     LocalContainer limiter_;
     std::vector<LocalContainer> tempXYplus_, tempXYminus_, temp_; 
-    Communicator commXYplus_, commXYminus_;
-    LocalMatrix plus, minus; //interpolation matrices
-    LocalMatrix plusT, minusT; //interpolation matrices
+    CommunicatorXY commXYplus_, commXYminus_;
+    LocalIMatrix plus, minus; //interpolation matrices
+    LocalIMatrix plusT, minusT; //interpolation matrices
 };
 ///@cond
 //////////////////////////////////////DEFINITIONS/////////////////////////////////////
-template<class MPIGeometry, class LocalMatrix, class CommunicatorXY, class LocalContainer>
+template<class MPIGeometry, class LocalIMatrix, class CommunicatorXY, class LocalContainer>
 template <class Field, class Limiter>
-FieldAligned<MPIGeometry, RowDistMat<LocalMatrix, CommunicatorXY>, MPI_Vector<LocalContainer> >::FieldAligned(Field field, MPIGeometry grid, double eps, Limiter limit, dg::bc globalbcz, double deltaPhi ): 
+FieldAligned<MPIGeometry, RowDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vector<LocalContainer> >::FieldAligned(Field field, MPIGeometry grid, double eps, Limiter limit, dg::bc globalbcz, double deltaPhi ): 
     hz_( dg::evaluate( dg::zero, grid)), hp_( hz_), hm_( hz_), 
     g_(grid), bcz_(grid.bcz()), 
     tempXYplus_(g_.Nz()), tempXYminus_(g_.Nz()), temp_(g_.Nz())
-- 
GitLab


From 7935f59b4f191904f44e9f892840a988f96857cf Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 18 Sep 2017 14:47:22 +0200
Subject: [PATCH 301/453] update documentation for MPI_Vector

---
 inc/dg/backend/mpi_vector.h | 78 ++++++++++++++-----------------------
 1 file changed, 30 insertions(+), 48 deletions(-)

diff --git a/inc/dg/backend/mpi_vector.h b/inc/dg/backend/mpi_vector.h
index dbb2cb68a..e3c4b5fa1 100644
--- a/inc/dg/backend/mpi_vector.h
+++ b/inc/dg/backend/mpi_vector.h
@@ -14,23 +14,23 @@ namespace dg
  * @brief mpi Vector class 
  *
  * @ingroup mpi_structures
- * The idea of this Vector class is to simply base it on an existing container class
- * that fully supports the blas functionality. In a computation we use mpi to 
- communicate (e.g. boundary points in matrix-vector multiplications) and use
- the existing blas functions for the local computations. 
- * (At the blas level 1 level communication is needed for scalar products)
- * @note Don't start looking for ghostcells, there aren't any
- * @tparam container underlying local container class
+ *
+ * This class is a simple wrapper around a container object and an MPI_Comm. 
+ * The blas1 and blas2 functionality is available iff it is available for the container type.
+ * We use mpi to communicate (e.g. boundary points in matrix-vector multiplications) 
+ * and use the existing blas functions for the local computations. 
+ * (At the blas level 1 communication is needed only for scalar products)
+ * @tparam container local container type. Must have a size() and a swap() member function.
  */
 template<class container>
 struct MPI_Vector
 {
     typedef container container_type;//!< typedef to acces underlying container
-    MPI_Vector(){}
+    ///no data is allocated, communicator is MPI_COMM_WORLD
+    MPI_Vector(){ comm_ = MPI_COMM_WORLD;}
     /**
      * @brief construct a vector
-     *
-     * @param data internal data
+     * @param data internal data copy 
      * @param comm MPI communicator
      */
     MPI_Vector( const container& data, MPI_Comm comm): 
@@ -40,53 +40,35 @@ struct MPI_Vector
     * @brief Conversion operator
     *
     * uses conversion between compatible containers
-    * @tparam OtherContainer Another container class (container must be copy constructible from OtherContainer)
+    * @tparam OtherContainer another container class (container must be copy constructible from OtherContainer)
     * @param src the source 
     */
     template<class OtherContainer>
     MPI_Vector( const MPI_Vector<OtherContainer>& src){ data_ = src.data(); comm_ = src.communicator();} 
 
-    /**
-     * @brief Set the communicator to which this vector belongs
-     *
-     * @return MPI communicator reference
-     */
-    MPI_Comm& communicator(){return comm_;}
-
-    /**
-     * @brief Set underlying data
-     *
-     * @return 
-     */
-    container& data() {return data_;}
-    /**
-     * @brief Get underlying data
-     *
-     * @return 
-     */
+    ///@brief Get underlying data
+    ///@return read access to data
     const container& data() const {return data_;}
-    /**
-     * @brief Return local size
-     * 
-     * @return local size
-     */
-    unsigned size() const{return data_.size();}
+    ///@brief Set underlying data
+    ///@return write access to data
+    container& data() {return data_;}
 
-    /**
-     * @brief The communicator to which this vector belongs
-     *
-     * @return MPI communicator
-     */
+    ///@brief Get the communicator to which this vector belongs
+    ///@return read access to MPI communicator
     MPI_Comm communicator() const{return comm_;}
+    ///@brief Set the communicator to which this vector belongs
+    ///@return write access to MPI communicator
+    MPI_Comm& communicator(){return comm_;}
 
-    /**
-     * @brief Swap data 
-     *
-     * @param that must have equal sizes and communicator
-     */
-    void swap( MPI_Vector& that){ 
-        assert( comm_ == that.comm_);
-        data_.swap(that.data_);
+    ///@brief Return the size of the data object
+    ///@return local size
+    unsigned size() const{return data_.size();}
+
+    ///@brief Swap data  and communicator
+    ///@param src communicator and data is swapped
+    void swap( MPI_Vector& src){ 
+        std::swap( comm_ == src.comm_);
+        data_.swap(src.data_);
     }
   private:
     container data_; 
-- 
GitLab


From 5ef5f4beaf5b97d1d19994d0216ae07b55cf5d30 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 18 Sep 2017 16:52:03 +0200
Subject: [PATCH 302/453] added mpi_communicator.h

---
 inc/dg/backend/evaluation.cuh     |   1 +
 inc/dg/backend/mpi_collective.h   |  93 +--------------------
 inc/dg/backend/mpi_communicator.h | 132 ++++++++++++++++++++++++++++++
 inc/geometries/fluxfunctions.h    |   2 +
 inc/geometries/generator.h        |   9 +-
 5 files changed, 141 insertions(+), 96 deletions(-)
 create mode 100644 inc/dg/backend/mpi_communicator.h

diff --git a/inc/dg/backend/evaluation.cuh b/inc/dg/backend/evaluation.cuh
index 7146d223e..a9e90fc53 100644
--- a/inc/dg/backend/evaluation.cuh
+++ b/inc/dg/backend/evaluation.cuh
@@ -11,6 +11,7 @@
 namespace dg
 {
 
+//maybe we should consider using boost::function as argument type as a generic function pointer typ
 
 ///@addtogroup evaluation
 ///@{
diff --git a/inc/dg/backend/mpi_collective.h b/inc/dg/backend/mpi_collective.h
index 9367d333f..85a397ced 100644
--- a/inc/dg/backend/mpi_collective.h
+++ b/inc/dg/backend/mpi_collective.h
@@ -11,10 +11,10 @@
 #include "thrust_vector_blas.cuh"
 #include "dg/blas1.h"
 #include "memory.h"
+#include "mpi_communicator.h"
 
 namespace dg{
 
-
 ///@cond
 
 /**
@@ -120,97 +120,6 @@ void Collective<Index, Device>::gather( const Device& gatherFrom, Device& values
 }
 //BijectiveComm ist der Spezialfall, dass jedes Element nur ein einziges Mal gebraucht wird. 
 ///@endcond
-//
-/**
- * @brief Struct that performs collective scatter and gather operations across processes
- * on distributed vectors using mpi
- *
- * In order to understand the issue you must first really(!) understand what 
- gather and scatter operations are, so grab pen and paper: 
-
- First we note that gather and scatter are most often used in the context
- of memory buffers. The buffer needs to be filled wih values (gather) or these
- values need to be written back into the original place (scatter).
-
- Gather: imagine a buffer vector w and a map that gives to every element in this vector w
- an index into a source vector v where the value of this element should be taken from
- i.e. \f$ w[i] = v[\text{idx}[i]] \f$ 
- Note that an index in the source vector can appear several times or not at all. 
- This is why the source vector w can have any size and even be smaller than w. 
-
- Scatter: imagine a buffer vector w and a map that gives to every element in the buffer w an
- index in a target vector v where this element should go to, 
- i.e. \f$ v[\text{idx}[i]] = w[i] \f$. This is ill-defined.
- Note again that an index in v can appear several times or never at all. 
- Then in our case we perform a reduction operation (we sum up all elements) beginning
- with 0 which remedies the defintion. 
-
-Since it is a vector operation the gather and scatter operation can 
-also be represented/defined by a matrix. The gather matrix is just a 
-(permutation) matrix of 1's and 0's with exactly one "1" in each line.
-In a "coo" formatted sparse matrix format the values array would consist only of "1"s, 
-row array is just the index and column array is the gather map.
-We uniquely define the corresponding scatter matrix as the transpose of the gather matrix. 
-The scatter matrix can have zero, one or more "1"s in each line.
-\f[ w = G v \\
-    v = S w \f]
-
-The scatter matrix S is the actual inverse of G if and only if the gather map is bijective.
-In this case the buffer and the vector can swap their roles. 
-
-Finally note that when v is filled with its indices, i.e. \f$ v[i] = i \f$, then
-the gather operation will reproduce the index map in the buffer w \f$ w[i] = \text{idx}[i]\f$ .
-
- * @ingroup mpi_structures
- */
-struct aCommunicator
-{
-
-    /**
-     * @brief Gather data across processes
-     *
-     * @param values data to gather from
-     * @tparam LocalContainer a container on a shared memory system
-     *
-     * @return the buffer vector of size size()
-     */
-    template< class LocalContainer>
-    LocalContainer global_gather( const LocalContainer& values)const;
-    template< class LocalContainer>
-    void global_gather( const LocalContainer& values, LocalContainer& gathered)const;
-    //actually the return type in NNC is const LocalContainer& 
-
-    /**
-     * @brief Scatters data accross processes and reduces on double indices
-     *
-     * @tparam LocalContainer a container on a shared memory system
-     * @param toScatter buffer vector (has to be of size given by size())
-     * @param values contains values from other processes sent back to the origin 
-     */
-    template< class LocalContainer>
-    void global_scatter_reduce( const LocalContainer& toScatter, LocalContainer& values) const;
-    template< class LocalContainer>
-    LocalContainer global_scatter_reduce( const LocalContainer& toScatter, LocalContainer& values) const;
-
-    /**
-    * @brief The size of the local buffer = local map size
-    *
- Consider that both the vector v and the buffer w are distributed across processes.
- In Feltor the vector v is distributed equally among processes and the local size
- of v is the same for all processes. However the buffer size might be different for each process. 
-    * @note may return 0 to indicate identity between v and w and that no MPI communication is needed 
-    * @note we assume that the vector size is always the local size of a dg::MPI_Vector
-    * @return buffer size
-    */
-    unsigned size() const;
-    /**
-    * @brief The internal mpi communicator used 
-    *
-    * used to assert that communicators of matrix and vector are the same
-    * @return MPI Communicator
-    */
-    MPI_Comm communicator() const;
-};
 
 /**
  * @ingroup mpi_structures
diff --git a/inc/dg/backend/mpi_communicator.h b/inc/dg/backend/mpi_communicator.h
new file mode 100644
index 000000000..bfdd5b383
--- /dev/null
+++ b/inc/dg/backend/mpi_communicator.h
@@ -0,0 +1,132 @@
+#pragma once
+
+namespace dg
+{
+
+/**
+ * @brief Struct that performs collective scatter and gather operations across processes
+ * on distributed vectors using MPI
+ *
+ * In order to understand the issue you must first really(!) understand what 
+ gather and scatter operations are, so grab pen and paper: 
+
+ First we note that gather and scatter are most often used in the context
+ of memory buffers. The buffer needs to be filled wih values (gather) or these
+ values need to be written back into the original place (scatter).
+
+ Gather: imagine a buffer vector w and a map that gives to every element in this vector w
+ an index into a source vector v where the value of this element should be taken from
+ i.e. \f$ w[i] = v[\text{idx}[i]] \f$ 
+ Note that an index in the source vector can appear several times or not at all. 
+ This is why the source vector w can have any size and even be smaller than w. 
+
+ Scatter: imagine a buffer vector w and a map that gives to every element in the buffer w an
+ index in a target vector v where this element should go to, 
+ i.e. \f$ v[\text{idx}[i]] = w[i] \f$. This is ill-defined.
+ Note again that an index in v can appear several times or never at all. 
+ Then in our case we perform a reduction operation (we sum up all elements) beginning
+ with 0 which remedies the defintion. 
+
+Since it is a vector operation the gather and scatter operation can 
+also be represented/defined by a matrix. The gather matrix is just a 
+(permutation) matrix of 1's and 0's with exactly one "1" in each line.
+In a "coo" formatted sparse matrix format the values array would consist only of "1"s, 
+row array is just the index and column array is the gather map.
+We uniquely define the corresponding scatter matrix as the transpose of the gather matrix. 
+The scatter matrix can have zero, one or more "1"s in each line.
+\f[ w = G v \\
+    v = S w \f]
+
+The scatter matrix S is the actual inverse of G if and only if the gather map is bijective.
+In this case the buffer and the vector can swap their roles. 
+
+Finally note that when v is filled with its indices, i.e. \f$ v[i] = i \f$, then
+the gather operation will reproduce the index map in the buffer w \f$ w[i] = \text{idx}[i]\f$ .
+
+ * @tparam LocalContainer a container on a shared memory system
+ * @ingroup mpi_structures
+ */
+template< class LocalContainer>
+struct aCommunicator
+{
+
+    /**
+     * @brief Allocate a LocalContainer object of size size()
+     * @return an object on the stack
+     */
+    LocalContainer make_buffer( )const{
+        return do_make_buffer();
+    }
+
+
+    /**
+     * @brief Gather data across processes
+     * @param values data to gather from
+     * @param gather object to hold the gathered data ( must be of size size())
+     */
+    void global_gather( const LocalContainer& values, LocalContainer& gather)const
+    {
+        do_global_gather( values, gather);
+    }
+
+    /**
+     * @brief Gather data across processes (memory allocating version)
+     * @param values data to gather from
+     * @return object to hold the gathered data
+     */
+    LocalContainer global_gather( const LocalContainer& values) const
+    {
+        LocalContainer tmp = do_make_buffer();
+        do_global_gather( values, tmp);
+        return tmp;
+    }
+
+    /**
+     * @brief Scatters data accross processes and reduces on double indices
+     *
+     * @param toScatter buffer vector (has to be of size given by size())
+     * @param values contains values from other processes sent back to the origin 
+     */
+    void global_scatter_reduce( const LocalContainer& toScatter, LocalContainer& values) const{
+        do_global_scatter_reduce(toScatter, values);
+    }
+
+    /**
+    * @brief The size of the local buffer = local map size
+    *
+    * Consider that both the vector v and the buffer w are distributed across processes.
+    * In Feltor the vector v is distributed equally among processes and the local size
+    * of v is the same for all processes. However the buffer size might be different for each process. 
+    * @note may return 0 to indicate identity between v and w and that no MPI communication is needed 
+    * @note we assume that the vector size is always the local size of a dg::MPI_Vector
+    * @return buffer size
+    */
+    unsigned size() const;
+    /**
+    * @brief The internal MPI communicator used 
+    *
+    * e.g. used to assert that communicators of matrix and vector are the same
+    * @return MPI Communicator
+    */
+    MPI_Comm communicator() const{return m_comm;}
+    virtual aCommunicator* clone() const =0;
+    virtual ~aCommunicator(){}
+    protected:
+    /**
+     * @brief default constructor
+     * @param comm defaults to MPI_COMM_WORLD
+     */
+    aCommunicator(MPI_Comm comm = MPI_COMM_WORLD){m_comm = comm;}
+    aCommunicator(const aCommunicator& src){ m_comm = src.m_comm; }
+    aCommunicator& operator=(const aCommunicator& src){
+        m_comm = src.m_comm;
+        return *this 
+    }
+    private:
+    MPI_Comm m_comm;
+    virtual LocalContainer do_make_buffer( )const=0;
+    virtual void do_global_gather( const LocalContainer& values, LocalContainer& gathered)const=0;
+    virtual void do_global_scatter_reduce( const LocalContainer& toScatter, LocalContainer& values) const=0;
+};
+
+}//namespace dg
diff --git a/inc/geometries/fluxfunctions.h b/inc/geometries/fluxfunctions.h
index f8b0f930c..837e5d877 100644
--- a/inc/geometries/fluxfunctions.h
+++ b/inc/geometries/fluxfunctions.h
@@ -7,6 +7,8 @@ namespace geo
 {
 ///@addtogroup fluxfunctions
 ///@{
+//
+//In the next round of updates we should consider using boost::function 
 
 /**
 * @brief This functor represents functions written in cylindrical coordinates
diff --git a/inc/geometries/generator.h b/inc/geometries/generator.h
index 6f5f42c25..08fd9da9f 100644
--- a/inc/geometries/generator.h
+++ b/inc/geometries/generator.h
@@ -60,11 +60,12 @@ struct aGenerator2d
     virtual ~aGenerator2d(){}
 
     protected:
+    ///empty
     aGenerator2d(){}
-    aGenerator2d(const aGenerator2d& src){}
-    aGenerator2d& operator=(const aGenerator2d& src){
-        return *this;
-    }
+    ///empty
+    aGenerator2d(const aGenerator2d& ){}
+    ///return *this
+    aGenerator2d& operator=(const aGenerator2d& ){ return *this; }
     private:
     virtual void do_generate(
          const thrust::host_vector<double>& zeta1d, 
-- 
GitLab


From 7cbff91ba8ef55a7a3c7658e408a0d9732007d8d Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 18 Sep 2017 17:20:31 +0200
Subject: [PATCH 303/453] made Collective objects derive from aCommunicator

---
 inc/dg/backend/collective_mpit.cu |   2 +-
 inc/dg/backend/mpi_collective.h   | 161 +++++++++++++++---------------
 inc/dg/backend/mpi_communicator.h |  22 ++--
 3 files changed, 91 insertions(+), 94 deletions(-)

diff --git a/inc/dg/backend/collective_mpit.cu b/inc/dg/backend/collective_mpit.cu
index d2e7a4728..582526121 100644
--- a/inc/dg/backend/collective_mpit.cu
+++ b/inc/dg/backend/collective_mpit.cu
@@ -11,10 +11,10 @@ int main( int argc, char * argv[])
     int rank, size;
     MPI_Comm_rank( MPI_COMM_WORLD, &rank);
     MPI_Comm_size( MPI_COMM_WORLD, &size);
+    if(size!=4 ){std::cerr <<"You run with "<<size<<" processes. Run with 4 processes!\n"; MPI_Finalize(); return 0;}
 
     if(rank==0)std::cout <<"Nx " <<Nx << " and Ny "<<Ny<<std::endl;
     if(rank==0)std::cout <<"Size =  " <<size <<std::endl;
-    if(size!=4 && rank == 0){std::cerr <<"You run with "<<size<<" processes. Run with 4 processes!\n"; MPI_Finalize(); return -1;}
 
     if(rank==0)std::cout << "Test BijectiveComm: scatter followed by gather leaves the input vector intact\n";
     thrust::host_vector<double> v( Nx*Ny, 3*rank);
diff --git a/inc/dg/backend/mpi_collective.h b/inc/dg/backend/mpi_collective.h
index 85a397ced..60736bd43 100644
--- a/inc/dg/backend/mpi_collective.h
+++ b/inc/dg/backend/mpi_collective.h
@@ -144,7 +144,7 @@ void Collective<Index, Device>::gather( const Device& gatherFrom, Device& values
  @note models aCommunicator
  */
 template< class Index, class Vector>
-struct BijectiveComm 
+struct BijectiveComm : public aCommunicator<Vector>
 {
     /**
      * @brief Construct empty class
@@ -159,57 +159,23 @@ struct BijectiveComm
      * @param comm An MPI Communicator that contains the participants of the scatter/gather
      * @note The actual scatter/gather map is constructed from the given map so the result behaves as if pids was the actual scatter/gather map on the buffer
      */
-    BijectiveComm( const thrust::host_vector<int>& pids, MPI_Comm comm)
-    {
+    BijectiveComm( const thrust::host_vector<int>& pids, MPI_Comm comm) {
         construct( pids, comm);
     }
 
     template<class OtherIndex, class OtherVector>
-    BijectiveComm( const BijectiveComm<OtherIndex, OtherVector>& src)
-    {
+    BijectiveComm( const BijectiveComm<OtherIndex, OtherVector>& src) {
         construct( src.get_pids(), src.communicator());
     }
 
     /**
-     * @brief Globally gathers data into a buffer according to the map given in the Constructor
-     *
-     * The order of the received elements is according to their original array index 
-     * (i.e. a[0] appears before a[1]) and their process rank of origin ( i.e. values from rank 0 appear before values from rank 1)
-     * @param values data to send (must have the size given 
-     * by the map in the constructor, s.a. size())
-     *
-     * @return received data from other processes of size size()
-     * @note a scatter followed by a gather of the received values restores the original array
-     */
-    Vector global_gather( const Vector& values)const
-    {
-        //actually this is a scatter but we constructed it invertedly
-        //we could maybe transpose the Collective object!?
-        assert( values.size() == idx_.size());
-        //nach PID ordnen
-        thrust::gather( idx_.begin(), idx_.end(), values.begin(), values_.data().begin());
-        //senden
-        Vector store( p_.store_size());
-        p_.scatter( values_.data(), store);
-        return store;
-    }
-
-    /**
-     * @brief Scatter data according to the map given in the constructor 
-     *
-     * This method is the inverse of gather
-     * @param toScatter other processes collect data from this vector (has to be of size given by recv_size())
-     * @param values contains values from other processes sent back to the origin (must have the size of the map given in the constructor, or send_size())
-     * @note a scatter followed by a gather of the received values restores the original array
-     */
-    void global_scatter_reduce( const Vector& toScatter, Vector& values) const
-    {
-        //actually this is a gather but we constructed it invertedly
-        p_.gather( toScatter, values_.data());
-        //nach PID geordnete Werte wieder umsortieren
-        thrust::scatter( values_.data().begin(), values_.data().end(), idx_.begin(), values.begin());
-    }
-
+    * @brief These are the pids that were given in the Constructor
+    * @return the vector given in the constructor
+    */
+    const thrust::host_vector<int>& get_pids()const{return pids_;}
+    virtual BijectiveComm* clone() const {return new BijectiveComm(*this);}
+    private:
+    MPI_Comm do_communicator() const {return p_.communicator();}
     /**
      * @brief compute total # of elements the calling process receives in the scatter process (or sends in the gather process)
      *
@@ -217,20 +183,11 @@ struct BijectiveComm
      *
      * @return # of elements to receive
      */
-    unsigned size() const { return p_.store_size();}
-    /**
-    * @brief The internal communicator used 
-    *
-    * @return MPI Communicator
-    */
-    MPI_Comm communicator() const {return p_.communicator();}
-    /**
-    * @brief These are the pids that was given in the Constructor
-    *
-    * @return the vector given in the constructor
-    */
-    const thrust::host_vector<int> get_pids()const{return pids_;}
-    private:
+    unsigned do_size() const { return p_.store_size();}
+    Vector do_make_buffer()const{ 
+        Vector tmp( do_size() );
+        return tmp;
+    }
     void construct( thrust::host_vector<int> pids, MPI_Comm comm)
     {
         pids_ = pids;
@@ -261,6 +218,43 @@ struct BijectiveComm
         p_.construct( sendTo, comm);
         values_.data().resize( idx_.size());
     }
+    /**
+     * @brief Globally gathers data into a buffer according to the map given in the Constructor
+     *
+     * The order of the received elements is according to their original array index 
+     * (i.e. a[0] appears before a[1]) and their process rank of origin ( i.e. values from rank 0 appear before values from rank 1)
+     * @param values data to send (must have the size given 
+     * by the map in the constructor, s.a. size())
+     *
+     * @return received data from other processes of size size()
+     * @note a scatter followed by a gather of the received values restores the original array
+     */
+    void do_global_gather( const Vector& values, Vector& store)const
+    {
+        //actually this is a scatter but we constructed it invertedly
+        //we could maybe transpose the Collective object!?
+        assert( values.size() == idx_.size());
+        //nach PID ordnen
+        thrust::gather( idx_.begin(), idx_.end(), values.begin(), values_.data().begin());
+        //senden
+        p_.scatter( values_.data(), store);
+    }
+
+    /**
+     * @brief Scatter data according to the map given in the constructor 
+     *
+     * This method is the inverse of gather
+     * @param toScatter other processes collect data from this vector (has to be of size given by recv_size())
+     * @param values contains values from other processes sent back to the origin (must have the size of the map given in the constructor, or send_size())
+     * @note a scatter followed by a gather of the received values restores the original array
+     */
+    void do_global_scatter_reduce( const Vector& toScatter, Vector& values) const
+    {
+        //actually this is a gather but we constructed it invertedly
+        p_.gather( toScatter, values_.data());
+        //nach PID geordnete Werte wieder umsortieren
+        thrust::scatter( values_.data().begin(), values_.data().end(), idx_.begin(), values.begin());
+    }
     Buffer<Vector> values_;
     Index idx_;
     Collective<Index, Vector> p_;
@@ -280,7 +274,7 @@ struct BijectiveComm
  @note models aCommunicator
  */
 template< class Index, class Vector>
-struct SurjectiveComm
+struct SurjectiveComm : public aCommunicator<Vector>
 {
     /**
      * @brief Construct empty class
@@ -301,6 +295,7 @@ struct SurjectiveComm
         construct( localGatherMap, pidGatherMap, comm);
     }
 
+    ///use global2localIdx of MPI geometries
     template<class MPIGeometry>
     SurjectiveComm( const thrust::host_vector<int>& globalGatherMap, const MPIGeometry& g)
     {
@@ -319,16 +314,22 @@ struct SurjectiveComm
         construct( src.getLocalGatherMap(), src.getPidGatherMap(), src.communicator());
     }
 
-    Vector global_gather( const Vector& values)const
+    const thrust::host_vector<int>& getLocalGatherMap() const {return localGatherMap_;}
+    const thrust::host_vector<int>& getPidGatherMap() const {return pidGatherMap_;}
+    const Index& getSortedGatherMap() const {return sortedGatherMap_;}
+    virtual SurjectiveComm* clone() const {return new SurjectiveComm(*this);}
+    private:
+    Vector do_make_buffer()const{
+        Vector tmp(do_size());
+        return tmp;
+    }
+    void do_global_gather( const Vector& values, Vector& buffer)const
     {
         //gather values to store
         thrust::gather( gatherMap_.begin(), gatherMap_.end(), values.begin(), store_.data().begin());
-        //now gather from store into buffer
-        Vector buffer( buffer_size_);
         bijectiveComm_.global_scatter_reduce( store_.data(), buffer);
-        return buffer;
     }
-    void global_scatter_reduce( const Vector& toScatter, Vector& values)
+    void do_global_scatter_reduce( const Vector& toScatter, Vector& values)const
     {
         //first gather values into store
         Vector store_t = bijectiveComm_.global_gather( toScatter);
@@ -336,12 +337,8 @@ struct SurjectiveComm
         thrust::gather( sortMap_.begin(), sortMap_.end(), store_t.begin(), store_.data().begin());
         thrust::reduce_by_key( sortedGatherMap_.begin(), sortedGatherMap_.end(), store_.data().begin(), keys_.data().begin(), values.begin());
     }
-    unsigned size() const {return buffer_size_;}
-    const thrust::host_vector<int> getLocalGatherMap() const {return localGatherMap_;}
-    const thrust::host_vector<int> getPidGatherMap() const {return pidGatherMap_;}
-    MPI_Comm communicator()const{return bijectiveComm_.communicator();}
-    const Index& getSortedGatherMap() const {return sortedGatherMap_;}
-    private:
+    MPI_Comm do_communicator()const{return bijectiveComm_.communicator();}
+    unsigned do_size() const {return buffer_size_;}
     void construct( thrust::host_vector<int> localGatherMap, thrust::host_vector<int> pidGatherMap, MPI_Comm comm)
     {
         bijectiveComm_ = BijectiveComm<Index, Vector>( pidGatherMap, comm);
@@ -386,7 +383,7 @@ struct SurjectiveComm
  @note models aCommunicator
  */
 template< class Index, class Vector>
-struct GeneralComm
+struct GeneralComm : public aCommunicator<Vector>
 {
     GeneralComm(){}
     GeneralComm( const thrust::host_vector<int>& localGatherMap, const thrust::host_vector<int>& pidGatherMap, MPI_Comm comm) {
@@ -407,21 +404,27 @@ struct GeneralComm
         construct( local, pids, g.communicator());
     }
 
-    Vector global_gather( const Vector& values)const
+    const thrust::host_vector<int>& getLocalGatherMap() const {return surjectiveComm_.getLocalGatherMap();}
+    const thrust::host_vector<int>& getPidGatherMap() const {return surjectiveComm_.getPidGatherMap();}
+    virtual GeneralComm* clone() const {return new GeneralComm(*this);}
+    private:
+    Vector do_make_buffer() const{ 
+        Vector tmp(do_size());
+        return tmp;
+    }
+
+    MPI_Comm do_communicator()const{return surjectiveComm_.communicator();}
+    void do_global_gather( const Vector& values, Vector& sink)const
     {
-        return surjectiveComm_.global_gather( values);
+        surjectiveComm_.global_gather( values, sink);
     }
-    void global_scatter_reduce( const Vector& toScatter, Vector& values)
+    void do_global_scatter_reduce( const Vector& toScatter, Vector& values)const
     {
         surjectiveComm_.global_scatter_reduce( toScatter, store_.data());
         thrust::scatter( store_.data().begin(), store_.data().end(), scatterMap_.begin(), values.begin());
     }
 
-    unsigned size() const{return surjectiveComm_.size();}
-    const thrust::host_vector<int> getLocalGatherMap() const {return surjectiveComm_.getLocalGatherMap();}
-    const thrust::host_vector<int> getPidGatherMap() const {return surjectiveComm_.getPidGatherMap();}
-    MPI_Comm communicator()const{return surjectiveComm_.communicator();}
-    private:
+    unsigned do_size() const{return surjectiveComm_.size();}
     void construct( const thrust::host_vector<int>& localGatherMap, const thrust::host_vector<int>& pidGatherMap, MPI_Comm comm)
     {
         surjectiveComm_ = SurjectiveComm<Index,Vector>(localGatherMap, pidGatherMap, comm);
diff --git a/inc/dg/backend/mpi_communicator.h b/inc/dg/backend/mpi_communicator.h
index bfdd5b383..278006d06 100644
--- a/inc/dg/backend/mpi_communicator.h
+++ b/inc/dg/backend/mpi_communicator.h
@@ -54,7 +54,7 @@ struct aCommunicator
      * @brief Allocate a LocalContainer object of size size()
      * @return an object on the stack
      */
-    LocalContainer make_buffer( )const{
+    LocalContainer allocate_buffer( )const{
         return do_make_buffer();
     }
 
@@ -101,29 +101,23 @@ struct aCommunicator
     * @note we assume that the vector size is always the local size of a dg::MPI_Vector
     * @return buffer size
     */
-    unsigned size() const;
+    unsigned size() const{return do_size();}
     /**
     * @brief The internal MPI communicator used 
     *
     * e.g. used to assert that communicators of matrix and vector are the same
     * @return MPI Communicator
     */
-    MPI_Comm communicator() const{return m_comm;}
+    MPI_Comm communicator() const{return do_communicator();}
     virtual aCommunicator* clone() const =0;
     virtual ~aCommunicator(){}
     protected:
-    /**
-     * @brief default constructor
-     * @param comm defaults to MPI_COMM_WORLD
-     */
-    aCommunicator(MPI_Comm comm = MPI_COMM_WORLD){m_comm = comm;}
-    aCommunicator(const aCommunicator& src){ m_comm = src.m_comm; }
-    aCommunicator& operator=(const aCommunicator& src){
-        m_comm = src.m_comm;
-        return *this 
-    }
+    aCommunicator(){}
+    aCommunicator(const aCommunicator& src){ }
+    aCommunicator& operator=(const aCommunicator& src){ return *this; }
     private:
-    MPI_Comm m_comm;
+    virtual MPI_Comm do_communicator() const=0;
+    virtual unsigned do_size() const=0;
     virtual LocalContainer do_make_buffer( )const=0;
     virtual void do_global_gather( const LocalContainer& values, LocalContainer& gathered)const=0;
     virtual void do_global_scatter_reduce( const LocalContainer& toScatter, LocalContainer& values) const=0;
-- 
GitLab


From f6d2cbaea09b0b6e9c549e99eca2daba544dd653 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Mon, 18 Sep 2017 23:10:41 +0200
Subject: [PATCH 304/453] NNC survives being derived from aCommunicator

---
 inc/dg/backend/mpi_communicator.h |  66 ++++++++++------
 inc/dg/backend/mpi_derivatives.h  |  20 ++---
 inc/dg/backend/mpi_matrix.h       | 101 +++++++++++-------------
 inc/dg/backend/mpi_vector.h       | 127 ++++++++++++++----------------
 4 files changed, 154 insertions(+), 160 deletions(-)

diff --git a/inc/dg/backend/mpi_communicator.h b/inc/dg/backend/mpi_communicator.h
index 278006d06..ae960b13e 100644
--- a/inc/dg/backend/mpi_communicator.h
+++ b/inc/dg/backend/mpi_communicator.h
@@ -7,28 +7,29 @@ namespace dg
  * @brief Struct that performs collective scatter and gather operations across processes
  * on distributed vectors using MPI
  *
- * In order to understand the issue you must first really(!) understand what 
+ * In order to understand what this class does you should first really(!) understand what 
  gather and scatter operations are, so grab pen and paper: 
 
- First we note that gather and scatter are most often used in the context
+ First, we note that gather and scatter are most often used in the context
  of memory buffers. The buffer needs to be filled wih values (gather) or these
  values need to be written back into the original place (scatter).
 
- Gather: imagine a buffer vector w and a map that gives to every element in this vector w
+ @b Gather: imagine a buffer vector w and a map that gives to every element in this vector w
  an index into a source vector v where the value of this element should be taken from
  i.e. \f$ w[i] = v[\text{idx}[i]] \f$ 
- Note that an index in the source vector can appear several times or not at all. 
- This is why the source vector w can have any size and even be smaller than w. 
-
- Scatter: imagine a buffer vector w and a map that gives to every element in the buffer w an
- index in a target vector v where this element should go to, 
- i.e. \f$ v[\text{idx}[i]] = w[i] \f$. This is ill-defined.
- Note again that an index in v can appear several times or never at all. 
- Then in our case we perform a reduction operation (we sum up all elements) beginning
- with 0 which remedies the defintion. 
-
-Since it is a vector operation the gather and scatter operation can 
-also be represented/defined by a matrix. The gather matrix is just a 
+ Note that an index into the source vector v can appear several times or not at all. 
+ This is why the source vector v can have any size and even be smaller than w. 
+
+ @b Scatter: imagine a buffer vector w and a map that gives to every element in the buffer w an
+ index into a target vector v where this element should go to. 
+ Note again that an index into v can appear several times or never at all. 
+ If the index appears more than once, we perform a reduction operation  (we sum up all elements) 
+ on these indices initializing the sum with 0.
+ Note that \f$ v[\text{idx}[i]] = w[i] \f$ is NOT the correct definition of this, because
+ it does not show the reduction.
+
+It is more accurate to represent the gather and scatter operation 
+by a matrix. The gather matrix is just a 
 (permutation) matrix of 1's and 0's with exactly one "1" in each line.
 In a "coo" formatted sparse matrix format the values array would consist only of "1"s, 
 row array is just the index and column array is the gather map.
@@ -37,10 +38,15 @@ The scatter matrix can have zero, one or more "1"s in each line.
 \f[ w = G v \\
     v = S w \f]
 
-The scatter matrix S is the actual inverse of G if and only if the gather map is bijective.
+This class performs these operations for the case that v and w are distributed across processes.
+We always assume that the source vector v is distributed equally among processes, i.e. 
+each process holds a chunk of v of equal size. On the other hand the local size 
+of w may vary among processes depending on the gather/scatter map. 
+
+@note The scatter matrix S is the actual inverse of G if and only if the gather map is bijective.
 In this case the buffer and the vector can swap their roles. 
 
-Finally note that when v is filled with its indices, i.e. \f$ v[i] = i \f$, then
+@note Finally note that when v is filled with its indices, i.e. \f$ v[i] = i \f$, then
 the gather operation will reproduce the index map in the buffer w \f$ w[i] = \text{idx}[i]\f$ .
 
  * @tparam LocalContainer a container on a shared memory system
@@ -49,16 +55,16 @@ the gather operation will reproduce the index map in the buffer w \f$ w[i] = \te
 template< class LocalContainer>
 struct aCommunicator
 {
+    typedef LocalContainer container_type; //!< typedef to derive container type
 
     /**
-     * @brief Allocate a LocalContainer object of size size()
-     * @return an object on the stack
+     * @brief Allocate a buffer object of size size()
+     * @return a buffer object on the stack
      */
     LocalContainer allocate_buffer( )const{
         return do_make_buffer();
     }
 
-
     /**
      * @brief Gather data across processes
      * @param values data to gather from
@@ -82,8 +88,7 @@ struct aCommunicator
     }
 
     /**
-     * @brief Scatters data accross processes and reduces on double indices
-     *
+     * @brief Scatter data accross processes and reduce on multiple indices
      * @param toScatter buffer vector (has to be of size given by size())
      * @param values contains values from other processes sent back to the origin 
      */
@@ -92,14 +97,14 @@ struct aCommunicator
     }
 
     /**
-    * @brief The size of the local buffer = local map size
+    * @brief The size of the local buffer vector w = local map size
     *
-    * Consider that both the vector v and the buffer w are distributed across processes.
+    * Consider that both the source vector v and the buffer w are distributed across processes.
     * In Feltor the vector v is distributed equally among processes and the local size
     * of v is the same for all processes. However the buffer size might be different for each process. 
-    * @note may return 0 to indicate identity between v and w and that no MPI communication is needed 
-    * @note we assume that the vector size is always the local size of a dg::MPI_Vector
     * @return buffer size
+    * @note may return 0 to indicate that no MPI communication is needed 
+    * @note we assume that the vector size is always the local size of a dg::MPI_Vector
     */
     unsigned size() const{return do_size();}
     /**
@@ -109,11 +114,20 @@ struct aCommunicator
     * @return MPI Communicator
     */
     MPI_Comm communicator() const{return do_communicator();}
+    ///@brief Generic copy method
+    ///@return pointer to allocated object
     virtual aCommunicator* clone() const =0;
+    ///@brief vritual destructor
     virtual ~aCommunicator(){}
     protected:
+    ///@brief only derived classes can construct
     aCommunicator(){}
+    ///@brief only derived classes can copy
+    ///@param src source 
     aCommunicator(const aCommunicator& src){ }
+    ///@brief only derived classes can assign
+    ///@param src source 
+    ///@return *this
     aCommunicator& operator=(const aCommunicator& src){ return *this; }
     private:
     virtual MPI_Comm do_communicator() const=0;
diff --git a/inc/dg/backend/mpi_derivatives.h b/inc/dg/backend/mpi_derivatives.h
index c0282a5e4..763ae619c 100644
--- a/inc/dg/backend/mpi_derivatives.h
+++ b/inc/dg/backend/mpi_derivatives.h
@@ -122,7 +122,7 @@ EllSparseBlockMat<double> distribute_rows( const EllSparseBlockMat<double>& src,
 RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dx( const aMPITopology2d& g, bc bcx, direction dir = centered)
 {
     EllSparseBlockMat<double> matrix = dg::create::dx( g.global(), bcx, dir);
-    int vector_dimensions[] = {(int)(g.n()*g.Nx()), (int)(g.n()*g.Ny()), 1}; //x, y, z
+    unsigned vector_dimensions[] = {(unsigned)(g.n()*g.Nx()), (unsigned)(g.n()*g.Ny()), 1}; //x, y, z
     MPI_Comm comm = g.communicator();
     int ndims;
     MPI_Cartdim_get( comm, &ndims);
@@ -151,7 +151,7 @@ RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dx( c
 RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dy( const aMPITopology2d& g, bc bcy, direction dir = centered)
 {
     EllSparseBlockMat<double> matrix = dg::create::dy( g.global(), bcy, dir);
-    int vector_dimensions[] = {(int)(g.n()*g.Nx()), (int)(g.n()*g.Ny()), 1}; //x, y, z
+    unsigned vector_dimensions[] = {(unsigned)(g.n()*g.Nx()), (unsigned)(g.n()*g.Ny()), 1}; //x, y, z
     MPI_Comm comm = g.communicator();
     int ndims;
     MPI_Cartdim_get( comm, &ndims);
@@ -178,7 +178,7 @@ RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dy( c
 RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpX( const aMPITopology2d& g, bc bcx)
 {
     EllSparseBlockMat<double> matrix = dg::create::jumpX( g.global(), bcx);
-    int vector_dimensions[] = {(int)(g.n()*g.Nx()), (int)(g.n()*g.Ny()), 1}; //x, y, z
+    unsigned vector_dimensions[] = {(unsigned)(g.n()*g.Nx()), (unsigned)(g.n()*g.Ny()), 1}; //x, y, z
     MPI_Comm comm = g.communicator();
     int ndims;
     MPI_Cartdim_get( comm, &ndims);
@@ -204,7 +204,7 @@ RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpX
 RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpY( const aMPITopology2d& g, bc bcy)
 {
     EllSparseBlockMat<double> matrix = dg::create::jumpY( g.global(), bcy);
-    int vector_dimensions[] = {(int)(g.n()*g.Nx()), (int)(g.n()*g.Ny()), 1}; //x, y, z
+    unsigned vector_dimensions[] = {(unsigned)(g.n()*g.Nx()), (unsigned)(g.n()*g.Ny()), 1}; //x, y, z
     MPI_Comm comm = g.communicator();
     int ndims;
     MPI_Cartdim_get( comm, &ndims);
@@ -232,7 +232,7 @@ RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpY
 RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dx( const aMPITopology3d& g, bc bcx, direction dir = centered)
 {
     EllSparseBlockMat<double> matrix = dg::create::dx( g.global(), bcx, dir);
-    int vector_dimensions[] = {(int)(g.n()*g.Nx()), (int)(g.n()*g.Ny()), (int)(g.Nz())}; //x, y, z
+    unsigned vector_dimensions[] = {(unsigned)(g.n()*g.Nx()), (unsigned)(g.n()*g.Ny()), (unsigned)(g.Nz())}; //x, y, z
     MPI_Comm comm = g.communicator();
     int ndims;
     MPI_Cartdim_get( comm, &ndims);
@@ -259,7 +259,7 @@ RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dx( c
 RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dy( const aMPITopology3d& g, bc bcy, direction dir = centered)
 {
     EllSparseBlockMat<double> matrix = dg::create::dy( g.global(), bcy, dir);
-    int vector_dimensions[] = {(int)(g.n()*g.Nx()), (int)(g.n()*g.Ny()), (int)(g.Nz())}; //x, y, z
+    unsigned vector_dimensions[] = {(unsigned)(g.n()*g.Nx()), (unsigned)(g.n()*g.Ny()), (unsigned)(g.Nz())}; //x, y, z
     MPI_Comm comm = g.communicator();
     int ndims;
     MPI_Cartdim_get( comm, &ndims);
@@ -286,7 +286,7 @@ RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dy( c
 RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dz( const aMPITopology3d& g, bc bcz, direction dir = centered)
 {
     EllSparseBlockMat<double> matrix = dg::create::dz( g.global(), bcz, dir);
-    int vector_dimensions[] = {(int)(g.n()*g.Nx()), (int)(g.n()*g.Ny()), (int)(g.Nz())}; //x, y, z
+    unsigned vector_dimensions[] = {(unsigned)(g.n()*g.Nx()), (unsigned)(g.n()*g.Ny()), (unsigned)(g.Nz())}; //x, y, z
     MPI_Comm comm = g.communicator();
     int ndims;
     MPI_Cartdim_get( comm, &ndims);
@@ -313,7 +313,7 @@ RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dz( c
 RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpX( const aMPITopology3d& g, bc bcx)
 {
     EllSparseBlockMat<double> matrix = dg::create::jumpX( g.global(), bcx);
-    int vector_dimensions[] = {(int)(g.n()*g.Nx()), (int)(g.n()*g.Ny()), (int)(g.Nz())}; //x, y, z
+    unsigned vector_dimensions[] = {(unsigned)(g.n()*g.Nx()), (unsigned)(g.n()*g.Ny()), (unsigned)(g.Nz())}; //x, y, z
     MPI_Comm comm = g.communicator();
     int ndims;
     MPI_Cartdim_get( comm, &ndims);
@@ -340,7 +340,7 @@ RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpX
 RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpY( const aMPITopology3d& g, bc bcy)
 {
     EllSparseBlockMat<double> matrix = dg::create::jumpY( g.global(), bcy);
-    int vector_dimensions[] = {(int)(g.n()*g.Nx()), (int)(g.n()*g.Ny()), (int)(g.Nz())}; //x, y, z
+    unsigned vector_dimensions[] = {(unsigned)(g.n()*g.Nx()), (unsigned)(g.n()*g.Ny()), (unsigned)(g.Nz())}; //x, y, z
     MPI_Comm comm = g.communicator();
     int ndims;
     MPI_Cartdim_get( comm, &ndims);
@@ -366,7 +366,7 @@ RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpY
 RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpZ( const aMPITopology3d& g, bc bcz)
 {
     EllSparseBlockMat<double> matrix = dg::create::jumpZ( g.global(), bcz);
-    int vector_dimensions[] = {(int)(g.n()*g.Nx()), (int)(g.n()*g.Ny()), (int)(g.Nz())}; //x, y, z
+    unsigned vector_dimensions[] = {(unsigned)(g.n()*g.Nx()), (unsigned)(g.n()*g.Ny()), (unsigned)(g.Nz())}; //x, y, z
     MPI_Comm comm = g.communicator();
     int ndims;
     MPI_Cartdim_get( comm, &ndims);
diff --git a/inc/dg/backend/mpi_matrix.h b/inc/dg/backend/mpi_matrix.h
index 3e5dde024..e6ab59264 100644
--- a/inc/dg/backend/mpi_matrix.h
+++ b/inc/dg/backend/mpi_matrix.h
@@ -1,6 +1,7 @@
 #pragma once
 
 #include "mpi_vector.h"
+#include "memory.h"
 
 /*!@file
 
@@ -40,17 +41,19 @@ only the inner matrix is applied.
 template<class LocalMatrixInner, class LocalMatrixOuter, class Collective >
 struct RowColDistMat
 {
+    ///@brief no memory allocation
     RowColDistMat(){}
 
-
     /**
     * @brief Constructor 
     *
-    * @param m_inside The local matrix for the inner elements
-    * @param m_outside A local matrix for the elements from other processes
+    * @param inside The local matrix for the inner elements
+    * @param outside A local matrix for the elements from other processes
     * @param c The communication object
     */
-    RowColDistMat( const LocalMatrixInner& m_inside, const LocalMatrixOuter& m_outside, const Collective& c):m_i(m_inside), m_o(m_outside), c_(c) { }
+    RowColDistMat( const LocalMatrixInner& inside, const LocalMatrixOuter& outside, const Collective& c):
+        m_i(inside), m_o(outside), m_c(c), m_buffer( c.allocate_buffer()) { }
+
 
     /**
     * @brief Copy constructor 
@@ -65,8 +68,8 @@ struct RowColDistMat
     * @param src another Matrix
     */
     template< class OtherMatrixInner, class OtherMatrixOuter, class OtherCollective>
-    RowColDistMat( const RowColDistMat<OtherMatrixInner, OtherMatrixOuter, OtherCollective>& src):m_i(src.inner_matrix()), m_o( src.outer_matrix()), c_(src.collective())
-    { }
+    RowColDistMat( const RowColDistMat<OtherMatrixInner, OtherMatrixOuter, OtherCollective>& src):
+        m_i(src.inner_matrix()), m_o( src.outer_matrix()), m_c(src.collective()), m_buffer( m_c.get().allocate_buffer()) { }
     /**
     * @brief Read access to the inner matrix
     *
@@ -84,7 +87,7 @@ struct RowColDistMat
     *
     * @return 
     */
-    const Collective& collective() const{return c_;}
+    const Collective& collective() const{return m_c.get();}
     
     /**
     * @brief Matrix Vector product
@@ -101,7 +104,7 @@ struct RowColDistMat
     template<class container> 
     void symv( double alpha, const MPI_Vector<container>& x, double beta, MPI_Vector<container>& y) const
     {
-        if( c_.size() == 0) //no communication needed
+        if( m_c.get().size() == 0) //no communication needed
         {
             dg::blas2::detail::doSymv( alpha, m_i, x.data(), beta, y.data(), 
                        typename dg::MatrixTraits<LocalMatrixInner>::matrix_category(), 
@@ -110,13 +113,13 @@ struct RowColDistMat
 
         }
         assert( x.communicator() == y.communicator());
-        assert( x.communicator() == c_.communicator());
+        assert( x.communicator() == m_c.get().communicator());
         //1. compute inner points
         dg::blas2::detail::doSymv( alpha, m_i, x.data(), beta, y.data(), 
                        typename dg::MatrixTraits<LocalMatrixInner>::matrix_category(), 
                        typename dg::VectorTraits<container>::vector_category() );
         //2. communicate outer points
-        const container& temp = c_.global_gather( x.data());
+        const container& temp = m_c.get().global_gather( x.data());
         //3. compute and add outer points
         dg::blas2::detail::doSymv(alpha, m_o, temp, 1., y.data(), 
                        typename dg::MatrixTraits<LocalMatrixOuter>::matrix_category(), 
@@ -136,7 +139,7 @@ struct RowColDistMat
     template<class container> 
     void symv( const MPI_Vector<container>& x, MPI_Vector<container>& y) const
     {
-        if( c_.size() == 0) //no communication needed
+        if( m_c.get().size() == 0) //no communication needed
         {
             dg::blas2::detail::doSymv( m_i, x.data(), y.data(), 
                        typename dg::MatrixTraits<LocalMatrixInner>::matrix_category(), 
@@ -146,16 +149,16 @@ struct RowColDistMat
 
         }
         assert( x.communicator() == y.communicator());
-        assert( x.communicator() == c_.communicator());
+        assert( x.communicator() == m_c.get().communicator());
         //1. compute inner points
         dg::blas2::detail::doSymv( m_i, x.data(), y.data(), 
                        typename dg::MatrixTraits<LocalMatrixInner>::matrix_category(), 
                        typename dg::VectorTraits<container>::vector_category(),
                        typename dg::VectorTraits<container>::vector_category() );
         //2. communicate outer points
-        const container& temp = c_.global_gather( x.data());
+        m_c.get().global_gather( x.data(), m_buffer.data());
         //3. compute and add outer points
-        dg::blas2::detail::doSymv(1., m_o, temp, 1., y.data(), 
+        dg::blas2::detail::doSymv(1., m_o, m_buffer.data(), 1., y.data(), 
                        typename dg::MatrixTraits<LocalMatrixOuter>::matrix_category(), 
                        typename dg::VectorTraits<container>::vector_category() );
     }
@@ -163,15 +166,16 @@ struct RowColDistMat
     private:
     LocalMatrixInner m_i;
     LocalMatrixOuter m_o;
-    Collective c_;
+    Handle<Collective> m_c;
+    Buffer< typename Collective::container_type>  m_buffer;
 };
 
 
 ///Type of distribution of MPI distributed matrices
 enum dist_type
 {
-    row_dist; //!< Row distributed
-    col_dist; //!< Column distributed
+    row_dist=0, //!< Row distributed
+    col_dist=1 //!< Column distributed
 };
 
 /**
@@ -193,6 +197,7 @@ only the local matrix is applied.
 template<class LocalMatrix, class Collective >
 struct MPIDistMat
 {
+    ///@brief no memory allocation
     MPIDistMat( ) { }
     /**
     * @brief Constructor 
@@ -201,7 +206,8 @@ struct MPIDistMat
     * @param c The communication object
     * @param dist either row or column distributed
     */
-    MPIDistMat( const LocalMatrix& m, const Collective& c, enum dist_type dist = row_dist):m_(m), c_(c), m_dist( dist) { }
+    MPIDistMat( const LocalMatrix& m, const Collective& c, enum dist_type dist = row_dist):
+        m_m(m), m_c(c), m_buffer( c.allocate_buffer()), m_dist( dist) { }
 
     /**
     * @brief Copy Constructor 
@@ -211,20 +217,20 @@ struct MPIDistMat
     * @param src The other matrix
     */
     template< class OtherMatrix, class OtherCollective>
-    MPIDistMat( const MPIDistMat<OtherMatrix, OtherCollective>& src):m_(src.matrix()), c_(src.collective())
-    { }
+    MPIDistMat( const MPIDistMat<OtherMatrix, OtherCollective>& src):
+        m_m(src.matrix()), m_c(src.collective()), m_buffer( m_c.get().allocate_buffer()), m_dist(src.get_dist()) { }
     /**
     * @brief Access to the local matrix
     *
     * @return Reference to the local matrix
     */
-    const LocalMatrix& matrix() const{return m_;}
+    const LocalMatrix& matrix() const{return m_m;}
     /**
     * @brief Access to the communication object
     *
     * @return Reference to the collective object
     */
-    const Collective& collective() const{return c_;}
+    const Collective& collective() const{return m_c.get();}
 
     enum dist_type get_dist() const {return m_dist;}
     void set_dist(enum dist_type dist){m_dist=dist;}
@@ -233,36 +239,35 @@ struct MPIDistMat
     template<class container> 
     void symv( double alpha, const MPI_Vector<container>& x, double beta, MPI_Vector<container>& y)
     {
-        if( c_.size() == 0) //no communication needed
+        if( m_c.size() == 0) //no communication needed
         {
-            dg::blas2::detail::doSymv( alpha, m_, x.data(), beta, y.data(), 
+            dg::blas2::detail::doSymv( alpha, m_m, x.data(), beta, y.data(), 
                        typename dg::MatrixTraits<LocalMatrix>::matrix_category(), 
                        typename dg::VectorTraits<container>::vector_category() );
             return;
 
         }
         assert( x.communicator() == y.communicator());
-        assert( x.communicator() == c_.communicator());
+        assert( x.communicator() == m_c.communicator());
         if( m_dist == row_dist){
-            container temp = c_.global_gather( x.data());
-            dg::blas2::detail::doSymv( alpha, m_, temp, beta, y.data(), 
+            m_c.global_gather( x.data(), m_buffer.data());
+            dg::blas2::detail::doSymv( alpha, m_m, m_buffer.data(), beta, y.data(), 
                        typename dg::MatrixTraits<LocalMatrix>::matrix_category(), 
                        typename dg::VectorTraits<container>::vector_category() );
         }
         if( m_dist == col_dist){
-            container temp( c_.size());
-            dg::blas2::detail::doSymv( alpha, m_, x.data(), beta, temp, 
+            dg::blas2::detail::doSymv( alpha, m_m, x.data(), beta, m_buffer.data(), 
                            typename dg::MatrixTraits<LocalMatrix>::matrix_category(), 
                            typename dg::VectorTraits<container>::vector_category() );
-            c_.global_scatter_reduce( temp, y.data());
+            m_c.get().global_scatter_reduce( m_buffer.data(), y.data());
         }
     }
     template<class container> 
     void symv( const MPI_Vector<container>& x, MPI_Vector<container>& y)
     {
-        if( c_.size() == 0) //no communication needed
+        if( m_c.get().size() == 0) //no communication needed
         {
-            dg::blas2::detail::doSymv( m_, x.data(), y.data(), 
+            dg::blas2::detail::doSymv( m_m, x.data(), y.data(), 
                        typename dg::MatrixTraits<LocalMatrix>::matrix_category(), 
                        typename dg::VectorTraits<container>::vector_category(),
                        typename dg::VectorTraits<container>::vector_category() );
@@ -270,29 +275,29 @@ struct MPIDistMat
 
         }
         assert( x.communicator() == y.communicator());
-        assert( x.communicator() == c_.communicator());
+        assert( x.communicator() == m_c.communicator());
         if( m_dist == row_dist){
-            container temp = c_.global_gather( x.data());
-            dg::blas2::detail::doSymv( m_, temp, y.data(), 
+            m_c.get().global_gather( x.data(), m_buffer.data());
+            dg::blas2::detail::doSymv( m_m, m_buffer.data(), y.data(), 
                        typename dg::MatrixTraits<LocalMatrix>::matrix_category(), 
                        typename dg::VectorTraits<container>::vector_category(),
                        typename dg::VectorTraits<container>::vector_category() );
         }
         if( m_dist == col_dist){
-            container temp( c_.size());
-            dg::blas2::detail::doSymv( m_, x.data(), temp, 
+            dg::blas2::detail::doSymv( m_m, x.data(), m_buffer.data(), 
                        typename dg::MatrixTraits<LocalMatrix>::matrix_category(), 
                        typename dg::VectorTraits<container>::vector_category(),
                        typename dg::VectorTraits<container>::vector_category() );
-            c_.global_scatter_reduce( temp, y.data());
+            m_c.get().global_scatter_reduce( m_buffer.data(), y.data());
         }
     }
 
         
     private:
+    LocalMatrix m_m;
+    Handle<Collective> m_c;
+    Buffer< typename Collective::container_type> m_buffer;
     enum dist_type m_dist;
-    LocalMatrix m_;
-    Collective c_;
 };
 
 ///@cond
@@ -310,25 +315,13 @@ struct MatrixTraits<const RowColDistMat<LI,LO, C> >
 };
 
 template<class L, class C>
-struct MatrixTraits<RowDistMat<L, C> >
-{
-    typedef typename MatrixTraits<L>::value_type value_type;//!< value type
-    typedef MPIMatrixTag matrix_category; //!< 
-};
-template<class L, class C>
-struct MatrixTraits<const RowDistMat<L, C> >
-{
-    typedef typename MatrixTraits<L>::value_type value_type;//!< value type
-    typedef MPIMatrixTag matrix_category; //!< 
-};
-template<class L, class C>
-struct MatrixTraits<ColDistMat<L, C> >
+struct MatrixTraits<MPIDistMat<L, C> >
 {
     typedef typename MatrixTraits<L>::value_type value_type;//!< value type
     typedef MPIMatrixTag matrix_category; //!< 
 };
 template<class L, class C>
-struct MatrixTraits<const ColDistMat<L, C> >
+struct MatrixTraits<const MPIDistMat<L, C> >
 {
     typedef typename MatrixTraits<L>::value_type value_type;//!< value type
     typedef MPIMatrixTag matrix_category; //!< 
diff --git a/inc/dg/backend/mpi_vector.h b/inc/dg/backend/mpi_vector.h
index e3c4b5fa1..79209a4a2 100644
--- a/inc/dg/backend/mpi_vector.h
+++ b/inc/dg/backend/mpi_vector.h
@@ -5,6 +5,7 @@
 #include <thrust/gather.h>
 #include "vector_traits.h"
 #include "thrust_vector_blas.cuh"
+#include "mpi_communicator.h"
 #include "memory.h"
 
 namespace dg
@@ -67,7 +68,7 @@ struct MPI_Vector
     ///@brief Swap data  and communicator
     ///@param src communicator and data is swapped
     void swap( MPI_Vector& src){ 
-        std::swap( comm_ == src.comm_);
+        std::swap( comm_ , src.comm_);
         data_.swap(src.data_);
     }
   private:
@@ -102,8 +103,9 @@ struct VectorTraits<const MPI_Vector<container> > {
 * @note models aCommunicator
 */
 template<class Index, class Vector>
-struct NearestNeighborComm
+struct NearestNeighborComm : public aCommunicator<Vector>
 {
+    ///@brief no communication
     NearestNeighborComm(){
         silent_ = true;
     }
@@ -115,7 +117,7 @@ struct NearestNeighborComm
     * @param comm the (cartesian) communicator 
     * @param direction 0 is x, 1 is y, 2 is z
     */
-    NearestNeighborComm( int n, const int vector_dimensions[3], MPI_Comm comm, int direction)
+    NearestNeighborComm( unsigned n, const unsigned vector_dimensions[3], MPI_Comm comm, unsigned direction)
     {
         construct( n, vector_dimensions, comm, direction);
     }
@@ -136,58 +138,43 @@ struct NearestNeighborComm
             construct( src.n(), src.dims(), src.communicator(), src.direction());
     }
 
-    /**
-    * @brief Construct a vector containing halo cells of neighboring processes
-    *
-    * No inner points are stored
-    * @param input local input vector
-    *
-    * @return new container
-    */
-    const Vector& global_gather( const Vector& input)const;
-    /**
-    * @brief local size of the output of global_gather
-    *
-    * @return size
-    */
-    int size()const; //size of values is size of input plus ghostcells
-    /**
-    * @brief The communicator used
-    *
-    * @return MPI communicator
-    */
-    MPI_Comm communicator() const {return comm_;}
     /**
     * @brief halo size
-    *
     * @return  halo size
     */
-    int n() const{return n_;}
+    unsigned n() const{return n_;}
     /**
     * @brief  The dimensionality of the input vector
-    *
-    * @return dimensions
+    * @return dimensions ( 3)
     */
-    const int* dims() const{return dim_;}
+    const unsigned* dims() const{return dim_;}
     /**
     * @brief The direction of communication
     *
     * @return direction
     */
-    int direction() const {return direction_;}
+    unsigned direction() const {return direction_;}
+    NearestNeighborComm* clone()const{return new NearestNeighborComm(*this);}
     private:
-    void construct( int n, const int vector_dimensions[3], MPI_Comm comm, int direction);
+    MPI_Comm do_communicator() const {return comm_;}
+    unsigned do_size()const; //size of values is size of input plus ghostcells
+    Vector do_make_buffer( )const{
+        Vector tmp( do_size());
+        return tmp;
+    }
+    void do_global_gather( const Vector& values, Vector& gather)const;
+    void do_global_scatter_reduce( const Vector& toScatter, Vector& values)const;
+    void construct( unsigned n, const unsigned vector_dimensions[3], MPI_Comm comm, unsigned direction);
 
-    int n_, dim_[3]; //deepness, dimensions
+    unsigned n_, dim_[3]; //deepness, dimensions
     MPI_Comm comm_;
-    int direction_;
+    unsigned direction_;
     bool silent_;
-    Index gather_map1, gather_map2, scatter_map1, scatter_map2;
-    //dynamically allocate buffer so that global_gather can be const
-    Buffer<Vector> values, buffer1, buffer2, rb1, rb2; 
+    Index gather_map1, gather_map2, scatter_map1, scatter_map2; //buffer_size
+    Buffer<Vector> buffer1, buffer2, rb1, rb2;  //buffer_size
 
     void sendrecv( Vector&, Vector&, Vector& , Vector&)const;
-    int buffer_size() const;
+    unsigned buffer_size() const;
 };
 
 typedef NearestNeighborComm<thrust::host_vector<int>, thrust::host_vector<double> > NNCH; //!< host Communicator for the use in an mpi matrix for derivatives
@@ -195,27 +182,28 @@ typedef NearestNeighborComm<thrust::host_vector<int>, thrust::host_vector<double
 ///@cond
 
 template<class I, class V>
-void NearestNeighborComm<I,V>::construct( int n, const int dimensions[3], MPI_Comm comm, int direction)
+void NearestNeighborComm<I,V>::construct( unsigned n, const unsigned dimensions[3], MPI_Comm comm, unsigned direction)
 {
     silent_=false;
+    n_=n;
+    dim_[0] = dimensions[0], dim_[1] = dimensions[1], dim_[2] = dimensions[2];
+    comm_ = comm;
+    direction_ = direction;
+    {
         int ndims;
         MPI_Cartdim_get( comm, &ndims);
         int dims[ndims], periods[ndims], coords[ndims];
         MPI_Cart_get( comm, ndims, dims, periods, coords);
         if( dims[direction] == 1) silent_ = true;
-    n_=n;
-    dim_[0] = dimensions[0], dim_[1] = dimensions[1], dim_[2] = dimensions[2];
-    comm_ = comm;
-    direction_ = direction;
-    assert( 0<=direction);
+    }
     assert( direction <3);
     thrust::host_vector<int> hbgather1(buffer_size()), hbgather2(hbgather1), hbscattr1(buffer_size()), hbscattr2(hbscattr1);
     switch( direction)
     {
         case( 0):
-        for( int i=0; i<dim_[2]*dim_[1]; i++)
+        for( unsigned i=0; i<dim_[2]*dim_[1]; i++)
         {
-            for( int j=0; j<n_; j++)
+            for( unsigned j=0; j<n_; j++)
             {
                 hbgather1[i*n+j] = (i*dim_[0]               + j);
                 hbgather2[i*n+j] = (i*dim_[0] + dim_[0] - n + j);
@@ -225,10 +213,10 @@ void NearestNeighborComm<I,V>::construct( int n, const int dimensions[3], MPI_Co
         }
         break;
         case( 1):
-        for( int i=0; i<dim_[2]; i++)
+        for( unsigned i=0; i<dim_[2]; i++)
         {
-            for( int j=0; j<n; j++)
-                for( int k=0; k<dim_[0]; k++)
+            for( unsigned j=0; j<n; j++)
+                for( unsigned k=0; k<dim_[0]; k++)
                 {
                     hbgather1[(i*n+j)*dim_[0]+k] = 
                         (i*dim_[1] +               j)*dim_[0] + k;
@@ -242,9 +230,9 @@ void NearestNeighborComm<I,V>::construct( int n, const int dimensions[3], MPI_Co
         }
         break;
         case( 2):
-        for( int i=0; i<n; i++)
+        for( unsigned i=0; i<n; i++)
         {
-            for( int j=0; j<dim_[0]*dim_[1]; j++)
+            for( unsigned j=0; j<dim_[0]*dim_[1]; j++)
             {
                 hbgather1[i*dim_[0]*dim_[1]+j] =  i*dim_[0]*dim_[1]            + j;
                 hbgather2[i*dim_[0]*dim_[1]+j] = (i+dim_[2]-n)*dim_[0]*dim_[1] + j;
@@ -256,20 +244,19 @@ void NearestNeighborComm<I,V>::construct( int n, const int dimensions[3], MPI_Co
     }
     gather_map1 =hbgather1, gather_map2 =hbgather2;
     scatter_map1=hbscattr1, scatter_map2=hbscattr2;
-    values.data().resize( size());
     buffer1.data().resize( buffer_size()), buffer2.data().resize( buffer_size());
     rb1.data().resize( buffer_size()), rb2.data().resize( buffer_size());
 }
 
 template<class I, class V>
-int NearestNeighborComm<I,V>::size() const
+unsigned NearestNeighborComm<I,V>::do_size() const
 {
     if( silent_) return 0;
     return 2*buffer_size();
 }
 
 template<class I, class V>
-int NearestNeighborComm<I,V>::buffer_size() const
+unsigned NearestNeighborComm<I,V>::buffer_size() const
 {
     switch( direction_)
     {
@@ -285,30 +272,30 @@ int NearestNeighborComm<I,V>::buffer_size() const
 }
 
 template<class I, class V>
-const V& NearestNeighborComm<I,V>::global_gather( const V& input) const
+void NearestNeighborComm<I,V>::do_global_gather( const V& input, V& values) const
 {
-    if( silent_) return values.data();
-    //int rank;
-    //MPI_Comm_rank( MPI_COMM_WORLD, &rank);
-    //dg::Timer t;
-    //t.tic();
+    if( silent_) return;
     //gather values from input into sendbuffer
     thrust::gather( gather_map1.begin(), gather_map1.end(), input.begin(), buffer1.data().begin());
     thrust::gather( gather_map2.begin(), gather_map2.end(), input.begin(), buffer2.data().begin());
-    //t.toc();
-    //if(rank==0)std::cout << "Gather       took "<<t.diff()<<"s\n";
-    //t.tic();
     //mpi sendrecv
     sendrecv( buffer1.data(), buffer2.data(), rb1.data(), rb2.data());
-    //t.toc();
-    //if(rank==0)std::cout << "MPI sendrecv took "<<t.diff()<<"s\n";
-    //t.tic();
     //scatter received values into values array
-    thrust::scatter( rb1.data().begin(), rb1.data().end(), scatter_map1.begin(), values.data().begin());
-    thrust::scatter( rb2.data().begin(), rb2.data().end(), scatter_map2.begin(), values.data().begin());
-    //t.toc();
-    //if(rank==0)std::cout << "Scatter      took "<<t.diff()<<"s\n";
-    return values.data();
+    thrust::scatter( rb1.data().begin(), rb1.data().end(), scatter_map1.begin(), values.begin());
+    thrust::scatter( rb2.data().begin(), rb2.data().end(), scatter_map2.begin(), values.begin());
+}
+template<class I, class V>
+void NearestNeighborComm<I,V>::do_global_scatter_reduce( const V& values, V& input) const
+{
+    if( silent_) return;
+    //scatter received values into values array
+    thrust::gather( scatter_map1.begin(), scatter_map1.end(), values.begin(), rb1.data().begin());
+    thrust::gather( scatter_map2.begin(), scatter_map2.end(), values.begin(), rb2.data().begin());
+    //mpi sendrecv
+    sendrecv( rb1.data(), rb2.data(), buffer1.data(), buffer2.data());
+    //gather values from input into sendbuffer
+    thrust::scatter( buffer1.data().begin(), buffer1.data().end(), gather_map1.begin(), input.begin());
+    thrust::scatter( buffer2.data().begin(), buffer2.data().end(), gather_map2.begin(), input.begin());
 }
 
 template<class I, class V>
-- 
GitLab


From ac96bab7d4dd268ac52e59ed0624626cac2ac579 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Tue, 19 Sep 2017 11:30:04 +0200
Subject: [PATCH 305/453] introduced test on nearest neighbor comm

---
 inc/dg/backend/collective_mpit.cu | 29 ++++++++++++++++--
 inc/dg/backend/mpi_collective.h   | 51 +++++++++++++++----------------
 inc/dg/backend/mpi_vector.h       |  3 +-
 3 files changed, 53 insertions(+), 30 deletions(-)

diff --git a/inc/dg/backend/collective_mpit.cu b/inc/dg/backend/collective_mpit.cu
index 582526121..5e638b1f7 100644
--- a/inc/dg/backend/collective_mpit.cu
+++ b/inc/dg/backend/collective_mpit.cu
@@ -2,6 +2,7 @@
 
 #include <mpi.h>
 #include "mpi_collective.h"
+#include "mpi_vector.h"
 
 
 int main( int argc, char * argv[])
@@ -26,7 +27,7 @@ int main( int argc, char * argv[])
     }
     const thrust::host_vector<double> w(v);
     dg::BijectiveComm<thrust::host_vector<int>, thrust::host_vector<double> > c(m, MPI_COMM_WORLD);
-    thrust::host_vector<double> receive(c.size());
+    thrust::host_vector<double> receive = c.allocate_buffer();
     receive = c.global_gather( v);
     MPI_Barrier( MPI_COMM_WORLD);
     c.global_scatter_reduce( receive, v);
@@ -44,7 +45,7 @@ int main( int argc, char * argv[])
     }
     MPI_Barrier(MPI_COMM_WORLD);
     if(rank==0)std::cout << "Test SurjectiveComm and GeneralComm:\n";
-    Nx = 3, Ny = 3; 
+    Nx = 5, Ny = 5; 
     thrust::host_vector<double> vec( Nx*Ny, rank), result( Nx*Ny);
     thrust::host_vector<int> idx( (Nx+1)*(Ny+1)), pids( (Nx+1)*(Ny+1));
     for( unsigned i=0; i<(Nx+1)*(Ny+1); i++)
@@ -73,6 +74,30 @@ int main( int argc, char * argv[])
             std::cerr <<"Rank "<<rank<<" TEST FAILED"<<std::endl;
     }
 
+    MPI_Barrier(MPI_COMM_WORLD);
+    if(rank==0)std::cout<< "Test Nearest Neighbor Comm\n";
+    unsigned dims[3] = {Nx, Ny, 1}; 
+    int np[2] = {2,2}, periods[2] = { false, false};
+    MPI_Comm comm;
+    MPI_Cart_create( MPI_COMM_WORLD, 2, np, periods, true, &comm);
+
+    dg::NearestNeighborComm<thrust::host_vector<int>, thrust::host_vector<double> > nnch( 2, dims, comm, 1);
+    thrust::host_vector<double> tmp = nnch.allocate_buffer();
+    nnch.global_gather( vec, tmp);
+    thrust::host_vector<double> vec2(vec);
+    nnch.global_scatter_reduce( tmp, vec2);
+    for( unsigned i=0; i<(Nx)*(Ny); i++)
+    {
+        if( vec[i] != vec2[i]) equal = false;
+    }
+    {
+        if( equal) 
+            std::cout <<"Rank "<<rank<<" TEST PASSED"<<std::endl;
+        else
+            std::cerr <<"Rank "<<rank<<" TEST FAILED"<<std::endl;
+    }
+
+
 
     MPI_Finalize();
 
diff --git a/inc/dg/backend/mpi_collective.h b/inc/dg/backend/mpi_collective.h
index 60736bd43..4e06d2355 100644
--- a/inc/dg/backend/mpi_collective.h
+++ b/inc/dg/backend/mpi_collective.h
@@ -141,7 +141,6 @@ void Collective<Index, Device>::gather( const Device& gatherFrom, Device& values
  @endcode
  @tparam Index an integer Vector
  @tparam Vector a Vector (the data type of Vector must be double)
- @note models aCommunicator
  */
 template< class Index, class Vector>
 struct BijectiveComm : public aCommunicator<Vector>
@@ -263,23 +262,21 @@ struct BijectiveComm : public aCommunicator<Vector>
 
 /**
  * @ingroup mpi_structures
- * @brief Struct that performs surjective collective scatter and gather operations across processes on distributed vectors using mpi
+ * @brief Struct that performs injective collective scatter and gather operations across processes on distributed vectors using mpi
  *
- * This Communicator can perform general global gather and
- scatter operations. We only assume that the gather/scatter map
- is surjective, i.e. all elements in a vector get gathered. This
- is important only in the global_scatter_reduce function.
+ * This Communicator performs injective global gather and
+ scatter operations, which means that the gather/scatter map
+ is injective, i.e. all elements in a vector get gathered. This
+ is relevant only in the global_scatter_reduce function, since 
+ there is no reduction needed.
  @tparam Index an integer Vector
  @tparam Vector a Vector (the data type of Vector must be double)
- @note models aCommunicator
  */
 template< class Index, class Vector>
-struct SurjectiveComm : public aCommunicator<Vector>
+struct InjectiveComm : public aCommunicator<Vector>
 {
-    /**
-     * @brief Construct empty class
-     */
-    SurjectiveComm(){}
+    ///@brief Construct empty class
+    InjectiveComm(){}
     /**
     * @brief Construct from local indices and PIDs
     *
@@ -288,16 +285,16 @@ struct SurjectiveComm : public aCommunicator<Vector>
     Same size as localGatherMap.
      *   The rank needs to be element of the given communicator.
     * @param comm The MPI communicator participating in the scatter/gather operations
-    * @note we assume that the gather map is surjective
+    * @note we assume that the gather map is injective
     */
-    SurjectiveComm( const thrust::host_vector<int>& localGatherMap, const thrust::host_vector<int>& pidGatherMap, MPI_Comm comm)
+    InjectiveComm( const thrust::host_vector<int>& localGatherMap, const thrust::host_vector<int>& pidGatherMap, MPI_Comm comm)
     {
         construct( localGatherMap, pidGatherMap, comm);
     }
 
     ///use global2localIdx of MPI geometries
     template<class MPIGeometry>
-    SurjectiveComm( const thrust::host_vector<int>& globalGatherMap, const MPIGeometry& g)
+    InjectiveComm( const thrust::host_vector<int>& globalGatherMap, const MPIGeometry& g)
     {
         thrust::host_vector<int> local(globalGatherMap.size()), pids(globalGatherMap.size());
         bool success = true;
@@ -309,7 +306,7 @@ struct SurjectiveComm : public aCommunicator<Vector>
     }
 
     template<class OtherIndex, class OtherVector>
-    SurjectiveComm( const SurjectiveComm<OtherIndex, OtherVector>& src)
+    InjectiveComm( const InjectiveComm<OtherIndex, OtherVector>& src)
     {
         construct( src.getLocalGatherMap(), src.getPidGatherMap(), src.communicator());
     }
@@ -317,7 +314,7 @@ struct SurjectiveComm : public aCommunicator<Vector>
     const thrust::host_vector<int>& getLocalGatherMap() const {return localGatherMap_;}
     const thrust::host_vector<int>& getPidGatherMap() const {return pidGatherMap_;}
     const Index& getSortedGatherMap() const {return sortedGatherMap_;}
-    virtual SurjectiveComm* clone() const {return new SurjectiveComm(*this);}
+    virtual InjectiveComm* clone() const {return new InjectiveComm(*this);}
     private:
     Vector do_make_buffer()const{
         Vector tmp(do_size());
@@ -380,11 +377,11 @@ struct SurjectiveComm : public aCommunicator<Vector>
  scatter operations. 
  @tparam Index an integer Vector
  @tparam Vector a Vector (the data type of Vector must be double, and the Vector must have size() and resize() function)
- @note models aCommunicator
  */
 template< class Index, class Vector>
 struct GeneralComm : public aCommunicator<Vector>
 {
+    ///@brief no memory allocation
     GeneralComm(){}
     GeneralComm( const thrust::host_vector<int>& localGatherMap, const thrust::host_vector<int>& pidGatherMap, MPI_Comm comm) {
         construct( localGatherMap, pidGatherMap, comm);
@@ -404,8 +401,8 @@ struct GeneralComm : public aCommunicator<Vector>
         construct( local, pids, g.communicator());
     }
 
-    const thrust::host_vector<int>& getLocalGatherMap() const {return surjectiveComm_.getLocalGatherMap();}
-    const thrust::host_vector<int>& getPidGatherMap() const {return surjectiveComm_.getPidGatherMap();}
+    const thrust::host_vector<int>& getLocalGatherMap() const {return injectiveComm_.getLocalGatherMap();}
+    const thrust::host_vector<int>& getPidGatherMap() const {return injectiveComm_.getPidGatherMap();}
     virtual GeneralComm* clone() const {return new GeneralComm(*this);}
     private:
     Vector do_make_buffer() const{ 
@@ -413,23 +410,23 @@ struct GeneralComm : public aCommunicator<Vector>
         return tmp;
     }
 
-    MPI_Comm do_communicator()const{return surjectiveComm_.communicator();}
+    MPI_Comm do_communicator()const{return injectiveComm_.communicator();}
     void do_global_gather( const Vector& values, Vector& sink)const
     {
-        surjectiveComm_.global_gather( values, sink);
+        injectiveComm_.global_gather( values, sink);
     }
     void do_global_scatter_reduce( const Vector& toScatter, Vector& values)const
     {
-        surjectiveComm_.global_scatter_reduce( toScatter, store_.data());
+        injectiveComm_.global_scatter_reduce( toScatter, store_.data());
         thrust::scatter( store_.data().begin(), store_.data().end(), scatterMap_.begin(), values.begin());
     }
 
-    unsigned do_size() const{return surjectiveComm_.size();}
+    unsigned do_size() const{return injectiveComm_.size();}
     void construct( const thrust::host_vector<int>& localGatherMap, const thrust::host_vector<int>& pidGatherMap, MPI_Comm comm)
     {
-        surjectiveComm_ = SurjectiveComm<Index,Vector>(localGatherMap, pidGatherMap, comm);
+        injectiveComm_ = InjectiveComm<Index,Vector>(localGatherMap, pidGatherMap, comm);
 
-        const Index& sortedGatherMap_ = surjectiveComm_.getSortedGatherMap();
+        const Index& sortedGatherMap_ = injectiveComm_.getSortedGatherMap();
         thrust::host_vector<int> gatherMap;
         dg::blas1::transfer( sortedGatherMap_, gatherMap);
         thrust::host_vector<int> one( gatherMap.size(), 1), keys(one), number(one);
@@ -442,7 +439,7 @@ struct GeneralComm : public aCommunicator<Vector>
         scatterMap_.resize(distance);
         thrust::copy( keys.begin(), keys.begin() + distance, scatterMap_.begin());
     }
-    SurjectiveComm<Index, Vector> surjectiveComm_;
+    InjectiveComm<Index, Vector> injectiveComm_;
     Buffer<Vector> store_;
     Index scatterMap_;
 };
diff --git a/inc/dg/backend/mpi_vector.h b/inc/dg/backend/mpi_vector.h
index 79209a4a2..f4169141f 100644
--- a/inc/dg/backend/mpi_vector.h
+++ b/inc/dg/backend/mpi_vector.h
@@ -95,7 +95,8 @@ struct VectorTraits<const MPI_Vector<container> > {
 /**
 * @brief Communicator for nearest neighbor communication
 *
-* exchanges a halo of given size among next neighbors in a given direction
+* exchanges a halo of given depth among neighboring processes in a given direction 
+* (the corresponding index map is injective)
 * @ingroup mpi_structures
 * @tparam Index the type of index container (must be either thrust::host_vector<int> or thrust::device_vector<int>)
 * @tparam Vector the vector container type must have a resize() function and work 
-- 
GitLab


From 01561a7e080a769e2b5c1ce6ff85cf1fa3397e9a Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Tue, 19 Sep 2017 14:20:06 +0200
Subject: [PATCH 306/453] made global2local functions const and fumbled more
 with documentation

---
 inc/dg/backend/collective_mpit.cu | 19 ++++---
 inc/dg/backend/mpi_collective.h   | 91 ++++++++++++++++++-------------
 inc/dg/backend/mpi_grid.h         |  4 +-
 inc/dg/backend/mpi_vector.h       |  3 +-
 4 files changed, 68 insertions(+), 49 deletions(-)

diff --git a/inc/dg/backend/collective_mpit.cu b/inc/dg/backend/collective_mpit.cu
index 5e638b1f7..eb2b30575 100644
--- a/inc/dg/backend/collective_mpit.cu
+++ b/inc/dg/backend/collective_mpit.cu
@@ -39,9 +39,9 @@ int main( int argc, char * argv[])
     }
     {
         if( equal) 
-            std::cout <<"Rank "<<rank<<" TEST PASSED"<<std::endl;
+            std::cout <<"Rank "<<rank<<" PASSED"<<std::endl;
         else
-            std::cerr <<"Rank "<<rank<<" TEST FAILED"<<std::endl;
+            std::cerr <<"Rank "<<rank<<" FAILED"<<std::endl;
     }
     MPI_Barrier(MPI_COMM_WORLD);
     if(rank==0)std::cout << "Test SurjectiveComm and GeneralComm:\n";
@@ -58,20 +58,21 @@ int main( int argc, char * argv[])
     receive = s.global_gather( vec);
     //for( unsigned i=0; i<(Nx+1)*(Ny+1); i++)
     //    if(rank==0) std::cout << i<<"\t "<< receive[i] << std::endl;
-    s.global_scatter_reduce( receive, vec);
+    thrust::host_vector<double> vec2(vec.size());
+    s.global_scatter_reduce( receive, vec2);
     equal=true;
     for( unsigned i=0; i<(Nx)*(Ny); i++)
     {
         //if(rank==1) std::cout << i<<"\t "<< vec[i] << std::endl;
         result[i] = rank; 
         if( i < (Nx+1)*(Ny+1) - Nx*Ny) result[i] += (rank)%size;
-        if( vec[i] != result[i]) equal = false;
+        if( vec2[i] != result[i]) equal = false;
     }
     {
         if( equal) 
-            std::cout <<"Rank "<<rank<<" TEST PASSED"<<std::endl;
+            std::cout <<"Rank "<<rank<<" PASSED"<<std::endl;
         else
-            std::cerr <<"Rank "<<rank<<" TEST FAILED"<<std::endl;
+            std::cerr <<"Rank "<<rank<<" FAILED"<<std::endl;
     }
 
     MPI_Barrier(MPI_COMM_WORLD);
@@ -84,7 +85,7 @@ int main( int argc, char * argv[])
     dg::NearestNeighborComm<thrust::host_vector<int>, thrust::host_vector<double> > nnch( 2, dims, comm, 1);
     thrust::host_vector<double> tmp = nnch.allocate_buffer();
     nnch.global_gather( vec, tmp);
-    thrust::host_vector<double> vec2(vec);
+    vec2=vec;
     nnch.global_scatter_reduce( tmp, vec2);
     for( unsigned i=0; i<(Nx)*(Ny); i++)
     {
@@ -92,9 +93,9 @@ int main( int argc, char * argv[])
     }
     {
         if( equal) 
-            std::cout <<"Rank "<<rank<<" TEST PASSED"<<std::endl;
+            std::cout <<"Rank "<<rank<<" PASSED"<<std::endl;
         else
-            std::cerr <<"Rank "<<rank<<" TEST FAILED"<<std::endl;
+            std::cerr <<"Rank "<<rank<<" FAILED"<<std::endl;
     }
 
 
diff --git a/inc/dg/backend/mpi_collective.h b/inc/dg/backend/mpi_collective.h
index 4e06d2355..0d37b3bb4 100644
--- a/inc/dg/backend/mpi_collective.h
+++ b/inc/dg/backend/mpi_collective.h
@@ -150,7 +150,7 @@ struct BijectiveComm : public aCommunicator<Vector>
      */
     BijectiveComm( ){ }
     /**
-     * @brief Construct from a given map with respect to the data vector
+     * @brief Construct from a given map with respect to the source/data vector
      *
      * @param pids Gives to every point of the values/data vector (not the buffer vector!) 
      *   the rank to which to send this data element. 
@@ -262,39 +262,33 @@ struct BijectiveComm : public aCommunicator<Vector>
 
 /**
  * @ingroup mpi_structures
- * @brief Struct that performs injective collective scatter and gather operations across processes on distributed vectors using mpi
+ * @brief Struct that performs surjective collective scatter and gather operations across processes on distributed vectors using mpi
  *
- * This Communicator performs injective global gather and
+ * This Communicator performs surjective global gather and
  scatter operations, which means that the gather/scatter map
- is injective, i.e. all elements in a vector get gathered. This
- is relevant only in the global_scatter_reduce function, since 
- there is no reduction needed.
+ is surjective, i.e. all elements in a source vector get gathered. 
+ Compared to BijectiveComm in the global_gather function there is an additional 
+ gather and in the global_scatter_reduce function a reduction 
+ needs to be performed.
  @tparam Index an integer Vector
  @tparam Vector a Vector (the data type of Vector must be double)
  */
 template< class Index, class Vector>
-struct InjectiveComm : public aCommunicator<Vector>
+struct SurjectiveComm : public aCommunicator<Vector>
 {
-    ///@brief Construct empty class
-    InjectiveComm(){}
-    /**
-    * @brief Construct from local indices and PIDs
-    *
-    * @param localGatherMap The gather map containing local vector indices
-    * @param pidGatherMap The gather map containing the pids from where to gather the local index.
-    Same size as localGatherMap.
-     *   The rank needs to be element of the given communicator.
-    * @param comm The MPI communicator participating in the scatter/gather operations
-    * @note we assume that the gather map is injective
-    */
-    InjectiveComm( const thrust::host_vector<int>& localGatherMap, const thrust::host_vector<int>& pidGatherMap, MPI_Comm comm)
+    ///@copydoc GeneralComm::GeneralComm()
+    SurjectiveComm(){}
+    ///@copydoc GeneralComm::GeneralComm(const thrust::host_vector<int>&,const thrust::host_vector<int>&,MPI_Comm)
+    ///@note we assume that the gather map is surjective
+    SurjectiveComm( const thrust::host_vector<int>& localGatherMap, const thrust::host_vector<int>& pidGatherMap, MPI_Comm comm)
     {
         construct( localGatherMap, pidGatherMap, comm);
     }
 
-    ///use global2localIdx of MPI geometries
-    template<class MPIGeometry>
-    InjectiveComm( const thrust::host_vector<int>& globalGatherMap, const MPIGeometry& g)
+    ///@copydoc GeneralComm::GeneralComm(const thrust::host_vector<int>&,const MPITopology&)
+    ///@note we assume that the gather map is surjective
+    template<class MPITopology>
+    SurjectiveComm( const thrust::host_vector<int>& globalGatherMap, const MPITopology& g)
     {
         thrust::host_vector<int> local(globalGatherMap.size()), pids(globalGatherMap.size());
         bool success = true;
@@ -306,15 +300,16 @@ struct InjectiveComm : public aCommunicator<Vector>
     }
 
     template<class OtherIndex, class OtherVector>
-    InjectiveComm( const InjectiveComm<OtherIndex, OtherVector>& src)
+    SurjectiveComm( const SurjectiveComm<OtherIndex, OtherVector>& src)
     {
         construct( src.getLocalGatherMap(), src.getPidGatherMap(), src.communicator());
     }
 
+    ///@copydoc GeneralComm::getLocalGatherMap
     const thrust::host_vector<int>& getLocalGatherMap() const {return localGatherMap_;}
     const thrust::host_vector<int>& getPidGatherMap() const {return pidGatherMap_;}
     const Index& getSortedGatherMap() const {return sortedGatherMap_;}
-    virtual InjectiveComm* clone() const {return new InjectiveComm(*this);}
+    virtual SurjectiveComm* clone() const {return new SurjectiveComm(*this);}
     private:
     Vector do_make_buffer()const{
         Vector tmp(do_size());
@@ -374,7 +369,8 @@ struct InjectiveComm : public aCommunicator<Vector>
  * @brief Struct that performs general collective scatter and gather operations across processes on distributed vectors using mpi
  *
  * This Communicator can perform general global gather and
- scatter operations. 
+ scatter operations. Compared to SurjectiveComm the global_scatter_reduce function needs
+ to perform an additional scatter as some elements of the source vector might be left empty
  @tparam Index an integer Vector
  @tparam Vector a Vector (the data type of Vector must be double, and the Vector must have size() and resize() function)
  */
@@ -383,6 +379,16 @@ struct GeneralComm : public aCommunicator<Vector>
 {
     ///@brief no memory allocation
     GeneralComm(){}
+    /**
+    * @brief Construct from local indices and PIDs gather map
+    *
+    * The gather map shall be written with respect to the buffer vector (unlike in BijectiveComm, where it is given wrt the source vector) 
+    * @param localGatherMap The gather map containing local vector indices ( local buffer size)
+    * @param pidGatherMap The gather map containing the pids from where to gather the local index.
+    Same size as localGatherMap.
+     *   The rank needs to be element of the given communicator.
+    * @param comm The MPI communicator participating in the scatter/gather operations
+    */
     GeneralComm( const thrust::host_vector<int>& localGatherMap, const thrust::host_vector<int>& pidGatherMap, MPI_Comm comm) {
         construct( localGatherMap, pidGatherMap, comm);
     }
@@ -390,8 +396,17 @@ struct GeneralComm : public aCommunicator<Vector>
     GeneralComm( const GeneralComm<OtherIndex, OtherVector>& src){
         construct( src.getLocalGatherMap(), src.getPidGatherMap(), src.communicator());
     }
-    template<class MPIGeometry>
-    GeneralComm( const thrust::host_vector<int>& globalGatherMap, MPIGeometry& g)
+
+    /**
+     * @brief Construct from global indices gather map
+     *
+     * Use the global2localIdx() function to generate localGatherMap and pidGatherMap 
+     * @tparam MPITopology any implementation of an MPI Topology (aMPITopology2d, aMPITopology3d, ...)
+     * @param globalGatherMap The gather map containing global vector indices (local buffer size)
+     * @param g a grid object
+     */
+    template<class MPITopology>
+    GeneralComm( const thrust::host_vector<int>& globalGatherMap, const MPITopology& g)
     {
         thrust::host_vector<int> local(globalGatherMap.size()), pids(globalGatherMap.size());
         bool success = true;
@@ -401,8 +416,10 @@ struct GeneralComm : public aCommunicator<Vector>
         construct( local, pids, g.communicator());
     }
 
-    const thrust::host_vector<int>& getLocalGatherMap() const {return injectiveComm_.getLocalGatherMap();}
-    const thrust::host_vector<int>& getPidGatherMap() const {return injectiveComm_.getPidGatherMap();}
+    ///@brief read access to the local index gather map
+    const thrust::host_vector<int>& getLocalGatherMap() const {return surjectiveComm_.getLocalGatherMap();}
+    ///@brief read access to the pid gather map
+    const thrust::host_vector<int>& getPidGatherMap() const {return surjectiveComm_.getPidGatherMap();}
     virtual GeneralComm* clone() const {return new GeneralComm(*this);}
     private:
     Vector do_make_buffer() const{ 
@@ -410,23 +427,23 @@ struct GeneralComm : public aCommunicator<Vector>
         return tmp;
     }
 
-    MPI_Comm do_communicator()const{return injectiveComm_.communicator();}
+    MPI_Comm do_communicator()const{return surjectiveComm_.communicator();}
     void do_global_gather( const Vector& values, Vector& sink)const
     {
-        injectiveComm_.global_gather( values, sink);
+        surjectiveComm_.global_gather( values, sink);
     }
     void do_global_scatter_reduce( const Vector& toScatter, Vector& values)const
     {
-        injectiveComm_.global_scatter_reduce( toScatter, store_.data());
+        surjectiveComm_.global_scatter_reduce( toScatter, store_.data());
         thrust::scatter( store_.data().begin(), store_.data().end(), scatterMap_.begin(), values.begin());
     }
 
-    unsigned do_size() const{return injectiveComm_.size();}
+    unsigned do_size() const{return surjectiveComm_.size();}
     void construct( const thrust::host_vector<int>& localGatherMap, const thrust::host_vector<int>& pidGatherMap, MPI_Comm comm)
     {
-        injectiveComm_ = InjectiveComm<Index,Vector>(localGatherMap, pidGatherMap, comm);
+        surjectiveComm_ = SurjectiveComm<Index,Vector>(localGatherMap, pidGatherMap, comm);
 
-        const Index& sortedGatherMap_ = injectiveComm_.getSortedGatherMap();
+        const Index& sortedGatherMap_ = surjectiveComm_.getSortedGatherMap();
         thrust::host_vector<int> gatherMap;
         dg::blas1::transfer( sortedGatherMap_, gatherMap);
         thrust::host_vector<int> one( gatherMap.size(), 1), keys(one), number(one);
@@ -439,7 +456,7 @@ struct GeneralComm : public aCommunicator<Vector>
         scatterMap_.resize(distance);
         thrust::copy( keys.begin(), keys.begin() + distance, scatterMap_.begin());
     }
-    InjectiveComm<Index, Vector> injectiveComm_;
+    SurjectiveComm<Index, Vector> surjectiveComm_;
     Buffer<Vector> store_;
     Index scatterMap_;
 };
diff --git a/inc/dg/backend/mpi_grid.h b/inc/dg/backend/mpi_grid.h
index b31739f33..aa6aa334e 100644
--- a/inc/dg/backend/mpi_grid.h
+++ b/inc/dg/backend/mpi_grid.h
@@ -232,7 +232,7 @@ struct aMPITopology2d
     * @param globalIdx the corresponding global vector Index (contains result on output)
     * @return true if successful, false if localIdx or PID is not part of the grid
     */
-    bool local2globalIdx( int localIdx, int PID, int& globalIdx)
+    bool local2globalIdx( int localIdx, int PID, int& globalIdx)const
     {
         if( localIdx < 0 || localIdx >= (int)size()) return -1;
         int coords[2];
@@ -253,7 +253,7 @@ struct aMPITopology2d
     * @param PID contains corresponding PID in the communicator on output
     * @return true if successful, false if globalIdx is not part of the grid
     */
-    bool global2localIdx( int globalIdx, int& localIdx, int& PID)
+    bool global2localIdx( int globalIdx, int& localIdx, int& PID)const
     {
         if( globalIdx < 0 || globalIdx >= (int)g.size()) return -1;
         int coords[2];
diff --git a/inc/dg/backend/mpi_vector.h b/inc/dg/backend/mpi_vector.h
index f4169141f..c82f453e6 100644
--- a/inc/dg/backend/mpi_vector.h
+++ b/inc/dg/backend/mpi_vector.h
@@ -96,7 +96,8 @@ struct VectorTraits<const MPI_Vector<container> > {
 * @brief Communicator for nearest neighbor communication
 *
 * exchanges a halo of given depth among neighboring processes in a given direction 
-* (the corresponding index map is injective)
+* (the corresponding gather map is of general type and the communication 
+*  can also be modeled in GeneralComm, but not BijectiveComm or SurjectiveComm )
 * @ingroup mpi_structures
 * @tparam Index the type of index container (must be either thrust::host_vector<int> or thrust::device_vector<int>)
 * @tparam Vector the vector container type must have a resize() function and work 
-- 
GitLab


From 684d76850264b076edfff6f0e7c3a3ab991a6169 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Tue, 19 Sep 2017 15:08:44 +0200
Subject: [PATCH 307/453] update  documentation on mpi collective and mpi
 communicator

---
 inc/dg/backend/mpi_collective.h   | 68 +++++++++----------------------
 inc/dg/backend/mpi_communicator.h | 18 ++++----
 inc/dg/backend/mpi_grid.h         | 12 ++----
 3 files changed, 32 insertions(+), 66 deletions(-)

diff --git a/inc/dg/backend/mpi_collective.h b/inc/dg/backend/mpi_collective.h
index 0d37b3bb4..4c02bb1fd 100644
--- a/inc/dg/backend/mpi_collective.h
+++ b/inc/dg/backend/mpi_collective.h
@@ -128,26 +128,25 @@ void Collective<Index, Device>::gather( const Device& gatherFrom, Device& values
  *
  * @code
  int i = myrank;
- double values[10] = {i,i,i,i, 9,9,9,9};
- thrust::host_vector<double> hvalues( values, values+10);
- int pids[10] =      {0,1,2,3, 0,1,2,3};
- thrust::host_vector<int> hpids( pids, pids+10);
+ double values[8] = {i,i,i,i, 9,9,9,9};
+ thrust::host_vector<double> hvalues( values, values+8);
+ int pids[8] =      {0,1,2,3, 0,1,2,3};
+ thrust::host_vector<int> hpids( pids, pids+8);
  BijectiveComm coll( hpids, MPI_COMM_WORLD);
- thrust::host_vector<double> hrecv = coll.scatter( hvalues);
- //hrecv is now {0,9,1,9,2,9,3,9} e.g. for process 0 
- thrust::host_vector<double> hrecv2( coll.send_size());
- coll.gather( hrecv, hrecv2);
- //hrecv2 now equals hvalues independent of process rank
+ thrust::host_vector<double> hrecv = coll.global_gather( hvalues); //for e.g. process 0 hrecv is now {0,9,1,9,2,9,3,9} 
+ thrust::host_vector<double> hrecv2( hvalues.size());
+ coll.global_scatter_reduce( hrecv, hrecv2); //hrecv2 now equals hvalues independent of process rank
  @endcode
- @tparam Index an integer Vector
- @tparam Vector a Vector (the data type of Vector must be double)
+ * @tparam Index an integer thrust Vector (thrust::host_vector<int> or thrust::device_vector<int>) 
+ * @tparam Vector a thrust Vector (thrust::host_vector<double> or thrust::device_vector<double>)
+ * @note a scatter followed by a gather of the received values restores the original array
+ * @note The order of the received elements is according to their original array index 
+ *   (i.e. a[0] appears before a[1]) and their process rank of origin ( i.e. values from rank 0 appear before values from rank 1)
  */
 template< class Index, class Vector>
 struct BijectiveComm : public aCommunicator<Vector>
 {
-    /**
-     * @brief Construct empty class
-     */
+    ///@copydoc GeneralComm::GeneralComm()
     BijectiveComm( ){ }
     /**
      * @brief Construct from a given map with respect to the source/data vector
@@ -175,13 +174,6 @@ struct BijectiveComm : public aCommunicator<Vector>
     virtual BijectiveComm* clone() const {return new BijectiveComm(*this);}
     private:
     MPI_Comm do_communicator() const {return p_.communicator();}
-    /**
-     * @brief compute total # of elements the calling process receives in the scatter process (or sends in the gather process)
-     *
-     * (which might not equal the send size in each process)
-     *
-     * @return # of elements to receive
-     */
     unsigned do_size() const { return p_.store_size();}
     Vector do_make_buffer()const{ 
         Vector tmp( do_size() );
@@ -217,17 +209,6 @@ struct BijectiveComm : public aCommunicator<Vector>
         p_.construct( sendTo, comm);
         values_.data().resize( idx_.size());
     }
-    /**
-     * @brief Globally gathers data into a buffer according to the map given in the Constructor
-     *
-     * The order of the received elements is according to their original array index 
-     * (i.e. a[0] appears before a[1]) and their process rank of origin ( i.e. values from rank 0 appear before values from rank 1)
-     * @param values data to send (must have the size given 
-     * by the map in the constructor, s.a. size())
-     *
-     * @return received data from other processes of size size()
-     * @note a scatter followed by a gather of the received values restores the original array
-     */
     void do_global_gather( const Vector& values, Vector& store)const
     {
         //actually this is a scatter but we constructed it invertedly
@@ -239,14 +220,6 @@ struct BijectiveComm : public aCommunicator<Vector>
         p_.scatter( values_.data(), store);
     }
 
-    /**
-     * @brief Scatter data according to the map given in the constructor 
-     *
-     * This method is the inverse of gather
-     * @param toScatter other processes collect data from this vector (has to be of size given by recv_size())
-     * @param values contains values from other processes sent back to the origin (must have the size of the map given in the constructor, or send_size())
-     * @note a scatter followed by a gather of the received values restores the original array
-     */
     void do_global_scatter_reduce( const Vector& toScatter, Vector& values) const
     {
         //actually this is a gather but we constructed it invertedly
@@ -270,8 +243,8 @@ struct BijectiveComm : public aCommunicator<Vector>
  Compared to BijectiveComm in the global_gather function there is an additional 
  gather and in the global_scatter_reduce function a reduction 
  needs to be performed.
- @tparam Index an integer Vector
- @tparam Vector a Vector (the data type of Vector must be double)
+ * @tparam Index an integer thrust Vector (thrust::host_vector<int> or thrust::device_vector<int>) 
+ * @tparam Vector a thrust Vector (thrust::host_vector<double> or thrust::device_vector<double>)
  */
 template< class Index, class Vector>
 struct SurjectiveComm : public aCommunicator<Vector>
@@ -371,8 +344,8 @@ struct SurjectiveComm : public aCommunicator<Vector>
  * This Communicator can perform general global gather and
  scatter operations. Compared to SurjectiveComm the global_scatter_reduce function needs
  to perform an additional scatter as some elements of the source vector might be left empty
- @tparam Index an integer Vector
- @tparam Vector a Vector (the data type of Vector must be double, and the Vector must have size() and resize() function)
+ * @tparam Index an integer thrust Vector (thrust::host_vector<int> or thrust::device_vector<int>) 
+ * @tparam Vector a thrust Vector (thrust::host_vector<double> or thrust::device_vector<double>)
  */
 template< class Index, class Vector>
 struct GeneralComm : public aCommunicator<Vector>
@@ -426,14 +399,11 @@ struct GeneralComm : public aCommunicator<Vector>
         Vector tmp(do_size());
         return tmp;
     }
-
     MPI_Comm do_communicator()const{return surjectiveComm_.communicator();}
-    void do_global_gather( const Vector& values, Vector& sink)const
-    {
+    void do_global_gather( const Vector& values, Vector& sink)const {
         surjectiveComm_.global_gather( values, sink);
     }
-    void do_global_scatter_reduce( const Vector& toScatter, Vector& values)const
-    {
+    void do_global_scatter_reduce( const Vector& toScatter, Vector& values)const {
         surjectiveComm_.global_scatter_reduce( toScatter, store_.data());
         thrust::scatter( store_.data().begin(), store_.data().end(), scatterMap_.begin(), values.begin());
     }
diff --git a/inc/dg/backend/mpi_communicator.h b/inc/dg/backend/mpi_communicator.h
index ae960b13e..d30e4ac10 100644
--- a/inc/dg/backend/mpi_communicator.h
+++ b/inc/dg/backend/mpi_communicator.h
@@ -66,19 +66,19 @@ struct aCommunicator
     }
 
     /**
-     * @brief Gather data across processes
-     * @param values data to gather from
-     * @param gather object to hold the gathered data ( must be of size size())
+     * @brief Globally (across processes) gather data into a buffer 
+     * @param values data; other processes collect data from this vector
+     * @param buffer object to hold the gathered data ( must be of size size())
      */
-    void global_gather( const LocalContainer& values, LocalContainer& gather)const
+    void global_gather( const LocalContainer& values, LocalContainer& buffer)const
     {
         do_global_gather( values, gather);
     }
 
     /**
-     * @brief Gather data across processes (memory allocating version)
-     * @param values data to gather from
-     * @return object to hold the gathered data
+     * @brief Globally (across processes) gather data into a buffer (memory allocating version)
+     * @param values data; other processes collect data from this vector
+     * @return object that holds the gathered data
      */
     LocalContainer global_gather( const LocalContainer& values) const
     {
@@ -88,8 +88,8 @@ struct aCommunicator
     }
 
     /**
-     * @brief Scatter data accross processes and reduce on multiple indices
-     * @param toScatter buffer vector (has to be of size given by size())
+     * @brief Globally (across processes) scatter data accross processes and reduce on multiple indices
+     * @param toScatter buffer vector; (has to be of size given by size())
      * @param values contains values from other processes sent back to the origin 
      */
     void global_scatter_reduce( const LocalContainer& toScatter, LocalContainer& values) const{
diff --git a/inc/dg/backend/mpi_grid.h b/inc/dg/backend/mpi_grid.h
index aa6aa334e..1de4dda89 100644
--- a/inc/dg/backend/mpi_grid.h
+++ b/inc/dg/backend/mpi_grid.h
@@ -571,10 +571,8 @@ struct aMPITopology3d
      * @return pid of a process, or -1 if non of the grids matches
      */
     int pidOf( double x, double y, double z) const;
-    /**
-    * @copydoc aMPITopology2d::local2globalIdx(int,int,int&)
-    */
-    bool local2globalIdx( int localIdx, int PID, int& globalIdx)
+    ///@copydoc aMPITopology2d::local2globalIdx(int,int,int&)const
+    bool local2globalIdx( int localIdx, int PID, int& globalIdx)const
     {
         if( localIdx < 0 || localIdx >= (int)size()) return false;
         int coords[3];
@@ -589,10 +587,8 @@ struct aMPITopology3d
         globalIdx = (gIdx2*g.n()*g.Ny() + gIdx1)*g.n()*g.Nx() + gIdx0;
         return true;
     }
-    /**
-    * @copydoc aMPITopology2d::global2localIdx(int,int&,int&)
-    */
-    bool global2localIdx( int globalIdx, int& localIdx, int& PID)
+    ///@copydoc aMPITopology2d::global2localIdx(int,int&,int&)const
+    bool global2localIdx( int globalIdx, int& localIdx, int& PID)const
     {
         if( globalIdx < 0 || globalIdx >= (int)g.size()) return false;
         int coords[3];
-- 
GitLab


From 125684beffff3e1a8df25a84d93ca4ddd7f0c976 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Tue, 19 Sep 2017 22:24:15 +0200
Subject: [PATCH 308/453] checked size()==0 case in communicator

---
 inc/dg/backend/mpi_collective.h   |  18 ++--
 inc/dg/backend/mpi_communicator.h |   9 +-
 inc/dg/backend/mpi_matrix.h       |   8 +-
 inc/dg/backend/mpi_vector.h       |   2 -
 inc/geometries/fieldaligned.h     |  21 ++---
 inc/geometries/fluxfunctions.h    |   2 +-
 inc/geometries/mpi_fieldaligned.h | 145 +++---------------------------
 7 files changed, 43 insertions(+), 162 deletions(-)

diff --git a/inc/dg/backend/mpi_collective.h b/inc/dg/backend/mpi_collective.h
index 4c02bb1fd..5626e5583 100644
--- a/inc/dg/backend/mpi_collective.h
+++ b/inc/dg/backend/mpi_collective.h
@@ -66,7 +66,7 @@ struct Collective
      *
      * @return Total number of processes
      */
-    unsigned size() const {return sendTo_.size();}
+    unsigned size() const {return values_size();}
     MPI_Comm comm() const {return comm_;}
 
     void transpose(){ sendTo_.swap( recvFrom_);}
@@ -74,8 +74,12 @@ struct Collective
 
     void scatter( const Vector& values, Vector& store) const;
     void gather( const Vector& store, Vector& values) const;
-    unsigned store_size() const{ return thrust::reduce( recvFrom_.begin(), recvFrom_.end() );}
-    unsigned values_size() const{ return thrust::reduce( sendTo_.begin(), sendTo_.end() );}
+    unsigned store_size() const{ 
+        if( recvFrom_.empty()) return 0;
+        return thrust::reduce( recvFrom_.begin(), recvFrom_.end() );}
+    unsigned values_size() const{ 
+        if( sendTo_.empty()) return 0;
+        return thrust::reduce( sendTo_.begin(), sendTo_.end() );}
     MPI_Comm communicator() const{return comm_;}
     private:
     unsigned sendTo( unsigned pid) const {return sendTo_[pid];}
@@ -161,6 +165,7 @@ struct BijectiveComm : public aCommunicator<Vector>
         construct( pids, comm);
     }
 
+    ///@copydoc GeneralComm::GeneralComm(const GeneralComm<OtherIndex,OtherVector>&)
     template<class OtherIndex, class OtherVector>
     BijectiveComm( const BijectiveComm<OtherIndex, OtherVector>& src) {
         construct( src.get_pids(), src.communicator());
@@ -272,6 +277,7 @@ struct SurjectiveComm : public aCommunicator<Vector>
         construct( local, pids, g.communicator());
     }
 
+    ///@copydoc GeneralComm::GeneralComm(const GeneralComm<OtherIndex,OtherVector>&)
     template<class OtherIndex, class OtherVector>
     SurjectiveComm( const SurjectiveComm<OtherIndex, OtherVector>& src)
     {
@@ -350,7 +356,7 @@ struct SurjectiveComm : public aCommunicator<Vector>
 template< class Index, class Vector>
 struct GeneralComm : public aCommunicator<Vector>
 {
-    ///@brief no memory allocation
+    /// no memory allocation; the size shall be 0
     GeneralComm(){}
     /**
     * @brief Construct from local indices and PIDs gather map
@@ -365,9 +371,11 @@ struct GeneralComm : public aCommunicator<Vector>
     GeneralComm( const thrust::host_vector<int>& localGatherMap, const thrust::host_vector<int>& pidGatherMap, MPI_Comm comm) {
         construct( localGatherMap, pidGatherMap, comm);
     }
+    ///@brief reconstruct from another type if non-empty; else same as default constructor
     template<class OtherIndex, class OtherVector>
     GeneralComm( const GeneralComm<OtherIndex, OtherVector>& src){
-        construct( src.getLocalGatherMap(), src.getPidGatherMap(), src.communicator());
+        if( src.size() > 0)
+            construct( src.getLocalGatherMap(), src.getPidGatherMap(), src.communicator());
     }
 
     /**
diff --git a/inc/dg/backend/mpi_communicator.h b/inc/dg/backend/mpi_communicator.h
index d30e4ac10..3b9b63860 100644
--- a/inc/dg/backend/mpi_communicator.h
+++ b/inc/dg/backend/mpi_communicator.h
@@ -60,8 +60,10 @@ struct aCommunicator
     /**
      * @brief Allocate a buffer object of size size()
      * @return a buffer object on the stack
+     * @note if size()==0 the default constructor of LocalContainer is called
      */
     LocalContainer allocate_buffer( )const{
+        if( do_size() == 0 ) return LocalContainer();
         return do_make_buffer();
     }
 
@@ -69,9 +71,11 @@ struct aCommunicator
      * @brief Globally (across processes) gather data into a buffer 
      * @param values data; other processes collect data from this vector
      * @param buffer object to hold the gathered data ( must be of size size())
+     * @note if size()==0 nothing happens
      */
     void global_gather( const LocalContainer& values, LocalContainer& buffer)const
     {
+        if( do_size() == 0 ) return;
         do_global_gather( values, gather);
     }
 
@@ -79,6 +83,7 @@ struct aCommunicator
      * @brief Globally (across processes) gather data into a buffer (memory allocating version)
      * @param values data; other processes collect data from this vector
      * @return object that holds the gathered data
+     * @note if size()==0 the default constructor of LocalContainer is called
      */
     LocalContainer global_gather( const LocalContainer& values) const
     {
@@ -91,8 +96,10 @@ struct aCommunicator
      * @brief Globally (across processes) scatter data accross processes and reduce on multiple indices
      * @param toScatter buffer vector; (has to be of size given by size())
      * @param values contains values from other processes sent back to the origin 
+     * @note if size()==0 nothing happens
      */
     void global_scatter_reduce( const LocalContainer& toScatter, LocalContainer& values) const{
+        if( do_size() == 0 ) return;
         do_global_scatter_reduce(toScatter, values);
     }
 
@@ -104,7 +111,7 @@ struct aCommunicator
     * of v is the same for all processes. However the buffer size might be different for each process. 
     * @return buffer size
     * @note may return 0 to indicate that no MPI communication is needed 
-    * @note we assume that the vector size is always the local size of a dg::MPI_Vector
+    * @note we assume that, contrary to size(), the vector size is always the local size of a dg::MPI_Vector
     */
     unsigned size() const{return do_size();}
     /**
diff --git a/inc/dg/backend/mpi_matrix.h b/inc/dg/backend/mpi_matrix.h
index e6ab59264..bc0684f0b 100644
--- a/inc/dg/backend/mpi_matrix.h
+++ b/inc/dg/backend/mpi_matrix.h
@@ -29,9 +29,9 @@ namespace dg
  \f$ M_o\f$ is the outer matrix containing all elements which require 
  communication from the Collective object.
 * @tparam LocalMatrixInner The class of the matrix for local computations of the inner points. 
- doSymv(m,x,y) needs to be callable on the container class of the MPI_Vector
+ symv(m,x,y) needs to be callable on the container class of the MPI_Vector
 * @tparam LocalMatrixOuter The class of the matrix for local computations of the outer points. 
- doSymv(1,m,x,1,y) needs to be callable on the container class of the MPI_Vector
+ symv(1,m,x,1,y) needs to be callable on the container class of the MPI_Vector
 * @tparam Collective models aCommunicator The Communication class needs to gather values across processes. Only the global_gather and size
 member functions are needed. 
 Gather points from other processes that are necessary for the outer computations.
@@ -187,11 +187,9 @@ enum dist_type
 * @tparam LocalMatrix The class of the matrix for local computations. 
  symv needs to be callable on the container class of the MPI_Vector
 * @tparam Collective models aCommunicator The Communication class needs to scatter and gather values across processes. 
-container global_gather( const container& input);
 Gather all points (including the ones that the process already has) necessary for the local matrix-vector
 product into one vector, such that the local matrix can be applied.
-int size(); 
-should give the size of the vector that global_gather returns. If size()==0 the global_gather() function won't be called and
+If size()==0 the global_gather and global_scatter_reduce functions won't be called and
 only the local matrix is applied.
 */
 template<class LocalMatrix, class Collective >
diff --git a/inc/dg/backend/mpi_vector.h b/inc/dg/backend/mpi_vector.h
index c82f453e6..2df718ab6 100644
--- a/inc/dg/backend/mpi_vector.h
+++ b/inc/dg/backend/mpi_vector.h
@@ -276,7 +276,6 @@ unsigned NearestNeighborComm<I,V>::buffer_size() const
 template<class I, class V>
 void NearestNeighborComm<I,V>::do_global_gather( const V& input, V& values) const
 {
-    if( silent_) return;
     //gather values from input into sendbuffer
     thrust::gather( gather_map1.begin(), gather_map1.end(), input.begin(), buffer1.data().begin());
     thrust::gather( gather_map2.begin(), gather_map2.end(), input.begin(), buffer2.data().begin());
@@ -289,7 +288,6 @@ void NearestNeighborComm<I,V>::do_global_gather( const V& input, V& values) cons
 template<class I, class V>
 void NearestNeighborComm<I,V>::do_global_scatter_reduce( const V& values, V& input) const
 {
-    if( silent_) return;
     //scatter received values into values array
     thrust::gather( scatter_map1.begin(), scatter_map1.end(), values.begin(), rb1.data().begin());
     thrust::gather( scatter_map2.begin(), scatter_map2.end(), values.begin(), rb2.data().begin());
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index 556355cc9..5d9640fa8 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -313,6 +313,7 @@ struct FieldAligned
 {
 
     typedef IMatrix InterpolationMatrix;
+    ///@brief do not allocate memory
     FieldAligned(){}
 
     /**
@@ -388,6 +389,7 @@ struct FieldAligned
      */
     template< class BinaryOp>
     container evaluate( BinaryOp f, unsigned plane=0) const;
+
     /**
      * @brief Evaluate a 2d functor and transform to all planes along the fieldlines
      *
@@ -410,27 +412,20 @@ struct FieldAligned
 
     /**
     * @brief Applies the interpolation 
-    *
     * @param which specify what interpolation should be applied
     * @param in input 
     * @param out output may not equal input
     */
     void operator()(enum whichMatrix which, const container& in, container& out);
 
-    /**
-    * @brief hz is the distance between the plus and minus planes
-    * @return three-dimensional vector
-    */
+    ///@brief hz is the distance between the plus and minus planes
+    ///@return three-dimensional vector
     const container& hz()const {return hz_;}
-    /**
-    * @brief hp is the distance between the plus and current planes
-    * @return three-dimensional vector
-    */
+    ///@brief hp is the distance between the plus and current planes
+    ///@return three-dimensional vector
     const container& hp()const {return hp_;}
-    /**
-    * @brief hm is the distance between the current and minus planes
-    * @return three-dimensional vector
-    */
+    ///@brief hm is the distance between the current and minus planes
+    ///@return three-dimensional vector
     const container& hm()const {return hm_;}
     private:
     void ePlus( enum whichMatrix which, const container& in, container& out);
diff --git a/inc/geometries/fluxfunctions.h b/inc/geometries/fluxfunctions.h
index 837e5d877..9b9d8dd3b 100644
--- a/inc/geometries/fluxfunctions.h
+++ b/inc/geometries/fluxfunctions.h
@@ -8,7 +8,7 @@ namespace geo
 ///@addtogroup fluxfunctions
 ///@{
 //
-//In the next round of updates we should consider using boost::function 
+//In the next round of updates (C++11) we should consider using std::function maybe?
 
 /**
 * @brief This functor represents functions written in cylindrical coordinates
diff --git a/inc/geometries/mpi_fieldaligned.h b/inc/geometries/mpi_fieldaligned.h
index b5a49f537..39697283c 100644
--- a/inc/geometries/mpi_fieldaligned.h
+++ b/inc/geometries/mpi_fieldaligned.h
@@ -51,47 +51,14 @@ void sendBackward( InputIterator begin, InputIterator end, OutputIterator result
                     comm, &status);
 }
 }//namespace detail
-///@endcond
-
-
 
-/**
- * @brief Class for the evaluation of a parallel derivative (MPI Version)
- *
- * @ingroup fieldaligned
- * @tparam LocalIMatrix The matrix class of the interpolation matrix
- * @tparam Communicator The communicator used to exchange data in the RZ planes
- * @tparam LocalContainer The container-class to on which the interpolation matrix operates on (does not need to be dg::HVec)
- */
 template <class Geometry, class LocalIMatrix, class CommunicatorXY, class LocalContainer>
-struct FieldAligned< Geometry, RowDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vector<LocalContainer> > 
+struct FieldAligned< Geometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vector<LocalContainer> > 
 {
-    /**
-    * @brief Construct from a field and a grid
-    *
-    * @tparam Field The Fieldlines to be integrated: Has to provide void operator()( const std::vector<dg::HVec>&, std::vector<dg::HVec>&) where the first index is R, the second Z and the last s (the length of the field line)
-    * @tparam Limiter Class that can be evaluated on a 2d grid, returns 1 if there
-    is a limiter and 0 if there isn't. If a field line crosses the limiter in the plane \f$ \phi=0\f$ then the limiter boundary conditions apply. 
-    * @param field The field to integrate
-    * @param grid The grid on which to operate
-    * @param eps Desired accuracy of runge kutta
-    * @param limit Instance of the limiter class (Default is a limiter everywhere, note that if bcz is periodic it doesn't matter if there is a limiter or not)
-    * @param globalbcz Choose NEU or DIR. Defines BC in parallel on box
-    * @param deltaPhi Is either <0 (then it's ignored), may differ from hz() only if Nz() == 1
-    * @note If there is a limiter, the boundary condition is set by the bcz variable from the grid and can be changed by the set_boundaries function. If there is no limiter the boundary condition is periodic.
-    */
-    template <class Field, class Limiter>
-    FieldAligned(Field field, Geometry grid, double eps = 1e-4, Limiter limit = DefaultLimiter(), dg::bc globalbcz = dg::DIR, double deltaPhi = -1 );
+    FieldAligned(){}
+    template <class Limiter>
+    FieldAligned(const dg::geo::BinaryVectorLvl0& vec, const Geometry& grid, unsigned multiplyX, unsigned multiplyY, double eps = 1e-4, Limiter limit = FullLimiter(), dg::bc globalbcx = dg::DIR, dg::bc globalbcy = dg::DIR, double deltaPhi = -1);
 
-    /**
-     * @brief Set boundary conditions
-     *
-     * if Dirichlet boundaries are used the left value is the left function
-     value, if Neumann boundaries are used the left value is the left derivative value
-     * @param bcz boundary condition
-     * @param left left boundary value 
-     * @param right right boundary value
-     */
     void set_boundaries( dg::bc bcz, double left, double right)
     {
         bcz_ = bcz; 
@@ -100,15 +67,6 @@ struct FieldAligned< Geometry, RowDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vec
         right_ = dg::evaluate( dg::CONSTANT(right),g2d);
     }
 
-    /**
-     * @brief Set boundary conditions
-     *
-     * if Dirichlet boundaries are used the left value is the left function
-     value, if Neumann boundaries are used the left value is the left derivative value
-     * @param bcz boundary condition
-     * @param left left boundary value 
-     * @param right right boundary value
-     */
     void set_boundaries( dg::bc bcz, const MPI_Vector<LocalContainer>& left, const MPI_Vector<LocalContainer>& right)
     {
         bcz_ = bcz; 
@@ -116,16 +74,6 @@ struct FieldAligned< Geometry, RowDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vec
         right_ = right.data();
     }
 
-    /**
-     * @brief Set boundary conditions in the limiter region
-     *
-     * if Dirichlet boundaries are used the left value is the left function
-     value, if Neumann boundaries are used the left value is the left derivative value
-     * @param bcz boundary condition
-     * @param global 3D vector containing boundary values
-     * @param scal_left left scaling factor
-     * @param scal_right right scaling factor
-     */
     void set_boundaries( dg::bc bcz, const MPI_Vector<LocalContainer>& global, double scal_left, double scal_right)
     {
         bcz_ = bcz;
@@ -146,97 +94,24 @@ struct FieldAligned< Geometry, RowDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vec
         }
     }
 
-    /**
-     * @brief Evaluate a 2d functor and transform to all planes along the fieldlines
-     *
-     * Evaluates the given functor on a 2d plane and then follows fieldlines to 
-     * get the values in the 3rd dimension. Uses the grid given in the constructor.
-     * @tparam BinaryOp Binary Functor 
-     * @param f Functor to evaluate
-     * @param plane The number of the plane to start
-     *
-     * @return Returns an instance of container
-     */
     template< class BinaryOp>
     MPI_Vector<LocalContainer> evaluate( BinaryOp f, unsigned plane=0) const;
 
-    /**
-     * @brief Evaluate a 2d functor and transform to all planes along the fieldlines
-     *
-     * Evaluates the given functor on a 2d plane and then follows fieldlines to 
-     * get the values in the 3rd dimension. Uses the grid given in the constructor.
-     * The second functor is used to scale the values along the fieldlines.
-     * The fieldlines are assumed to be periodic.
-     * @tparam BinaryOp Binary Functor 
-     * @tparam UnaryOp Unary Functor 
-     * @param f Functor to evaluate in x-y
-     * @param g Functor to evaluate in z
-     * @param p0 The number of the plane to start
-     * @param rounds The number of rounds to follow a fieldline
-     *
-     * @return Returns an instance of container
-     */
     template< class BinaryOp, class UnaryOp>
     MPI_Vector<LocalContainer> evaluate( BinaryOp f, UnaryOp g, unsigned p0, unsigned rounds) const;
 
-    /**
-    * @brief Applies the interpolation to the next planes 
-    *
-    * @param in input 
-    * @param out output may not equal intpu
-    */
-    void einsPlus( const MPI_Vector<LocalContainer>& in, MPI_Vector<LocalContainer>& out);
-    /**
-    * @brief Applies the interpolation to the previous planes
-    *
-    * @param in input 
-    * @param out output may not equal intpu
-    */
-    void einsMinus( const MPI_Vector<LocalContainer>& in, MPI_Vector<LocalContainer>& out);
-    /**
-    * @brief Applies the transposed interpolation to the previous plane 
-    *
-    * @param in input 
-    * @param out output may not equal intpu
-    */
-    void einsPlusT( const MPI_Vector<LocalContainer>& in, MPI_Vector<LocalContainer>& out);
-    /**
-    * @brief Applies the transposed interpolation to the next plane 
-    *
-    * @param in input 
-    * @param out output may not equal intpu
-    */
-    void einsMinusT( const MPI_Vector<LocalContainer>& in, MPI_Vector<LocalContainer>& out);
-    /**
-    * @brief hz is the distance between the plus and minus planes
-    *
-    * @return three-dimensional vector
-    */
+    void operator()(enum whichMatrix which, const MPI_Vector<LocalContainer>& in, MPI_Vector<LocalContainer>& out);
+
     const MPI_Vector<LocalContainer>& hz()const {return hz_;}
-    /**
-    * @brief hp is the distance between the plus and current planes
-    *
-    * @return three-dimensional vector
-    */
     const MPI_Vector<LocalContainer>& hp()const {return hp_;}
-    /**
-    * @brief hm is the distance between the current and minus planes
-    *
-    * @return three-dimensional vector
-    */
     const MPI_Vector<LocalContainer>& hm()const {return hm_;}
-    /**
-    * @brief Access the underlying grid
-    *
-    * @return the grid
-    */
     const Geometry& grid() const{return g_;}
   private:
     typedef cusp::array1d_view< typename LocalContainer::iterator> View;
     typedef cusp::array1d_view< typename LocalContainer::const_iterator> cView;
     MPI_Vector<LocalContainer> hz_, hp_, hm_; 
     LocalContainer ghostM, ghostP;
-    Geometry g_;
+    dg::Handle<Geometry> g_;
     dg::bc bcz_;
     LocalContainer left_, right_;
     LocalContainer limiter_;
@@ -245,11 +120,11 @@ struct FieldAligned< Geometry, RowDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vec
     LocalIMatrix plus, minus; //interpolation matrices
     LocalIMatrix plusT, minusT; //interpolation matrices
 };
-///@cond
 //////////////////////////////////////DEFINITIONS/////////////////////////////////////
 template<class MPIGeometry, class LocalIMatrix, class CommunicatorXY, class LocalContainer>
-template <class Field, class Limiter>
-FieldAligned<MPIGeometry, RowDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vector<LocalContainer> >::FieldAligned(Field field, MPIGeometry grid, double eps, Limiter limit, dg::bc globalbcz, double deltaPhi ): 
+template <class Limiter>
+FieldAligned<MPIGeometry, RowDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vector<LocalContainer> >::FieldAligned(
+    const dg::geo::BinaryVectorLvl0& vec, const MPIGeometry& grid, unsigned mx, unsigned my, double eps, Limiter limit, dg::bc globalbcx, dg::bc globalbcy, double deltaPhi):
     hz_( dg::evaluate( dg::zero, grid)), hp_( hz_), hm_( hz_), 
     g_(grid), bcz_(grid.bcz()), 
     tempXYplus_(g_.Nz()), tempXYminus_(g_.Nz()), temp_(g_.Nz())
-- 
GitLab


From f6d92464f46fced80739c39b2024714917b8b4be Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Tue, 19 Sep 2017 22:42:47 +0200
Subject: [PATCH 309/453] update fieldaligned.h a little

---
 inc/geometries/fieldaligned.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index 5d9640fa8..bde803a43 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -455,10 +455,10 @@ FieldAligned<Geometry, IMatrix, container>::FieldAligned(const dg::geo::BinaryVe
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     //%%%%%%%%%%%downcast grid since we don't have a virtual function perp_grid%%%%%%%%%%%%%
     const aGeometry3d* grid_ptr = &grid;
-    aGeometry2d* g2dCoarse_ptr;
     const dg::CartesianGrid3d* grid_cart = dynamic_cast<const dg::CartesianGrid3d*>(grid_ptr);
     const dg::CylindricalGrid3d* grid_cyl = dynamic_cast<const dg::CylindricalGrid3d*>(grid_ptr);
     const dg::CurvilinearProductGrid3d*  grid_curvi = dynamic_cast<const dg::CurvilinearProductGrid3d*>(grid_ptr);
+    aGeometry2d* g2dCoarse_ptr;
     if( grid_cart) 
     {
         dg::CartesianGrid2d cart = grid_cart->perp_grid();
@@ -645,13 +645,13 @@ void FieldAligned<G, I, container>::ePlus( enum whichMatrix which, const contain
         unsigned ip = (i0==Nz_-1) ? 0:i0+1;
 
         cView fp( f.cbegin() + ip*perp_size_, f.cbegin() + (ip+1)*perp_size_);
-        cView f0( f.cbegin() + i0*perp_size_, f.cbegin() + (i0+1)*perp_size_);
         View fP( fpe.begin() + i0*perp_size_, fpe.begin() + (i0+1)*perp_size_);
         if(which == einsPlus) cusp::multiply( plus, fp, fP);
         else if(which == einsMinusT) cusp::multiply( minusT, fp, fP );
         //make ghostcells i.e. modify fpe in the limiter region
         if( i0==Nz_-1 && bcz_ != dg::PER)
         {
+            cView f0( f.cbegin() + i0*perp_size_, f.cbegin() + (i0+1)*perp_size_);
             if( bcz_ == dg::DIR || bcz_ == dg::NEU_DIR)
             {
                 cusp::blas::axpby( rightV, f0, ghostPV, 2., -1.);
@@ -680,13 +680,13 @@ void FieldAligned<G, I, container>::eMinus( enum whichMatrix which, const contai
     {
         unsigned im = (i0==0) ? Nz_-1:i0-1;
         cView fm( f.cbegin() + im*perp_size_, f.cbegin() + (im+1)*perp_size_);
-        cView f0( f.cbegin() + i0*perp_size_, f.cbegin() + (i0+1)*perp_size_);
         View fM( fme.begin() + i0*perp_size_, fme.begin() + (i0+1)*perp_size_);
         if(which == einsPlusT) cusp::multiply( plusT, fm, fM );
         else if (which == einsMinus) cusp::multiply( minus, fm, fM );
         //make ghostcells
         if( i0==0 && bcz_ != dg::PER)
         {
+            cView f0( f.cbegin() + i0*perp_size_, f.cbegin() + (i0+1)*perp_size_);
             if( bcz_ == dg::DIR || bcz_ == dg::DIR_NEU)
             {
                 cusp::blas::axpby( leftV,  f0, ghostMV, 2., -1.);
-- 
GitLab


From 0e2555af7bbf8aaaac780708ab4a7686cb18c0d1 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 21 Sep 2017 22:56:41 +0200
Subject: [PATCH 310/453] use new blas1 function in elliptic, use symv in
 context with weights, make multistep assume const container& in Functor,
 update docu in multistep

---
 inc/dg/cg.h                  |  13 ++-
 inc/dg/elliptic.h            |  78 ++++++++--------
 inc/dg/elliptic2d_b.cu       |   4 +-
 inc/dg/helmholtz.h           |  28 +++---
 inc/dg/multigrid.h           |   4 +-
 inc/dg/multistep.h           | 170 +++++++++++++++--------------------
 inc/dg/multistep_t.cu        |  12 +--
 src/toefl/input/default.json |   2 +-
 src/toefl/toeflR.cuh         |  15 ++--
 9 files changed, 143 insertions(+), 183 deletions(-)

diff --git a/inc/dg/cg.h b/inc/dg/cg.h
index 1524b22ba..1d6260716 100644
--- a/inc/dg/cg.h
+++ b/inc/dg/cg.h
@@ -33,9 +33,7 @@ class CG
 {
   public:
     typedef typename VectorTraits<container>::value_type value_type;//!< value type of the container class
-    /**
-     * @brief Allocate nothing, 
-     */
+    ///@brief Allocate nothing, 
     CG(){}
       /**
        * @brief Reserve memory for the pcg method
@@ -470,7 +468,7 @@ struct Invert
     template< class SymmetricOp >
     unsigned operator()( SymmetricOp& op, container& phi, const container& rho)
     {
-        return this->operator()(op, phi, rho, op.inv_weights(), op.precond());
+        return this->operator()(op, phi, rho, op.weights(), op.inv_weights(), op.precond());
     }
 
     /**
@@ -484,14 +482,15 @@ struct Invert
      * @param op symmetric Matrix operator class
      * @param phi solution (write only)
      * @param rho right-hand-side
-     * @param inv_weights The inverse weights that normalize the symmetric operator
+     * @param weights The weights that normalize the symmetric operator
+     * @param inv_weights The inverse of the weights that normalize the symmetric operator
      * @param p The preconditioner  
      * @note If the Macro DG_BENCHMARK is defined this function will write timings to std::cout
      *
      * @return number of iterations used 
      */
     template< class Matrix, class Preconditioner >
-    unsigned operator()( Matrix& op, container& phi, const container& rho, const container& inv_weights, Preconditioner& p)
+    unsigned operator()( Matrix& op, container& phi, const container& rho, const container& weights, const container& inv_weights, Preconditioner& p)
     {
         assert( phi.size() != 0);
         assert( &rho != &phi);
@@ -504,7 +503,7 @@ struct Invert
         unsigned number;
         if( multiplyWeights_ ) 
         {
-            dg::blas1::pointwiseDivide( rho, inv_weights, m_ex.tail());
+            dg::blas2::symv( rho, weights, m_ex.tail());
             number = cg( op, phi, m_ex.tail(), p, inv_weights, eps_, nrmb_correction_);
         }
         else
diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index e674eec1e..23341af78 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -96,6 +96,7 @@ class Elliptic
         dg::blas2::transfer( dg::create::jumpY( g, bcy),   jumpY);
 
         dg::blas1::transfer( dg::create::inv_volume(g),    inv_weights_);
+        dg::blas1::transfer( dg::create::volume(g),        weights_);
         dg::blas1::transfer( dg::create::inv_weights(g),   precond_); 
         tempx = tempy = gradx = inv_weights_;
         chi_=g.metric();
@@ -131,14 +132,21 @@ class Elliptic
     }
 
     /**
-     * @brief Returns the vector missing in the un-normed symmetric matrix 
+     * @brief Return the vector missing in the un-normed symmetric matrix 
      *
-     * i.e. the inverse volume form 
-     * @return inverse volume form including weights 
+     * i.e. the inverse of the weights() function
+     * @return inverse volume form including inverse weights 
      */
     const container& inv_weights()const {return inv_weights_;}
     /**
-     * @brief Returns the default preconditioner to use in conjugate gradient
+     * @brief Return the vector making the matrix symmetric
+     *
+     * i.e. the volume form 
+     * @return volume form including weights 
+     */
+    const container& weights()const {return weights_;}
+    /**
+     * @brief Return the default preconditioner to use in conjugate gradient
      *
      * Currently returns the inverse of the weights without volume elment multiplied by the inverse of \f$ \chi\f$. 
      * This is especially good when \f$ \chi\f$ exhibits large amplitudes or variations
@@ -203,7 +211,7 @@ class Elliptic
         return centered;
     }
     Matrix leftx, lefty, rightx, righty, jumpX, jumpY;
-    container inv_weights_, precond_, weights_wo_vol; 
+    container weights_, inv_weights_, precond_, weights_wo_vol; 
     container tempx, tempy, gradx;
     norm no_;
     SparseTensor<container> chi_;
@@ -252,7 +260,7 @@ struct GeneralElliptic
         rightz( dg::create::dz( g, g.bcz(), dir)),
         jumpX ( dg::create::jumpX( g, g.bcx())),
         jumpY ( dg::create::jumpY( g, g.bcy())),
-        inv_weights_(dg::create::inv_volume(g)), precond_(dg::create::inv_weights(g)), 
+        weights_(dg::create::volume(g)), inv_weights_(dg::create::inv_volume(g)), precond_(dg::create::inv_weights(g)), 
         xchi( dg::evaluate( one, g) ), ychi( xchi), zchi( xchi), 
         xx(xchi), yy(xx), zz(xx), temp0( xx), temp1(temp0),
         no_(no)
@@ -280,7 +288,7 @@ struct GeneralElliptic
         rightz( dg::create::dz( g, bcz, dir)),
         jumpX ( dg::create::jumpX( g, bcx)),
         jumpY ( dg::create::jumpY( g, bcy)),
-        inv_weights_(dg::create::inv_volume(g)), precond_(dg::create::inv_weights(g)), 
+        weights_(dg::create::volume(g)), inv_weights_(dg::create::inv_volume(g)), precond_(dg::create::inv_weights(g)), 
         xchi( dg::evaluate( one, g) ), ychi( xchi), zchi( xchi), 
         xx(xchi), yy(xx), zz(xx), temp0( xx), temp1(temp0),
         no_(no)
@@ -343,37 +351,30 @@ struct GeneralElliptic
     void symv( const container& x, container& y) 
     {
         dg::blas2::gemv( rightx, x, temp0); //R_x*x 
-        dg::blas1::pointwiseDot( xchi, temp0, xx); //Chi_x*R_x*x 
+        dg::blas1::pointwiseDot( 1., xchi, temp0, 0., xx);//Chi_x*R_x*x
 
         dg::blas2::gemv( righty, x, temp0);//R_y*x
-        dg::blas1::pointwiseDot( ychi, temp0, yy);//Chi_y*R_y*x
+        dg::blas1::pointwiseDot( 1., ychi, temp0, 1., xx);//Chi_y*R_y*x
 
         dg::blas2::gemv( rightz, x, temp0); // R_z*x
-        dg::blas1::pointwiseDot( zchi, temp0, zz); //Chi_z*R_z*x
+        dg::blas1::pointwiseDot( 1., zchi, temp0, 1., xx);//Chi_z*R_z*x
 
-        dg::blas1::axpby( 1., xx, 1., yy, temp0);
-        dg::blas1::axpby( 1., zz, 1., temp0, temp0); //gradpar x 
-
-        dg::tensor::pointwiseDot( vol_, temp0, temp0);
+        dg::tensor::pointwiseDot( vol_, xx, temp0);
 
         dg::blas1::pointwiseDot( xchi, temp0, temp1); 
-        dg::blas2::gemv( leftx, temp1, xx); 
+        dg::blas2::gemv( -1., leftx, temp1, 0., y); 
 
         dg::blas1::pointwiseDot( ychi, temp0, temp1);
-        dg::blas2::gemv( lefty, temp1, yy);
+        dg::blas2::gemv( -1., lefty, temp1, 1., y);
 
         dg::blas1::pointwiseDot( zchi, temp0, temp1); 
-        dg::blas2::gemv( leftz, temp1, zz); 
+        dg::blas2::gemv( -1., leftz, temp1, 1., y); 
 
-        dg::blas1::axpby( -1., xx, -1., yy, y);
-        dg::blas1::axpby( -1., zz, +1., y, y); 
         if( no_==normed) 
             dg::tensor::pointwiseDivide( temp0, vol_, temp0);
         
-        dg::blas2::symv( jumpX, x, temp0);
-        dg::blas1::axpby( +1., temp0, 1., y, y); 
-        dg::blas2::symv( jumpY, x, temp0);
-        dg::blas1::axpby( +1., temp0, 1., y, y); 
+        dg::blas2::symv( +1., jumpX, x, 1., y);
+        dg::blas2::symv( +1., jumpY, x, 1., y);
         if( no_==not_normed)//multiply weights w/o volume
         {
             dg::tensor::pointwiseDivide( y, vol_, y);
@@ -396,7 +397,7 @@ struct GeneralElliptic
         return centered;
     }
     Matrix leftx, lefty, leftz, rightx, righty, rightz, jumpX, jumpY;
-    container inv_weights_, precond_; //contain coeffs for chi multiplication
+    container weights_, inv_weights_, precond_; //contain coeffs for chi multiplication
     container xchi, ychi, zchi, xx, yy, zz, temp0, temp1;
     norm no_;
     SparseElement<container> vol_;
@@ -436,7 +437,6 @@ struct GeneralEllipticSym
      */
     GeneralEllipticSym( const Geometry& g, norm no = not_normed, direction dir = forward): 
         ellipticForward_( g, no, dir), ellipticBackward_(g,no,inverse(dir)),
-        inv_weights_(dg::create::inv_volume(g)), precond_(dg::create::inv_weights(g)), 
         temp_( dg::evaluate( one, g) )
     { }
 
@@ -452,7 +452,6 @@ struct GeneralEllipticSym
      */
     GeneralEllipticSym( const Geometry& g, bc bcx, bc bcy,bc bcz, norm no = not_normed, direction dir = forward): 
         ellipticForward_( g, bcx, bcy, no, dir), ellipticBackward_(g,bcx,bcy,no,inverse(dir)),
-        inv_weights_(dg::create::inv_volume(g)), precond_(dg::create::inv_weights(g)), 
         temp_( dg::evaluate( one, g) )
     { 
     }
@@ -498,10 +497,12 @@ struct GeneralEllipticSym
         ellipticBackward_.set( chi);
     }
 
+    ///@copydoc Elliptic::weights()
+    const container& weights()const {return ellipticForward_.weights();}
     ///@copydoc Elliptic::inv_weights()
-    const container& inv_weights()const {return inv_weights_;}
+    const container& inv_weights()const {return ellipticForward_.inv_weights();}
     ///@copydoc GeneralElliptic::precond()
-    const container& precond()const {return precond_;}
+    const container& precond()const {return ellipticForward_.precond();}
 
     ///@copydoc Elliptic::symv()
     void symv( const container& x, container& y) 
@@ -518,7 +519,6 @@ struct GeneralEllipticSym
         return centered;
     }
     dg::GeneralElliptic<Geometry, Matrix, container> ellipticForward_, ellipticBackward_;
-    container inv_weights_, precond_; //contain coeffs for chi multiplication
     container temp_;
 };
 
@@ -614,23 +614,18 @@ struct TensorElliptic
         dg::blas2::gemv( righty, x, tempy_); //R_y*f
 
         //multiply with chi 
-        dg::blas1::pointwiseDot( chixx_, tempx_, gradx_); //gxx*v_x
-        dg::blas1::pointwiseDot( chixy_, tempx_, y); //gyx*v_x
-        dg::blas1::pointwiseDot( 1., chixy_, tempy_, 1., gradx_);//gxy*v_y
-        dg::blas1::pointwiseDot( 1., chiyy_, tempy_, 1., y); //gyy*v_y
+        dg::blas1::pointwiseDot( 1., chixx_, tempx_, 1., chixy_, tempy_, 0., gradx_);//gxy*v_y
+        dg::blas1::pointwiseDot( 1., chixy_, tempx_, 1., chiyy_, tempy_, 1., tempy_); //gyy*v_y
 
         //now take divergence
-        dg::blas2::gemv( leftx, gradx_, tempx_);  
-        dg::blas2::gemv( lefty, y, tempy_);  
-        dg::blas1::axpby( -1., tempx_, -1., tempy_, y); //-D_xx - D_yy 
+        dg::blas2::gemv( -1., leftx, gradx_, 0., y);  
+        dg::blas2::gemv( -1., lefty, tempy_, 1., y);  
         if( no_ == normed)
             dg::tensor::pointwiseDivide( y, vol_,y);
 
         //add jump terms
-        dg::blas2::symv( jumpX, x, tempx_);
-        dg::blas1::axpby( +1., tempx_, 1., y, y); 
-        dg::blas2::symv( jumpY, x, tempy_);
-        dg::blas1::axpby( +1., tempy_, 1., y, y); 
+        dg::blas2::symv( +1., jumpX, x, 1., y);
+        dg::blas2::symv( +1., jumpY, x, 1., y);
         if( no_ == not_normed)//multiply weights without volume
             dg::blas2::symv( weights_wo_vol, y, y);
     }
@@ -643,6 +638,7 @@ struct TensorElliptic
         dg::blas2::transfer( dg::create::dy( g, bcy, dir), righty);
         dg::blas2::transfer( dg::create::jumpX( g, bcx),   jumpX);
         dg::blas2::transfer( dg::create::jumpY( g, bcy),   jumpY);
+        dg::blas1::transfer( dg::create::volume(g),        weights_);
         dg::blas1::transfer( dg::create::inv_volume(g),    inv_weights_);
         dg::blas1::transfer( dg::create::inv_weights(g),   precond_); //weights are better preconditioners than volume
         dg::blas1::transfer( dg::evaluate( dg::one, g),    chixx_);
@@ -671,7 +667,7 @@ struct TensorElliptic
         return centered;
     }
     Matrix leftx, lefty, rightx, righty, jumpX, jumpY;
-    container inv_weights_, weights_wo_vol, precond_; //contain coeffs for chi multiplication
+    container weights_, inv_weights_, weights_wo_vol, precond_; //contain coeffs for chi multiplication
     container chixx_, chixy_, chiyy_, tempx_, tempy_, gradx_;
     SparseElement<container> vol_;
     norm no_;
diff --git a/inc/dg/elliptic2d_b.cu b/inc/dg/elliptic2d_b.cu
index e0ebe99f1..3510e9eb1 100644
--- a/inc/dg/elliptic2d_b.cu
+++ b/inc/dg/elliptic2d_b.cu
@@ -118,7 +118,7 @@ int main()
 		pol_forward.set_chi( chi);
 		x = temp;
 		dg::Invert<dg::DVec > invert_fw( x, n*n*Nx*Ny, eps);
-		std::cout << " "<< invert_fw( pol_forward, x, b, v2d, chi_inv);
+		std::cout << " "<< invert_fw( pol_forward, x, b, w2d, v2d, chi_inv);
 		dg::blas1::axpby( 1.,x,-1., solution, error);
 		err = dg::blas2::dot( w2d, error);
 		std::cout << " "<<sqrt( err/norm);
@@ -129,7 +129,7 @@ int main()
 		pol_backward.set_chi( chi);
 		x = temp;
 		dg::Invert<dg::DVec > invert_bw( x, n*n*Nx*Ny, eps);
-		std::cout << " "<< invert_bw( pol_backward, x, b, v2d, chi_inv);
+		std::cout << " "<< invert_bw( pol_backward, x, b, w2d, v2d, chi_inv);
 		dg::blas1::axpby( 1.,x,-1., solution, error);
 		err = dg::blas2::dot( w2d, error);
 		std::cout << " "<<sqrt( err/norm)<<std::endl;
diff --git a/inc/dg/helmholtz.h b/inc/dg/helmholtz.h
index 89a02f601..7fa0243a8 100644
--- a/inc/dg/helmholtz.h
+++ b/inc/dg/helmholtz.h
@@ -86,19 +86,17 @@ struct Helmholtz
             dg::blas1::pointwiseDot( 1., chi_.value(), x, -alpha_, y);
         else
             blas1::axpby( 1., x, -alpha_, y);
-        blas1::pointwiseDivide(y, laplaceM_.inv_weights(), y);
+        blas2::symv(laplaceM_.weights(), y, y);
 
     }
-    /**
-     * @brief These are the weights that made the operator symmetric
-     *
-     * @return weights
-     */
+    ///@copydoc Elliptic::weights()const
+    const container& weights()const {return laplaceM_.weights();}
+    ///@copydoc Elliptic::inv_weights()const
     const container& inv_weights()const {return laplaceM_.inv_weights();}
     /**
-     * @brief container to use in conjugate gradient solvers
+     * @brief Preconditioner to use in conjugate gradient solvers
      *
-     * @return container
+     * @return inverse weights without volume
      */
     const container& precond()const {return laplaceM_.precond();}
     /**
@@ -205,19 +203,17 @@ struct Helmholtz2
         tensor::pointwiseDot( chi_, x, y); //y = chi*x
         blas1::axpby( 1., y, -2.*alpha_, temp1_, y); 
         blas1::axpby( alpha_*alpha_, temp2_, 1., y, y);
-        blas1::pointwiseDivide( y, laplaceM_.inv_weights(), y);//Helmholtz is never normed
+        blas2::symv( laplaceM_.weights(), y, y);//Helmholtz is never normed
     }
-    /**
-     * @brief These are the weights that made the operator symmetric
-     *
-     * @return weights
-     */
+    ///@copydoc Elliptic::weights()const
+    const container& weights()const {return laplaceM_.weights();}
+    ///@copydoc Elliptic::inv_weights()const
     const container& inv_weights()const {return laplaceM_.inv_weights();}
     /**
-     * @brief container to use in conjugate gradient solvers
+     * @brief Preconditioner to use in conjugate gradient solvers
      *
      * multiply result by these coefficients to get the normed result
-     * @return container
+     * @return the inverse weights without volume
      */
     const container& precond()const {return laplaceM_.precond();}
     /**
diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index d28ac804d..cbde669f9 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -73,7 +73,7 @@ struct MultigridCG2d
 	{
         //project initial guess down to coarse grid
         project(x, x_);
-        dg::blas1::pointwiseDivide(b, op[0].inv_weights(), b_[0]);
+        dg::blas2::symv(op[0].weights(), b, b_[0]);
         // project b down to coarse grid
         for( unsigned u=0; u<stages_-1; u++)
             dg::blas2::gemv( interT_[u], b_[u], b_[u+1]);
@@ -142,7 +142,7 @@ struct MultigridCG2d
     template<class SymmetricOp>
     std::vector<unsigned> direct_solve( std::vector<SymmetricOp>& op, container&  x, const container& b, double eps)
     {
-        dg::blas1::pointwiseDivide(b, op[0].inv_weights(), b_[0]);
+        dg::blas2::symv(op[0].weights(), b, b_[0]);
         // compute residual r = Wb - A x
         dg::blas2::symv(op[0], x, m_r[0]);
         dg::blas1::axpby(-1.0, m_r[0], 1.0, b_[0], m_r[0]);
diff --git a/inc/dg/multistep.h b/inc/dg/multistep.h
index a9a40cfce..cb5246c7a 100644
--- a/inc/dg/multistep.h
+++ b/inc/dg/multistep.h
@@ -8,6 +8,25 @@
   */
 namespace dg{
 
+
+/*! @class hide_explicit_implicit
+ * @tparam Explicit 
+    models BinaryFunction with no return type (subroutine): 
+    void operator()(const container&, container&);
+    The first argument is the actual argument, The second contains
+    the return value, i.e. y' = f(y) translates to f( y, y').
+ * @tparam Implicit 
+    models BinaryFunction with no return type (subroutine): 
+    void operator()(const container&, container&);
+    The first argument is the actual argument, The second contains
+    the return value, i.e. y' = I(y) translates to I( y, y').
+    Furthermore the routines %weights(), %inv_weights() and %precond() must be callable
+    and return diagonal weights, inverse weights and the preconditioner for the conjugate gradient. 
+    The return type of these member functions must be useable in blas2 functions together with the container type.
+ * @param exp explic part
+ * @param imp implicit part ( must be linear and symmetric up to weights)
+ */
+
 ///@cond
 template< size_t k>
 struct ab_coeff
@@ -140,25 +159,19 @@ namespace detail{
 template< class LinearOp, class container>
 struct Implicit
 {
-    Implicit( double alpha, LinearOp& f, container& reference): f_(f), alpha_(alpha), temp_(reference){}
+    Implicit( double alpha, LinearOp& f): f_(f), alpha_(alpha){}
     void symv( const container& x, container& y) 
     {
         if( alpha_ != 0)
-        {
-            blas1::copy( x, temp_);//f_ might destroy x
-            f_( temp_,y);
-        }
+            f_( x,y);
         blas1::axpby( 1., x, alpha_, y, y);
-        blas1::pointwiseDivide( y, f_.inv_weights(),  y);
+        blas2::symv( f_.weights(), y, y);
     }
     //compute without weights
     void operator()( const container& x, container& y) 
     {
         if( alpha_ != 0)
-        {
-            blas1::copy( x, temp_);
-            f_( temp_,y);
-        }
+            f_( x,y);
         blas1::axpby( 1., x, alpha_, y, y);
     }
     double& alpha( ){  return alpha_;}
@@ -166,8 +179,6 @@ struct Implicit
   private:
     LinearOp& f_;
     double alpha_;
-    container& temp_;
-
 };
 
 }//namespace detail
@@ -183,12 +194,12 @@ struct MatrixTraits< detail::Implicit<M, V> >
 * @brief Struct for Karniadakis semi-implicit multistep time-integration
 * \f[
 * \begin{align}
-    {\bar v}^n &= \frac{1}{\gamma_0}\left(\sum_{q=0}^2 \alpha_q v^{n-q} + \Delta t\sum_{q=0}^2\beta_q  N( v^{n-q})\right) \\
-    \left( 1  - \frac{\Delta t}{\gamma_0}  \hat L\right)  v^{n+1} &= {\bar v}^n  
+    {\bar v}^n &= \frac{1}{\gamma_0}\left(\sum_{q=0}^2 \alpha_q v^{n-q} + \Delta t\sum_{q=0}^2\beta_q  \hat E( v^{n-q})\right) \\
+    \left( 1  - \frac{\Delta t}{\gamma_0}  \hat I\right)  v^{n+1} &= {\bar v}^n  
     \end{align}
     \f]
 
-    with
+    where \f$ \hat E \f$ constains the explicit and \f$ \hat I \f$ the implicit part of the equations. The coefficients are
     \f[
     \alpha_0 = \frac{18}{11}\ \alpha_1 = -\frac{9}{11}\ \alpha_2 = \frac{2}{11} \\
     \beta_0 = \frac{18}{11}\ \beta_1 = -\frac{18}{11}\ \beta_2 = \frac{6}{11} \\
@@ -225,46 +236,21 @@ struct Karniadakis
     /**
      * @brief Initialize with initial value
      *
-     * @tparam Functor models BinaryFunction with no return type (subroutine)
-        Its arguments both have to be of type container.
-        The first argument is the actual argument, The second contains
-        the return value, i.e. y' = f(y) translates to f( y, y').
-     * @tparam LinearOp models BinaryFunction with no return type (subroutine)
-        Its arguments both have to be of type container. 
-        The first argument is the actual argument, The second contains
-        the return value, i.e. y' = L(y) translates to diff( y, y').
-        Furthermore the routines weights() and precond() must be callable
-        and return diagonal weights and the preconditioner for the conjugate gradient. 
-     * @param f right hand side function or functor
-     * @param diff diffusion operator treated implicitely 
-     * @param u0 The initial value you later use 
+     * @copydoc hide_explicit_implicit
+     * @param u0 The initial value 
      * @param dt The timestep saved for later use
-     * @note Both Functor and LinearOp may change their first (input) argument, i.e. the first argument need not be const
      */
-    template< class Functor, class LinearOp>
-    void init( Functor& f, LinearOp& diff, const container& u0, double dt);
+    template< class Explicit, class Implicit>
+    void init( Explicit& exp, Implicit& imp, const container& u0, double dt);
 
     /**
-    * @brief Advance u for one timestep
+    * @brief Advance one timestep
     *
-    * @tparam Functor models BinaryFunction with no return type (subroutine)
-        Its arguments both have to be of type container.
-        The first argument is the actual argument, The second contains
-        the return value, i.e. y' = f(y) translates to f( y, y').
-    * @tparam LinearOp models BinaryFunction with no return type (subroutine)
-        Its arguments both have to be of type container.
-        The first argument is the actual argument, The second contains
-        the return value, i.e. y' = L(y) translates to diff( y, y').
-        Furthermore the routines weights() and precond() must be callable
-        and return diagonal weights and the preconditioner for the conjugate gradient. 
-        The Operator itself need not be symmetric. Symmetrization is done by the class itself.
-    * @param f right hand side function or functor (is called for u)
-    * @param diff diffusion operator treated implicitely 
+    * @copydoc hide_explicit_implicit
     * @param u (write-only), contains next step of time-integration on output
-     * @note Both Functor and LinearOp may change their first (input) argument, i.e. the first argument need not be const
     */
-    template< class Functor, class LinearOp>
-    void operator()( Functor& f, LinearOp& diff, container& u);
+    template< class Explicit, class Implicit>
+    void operator()( Explicit& exp, Implicit& imp, container& u);
 
 
     /**
@@ -294,20 +280,15 @@ template< class Functor, class Diffusion>
 void Karniadakis<container>::init( Functor& f, Diffusion& diff,  const container& u0,  double dt)
 {
     dt_ = dt;
-    container temp_(u0);
-    //implicit may write into temp
-    detail::Implicit<Diffusion, container> implicit( -dt, diff, temp_);
-    blas1::copy( u0, temp_); //copy u0
-    f( temp_, f_[0]);
     blas1::copy(  u0, u_[0]); 
+    detail::Implicit<Diffusion, container> implicit( -dt, diff);
+    f( u0, f_[0]);
     blas1::axpby( 1., u_[0], -dt, f_[0], f_[1]); //Euler step
-    implicit( f_[1], u_[1]); //explicit Euler step backwards, might destroy f_[1]
-    blas1::copy( u_[1], temp_); 
-    f( temp_, f_[1]);
+    implicit( f_[1], u_[1]); //explicit Euler step backwards
+    f( u_[1], f_[1]);
     blas1::axpby( 1.,u_[1], -dt, f_[1], f_[2]);
     implicit( f_[2], u_[2]);
-    blas1::copy( u_[2], temp_); 
-    f( temp_, f_[2]);
+    f( u_[2], f_[2]);
 }
 
 template<class container>
@@ -315,8 +296,7 @@ template< class Functor, class Diffusion>
 void Karniadakis<container>::operator()( Functor& f, Diffusion& diff, container& u)
 {
 
-    blas1::copy( u_[0], u); //save u_[0]
-    f( u, f_[0]);
+    f( u_[0], f_[0]);
     blas1::axpbypgz( dt_*b[0], f_[0], dt_*b[1], f_[1], dt_*b[2], f_[2]);
     blas1::axpbypgz( a[0], u_[0], a[1], u_[1], a[2], u_[2]);
     //permute f_[2], u_[2]  to be the new f_[0], u_[0]
@@ -330,8 +310,8 @@ void Karniadakis<container>::operator()( Functor& f, Diffusion& diff, container&
     double alpha[2] = {2., -1.};
     //double alpha[2] = {1., 0.};
     blas1::axpby( alpha[0], u_[1], alpha[1],  u_[2], u); //extrapolate previous solutions
-    blas1::pointwiseDivide( u_[0], diff.inv_weights(), u_[0]);
-    detail::Implicit<Diffusion, container> implicit( -dt_/11.*6., diff, f_[0]);
+    blas2::symv( diff.weights(), u_[0], u_[0]);
+    detail::Implicit<Diffusion, container> implicit( -dt_/11.*6., diff);
 #ifdef DG_BENCHMARK
 #ifdef MPI_VERSION
     int rank;
@@ -339,14 +319,14 @@ void Karniadakis<container>::operator()( Functor& f, Diffusion& diff, container&
 #endif//MPI
     Timer t;
     t.tic(); 
-    unsigned number = pcg( implicit, u, u_[0], diff.precond(), eps_);
+    unsigned number = pcg( implicit, u, u_[0], diff.precond(), diff.inv_weights(), eps_);
     t.toc();
 #ifdef MPI_VERSION
     if(rank==0)
 #endif//MPI
     std::cout << "# of pcg iterations for timestep: "<<number<<"/"<<pcg.get_max()<<" took "<<t.diff()<<"s\n";
 #else
-    pcg( implicit, u, u_[0], diff.precond(), eps_);
+    pcg( implicit, u, u_[0], diff.precond(), diff.inv_weights(), eps_);
 #endif //BENCHMARK
     blas1::copy( u, u_[0]); //store result
 }
@@ -379,48 +359,43 @@ struct SIRK
     /**
      * @brief integrate one step
      *
-     * @tparam Explicit Object containing explicit part 
-     * @tparam Imp Object containing implicit part ( must return precond() and weights())
-     * @param f explicit part of the equations
-     * @param g implicit part of the equations
+     * @copydoc hide_explicit_implicit
      * @param u0 start point
      * @param u1 end point (write only)
      * @param dt timestep
      */
-    template <class Explicit, class Imp>
-    void operator()( Explicit& f, Imp& g, const container& u0, container& u1, double dt)
+    template <class Explicit, class Implicit>
+    void operator()( Explicit& exp, Implicit& imp, const container& u0, container& u1, double dt)
     {
-        container u0_ = u0;
-        detail::Implicit<Imp, container> implicit( -dt*d[0], g, f_);
-        f(u0_, f_);
-        u0_ = u0;
-        g(u0_, g_);
+        detail::Implicit<Implicit, container> implicit( -dt*d[0], imp);
+        exp(u0, f_);
+        imp(u0, g_);
         dg::blas1::axpby( dt, f_, dt, g_, rhs);
-        blas1::pointwiseDivide( rhs, g.inv_weights(), rhs);
+        blas2::symv( imp.weights(), rhs, rhs);
         implicit.alpha() = -dt*d[0];
-        pcg( implicit, k_[0], rhs, g.precond(), eps_);
+        pcg( implicit, k_[0], rhs, imp.precond(), imp.inv_weights(), eps_);
 
-        dg::blas1::axpby( 1., u0_, b[1][0], k_[0], u1);
-        f(u1, f_);
-        dg::blas1::axpby( 1., u0_, c[1][0], k_[0], u1);
-        g(u1, g_);
+        dg::blas1::axpby( 1., u0, b[1][0], k_[0], u1);
+        exp(u1, f_);
+        dg::blas1::axpby( 1., u0, c[1][0], k_[0], u1);
+        imp(u1, g_);
         dg::blas1::axpby( dt, f_, dt, g_, rhs);
-        blas1::pointwiseDivide( rhs, g.inv_weights(), rhs);
+        blas2::symv( imp.weights(), rhs, rhs);
         implicit.alpha() = -dt*d[1];
-        pcg( implicit, k_[1], rhs, g.precond(), eps_);
+        pcg( implicit, k_[1], rhs, imp.precond(), imp.inv_weights(), eps_);
 
-        dg::blas1::axpby( 1., u0_, b[2][0], k_[0], u1);
+        dg::blas1::axpby( 1., u0, b[2][0], k_[0], u1);
         dg::blas1::axpby( b[2][1], k_[1], 1., u1);
-        f(u1, f_);
-        dg::blas1::axpby( 1., u0_, c[2][0], k_[0], u1);
+        exp(u1, f_);
+        dg::blas1::axpby( 1., u0, c[2][0], k_[0], u1);
         dg::blas1::axpby( c[2][1], k_[1], 1., u1);
-        g(u1, g_);
+        imp(u1, g_);
         dg::blas1::axpby( dt, f_, dt, g_, rhs);
-        blas1::pointwiseDivide( rhs, g.inv_weights(), rhs);
+        blas2::symv( imp.weights(), rhs, rhs);
         implicit.alpha() = -dt*d[2];
-        pcg( implicit, k_[2], rhs, g.precond(), eps_);
+        pcg( implicit, k_[2], rhs, imp.precond(), imp.inv_weights(), eps_);
         //sum up results
-        dg::blas1::axpby( 1., u0_, w[0], k_[0], u1);
+        dg::blas1::axpby( 1., u0, w[0], k_[0], u1);
         dg::blas1::axpby( w[1], k_[1], 1., u1);
         dg::blas1::axpby( w[2], k_[2], 1., u1);
     }
@@ -430,22 +405,19 @@ struct SIRK
      *
      * Make same timestep twice, once with half timestep. The resulting error should be smaller than some given tolerance
      *
-     * @tparam Explicit Object containing explicit part 
-     * @tparam Imp Object containing implicit part ( must return precond() and weights())
-     * @param f explicit part of the equations
-     * @param g implicit part of the equations
+     * @copydoc hide_explicit_implicit
      * @param u0 start point
      * @param u1 end point (write only)
      * @param dt timestep ( read and write) contains new recommended timestep afterwards
      * @param tolerance tolerable error
      */
-    template <class Explicit, class Imp>
-    void adaptive_step( Explicit& f, Imp& g, const container& u0, container& u1, double& dt, double tolerance)
+    template <class Explicit, class Implicit>
+    void adaptive_step( Explicit& exp, Implicit& imp, const container& u0, container& u1, double& dt, double tolerance)
     {
         container temp = u0;
-        this->operator()( f, g, u0, u1, dt/2.);
-        this->operator()( f, g, u1, temp, dt/2.);
-        this->operator()( f, g, u0, u1, dt);
+        this->operator()( exp, imp, u0, u1, dt/2.);
+        this->operator()( exp, imp, u1, temp, dt/2.);
+        this->operator()( exp, imp, u0, u1, dt);
         dg::blas1::axpby( 1., u1, -1., temp);
         double error = dg::blas1::dot( temp, temp);
         std::cout << "ERROR " << error<< std::endl;
diff --git a/inc/dg/multistep_t.cu b/inc/dg/multistep_t.cu
index 9a7d7bc06..e14575309 100644
--- a/inc/dg/multistep_t.cu
+++ b/inc/dg/multistep_t.cu
@@ -22,8 +22,8 @@ template< class Matrix, class container>
 struct Diffusion
 {
     Diffusion( const dg::Grid2d& g, double nu): nu_(nu),
-        precon( dg::create::inv_weights(g)), 
-        v2d(2, dg::create::inv_weights(g)),
+        w2d( dg::create::weights(g)), 
+        v2d( dg::create::inv_weights(g)),
         LaplacianM( g, dg::normed) 
         { }
 
@@ -35,12 +35,12 @@ struct Diffusion
         }
         dg::blas1::axpby( 0.,y, -nu_, y);
     }
-    const std::vector<container>& inv_weights(){return v2d;}
-    const container& precond(){return precon;}
+    const container& inv_weights(){return v2d;}
+    const container& weights(){return w2d;}
+    const container& precond(){return v2d;}
   private:
     double nu_;
-    const container precon;
-    const std::vector<container> v2d;
+    const container w2d, v2d;
     dg::Elliptic<dg::CartesianGrid2d, Matrix, container> LaplacianM;
 };
 
diff --git a/src/toefl/input/default.json b/src/toefl/input/default.json
index de0f55c80..042639d21 100644
--- a/src/toefl/input/default.json
+++ b/src/toefl/input/default.json
@@ -2,7 +2,7 @@
     "n" :  3,       
     "Nx" : 100,    
     "Ny" : 100,   
-    "dt" : 3.0,  
+    "dt" : 1.5,  
     "n_out"  : 3,  
     "Nx_out" : 100,
     "Ny_out" : 100, 
diff --git a/src/toefl/toeflR.cuh b/src/toefl/toeflR.cuh
index 84e57c2a3..a64c46d14 100644
--- a/src/toefl/toeflR.cuh
+++ b/src/toefl/toeflR.cuh
@@ -17,11 +17,8 @@ template<class Geometry, class Matrix, class container>
 struct Implicit
 {
     Implicit( const Geometry& g, double nu):
-        nu_(nu), 
-        temp( dg::create::inv_weights( g) ), inv_weights_(2,temp),
-        LaplacianM_perp( g, dg::normed, dg::centered){
-    }
-    void operator()( std::vector<container>& x, std::vector<container>& y)
+        nu_(nu), LaplacianM_perp( g, dg::normed, dg::centered){ }
+    void operator()( const std::vector<container>& x, std::vector<container>& y)
     {
         /* x[0] := N_e - 1
          * x[2] := N_i - 1 
@@ -36,13 +33,13 @@ struct Implicit
         }
     }
     dg::Elliptic<Geometry, Matrix, container>& laplacianM() {return LaplacianM_perp;}
-    const std::vector<container>& inv_weights(){return inv_weights_;}
+    const container& weights(){return LaplacianM_perp.weights();}
+    const container& inv_weights(){return LaplacianM_perp.inv_weights();}
     const container& precond(){return LaplacianM_perp.precond();}
 
   private:
     double nu_;
     container temp;
-    std::vector<container> inv_weights_;
     dg::Elliptic<Geometry, Matrix, container> LaplacianM_perp;
 };
 
@@ -88,7 +85,7 @@ struct Explicit
      * @param y input vector
      * @param yp the rhs yp = f(y)
      */
-    void operator()( std::vector<container>& y, std::vector<container>& yp);
+    void operator()( const std::vector<container>& y, std::vector<container>& yp);
 
     /**
      * @brief Return the mass of the last field in operator() in a global computation
@@ -288,7 +285,7 @@ const container& Explicit<G, M, container>::polarisation( const std::vector<cont
 }
 
 template< class G, class M, class container>
-void Explicit<G, M, container>::operator()( std::vector<container>& y, std::vector<container>& yp)
+void Explicit<G, M, container>::operator()( const std::vector<container>& y, std::vector<container>& yp)
 {
     //y[0] = N_e - 1
     //y[1] = N_i - 1 || y[1] = Omega
-- 
GitLab


From 094bd5d778adb5c7d88c6c86fa7b92dca741ef9e Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Fri, 22 Sep 2017 12:11:55 +0200
Subject: [PATCH 311/453] implementing perp_grid in MPI base grids

---
 inc/dg/backend/grid.h            |  2 +-
 inc/dg/geometry/base_geometry.h  |  7 ++++
 inc/dg/geometry/mpi_base.h       | 23 ++++++++++-
 inc/geometries/curvilinear.h     | 65 +++++++++++++++++++++-----------
 inc/geometries/fieldaligned.h    | 52 ++++++++++++++-----------
 inc/geometries/mpi_curvilinear.h | 27 ++++++++++---
 6 files changed, 125 insertions(+), 51 deletions(-)

diff --git a/inc/dg/backend/grid.h b/inc/dg/backend/grid.h
index 048c60e8b..871f5c56f 100644
--- a/inc/dg/backend/grid.h
+++ b/inc/dg/backend/grid.h
@@ -28,7 +28,7 @@
  */
 
 /*!@class hide_grid_parameters3d
- * @brief Construct a 3D topology
+ * @brief Construct a 3D grid
  *
  * @param x0 left boundary in x
  * @param x1 right boundary in x 
diff --git a/inc/dg/geometry/base_geometry.h b/inc/dg/geometry/base_geometry.h
index dc81e5834..4973fad79 100644
--- a/inc/dg/geometry/base_geometry.h
+++ b/inc/dg/geometry/base_geometry.h
@@ -206,6 +206,12 @@ struct CartesianGrid3d: public dg::aGeometry3d
      */
     CartesianGrid3d( const dg::Grid3d& g):dg::aGeometry3d(g.x0(), g.x1(), g.y0(), g.y1(), g.z0(), g.z1(),g.n(),g.Nx(),g.Ny(),g.Nz(),g.bcx(),g.bcy(),g.bcz()){}
     virtual CartesianGrid3d* clone()const{return new CartesianGrid3d(*this);}
+    /*!
+     * @brief The grid made up by the first two dimensions
+     *
+     * This is possible because the 3d grid is a product grid of a 2d perpendicular grid and a 1d parallel grid
+     * @return A newly constructed perpendicular grid
+     */
     CartesianGrid2d perp_grid() const{ return CartesianGrid2d(x0(),x1(),y0(),y1(),n(),Nx(),Ny(),bcx(),bcy());}
     private:
     virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz){
@@ -224,6 +230,7 @@ struct CylindricalGrid3d: public dg::aGeometry3d
     ///@note x corresponds to R, y to Z and z to phi, the volume element is R
     CylindricalGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = PER, bc bcy = PER, bc bcz = PER): dg::aGeometry3d(x0,x1,y0,y1,z0,z1,n,Nx,Ny,Nz,bcx,bcy,bcz){}
     virtual CylindricalGrid3d* clone()const{return new CylindricalGrid3d(*this);}
+    ///@copydoc  CartesianGrid3d::perp_grid()const
     CartesianGrid2d perp_grid() const{ return CartesianGrid2d(x0(),x1(),y0(),y1(),n(),Nx(),Ny(),bcx(),bcy());}
     private:
     virtual SparseTensor<thrust::host_vector<double> > do_compute_metric()const{
diff --git a/inc/dg/geometry/mpi_base.h b/inc/dg/geometry/mpi_base.h
index 897c2a791..491f8c6bf 100644
--- a/inc/dg/geometry/mpi_base.h
+++ b/inc/dg/geometry/mpi_base.h
@@ -141,7 +141,7 @@ struct CartesianMPIGrid2d : public aMPIGeometry2d
  */
 struct CartesianMPIGrid3d : public aMPIGeometry3d
 {
-    typedef CartesianGrid2d perpendicular_grid;
+    typedef CartesianMPIGrid2d perpendicular_grid;
     ///@copydoc hide_grid_parameters3d
     ///@copydoc hide_comm_parameters3d
     CartesianMPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, MPI_Comm comm): aMPIGeometry3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, dg::PER,dg::PER,dg::PER, comm){}
@@ -155,7 +155,24 @@ struct CartesianMPIGrid3d : public aMPIGeometry3d
     ///@param g existing grid object
     CartesianMPIGrid3d( const dg::MPIGrid3d& g): aMPIGeometry3d( g.global().x0(),g.global().x1(),g.global().y0(),g.global().y1(),g.global().z0(),g.global().z1(),g.global().n(),g.global().Nx(),g.global().Ny(),g.global().Nz(),g.global().bcx(),g.global().bcy(),g.global().bcz(),g.communicator()){}
     virtual CartesianMPIGrid3d* clone()const{return new CartesianMPIGrid3d(*this);}
+    /*!
+     * @brief The grid made up by the first two dimensions in space and process topology
+     *
+     * This is possible because the 3d grid is a product grid of a 2d perpendicular grid and a 1d parallel grid
+     * @return A newly constructed perpendicular grid with the perpendicular communicator
+     */
+    CartesianMPIGrid2d perp_grid()const{ 
+        return CartesianMPIGrid2d( global().x0(), global().x1(), global().y0(), global().y1(), global().n(), global().Nx(), global().Ny(), global().bcx(), global().bcy(), get_perp_comm( communicator() ));
+    }
+
     private:
+    MPI_Comm get_perp_comm( MPI_Comm src)
+    {
+        MPI_Comm planeComm;
+        int remain_dims[] = {true,true,false}; //true true false
+        MPI_Cart_sub( src, remain_dims, &planeComm);
+        return planeComm;
+    }
     virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz){
         aMPITopology3d::do_set(new_n,new_Nx,new_Ny,new_Nz);
     }
@@ -180,6 +197,10 @@ struct CylindricalMPIGrid3d: public aMPIGeometry3d
     CylindricalMPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, MPI_Comm comm):aMPIGeometry3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, bcx, bcy, dg::PER, comm){}
 
     virtual CylindricalMPIGrid3d* clone()const{return new CylindricalMPIGrid3d(*this);}
+    ///@copydoc CartesianMPIGrid3d::perp_grid()const
+    CartesianMPIGrid2d perp_grid()const{ 
+        return CartesianMPIGrid2d( global().x0(), global().x1(), global().y0(), global().y1(), global().n(), global().Nx(), global().Ny(), global().bcx(), global().bcy(), get_perp_comm( communicator() ));
+    }
     private:
     virtual SparseTensor<host_vector > do_compute_metric()const{
         SparseTensor<host_vector> metric(1);
diff --git a/inc/geometries/curvilinear.h b/inc/geometries/curvilinear.h
index ca22ca3fa..e7e733536 100644
--- a/inc/geometries/curvilinear.h
+++ b/inc/geometries/curvilinear.h
@@ -7,6 +7,35 @@
 
 namespace dg
 {
+    /*!@class hide_grid_parameters3d
+     * @brief Construct a 3D grid
+     *
+     * the coordinates of the computational space are called x,y,z
+     * @param generator generates the perpendicular grid
+     * @param n number of %Gaussian nodes in x and y
+     *  (1<=n<=20, note that the library is optimized for n=3 )
+     * @attention # of polynomial coefficients in z direction is always 1
+     * @param Nx number of cells in x
+     * @param Ny number of cells in y 
+     * @param Nz  number of cells z
+     * @param bcx boundary condition in x
+     * @param bcy boundary condition in y
+     * @param bcz boundary condition in z
+     */
+    /*!@class hide_grid_parameters2d
+     * @brief Construct a 2D grid
+     *
+     * the coordinates of the computational space are called x,y,z
+     * @param generator generates the grid
+     * @param n number of %Gaussian nodes in x and y
+     *  (1<=n<=20, note that the library is optimized for n=3 )
+     * @param Nx number of cells in x
+     * @param Ny number of cells in y 
+     * @param bcx boundary condition in x
+     * @param bcy boundary condition in y
+     */
+
+
 ///@addtogroup grids
 ///@{
 
@@ -49,18 +78,7 @@ struct CurvilinearProductGrid3d : public dg::aGeometry3d
 {
     typedef CurvilinearGrid2d perpendicular_grid;
 
-    /*!@brief Constructor
-    
-     * the coordinates of the computational space are called x,y,z
-     * @param generator must generate a grid
-     * @param n number of %Gaussian nodes in x and y
-     * @param Nx number of cells in x
-     * @param Ny number of cells in y 
-     * @param Nz  number of cells z
-     * @param bcx boundary condition in x
-     * @param bcy boundary condition in y
-     * @param bcz boundary condition in z
-     */
+    ///@copydoc hide_grid_parameters3d
     CurvilinearProductGrid3d( const aGenerator2d& generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx=dg::DIR, bc bcy=dg::PER, bc bcz=dg::PER):
         dg::aGeometry3d( 0, generator.width(), 0., generator.height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, bcy, bcz)
     { 
@@ -70,8 +88,15 @@ struct CurvilinearProductGrid3d : public dg::aGeometry3d
         constructParallel(Nz);
     }
 
+    /*!
+     * @brief The grid made up by the first two dimensions
+     *
+     * This is possible because the 3d grid is a product grid of a 2d perpendicular grid and a 1d parallel grid
+     * @return A newly constructed perpendicular grid
+     */
     perpendicular_grid perp_grid() const;// { return perpendicular_grid(*this);}
 
+    ///@copydoc CurvilinearGrid2d::generator()const
     const aGenerator2d & generator() const{return handle_.get();}
     virtual CurvilinearProductGrid3d* clone()const{return new CurvilinearProductGrid3d(*this);}
     private:
@@ -132,20 +157,17 @@ struct CurvilinearProductGrid3d : public dg::aGeometry3d
  */
 struct CurvilinearGrid2d : public dg::aGeometry2d
 {
-    /*!@brief Constructor
-    
-     * @param generator must generate an orthogonal grid (class takes ownership of the pointer)
-     * @param n number of polynomial coefficients
-     * @param Nx number of cells in first coordinate
-     * @param Ny number of cells in second coordinate
-     * @param bcx boundary condition in first coordinate
-     * @param bcy boundary condition in second coordinate
-     */
+    ///@copydoc hide_grid_parameters2d
     CurvilinearGrid2d( const aGenerator2d& generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx=dg::DIR, bc bcy=dg::PER):
         dg::aGeometry2d( 0, generator.width(), 0., generator.height(), n, Nx, Ny, bcx, dg::PER), handle_(generator)
     {
         construct( n,Nx,Ny);
     }
+
+    /**
+     * @brief Explicitly convert 3d product grid to the perpendicular grid
+     * @param g 3d product grid
+     */
     explicit CurvilinearGrid2d( CurvilinearProductGrid3d g):
         dg::aGeometry2d( g.x0(), g.x1(), g.y0(), g.y1(), g.n(), g.Nx(), g.Ny(), g.bcx(), g.bcy() ), handle_(g.generator())
     {
@@ -156,6 +178,7 @@ struct CurvilinearGrid2d : public dg::aGeometry2d
         map_.pop_back();
     }
 
+    ///read access to the generator 
     const aGenerator2d& generator() const{return handle_.get();}
     virtual CurvilinearGrid2d* clone()const{return new CurvilinearGrid2d(*this);}
     private:
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index bde803a43..cbd3a52bc 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -292,6 +292,32 @@ void integrate_all_fieldlines2d( const dg::geo::BinaryVectorLvl0& vec, const aGe
     ym_result=ym;
     delete g2dField_ptr;
 }
+
+aGeometry2d* clone_3d_to_perp( const aGeometry3d* grid_ptr)
+{
+    const dg::CartesianGrid3d* grid_cart = dynamic_cast<const dg::CartesianGrid3d*>(grid_ptr);
+    const dg::CylindricalGrid3d* grid_cyl = dynamic_cast<const dg::CylindricalGrid3d*>(grid_ptr);
+    const dg::CurvilinearProductGrid3d*  grid_curvi = dynamic_cast<const dg::CurvilinearProductGrid3d*>(grid_ptr);
+    aGeometry2d* g2d_ptr;
+    if( grid_cart) 
+    {
+        dg::CartesianGrid2d cart = grid_cart->perp_grid();
+        g2d_ptr = cart.clone();
+    }
+    else if( grid_cyl) 
+    {
+        dg::CartesianGrid2d cart = grid_cyl->perp_grid();
+        g2d_ptr = cart.clone();
+    }
+    else if( grid_curvi) 
+    {
+        dg::CurvilinearGrid2d curv = grid_curvi->perp_grid();
+        g2d_ptr = curv.clone();
+    }
+    else
+        throw dg::Error( dg::Message(_ping_)<<"Grid class not recognized!");
+    return g2d_ptr;
+}
 }//namespace detail
 ///@endcond
 
@@ -443,6 +469,8 @@ struct FieldAligned
 
 ///@cond 
 ////////////////////////////////////DEFINITIONS////////////////////////////////////////
+//
+
 
 template<class Geometry, class IMatrix, class container>
 template <class Limiter>
@@ -454,28 +482,7 @@ FieldAligned<Geometry, IMatrix, container>::FieldAligned(const dg::geo::BinaryVe
     else assert( grid.Nz() == 1 || grid.hz()==deltaPhi);
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     //%%%%%%%%%%%downcast grid since we don't have a virtual function perp_grid%%%%%%%%%%%%%
-    const aGeometry3d* grid_ptr = &grid;
-    const dg::CartesianGrid3d* grid_cart = dynamic_cast<const dg::CartesianGrid3d*>(grid_ptr);
-    const dg::CylindricalGrid3d* grid_cyl = dynamic_cast<const dg::CylindricalGrid3d*>(grid_ptr);
-    const dg::CurvilinearProductGrid3d*  grid_curvi = dynamic_cast<const dg::CurvilinearProductGrid3d*>(grid_ptr);
-    aGeometry2d* g2dCoarse_ptr;
-    if( grid_cart) 
-    {
-        dg::CartesianGrid2d cart = grid_cart->perp_grid();
-        g2dCoarse_ptr = cart.clone();
-    }
-    else if( grid_cyl) 
-    {
-        dg::CartesianGrid2d cart = grid_cyl->perp_grid();
-        g2dCoarse_ptr = cart.clone();
-    }
-    else if( grid_curvi) 
-    {
-        dg::CurvilinearGrid2d curv = grid_curvi->perp_grid();
-        g2dCoarse_ptr = curv.clone();
-    }
-    else
-        throw dg::Error( dg::Message(_ping_)<<"Grid class not recognized!");
+    const aGeometry2d* g2dCoarse_ptr = detail::clone_3d_to_perp(&grid);
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     //Resize vector to 2D grid size
     perp_size_ = g2dCoarse_ptr->size();
@@ -705,6 +712,7 @@ void FieldAligned<G, I, container>::eMinus( enum whichMatrix which, const contai
     }
 }
 
+
 ///@endcond 
 
 
diff --git a/inc/geometries/mpi_curvilinear.h b/inc/geometries/mpi_curvilinear.h
index da79600dc..7390e21a7 100644
--- a/inc/geometries/mpi_curvilinear.h
+++ b/inc/geometries/mpi_curvilinear.h
@@ -25,6 +25,9 @@ struct CurvilinearProductMPIGrid3d;
  */
 struct CurvilinearMPIGrid2d : public dg::aMPIGeometry2d
 {
+    /// @opydoc hide_grid_parameters2d
+    /// @param comm a two-dimensional Cartesian communicator
+    /// @note the paramateres given in the constructor are global parameters 
     CurvilinearMPIGrid2d( const aGenerator2d& generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx, dg::bc bcy, MPI_Comm comm2d): 
         dg::aMPIGeometry2d( 0, generator.width(), 0., generator.height(), n, Nx, Ny, bcx, bcy, comm2d), handle_(generator)
     {
@@ -32,8 +35,10 @@ struct CurvilinearMPIGrid2d : public dg::aMPIGeometry2d
         dg::CurvilinearGrid2d g(generator, n, Nx, Ny);
         divide_and_conquer(g);
     }
+    ///explicit conversion of 3d product grid to the perpendicular grid
     explicit CurvilinearMPIGrid2d( const CurvilinearProductMPIGrid3d& g);
 
+    ///read access to the generator 
     const aGenerator2d& generator() const{return handle_.get();}
     virtual CurvilinearMPIGrid2d* clone()const{return new CurvilinearMPIGrid2d(*this);}
     private:
@@ -43,7 +48,7 @@ struct CurvilinearMPIGrid2d : public dg::aMPIGeometry2d
         dg::CurvilinearGrid2d g( handle_.get(), new_n, new_Nx, new_Ny);
         divide_and_conquer(g);//distribute to processes
     }
-    MPI_Comm get_reduced_comm( MPI_Comm src)
+    MPI_Comm get_perp_comm( MPI_Comm src)
     {
         MPI_Comm planeComm;
         int remain_dims[] = {true,true,false}; //true true false
@@ -83,27 +88,37 @@ struct CurvilinearMPIGrid2d : public dg::aMPIGeometry2d
 };
 
 /**
- * This is s 2x1 product space MPI grid
+ * This is s 2x1 curvilinear product space MPI grid
  */
 struct CurvilinearProductMPIGrid3d : public dg::aMPIGeometry3d
 {
     typedef dg::CurvilinearMPIGrid2d perpendicular_grid; //!< the two-dimensional grid
+    /// @opydoc hide_grid_parameters3d
+    /// @param comm a three-dimensional Cartesian communicator
+    /// @note the paramateres given in the constructor are global parameters 
     CurvilinearProductMPIGrid3d( const aGenerator2d& generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz, MPI_Comm comm): 
         dg::aMPIGeometry3d( 0, generator.width(), 0., generator.height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, bcy, bcz, comm),
         handle_( generator)
     {
         map_.resize(3);
-        CurvilinearMPIGrid2d g(generator,n,Nx,Ny, bcx, bcy, get_reduced_comm(comm));
+        CurvilinearMPIGrid2d g(generator,n,Nx,Ny, bcx, bcy, get_perp_comm(comm));
         constructPerp( g);
         constructParallel(this->Nz());
     }
 
+    /*!
+     * @brief The grid made up by the first two dimensions in space and process topology
+     *
+     * This is possible because the 3d grid is a product grid of a 2d perpendicular grid and a 1d parallel grid
+     * @return A newly constructed perpendicular grid with the perpendicular communicator
+     */
     perpendicular_grid perp_grid() const { return perpendicular_grid(*this);}
 
+    ///read access to the generator
     const aGenerator2d& generator() const{return handle_.get();}
     virtual CurvilinearProductMPIGrid3d* clone()const{return new CurvilinearProductMPIGrid3d(*this);}
     private:
-    MPI_Comm get_reduced_comm( MPI_Comm src)
+    MPI_Comm get_perp_comm( MPI_Comm src)
     {
         MPI_Comm planeComm;
         int remain_dims[] = {true,true,false}; //true true false
@@ -115,7 +130,7 @@ struct CurvilinearProductMPIGrid3d : public dg::aMPIGeometry3d
         dg::aMPITopology3d::do_set(new_n, new_Nx, new_Ny, new_Nz);
         if( !( new_n == n() && new_Nx == Nx() && new_Ny == Ny() ) )
         {
-            CurvilinearMPIGrid2d g(handle_.get(),new_n,new_Nx,new_Ny, this->bcx(), this->bcy(), get_reduced_comm(communicator()));
+            CurvilinearMPIGrid2d g(handle_.get(),new_n,new_Nx,new_Ny, this->bcx(), this->bcy(), get_perp_comm(communicator()));
             constructPerp( g);
         }
         constructParallel(this->Nz());
@@ -180,7 +195,7 @@ struct CurvilinearProductMPIGrid3d : public dg::aMPIGeometry3d
 };
 ///@cond
 CurvilinearMPIGrid2d::CurvilinearMPIGrid2d( const CurvilinearProductMPIGrid3d& g):
-    dg::aMPIGeometry2d( g.global().x0(), g.global().x1(), g.global().y0(), g.global().y1(), g.global().n(), g.global().Nx(), g.global().Ny(), g.global().bcx(), g.global().bcy(), get_reduced_comm( g.communicator() )),
+    dg::aMPIGeometry2d( g.global().x0(), g.global().x1(), g.global().y0(), g.global().y1(), g.global().n(), g.global().Nx(), g.global().Ny(), g.global().bcx(), g.global().bcy(), get_perp_comm( g.communicator() )),
     handle_(g.generator())
 {
     map_=g.map();
-- 
GitLab


From 44cdd6bfb712ba15062fa89cbe97e6b80448ae49 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sat, 23 Sep 2017 16:29:44 +0200
Subject: [PATCH 312/453] update mathjax relpath to cdnjs, resolve tag file
 dependencies in doc/Makefile, create clone3dtoperp MPI version

---
 inc/dg/Doxyfile                   |  4 +--
 inc/dg/dg_doc.h                   |  2 +-
 inc/doc/Makefile                  | 48 +++++++++++++++++++------------
 inc/file/Doxyfile                 |  4 +--
 inc/geometries/Doxyfile           |  4 +--
 inc/geometries/mpi_fieldaligned.h | 26 +++++++++++++++++
 inc/geometries/solovev_doc.h      |  2 +-
 7 files changed, 64 insertions(+), 26 deletions(-)

diff --git a/inc/dg/Doxyfile b/inc/dg/Doxyfile
index 45c9eb5ee..11667eb36 100644
--- a/inc/dg/Doxyfile
+++ b/inc/dg/Doxyfile
@@ -1514,7 +1514,7 @@ MATHJAX_FORMAT         = HTML-CSS
 # The default value is: http://cdn.mathjax.org/mathjax/latest.
 # This tag requires that the tag USE_MATHJAX is set to YES.
 
-MATHJAX_RELPATH        = /usr/share/javascript/mathjax
+MATHJAX_RELPATH        = https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?... 
 
 # The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax
 # extension names that should be enabled during MathJax rendering. For example
@@ -2101,7 +2101,7 @@ TAGFILES               =
 # tag file that is based on the input files it reads. See section "Linking to
 # external documentation" for more information about the usage of tag files.
 
-GENERATE_TAGFILE       = doc/dg.tag
+GENERATE_TAGFILE       = 
 
 # If the ALLEXTERNALS tag is set to YES, all external class will be listed in
 # the class index. If set to NO, only the inherited external classes will be
diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index 3899c675b..b88f75645 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -84,7 +84,7 @@
  *     @defgroup arakawa Discretization of Poisson bracket
  *     @defgroup matrixoperators Elliptic and Helmholtz operators
  * @}
- * @defgroup misc Level 99: Miscellaneous additions
+ * @defgroup misc Level 00: Miscellaneous additions
  * @{
  *     @defgroup timer Timer class
  *     @defgroup functions Functions and Functors
diff --git a/inc/doc/Makefile b/inc/doc/Makefile
index e9a2cf000..f695bc957 100644
--- a/inc/doc/Makefile
+++ b/inc/doc/Makefile
@@ -1,5 +1,5 @@
 tagfiles="./dg.tag=../../dg/html ./file.tag=../../file/html"
-.PHONY: dg file geometries
+.PHONY: clean doc dg.tag geometries.tag dg file geometries
 
 ###Packages needed on linux:
 ###doxygen
@@ -9,41 +9,53 @@ tagfiles="./dg.tag=../../dg/html ./file.tag=../../file/html"
 
 all: doc
 
-dg:
+dg.tag:
 	(cat ../dg/Doxyfile; \
-    mkdir -p dg; \
 	echo "INPUT = ../dg/"; \
 	echo "OUTPUT_DIRECTORY = ./dg"; \
-	echo "HTML_HEADER = header.html"; \
-	echo "MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML";  \
+	echo "GENERATE_HTML = NO"; \
 	echo "GENERATE_TAGFILE = ./dg.tag" ) | doxygen - ;
 
+geometries.tag: dg.tag
+	(cat ../geometries/Doxyfile; \
+	echo "INPUT = ../geometries/"; \
+	echo "OUTPUT_DIRECTORY = ./geometries"; \
+	echo "GENERATE_HTML = NO"; \
+	echo "GENERATE_TAGFILE = ./geometries.tag"; \
+    echo "TAGFILES = dg.tag=../../dg/html") | doxygen - ; 
 
-file:
-	(cat ../file/Doxyfile; \
-    mkdir -p file; \
-	echo "INPUT = ../file/"; \
-	echo "OUTPUT_DIRECTORY = ./file"; \
+dg: geometries.tag
+	(cat ../dg/Doxyfile; \
+	echo "INPUT = ../dg/"; \
+	echo "OUTPUT_DIRECTORY = ./dg"; \
 	echo "HTML_HEADER = header.html"; \
-	echo "MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"; \
-	echo "GENERATE_TAGFILE = ./file.tag" ) | doxygen - ; 
+    echo "EXTERNAL_GROUPS=NO" ;\
+    echo "EXTERNAL_PAGES=NO" ;\
+    echo "TAGFILES = geometries.tag=../../geometries/html") | doxygen - ; 
 
-geometries:
+geometries: dg.tag
 	(cat ../geometries/Doxyfile; \
 	echo "INPUT = ../geometries/"; \
-    echo "OUTPUT_DIRECTORY = ./geometries/";  \
+	echo "OUTPUT_DIRECTORY = ./geometries"; \
 	echo "HTML_HEADER = header.html"; \
     echo "EXTERNAL_GROUPS=NO" ;\
     echo "EXTERNAL_PAGES=NO" ;\
-	echo "MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML";  \
-    echo "TAGFILES = $(tagfiles)") | doxygen - ; 
+    echo "TAGFILES = dg.tag=../../dg/html") | doxygen - ; 
 
-doc: dg file geometries
+file:
+	(cat ../file/Doxyfile; \
+	echo "INPUT = ../file/"; \
+	echo "OUTPUT_DIRECTORY = ./file"; \
+	echo "HTML_HEADER = header.html"; \
+	echo ) | doxygen - ; 
+
+doc: dg geometries file
 	ln -sf dg/html/modules.html index.html
 	#Open with: firefox index.html or on Windows: firefox dg/html/modules.html
 
 
+
 clean:
-	rm -rf dg file geometries dg.tag file.tag index.html
+	rm -rf dg file geometries dg.tag file.tag geometries.tag index.html
 
 
diff --git a/inc/file/Doxyfile b/inc/file/Doxyfile
index ce14304ec..9e297d8a9 100644
--- a/inc/file/Doxyfile
+++ b/inc/file/Doxyfile
@@ -1504,7 +1504,7 @@ MATHJAX_FORMAT         = HTML-CSS
 # The default value is: http://cdn.mathjax.org/mathjax/latest.
 # This tag requires that the tag USE_MATHJAX is set to YES.
 
-MATHJAX_RELPATH        = http://cdn.mathjax.org/mathjax/latest
+MATHJAX_RELPATH        = https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?...
 
 # The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax
 # extension names that should be enabled during MathJax rendering. For example
@@ -2091,7 +2091,7 @@ TAGFILES               =
 # tag file that is based on the input files it reads. See section "Linking to
 # external documentation" for more information about the usage of tag files.
 
-GENERATE_TAGFILE       = doc/file.tag
+GENERATE_TAGFILE       =
 
 # If the ALLEXTERNALS tag is set to YES, all external class will be listed in
 # the class index. If set to NO, only the inherited external classes will be
diff --git a/inc/geometries/Doxyfile b/inc/geometries/Doxyfile
index 97effe2a2..9872c17e0 100644
--- a/inc/geometries/Doxyfile
+++ b/inc/geometries/Doxyfile
@@ -1505,7 +1505,7 @@ MATHJAX_FORMAT         = HTML-CSS
 # The default value is: http://cdn.mathjax.org/mathjax/latest.
 # This tag requires that the tag USE_MATHJAX is set to YES.
 
-MATHJAX_RELPATH        = /usr/share/javascript/mathjax/
+MATHJAX_RELPATH        = https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?...
 
 # The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax
 # extension names that should be enabled during MathJax rendering. For example
@@ -2086,7 +2086,7 @@ SKIP_FUNCTION_MACROS   = YES
 # the path). If a tag file is not located in the directory in which doxygen is
 # run, you must also specify the path to the tagfile here.
 
-TAGFILES               = ../dg/doc/dg.tag=../../../dg/doc/html
+TAGFILES               = 
 
 # When a file name is specified after GENERATE_TAGFILE, doxygen will create a
 # tag file that is based on the input files it reads. See section "Linking to
diff --git a/inc/geometries/mpi_fieldaligned.h b/inc/geometries/mpi_fieldaligned.h
index 39697283c..b80d6b042 100644
--- a/inc/geometries/mpi_fieldaligned.h
+++ b/inc/geometries/mpi_fieldaligned.h
@@ -50,6 +50,32 @@ void sendBackward( InputIterator begin, InputIterator end, OutputIterator result
                     source, 3, //source
                     comm, &status);
 }
+
+aMPIGeometry2d* clone_MPI3d_to_perp( const aMPIGeometry3d* grid_ptr)
+{
+    const dg::CartesianMPIGrid3d* grid_cart = dynamic_cast<const dg::CartesianMPIGrid3d*>(grid_ptr);
+    const dg::CylindricalMPIGrid3d* grid_cyl = dynamic_cast<const dg::CylindricalMPIGrid3d*>(grid_ptr);
+    const dg::CurvilinearProductMPIGrid3d*  grid_curvi = dynamic_cast<const dg::CurvilinearProductMPIGrid3d*>(grid_ptr);
+    aGeometry2d* g2d_ptr;
+    if( grid_cart) 
+    {
+        dg::CartesianMPIGrid2d cart = grid_cart->perp_grid();
+        g2d_ptr = cart.clone();
+    }
+    else if( grid_cyl) 
+    {
+        dg::CartesianMPIGrid2d cart = grid_cyl->perp_grid();
+        g2d_ptr = cart.clone();
+    }
+    else if( grid_curvi) 
+    {
+        dg::CurvilinearMPIGrid2d curv = grid_curvi->perp_grid();
+        g2d_ptr = curv.clone();
+    }
+    else
+        throw dg::Error( dg::Message(_ping_)<<"Grid class not recognized!");
+    return g2d_ptr;
+}
 }//namespace detail
 
 template <class Geometry, class LocalIMatrix, class CommunicatorXY, class LocalContainer>
diff --git a/inc/geometries/solovev_doc.h b/inc/geometries/solovev_doc.h
index b2387e4ee..45fa73b94 100644
--- a/inc/geometries/solovev_doc.h
+++ b/inc/geometries/solovev_doc.h
@@ -1,7 +1,7 @@
 #error Documentation only
 /*! 
  * 
- * @defgroup grids 1. New Geometry classes
+ * @defgroup grids 1. New geometric grids
  * @defgroup generators_geo 2. Grid generators
  *
       All the grids introduced by this extension can be constructed with 
-- 
GitLab


From 872e93e515289888eb97a2cc9f7300aea650ce1e Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sat, 23 Sep 2017 16:47:43 +0200
Subject: [PATCH 313/453] place grids and generator in geo namespace

---
 inc/geometries/curvilinear.h      | 3 +++
 inc/geometries/generator.h        | 3 +++
 inc/geometries/mpi_curvilinear.h  | 3 +++
 inc/geometries/mpi_fieldaligned.h | 2 +-
 4 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/inc/geometries/curvilinear.h b/inc/geometries/curvilinear.h
index e7e733536..e49e877df 100644
--- a/inc/geometries/curvilinear.h
+++ b/inc/geometries/curvilinear.h
@@ -6,6 +6,8 @@
 #include "generator.h"
 
 namespace dg
+{
+namespace geo
 {
     /*!@class hide_grid_parameters3d
      * @brief Construct a 3D grid
@@ -209,4 +211,5 @@ struct CurvilinearGrid2d : public dg::aGeometry2d
 CurvilinearProductGrid3d::perpendicular_grid CurvilinearProductGrid3d::perp_grid() const { return CurvilinearProductGrid3d::perpendicular_grid(*this);}
 ///@endcond
 
+}//namespace geo
 }//namespace dg
diff --git a/inc/geometries/generator.h b/inc/geometries/generator.h
index 08fd9da9f..f42eb20fc 100644
--- a/inc/geometries/generator.h
+++ b/inc/geometries/generator.h
@@ -2,6 +2,8 @@
 
 namespace dg
 {
+namespace geo
+{
 
 /**
 * @brief The abstract generator base class 
@@ -83,4 +85,5 @@ struct aGenerator2d
 
 };
 
+}//namespace geo
 }//namespace dg
diff --git a/inc/geometries/mpi_curvilinear.h b/inc/geometries/mpi_curvilinear.h
index 7390e21a7..df5e9ac1a 100644
--- a/inc/geometries/mpi_curvilinear.h
+++ b/inc/geometries/mpi_curvilinear.h
@@ -13,6 +13,8 @@
 
 namespace dg
 {
+namespace geo
+{
 
 ///@cond
 struct CurvilinearProductMPIGrid3d; 
@@ -213,5 +215,6 @@ CurvilinearMPIGrid2d::CurvilinearMPIGrid2d( const CurvilinearProductMPIGrid3d& g
 ///@endcond
 
 ///@}
+}//namespace geo
 }//namespace dg
 
diff --git a/inc/geometries/mpi_fieldaligned.h b/inc/geometries/mpi_fieldaligned.h
index b80d6b042..1ed524b59 100644
--- a/inc/geometries/mpi_fieldaligned.h
+++ b/inc/geometries/mpi_fieldaligned.h
@@ -56,7 +56,7 @@ aMPIGeometry2d* clone_MPI3d_to_perp( const aMPIGeometry3d* grid_ptr)
     const dg::CartesianMPIGrid3d* grid_cart = dynamic_cast<const dg::CartesianMPIGrid3d*>(grid_ptr);
     const dg::CylindricalMPIGrid3d* grid_cyl = dynamic_cast<const dg::CylindricalMPIGrid3d*>(grid_ptr);
     const dg::CurvilinearProductMPIGrid3d*  grid_curvi = dynamic_cast<const dg::CurvilinearProductMPIGrid3d*>(grid_ptr);
-    aGeometry2d* g2d_ptr;
+    aMPIGeometry2d* g2d_ptr;
     if( grid_cart) 
     {
         dg::CartesianMPIGrid2d cart = grid_cart->perp_grid();
-- 
GitLab


From 9af7a549db66022809cb4c64766ab665e189cf59 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 25 Sep 2017 11:30:06 +0200
Subject: [PATCH 314/453] added mpi typedefs for cusp csr matrices in new file
 mpi_projection.h

---
 inc/dg/backend/mpi_projection.h | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 inc/dg/backend/mpi_projection.h

diff --git a/inc/dg/backend/mpi_projection.h b/inc/dg/backend/mpi_projection.h
new file mode 100644
index 000000000..9066b6d10
--- /dev/null
+++ b/inc/dg/backend/mpi_projection.h
@@ -0,0 +1,16 @@
+#pragma once
+
+#include "projection.cuh"
+
+namespace dg
+{
+///@addtogroup typedefs
+///@{
+//interpolation matrices
+typedef MPIDistMat< cusp::csr_matrix<int, double, cusp::host_memory>, GeneralComm< thrust::host_vector<int>, thrust::host_vector<double> >   MIHMatrix; //!< MPI distributed CSR host Matrix
+typedef MPIDistMat< cusp::csr_matrix<int, double, cusp::device_memory>, GeneralComm< thrust::host_vector<int>, thrust::host_vector<double> > MIDMatrix; //!< MPI distributed CSR device Matrix
+///@}
+
+
+
+}//namespace dg
-- 
GitLab


From a04f5d938f3dd0aafa85cbe0035367a1f7154ea1 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 25 Sep 2017 16:43:24 +0200
Subject: [PATCH 315/453] tried to solve the recursion problem in doxygen but
 failed and restored old Makefile

---
 inc/dg/backend/mpi_collective.h |  4 +--
 inc/dg/backend/mpi_projection.h | 44 +++++++++++++++++++++++++++++++--
 inc/dg/backend/typedefs.cuh     | 24 +++++++++---------
 inc/dg/dg_doc.h                 |  1 -
 inc/doc/Makefile                | 27 ++++++--------------
 inc/geometries/Doxyfile         |  2 +-
 6 files changed, 65 insertions(+), 37 deletions(-)

diff --git a/inc/dg/backend/mpi_collective.h b/inc/dg/backend/mpi_collective.h
index 5626e5583..460969e4d 100644
--- a/inc/dg/backend/mpi_collective.h
+++ b/inc/dg/backend/mpi_collective.h
@@ -356,12 +356,12 @@ struct SurjectiveComm : public aCommunicator<Vector>
 template< class Index, class Vector>
 struct GeneralComm : public aCommunicator<Vector>
 {
-    /// no memory allocation; the size shall be 0
+    /// no memory allocation; size 0
     GeneralComm(){}
     /**
     * @brief Construct from local indices and PIDs gather map
     *
-    * The gather map shall be written with respect to the buffer vector (unlike in BijectiveComm, where it is given wrt the source vector) 
+    * The indices in the gather map is written with respect to the buffer vector (unlike in BijectiveComm, where it is given wrt the source vector) 
     * @param localGatherMap The gather map containing local vector indices ( local buffer size)
     * @param pidGatherMap The gather map containing the pids from where to gather the local index.
     Same size as localGatherMap.
diff --git a/inc/dg/backend/mpi_projection.h b/inc/dg/backend/mpi_projection.h
index 9066b6d10..99d13a72b 100644
--- a/inc/dg/backend/mpi_projection.h
+++ b/inc/dg/backend/mpi_projection.h
@@ -1,16 +1,56 @@
 #pragma once
 
 #include "projection.cuh"
+#include "typedefs.cuh"
 
 namespace dg
 {
 ///@addtogroup typedefs
 ///@{
 //interpolation matrices
-typedef MPIDistMat< cusp::csr_matrix<int, double, cusp::host_memory>, GeneralComm< thrust::host_vector<int>, thrust::host_vector<double> >   MIHMatrix; //!< MPI distributed CSR host Matrix
-typedef MPIDistMat< cusp::csr_matrix<int, double, cusp::device_memory>, GeneralComm< thrust::host_vector<int>, thrust::host_vector<double> > MIDMatrix; //!< MPI distributed CSR device Matrix
+typedef MPIDistMat< dg::IHMatrix, GeneralComm< dg::iHVec, dg::HVec > MIHMatrix; //!< MPI distributed CSR host Matrix
+typedef MPIDistMat< dg::IDMatrix, GeneralComm< dg::iDVec, dg::DVec > MIDMatrix; //!< MPI distributed CSR device Matrix
 ///@}
 
+namespace create
+{
+
+///@addtogroup interpolation
+///@{
+///@copydoc interpolation
+dg::MIHMatrix interpolation( const aMPITopology2d& g_new, const aMPITopology2d& g_old)
+{
+    return MIHMatrix( interpolation( g_new.local(), g_old.local()), GeneralComm<iHVec, HVec>());
+}
+
+///@copydoc interpolation
+dg::MIHMatrix interpolation( const aMPITopology3d& g_new, const aMPITopology3d& g_old)
+{
+    return MIHMatrix( interpolation( g_new.local(), g_old.local()), GeneralComm<iHVec, HVec>());
+}
+///@copydoc interpolationT
+dg::MIHMatrix interpolationT( const aMPITopology2d& g_new, const aMPITopology2d& g_old)
+{
+    return MIHMatrix( interpolationT( g_new.local(), g_old.local()), GeneralComm<iHVec, HVec>());
+}
+
+///@copydoc interpolationT
+dg::MIHMatrix interpolationT( const aMPITopology3d& g_new, const aMPITopology3d& g_old)
+{
+    return MIHMatrix( interpolationT( g_new.local(), g_old.local()), GeneralComm<iHVec, HVec>());
+}
+///@copydoc projection
+dg::MIHMatrix projection( const aMPITopology2d& g_new, const aMPITopology2d& g_old)
+{
+    return MIHMatrix( projection( g_new.local(), g_old.local()), GeneralComm<iHVec, HVec>());
+}
 
+///@copydoc projection
+dg::MIHMatrix projection( const aMPITopology3d& g_new, const aMPITopology3d& g_old)
+{
+    return MIHMatrix( projection( g_new.local(), g_old.local()), GeneralComm<iHVec, HVec>());
+}
+///@}
 
+}//namespace create
 }//namespace dg
diff --git a/inc/dg/backend/typedefs.cuh b/inc/dg/backend/typedefs.cuh
index fc9afdf9a..96739488b 100644
--- a/inc/dg/backend/typedefs.cuh
+++ b/inc/dg/backend/typedefs.cuh
@@ -22,15 +22,15 @@ typedef EllSparseBlockMat<double> HMatrix; //!< Host Matrix for derivatives
 
 #ifdef MPI_VERSION
 //typedef MPI_Vector<thrust::device_vector<double> >  MDVec; //!< MPI Device Vector s.a. dg::DVec
-typedef MPI_Vector<DVec >  MDVec; //!< MPI Device Vector s.a. dg::DVec
-typedef MPI_Vector<thrust::host_vector<double>  >   MHVec; //!< MPI Host Vector s.a. dg::HVec
+typedef MPI_Vector<dg::DVec >  MDVec; //!< MPI Device Vector s.a. dg::DVec
+typedef MPI_Vector<dg::HVec >  MHVec; //!< MPI Host Vector s.a. dg::HVec
 
-typedef NearestNeighborComm<thrust::host_vector<int>, thrust::host_vector<double> > NNCH; //!< host Communicator for the use in an mpi matrix for derivatives
+typedef NearestNeighborComm<dg::iHVec, dg::HVec > NNCH; //!< host Communicator for the use in an mpi matrix for derivatives
 //typedef NearestNeighborComm<thrust::device_vector<int>, thrust::device_vector<double> > NNCD; //!< device Communicator for the use in an mpi matrix for derivatives
-typedef NearestNeighborComm<iDVec, DVec > NNCD; //!< device Communicator for the use in an mpi matrix for derivatives
+typedef NearestNeighborComm<dg::iDVec, dg::DVec > NNCD; //!< device Communicator for the use in an mpi matrix for derivatives
 
-typedef dg::RowColDistMat<dg::EllSparseBlockMat<double>, dg::CooSparseBlockMat<double>, dg::NNCH> MHMatrix; //!< MPI Host Matrix for derivatives
-typedef dg::RowColDistMat<dg::EllSparseBlockMatDevice<double>, dg::CooSparseBlockMatDevice<double>, dg::NNCD> MDMatrix; //!< MPI Device Matrix for derivatives
+typedef dg::RowColDistMat<dg::HMatrix, dg::CooSparseBlockMat<double>, dg::NNCH> MHMatrix; //!< MPI Host Matrix for derivatives
+typedef dg::RowColDistMat<dg::DMatrix, dg::CooSparseBlockMatDevice<double>, dg::NNCD> MDMatrix; //!< MPI Device Matrix for derivatives
 #endif
 //////////////////////////////////////////////FLOAT VERSIONS////////////////////////////////////////////////////
 //vectors
@@ -42,14 +42,14 @@ typedef EllSparseBlockMatDevice<float> fDMatrix; //!< Device Matrix for derivati
 typedef EllSparseBlockMat<float> fHMatrix; //!< Host Matrix for derivatives
 
 #ifdef MPI_VERSION
-typedef MPI_Vector<thrust::device_vector<float> >  fMDVec; //!< MPI Device Vector s.a. dg::DVec
-typedef MPI_Vector<thrust::host_vector<float>  >   fMHVec; //!< MPI Host Vector
+typedef MPI_Vector<dg::fDVec > fMDVec; //!< MPI Device Vector s.a. dg::DVec
+typedef MPI_Vector<dg::fHVec > fMHVec; //!< MPI Host Vector
 
-typedef NearestNeighborComm<thrust::host_vector<int>, thrust::host_vector<float> > fNNCH; //!< host Communicator for the use in an mpi matrix for derivatives
-typedef NearestNeighborComm<thrust::device_vector<int>, thrust::device_vector<float> > fNNCD; //!< device Communicator for the use in an mpi matrix for derivatives
+typedef NearestNeighborComm<dg::iHVec, dg::fHVec > fNNCH; //!< host Communicator for the use in an mpi matrix for derivatives
+typedef NearestNeighborComm<dg::iDVec, dg::fDVec > fNNCD; //!< device Communicator for the use in an mpi matrix for derivatives
 
-typedef dg::RowColDistMat<dg::EllSparseBlockMat<float>, dg::CooSparseBlockMat<float>, dg::fNNCH> fMHMatrix; //!< MPI Host Matrix for derivatives
-typedef dg::RowColDistMat<dg::EllSparseBlockMatDevice<float>, dg::CooSparseBlockMatDevice<float>, dg::fNNCD> fMDMatrix; //!< MPI Device Matrix for derivatives
+typedef dg::RowColDistMat<dg::fHMatrix, dg::CooSparseBlockMat<float>, dg::fNNCH> fMHMatrix; //!< MPI Host Matrix for derivatives
+typedef dg::RowColDistMat<dg::fDMatrix, dg::CooSparseBlockMatDevice<float>, dg::fNNCD> fMDMatrix; //!< MPI Device Matrix for derivatives
 #endif
 ///@}
 
diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index b88f75645..0d83a433c 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -173,4 +173,3 @@
  @note you can make your own SymmetricOp by providing the member function void symv(const container&, container&);
   and specializing MatrixTraits with the SelfMadeMatrixTag as the matrix_category
   */
-
diff --git a/inc/doc/Makefile b/inc/doc/Makefile
index f695bc957..6eee17e96 100644
--- a/inc/doc/Makefile
+++ b/inc/doc/Makefile
@@ -1,6 +1,3 @@
-tagfiles="./dg.tag=../../dg/html ./file.tag=../../file/html"
-.PHONY: clean doc dg.tag geometries.tag dg file geometries
-
 ###Packages needed on linux:
 ###doxygen
 ###libjs-mathjax
@@ -9,6 +6,8 @@ tagfiles="./dg.tag=../../dg/html ./file.tag=../../file/html"
 
 all: doc
 
+.PHONY: clean doc dg.tag dg file geometries
+
 dg.tag:
 	(cat ../dg/Doxyfile; \
 	echo "INPUT = ../dg/"; \
@@ -16,22 +15,14 @@ dg.tag:
 	echo "GENERATE_HTML = NO"; \
 	echo "GENERATE_TAGFILE = ./dg.tag" ) | doxygen - ;
 
-geometries.tag: dg.tag
-	(cat ../geometries/Doxyfile; \
-	echo "INPUT = ../geometries/"; \
-	echo "OUTPUT_DIRECTORY = ./geometries"; \
-	echo "GENERATE_HTML = NO"; \
-	echo "GENERATE_TAGFILE = ./geometries.tag"; \
-    echo "TAGFILES = dg.tag=../../dg/html") | doxygen - ; 
-
-dg: geometries.tag
+dg:
 	(cat ../dg/Doxyfile; \
 	echo "INPUT = ../dg/"; \
 	echo "OUTPUT_DIRECTORY = ./dg"; \
 	echo "HTML_HEADER = header.html"; \
     echo "EXTERNAL_GROUPS=NO" ;\
     echo "EXTERNAL_PAGES=NO" ;\
-    echo "TAGFILES = geometries.tag=../../geometries/html") | doxygen - ; 
+	echo ) | doxygen - ; 
 
 geometries: dg.tag
 	(cat ../geometries/Doxyfile; \
@@ -40,7 +31,7 @@ geometries: dg.tag
 	echo "HTML_HEADER = header.html"; \
     echo "EXTERNAL_GROUPS=NO" ;\
     echo "EXTERNAL_PAGES=NO" ;\
-    echo "TAGFILES = dg.tag=../../dg/html") | doxygen - ; 
+	echo "TAGFILES = dg.tag=../../dg/html") | doxygen - ; 
 
 file:
 	(cat ../file/Doxyfile; \
@@ -51,11 +42,9 @@ file:
 
 doc: dg geometries file
 	ln -sf dg/html/modules.html index.html
-	#Open with: firefox index.html or on Windows: firefox dg/html/modules.html
-
-
+	#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%#
+	#Open with: firefox index.html or on Windows: firefox dg/html/modules.html#
+	#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%#
 
 clean:
 	rm -rf dg file geometries dg.tag file.tag geometries.tag index.html
-
-
diff --git a/inc/geometries/Doxyfile b/inc/geometries/Doxyfile
index 9872c17e0..803732e04 100644
--- a/inc/geometries/Doxyfile
+++ b/inc/geometries/Doxyfile
@@ -849,7 +849,7 @@ EXCLUDE_PATTERNS       =
 # Note that the wildcards are matched against the file with absolute path, so to
 # exclude all test directories use the pattern */test/*
 
-EXCLUDE_SYMBOLS        =
+EXCLUDE_SYMBOLS        = hide_*
 
 # The EXAMPLE_PATH tag can be used to specify one or more files or directories
 # that contain example code fragments that are included (see the \include
-- 
GitLab


From e6d27bc72606a217adcafcecb4c3dc2385074c96 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 25 Sep 2017 16:55:41 +0200
Subject: [PATCH 316/453] finish documentation for mpi_projection.h

---
 inc/dg/backend/interpolation.cuh |  8 ++------
 inc/dg/backend/mpi_projection.h  | 27 ++++++++++++++++-----------
 inc/dg/backend/projection.cuh    | 25 +++++++------------------
 inc/dg/backend/typedefs.cuh      |  2 +-
 4 files changed, 26 insertions(+), 36 deletions(-)

diff --git a/inc/dg/backend/interpolation.cuh b/inc/dg/backend/interpolation.cuh
index 5cae73f7b..7ce9646dc 100644
--- a/inc/dg/backend/interpolation.cuh
+++ b/inc/dg/backend/interpolation.cuh
@@ -419,9 +419,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const Grid1d& g_
     return interpolation( pointsX, g_old);
 
 }
-/**
- * @copydoc interpolation(const Grid1d&,const Grid1d&)
- */
+///@copydoc interpolation(const Grid1d&,const Grid1d&)
 cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const aTopology2d& g_new, const aTopology2d& g_old)
 {
     //assert both grids are on the same box
@@ -436,9 +434,7 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const aTopology2
 
 }
 
-/**
- * @copydoc interpolation(const Grid1d&,const Grid1d&)
- */
+///@copydoc interpolation(const Grid1d&,const Grid1d&)
 cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const aTopology3d& g_new, const aTopology3d& g_old)
 {
     //assert both grids are on the same box
diff --git a/inc/dg/backend/mpi_projection.h b/inc/dg/backend/mpi_projection.h
index 99d13a72b..2a42d8766 100644
--- a/inc/dg/backend/mpi_projection.h
+++ b/inc/dg/backend/mpi_projection.h
@@ -1,15 +1,20 @@
 #pragma once
 
+#include "mpi_matrix.h"
 #include "projection.cuh"
 #include "typedefs.cuh"
 
+/*! @file
+  @brief Useful MPI typedefs and overloads of interpolation and projection
+  */
+
 namespace dg
 {
 ///@addtogroup typedefs
 ///@{
 //interpolation matrices
-typedef MPIDistMat< dg::IHMatrix, GeneralComm< dg::iHVec, dg::HVec > MIHMatrix; //!< MPI distributed CSR host Matrix
-typedef MPIDistMat< dg::IDMatrix, GeneralComm< dg::iDVec, dg::DVec > MIDMatrix; //!< MPI distributed CSR device Matrix
+typedef MPIDistMat< dg::IHMatrix, GeneralComm< dg::iHVec, dg::HVec > > MIHMatrix; //!< MPI distributed CSR host Matrix
+typedef MPIDistMat< dg::IDMatrix, GeneralComm< dg::iDVec, dg::DVec > > MIDMatrix; //!< MPI distributed CSR device Matrix
 ///@}
 
 namespace create
@@ -17,35 +22,35 @@ namespace create
 
 ///@addtogroup interpolation
 ///@{
-///@copydoc interpolation
+
+///@copydoc dg::create::interpolation(const Grid1d&,const Grid1d&)
 dg::MIHMatrix interpolation( const aMPITopology2d& g_new, const aMPITopology2d& g_old)
 {
     return MIHMatrix( interpolation( g_new.local(), g_old.local()), GeneralComm<iHVec, HVec>());
 }
-
-///@copydoc interpolation
+///@copydoc interpolation(const Grid1d&,const Grid1d&)
 dg::MIHMatrix interpolation( const aMPITopology3d& g_new, const aMPITopology3d& g_old)
 {
     return MIHMatrix( interpolation( g_new.local(), g_old.local()), GeneralComm<iHVec, HVec>());
 }
-///@copydoc interpolationT
+
+///@copydoc interpolationT(const Grid1d&,const Grid1d&)
 dg::MIHMatrix interpolationT( const aMPITopology2d& g_new, const aMPITopology2d& g_old)
 {
     return MIHMatrix( interpolationT( g_new.local(), g_old.local()), GeneralComm<iHVec, HVec>());
 }
-
-///@copydoc interpolationT
+///@copydoc interpolationT(const Grid1d&,const Grid1d&)
 dg::MIHMatrix interpolationT( const aMPITopology3d& g_new, const aMPITopology3d& g_old)
 {
     return MIHMatrix( interpolationT( g_new.local(), g_old.local()), GeneralComm<iHVec, HVec>());
 }
-///@copydoc projection
+
+///@copydoc projection(const Grid1d&,const Grid1d&)
 dg::MIHMatrix projection( const aMPITopology2d& g_new, const aMPITopology2d& g_old)
 {
     return MIHMatrix( projection( g_new.local(), g_old.local()), GeneralComm<iHVec, HVec>());
 }
-
-///@copydoc projection
+///@copydoc projection(const Grid1d&,const Grid1d&)
 dg::MIHMatrix projection( const aMPITopology3d& g_new, const aMPITopology3d& g_old)
 {
     return MIHMatrix( projection( g_new.local(), g_old.local()), GeneralComm<iHVec, HVec>());
diff --git a/inc/dg/backend/projection.cuh b/inc/dg/backend/projection.cuh
index 0a30740d2..a6293e2c2 100644
--- a/inc/dg/backend/projection.cuh
+++ b/inc/dg/backend/projection.cuh
@@ -66,18 +66,14 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const Grid1d& g
     cusp::transpose( temp, A);
     return A;
 }
-/**
- * @copydoc interpolationT
- */
+///@copydoc interpolationT(const Grid1d&,const Grid1d&)
 cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const aTopology2d& g_new, const aTopology2d& g_old)
 {
     cusp::coo_matrix<int, double, cusp::host_memory> temp = interpolation( g_old, g_new), A;
     cusp::transpose( temp, A);
     return A;
 }
-/**
- * @copydoc interpolationT
- */
+///@copydoc interpolationT(const Grid1d&,const Grid1d&)
 cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const aTopology3d& g_new, const aTopology3d& g_old)
 {
     cusp::coo_matrix<int, double, cusp::host_memory> temp = interpolation( g_old, g_new), A;
@@ -130,9 +126,7 @@ cusp::coo_matrix< int, double, cusp::host_memory> projection( const Grid1d& g_ne
 }
 
 
-/**
- * @copydoc projection
- */
+///@copydoc projection(const Grid1d&,const Grid1d&)
 cusp::coo_matrix< int, double, cusp::host_memory> projection( const aTopology2d& g_new, const aTopology2d& g_old)
 {
     if( g_old.Nx() % g_new.Nx() != 0) std::cerr << "ATTENTION: you project between incompatible grids in x!! old N: "<<g_old.Nx()<<" new N: "<<g_new.Nx()<<"\n";
@@ -159,9 +153,7 @@ cusp::coo_matrix< int, double, cusp::host_memory> projection( const aTopology2d&
     return A;
 }
 
-/**
- * @copydoc projection
- */
+///@copydoc projection(const Grid1d&,const Grid1d&)
 cusp::coo_matrix< int, double, cusp::host_memory> projection( const aTopology3d& g_new, const aTopology3d& g_old)
 {
     if( g_old.Nx() % g_new.Nx() != 0) std::cerr << "ATTENTION: you project between incompatible grids in x!! old N: "<<g_old.Nx()<<" new N: "<<g_new.Nx()<<"\n";
@@ -220,9 +212,8 @@ cusp::coo_matrix< int, double, cusp::host_memory> transformation( const aTopolog
     Y.sort_by_row_and_column();
     return Y;
 }
-/**
- * @copydoc transformation
- */
+
+///@copydoc transformation(const aTopology3d&,const aTopology3d&)
 cusp::coo_matrix< int, double, cusp::host_memory> transformation( const aTopology2d& g_new, const aTopology2d& g_old)
 {
     Grid2d g_lcm(g_new.x0(), g_new.x1(), g_new.y0(), g_new.y1(), 
@@ -233,9 +224,7 @@ cusp::coo_matrix< int, double, cusp::host_memory> transformation( const aTopolog
     Y.sort_by_row_and_column();
     return Y;
 }
-/**
- * @copydoc transformation
- */
+///@copydoc transformation(const aTopology3d&,const aTopology3d&)
 cusp::coo_matrix< int, double, cusp::host_memory> transformation( const Grid1d& g_new, const Grid1d& g_old)
 {
     Grid1d g_lcm(g_new.x0(), g_new.x1(), lcm(g_new.n(), g_old.n()), lcm(g_new.N(), g_old.N()));
diff --git a/inc/dg/backend/typedefs.cuh b/inc/dg/backend/typedefs.cuh
index 96739488b..37f01a1df 100644
--- a/inc/dg/backend/typedefs.cuh
+++ b/inc/dg/backend/typedefs.cuh
@@ -2,7 +2,7 @@
 #define _DG_TYPEDEFS_CUH_
 
 /*! @file
-  @brief This file contains useful typedefs of commonly used types.
+  @brief Useful typedefs of commonly used types.
   */
 
 namespace dg{
-- 
GitLab


From 437c5d31d8d2f8207a3a07a0abbef1f4d2ab3a76 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 25 Sep 2017 17:18:11 +0200
Subject: [PATCH 317/453] place X curvilinear grids also in geo namespace: we
 need to produce global perp MPI grids somehow

---
 inc/geometries/curvilinearX.h         |  3 ++
 inc/geometries/fieldaligned.h         | 22 +++++-----
 inc/geometries/mpi_fieldaligned.h     | 62 +++++++++++----------------
 inc/geometries/refined_curvilinearX.h |  3 ++
 4 files changed, 41 insertions(+), 49 deletions(-)

diff --git a/inc/geometries/curvilinearX.h b/inc/geometries/curvilinearX.h
index f0a928a75..ed7035ee3 100644
--- a/inc/geometries/curvilinearX.h
+++ b/inc/geometries/curvilinearX.h
@@ -10,6 +10,8 @@
 
 namespace dg
 {
+namespace geo
+{
 ///@addtogroup grids
 ///@{
 
@@ -140,4 +142,5 @@ struct CurvilinearGridX2d : public dg::aGeometryX2d
 
 ///@}
 
+} //namespace geo
 } //namespace dg
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index cbd3a52bc..b2b3bba92 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -482,12 +482,12 @@ FieldAligned<Geometry, IMatrix, container>::FieldAligned(const dg::geo::BinaryVe
     else assert( grid.Nz() == 1 || grid.hz()==deltaPhi);
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     //%%%%%%%%%%%downcast grid since we don't have a virtual function perp_grid%%%%%%%%%%%%%
-    const aGeometry2d* g2dCoarse_ptr = detail::clone_3d_to_perp(&grid);
+    const aGeometry2d* grid2d_ptr = detail::clone_3d_to_perp(&grid);
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     //Resize vector to 2D grid size
-    perp_size_ = g2dCoarse_ptr->size();
-    limiter_ = dg::evaluate( limit, *g2dCoarse_ptr);
-    right_ = left_ = dg::evaluate( zero, *g2dCoarse_ptr);
+    perp_size_ = grid2d_ptr->size();
+    limiter_ = dg::evaluate( limit, *grid2d_ptr);
+    right_ = left_ = dg::evaluate( zero, *grid2d_ptr);
     ghostM.resize( perp_size_); ghostP.resize( perp_size_);
     //%%%%%%%%%%%%%%%%%%%%%%%%%%Set starting points and integrate field lines%%%%%%%%%%%%%%
     std::cout << "Start fieldline integration!\n";
@@ -495,14 +495,14 @@ FieldAligned<Geometry, IMatrix, container>::FieldAligned(const dg::geo::BinaryVe
     std::vector<thrust::host_vector<double> > yp_coarse( 3), ym_coarse(yp_coarse); 
     t.tic();
     
-    dg::aGeometry2d* g2dField_ptr = g2dCoarse_ptr->clone();//INTEGRATE HIGH ORDER GRID
+    dg::aGeometry2d* g2dField_ptr = grid2d_ptr->clone();//INTEGRATE HIGH ORDER GRID
     g2dField_ptr->set( 7, g2dField_ptr->Nx(), g2dField_ptr->Ny());
     detail::integrate_all_fieldlines2d( vec, g2dField_ptr, yp_coarse, ym_coarse, deltaPhi, eps);
     delete g2dField_ptr;
 
-    dg::Grid2d g2dFine((dg::Grid2d(*g2dCoarse_ptr)));//FINE GRID
+    dg::Grid2d g2dFine((dg::Grid2d(*grid2d_ptr)));//FINE GRID
     g2dFine.multiplyCellNumbers((double)mx, (double)my);
-    IMatrix interpolate = dg::create::interpolation( g2dFine, *g2dCoarse_ptr);  //INTERPOLATE TO FINE GRID
+    IMatrix interpolate = dg::create::interpolation( g2dFine, *grid2d_ptr);  //INTERPOLATE TO FINE GRID
     std::vector<thrust::host_vector<double> > yp( 3, dg::evaluate(dg::zero, g2dFine)), ym(yp); 
     for( unsigned i=0; i<3; i++)
     {
@@ -520,9 +520,9 @@ FieldAligned<Geometry, IMatrix, container>::FieldAligned(const dg::geo::BinaryVe
     std::cout << "Fieldline integration took "<<t.diff()<<"s\n";
     //%%%%%%%%%%%%%%%%%%Create interpolation and projection%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     t.tic();
-    IMatrix plusFine  = dg::create::interpolation( yp[0], yp[1], *g2dCoarse_ptr, globalbcx, globalbcy);
-    IMatrix minusFine = dg::create::interpolation( ym[0], ym[1], *g2dCoarse_ptr, globalbcx, globalbcy);
-    IMatrix projection = dg::create::projection( *g2dCoarse_ptr, g2dFine);
+    IMatrix plusFine  = dg::create::interpolation( yp[0], yp[1], *grid2d_ptr, globalbcx, globalbcy);
+    IMatrix minusFine = dg::create::interpolation( ym[0], ym[1], *grid2d_ptr, globalbcx, globalbcy);
+    IMatrix projection = dg::create::projection( *grid2d_ptr, g2dFine);
     t.toc();
     std::cout <<"Creation of interpolation/projection took "<<t.diff()<<"s\n";
     t.tic();
@@ -544,7 +544,7 @@ FieldAligned<Geometry, IMatrix, container>::FieldAligned(const dg::geo::BinaryVe
     }
     dg::blas1::scal( hm_, -1.);
     dg::blas1::axpby(  1., hp_, +1., hm_, hz_);
-    delete g2dCoarse_ptr;
+    delete grid2d_ptr;
 }
 
 template<class G, class I, class container>
diff --git a/inc/geometries/mpi_fieldaligned.h b/inc/geometries/mpi_fieldaligned.h
index 1ed524b59..4a2d5eec1 100644
--- a/inc/geometries/mpi_fieldaligned.h
+++ b/inc/geometries/mpi_fieldaligned.h
@@ -149,50 +149,35 @@ struct FieldAligned< Geometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vec
 //////////////////////////////////////DEFINITIONS/////////////////////////////////////
 template<class MPIGeometry, class LocalIMatrix, class CommunicatorXY, class LocalContainer>
 template <class Limiter>
-FieldAligned<MPIGeometry, RowDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vector<LocalContainer> >::FieldAligned(
+FieldAligned<MPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vector<LocalContainer> >::FieldAligned(
     const dg::geo::BinaryVectorLvl0& vec, const MPIGeometry& grid, unsigned mx, unsigned my, double eps, Limiter limit, dg::bc globalbcx, dg::bc globalbcy, double deltaPhi):
     hz_( dg::evaluate( dg::zero, grid)), hp_( hz_), hm_( hz_), 
     g_(grid), bcz_(grid.bcz()), 
     tempXYplus_(g_.Nz()), tempXYminus_(g_.Nz()), temp_(g_.Nz())
 {
-    //create communicator with all processes in plane
-    typename MPIGeometry::perpendicular_grid g2d = grid.perp_grid();
-    unsigned localsize = g2d.size();
-    limiter_ = dg::evaluate( limit, g2d.local());
-    right_ = left_ = dg::evaluate( zero, g2d.local());
+    if( deltaPhi <=0) deltaPhi = grid.hz();
+    else assert( grid.Nz() == 1 || grid.hz()==deltaPhi);
+    //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+    //%%%%%%%%%%%downcast grid since we don't have a virtual function perp_grid%%%%%%%%%%%%%
+    const aMPIGeometry2d* g2dCoarse_ptr = detail::clone_MPI3d_to_perp(&grid);
+    //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+    unsigned localsize = grid2d_ptr->size();
+    limiter_ = dg::evaluate( limit, grid2d_ptr->local());
+    right_ = left_ = dg::evaluate( zero, grid2d_ptr->local());
     ghostM.resize( localsize); ghostP.resize( localsize);
-    //set up grid points as start for fieldline integrations 
-    std::vector<MPI_Vector<thrust::host_vector<double> > > y( 5, dg::evaluate(dg::zero, g2d));
-    y[0] = dg::evaluate( dg::cooX2d, g2d);
-    y[1] = dg::evaluate( dg::cooY2d, g2d);
-    y[2] = dg::evaluate( dg::zero, g2d);//distance (not angle)
-    y[3] = dg::pullback( dg::cooX2d, g2d);
-    y[4] = dg::pullback( dg::cooY2d, g2d);
-    //integrate to next z-planes
-    std::vector<thrust::host_vector<double> > yp(3, y[0].data()), ym(yp); 
-    if(deltaPhi<=0) deltaPhi = grid.hz();
-    else assert( g_.Nz() == 1 || grid.hz()==deltaPhi);
-#ifdef _OPENMP
-#pragma omp parallel for shared(field)
-#endif //_OPENMP
-    for( unsigned i=0; i<localsize; i++)
-    {
-        thrust::host_vector<double> coords(5), coordsP(5), coordsM(5);
-        coords[0] = y[0].data()[i], coords[1] = y[1].data()[i], coords[2] = y[2].data()[i], coords[3] = y[3].data()[i], coords[4] = y[4].data()[i];
-        double phi1 = deltaPhi;
-        boxintegrator( field, g2d.global(), coords, coordsP, phi1, eps, globalbcz);
-        phi1 = -deltaPhi;
-        boxintegrator( field, g2d.global(), coords, coordsM, phi1, eps, globalbcz);
-        yp[0][i] = coordsP[0], yp[1][i] = coordsP[1], yp[2][i] = coordsP[2];
-        ym[0][i] = coordsM[0], ym[1][i] = coordsM[1], ym[2][i] = coordsM[2];
-    }
-
+    //%%%%%%%%%%%%%%%%%%%%%%%%%%Set starting points and integrate field lines%%%%%%%%%%%%%%
+    std::vector<thrust::host_vector<double> > yp_coarse( 3), ym_coarse(yp_coarse); 
+    
+    dg::aGeometry2d* g2dField_ptr = grid2d_ptr->clone();//INTEGRATE HIGH ORDER GRID
+    g2dField_ptr->set( 7, g2dField_ptr->Nx(), g2dField_ptr->Ny());
+    detail::integrate_all_fieldlines2d( vec, g2dField_ptr, yp_coarse, ym_coarse, deltaPhi, eps);
+    delete g2dField_ptr;
 
     //determine pid of result 
     thrust::host_vector<int> pids( localsize);
     for( unsigned i=0; i<localsize; i++)
     {
-        pids[i]  = g2d.pidOf( yp[0][i], yp[1][i]);
+        pids[i]  = grid2d_ptr.pidOf( yp[0][i], yp[1][i]);
         if( pids[i]  == -1)
         {
             std::cerr << "ERROR: PID NOT FOUND!\n";
@@ -200,31 +185,31 @@ FieldAligned<MPIGeometry, RowDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vector<L
         }
     }
 
-    CommunicatorXY cp( pids, g2d.communicator());
+    CommunicatorXY cp( pids, grid2d_ptr.communicator());
     commXYplus_ = cp;
     thrust::host_vector<double> pX, pY;
     dg::blas1::transfer( cp.global_gather( yp[0]), pX);
     dg::blas1::transfer( cp.global_gather( yp[1]), pY);
 
     //construt interpolation matrix
-    plus = dg::create::interpolation( pX, pY, g2d.local(), globalbcz); //inner points hopefully never lie exactly on local boundary
+    plus = dg::create::interpolation( pX, pY, grid2d_ptr.local(), globalbcz); //inner points hopefully never lie exactly on local boundary
     cusp::transpose( plus, plusT);
 
     //do the same for the minus z-plane
     for( unsigned i=0; i<pids.size(); i++)
     {
-        pids[i]  = g2d.pidOf( ym[0][i], ym[1][i]);
+        pids[i]  = grid2d_ptr.pidOf( ym[0][i], ym[1][i]);
         if( pids[i] == -1)
         {
             std::cerr << "ERROR: PID NOT FOUND!\n";
             return;
         }
     }
-    CommunicatorXY cm( pids, g2d.communicator());
+    CommunicatorXY cm( pids, grid2d_ptr.communicator());
     commXYminus_ = cm;
     dg::blas1::transfer( cm.global_gather( ym[0]), pX);
     dg::blas1::transfer( cm.global_gather( ym[1]), pY);
-    minus = dg::create::interpolation( pX, pY, g2d.local(), globalbcz); //inner points hopefully never lie exactly on local boundary
+    minus = dg::create::interpolation( pX, pY, grid2d_ptr.local(), globalbcz); //inner points hopefully never lie exactly on local boundary
     cusp::transpose( minus, minusT);
     //copy to device
     for( unsigned i=0; i<g_.Nz(); i++)
@@ -240,6 +225,7 @@ FieldAligned<MPIGeometry, RowDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vector<L
         tempXYminus_[i].resize( commXYminus_.size());
         temp_[i].resize( localsize);
     }
+    delete grid2d_ptr;
 }
 
 template<class G, class M, class C, class container>
diff --git a/inc/geometries/refined_curvilinearX.h b/inc/geometries/refined_curvilinearX.h
index df73db911..521b33593 100644
--- a/inc/geometries/refined_curvilinearX.h
+++ b/inc/geometries/refined_curvilinearX.h
@@ -6,6 +6,8 @@
 
 namespace dg
 {
+namespace geo
+{
 ///@addtogroup grids
 ///@{
 
@@ -152,5 +154,6 @@ struct CurvilinearRefinedGridX2d : public dg::aGeometryX2d
 ///@}
 
 
+} //namespace geo
 } //namespace dg
 
-- 
GitLab


From ea7e87da05268440eda5f253c23fb06cf06ea30c Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Tue, 26 Sep 2017 13:23:49 +0200
Subject: [PATCH 318/453] update mpi 2d perp grid clone

---
 inc/geometries/fieldaligned.h     |  7 ++-----
 inc/geometries/mpi_fieldaligned.h | 17 ++++++++++-------
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index b2b3bba92..52c340e34 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -268,9 +268,6 @@ void integrate_all_fieldlines2d( const dg::geo::BinaryVectorLvl0& vec, const aGe
     std::cout << "Generation of interpolate grid took "<<t.diff()<<"s\n";
     //field in case of cartesian grid
     dg::detail::DSFieldCylindrical cyl_field(vec);
-#ifdef _OPENMP
-#pragma omp parallel for shared(field, cyl_field)
-#endif //_OPENMP
     for( unsigned i=0; i<g2dField_ptr->size(); i++)
     {
         thrust::host_vector<double> coords(3), coordsP(3), coordsM(3);
@@ -297,7 +294,7 @@ aGeometry2d* clone_3d_to_perp( const aGeometry3d* grid_ptr)
 {
     const dg::CartesianGrid3d* grid_cart = dynamic_cast<const dg::CartesianGrid3d*>(grid_ptr);
     const dg::CylindricalGrid3d* grid_cyl = dynamic_cast<const dg::CylindricalGrid3d*>(grid_ptr);
-    const dg::CurvilinearProductGrid3d*  grid_curvi = dynamic_cast<const dg::CurvilinearProductGrid3d*>(grid_ptr);
+    const dg::geo::CurvilinearProductGrid3d*  grid_curvi = dynamic_cast<const dg::geo::CurvilinearProductGrid3d*>(grid_ptr);
     aGeometry2d* g2d_ptr;
     if( grid_cart) 
     {
@@ -311,7 +308,7 @@ aGeometry2d* clone_3d_to_perp( const aGeometry3d* grid_ptr)
     }
     else if( grid_curvi) 
     {
-        dg::CurvilinearGrid2d curv = grid_curvi->perp_grid();
+        dg::geo::CurvilinearGrid2d curv = grid_curvi->perp_grid();
         g2d_ptr = curv.clone();
     }
     else
diff --git a/inc/geometries/mpi_fieldaligned.h b/inc/geometries/mpi_fieldaligned.h
index 4a2d5eec1..3706effff 100644
--- a/inc/geometries/mpi_fieldaligned.h
+++ b/inc/geometries/mpi_fieldaligned.h
@@ -51,26 +51,29 @@ void sendBackward( InputIterator begin, InputIterator end, OutputIterator result
                     comm, &status);
 }
 
-aMPIGeometry2d* clone_MPI3d_to_perp( const aMPIGeometry3d* grid_ptr)
+aGeometry2d* clone_MPI3d_to_global_perp( const aMPIGeometry3d* grid_ptr)
 {
     const dg::CartesianMPIGrid3d* grid_cart = dynamic_cast<const dg::CartesianMPIGrid3d*>(grid_ptr);
     const dg::CylindricalMPIGrid3d* grid_cyl = dynamic_cast<const dg::CylindricalMPIGrid3d*>(grid_ptr);
     const dg::CurvilinearProductMPIGrid3d*  grid_curvi = dynamic_cast<const dg::CurvilinearProductMPIGrid3d*>(grid_ptr);
-    aMPIGeometry2d* g2d_ptr;
+    aGeometry2d* g2d_ptr;
     if( grid_cart) 
     {
         dg::CartesianMPIGrid2d cart = grid_cart->perp_grid();
-        g2d_ptr = cart.clone();
+        dg::CartesianGrid2d global_cart( cart.global());
+        g2d_ptr = global_cart.clone();
     }
     else if( grid_cyl) 
     {
         dg::CartesianMPIGrid2d cart = grid_cyl->perp_grid();
-        g2d_ptr = cart.clone();
+        dg::CartesianGrid2d global_cart( cart.global());
+        g2d_ptr = global_cart.clone();
     }
     else if( grid_curvi) 
     {
-        dg::CurvilinearMPIGrid2d curv = grid_curvi->perp_grid();
-        g2d_ptr = curv.clone();
+        dg::geo::CurvilinearMPIGrid2d curv = grid_curvi->perp_grid();
+        dg::geo::CurvilinearGrid2d global_curv( curv.generator(), curv.global().n(), curv.global().Nx(), curv.global().Ny(), curv.bcx(), curv.bcy());
+        g2d_ptr = global_curv.clone();
     }
     else
         throw dg::Error( dg::Message(_ping_)<<"Grid class not recognized!");
@@ -159,7 +162,7 @@ FieldAligned<MPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vector<L
     else assert( grid.Nz() == 1 || grid.hz()==deltaPhi);
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     //%%%%%%%%%%%downcast grid since we don't have a virtual function perp_grid%%%%%%%%%%%%%
-    const aMPIGeometry2d* g2dCoarse_ptr = detail::clone_MPI3d_to_perp(&grid);
+    const aGeometry2d* g2d_ptr = detail::clone_MPI3d_to_global_perp(&grid);
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     unsigned localsize = grid2d_ptr->size();
     limiter_ = dg::evaluate( limit, grid2d_ptr->local());
-- 
GitLab


From 450da418209e708b17158735db5e0180e1507e86 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Tue, 26 Sep 2017 16:28:06 +0200
Subject: [PATCH 319/453] update Use->Uses

---
 inc/dg/backend/mpi_collective.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/inc/dg/backend/mpi_collective.h b/inc/dg/backend/mpi_collective.h
index 460969e4d..0994a027f 100644
--- a/inc/dg/backend/mpi_collective.h
+++ b/inc/dg/backend/mpi_collective.h
@@ -381,7 +381,7 @@ struct GeneralComm : public aCommunicator<Vector>
     /**
      * @brief Construct from global indices gather map
      *
-     * Use the global2localIdx() function to generate localGatherMap and pidGatherMap 
+     * Uses the global2localIdx() member of MPITopology to generate localGatherMap and pidGatherMap 
      * @tparam MPITopology any implementation of an MPI Topology (aMPITopology2d, aMPITopology3d, ...)
      * @param globalGatherMap The gather map containing global vector indices (local buffer size)
      * @param g a grid object
-- 
GitLab


From 645adacd39ca68f1f730e10408259c25402e0b4b Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Wed, 27 Sep 2017 16:23:17 +0200
Subject: [PATCH 320/453] added abstraction of MPI datatypes

---
 inc/dg/backend/mpi_collective.h   | 24 +++++++++---------
 inc/dg/backend/mpi_communicator.h | 41 +++++++++++++++++++++++++------
 inc/dg/backend/mpi_vector.h       |  8 +++---
 3 files changed, 49 insertions(+), 24 deletions(-)

diff --git a/inc/dg/backend/mpi_collective.h b/inc/dg/backend/mpi_collective.h
index 0994a027f..4155956cb 100644
--- a/inc/dg/backend/mpi_collective.h
+++ b/inc/dg/backend/mpi_collective.h
@@ -24,7 +24,6 @@ namespace dg{
  * to send (or gather from) and connects it to an intermediate "store"
  * In this way gather and scatter are defined with respect to the buffer and 
  * the store is the vector.
- @note the data type of the Vector class has to be double
  */
 template<class Index, class Vector>
 struct Collective
@@ -99,10 +98,10 @@ void Collective<Index, Device>::scatter( const Device& values, Device& store) co
     MPI_Alltoallv( 
             thrust::raw_pointer_cast( values.data()), 
             thrust::raw_pointer_cast( sendTo_.data()), 
-            thrust::raw_pointer_cast( accS_.data()), MPI_DOUBLE, 
+            thrust::raw_pointer_cast( accS_.data()), getMPIDataType<typename VectorTraits<Device>::value_type>(), 
             thrust::raw_pointer_cast( store.data()),
             thrust::raw_pointer_cast( recvFrom_.data()),
-            thrust::raw_pointer_cast( accR_.data()), MPI_DOUBLE, comm_);
+            thrust::raw_pointer_cast( accR_.data()), getMPIDataType<typename VectorTraits<Device>::value_type>(), comm_);
 }
 
 template< class Index, class Device>
@@ -117,10 +116,10 @@ void Collective<Index, Device>::gather( const Device& gatherFrom, Device& values
     MPI_Alltoallv( 
             thrust::raw_pointer_cast( gatherFrom.data()), 
             thrust::raw_pointer_cast( recvFrom_.data()),
-            thrust::raw_pointer_cast( accR_.data()), MPI_DOUBLE, 
+            thrust::raw_pointer_cast( accR_.data()), getMPIDataType<typename VectorTraits<Device>::value_type>(), 
             thrust::raw_pointer_cast( values.data()), 
             thrust::raw_pointer_cast( sendTo_.data()), 
-            thrust::raw_pointer_cast( accS_.data()), MPI_DOUBLE, comm_);
+            thrust::raw_pointer_cast( accS_.data()), getMPIDataType<typename VectorTraits<Device>::value_type>(), comm_);
 }
 //BijectiveComm ist der Spezialfall, dass jedes Element nur ein einziges Mal gebraucht wird. 
 ///@endcond
@@ -141,8 +140,8 @@ void Collective<Index, Device>::gather( const Device& gatherFrom, Device& values
  thrust::host_vector<double> hrecv2( hvalues.size());
  coll.global_scatter_reduce( hrecv, hrecv2); //hrecv2 now equals hvalues independent of process rank
  @endcode
- * @tparam Index an integer thrust Vector (thrust::host_vector<int> or thrust::device_vector<int>) 
- * @tparam Vector a thrust Vector (thrust::host_vector<double> or thrust::device_vector<double>)
+ * @tparam Index an integer thrust Vector
+ * @tparam Vector a thrust Vector
  * @note a scatter followed by a gather of the received values restores the original array
  * @note The order of the received elements is according to their original array index 
  *   (i.e. a[0] appears before a[1]) and their process rank of origin ( i.e. values from rank 0 appear before values from rank 1)
@@ -248,8 +247,8 @@ struct BijectiveComm : public aCommunicator<Vector>
  Compared to BijectiveComm in the global_gather function there is an additional 
  gather and in the global_scatter_reduce function a reduction 
  needs to be performed.
- * @tparam Index an integer thrust Vector (thrust::host_vector<int> or thrust::device_vector<int>) 
- * @tparam Vector a thrust Vector (thrust::host_vector<double> or thrust::device_vector<double>)
+ * @tparam Index an integer thrust Vector
+ * @tparam Vector a thrust Vector
  */
 template< class Index, class Vector>
 struct SurjectiveComm : public aCommunicator<Vector>
@@ -286,6 +285,7 @@ struct SurjectiveComm : public aCommunicator<Vector>
 
     ///@copydoc GeneralComm::getLocalGatherMap
     const thrust::host_vector<int>& getLocalGatherMap() const {return localGatherMap_;}
+    ///@copydoc GeneralComm::getPidGatherMap
     const thrust::host_vector<int>& getPidGatherMap() const {return pidGatherMap_;}
     const Index& getSortedGatherMap() const {return sortedGatherMap_;}
     virtual SurjectiveComm* clone() const {return new SurjectiveComm(*this);}
@@ -350,8 +350,8 @@ struct SurjectiveComm : public aCommunicator<Vector>
  * This Communicator can perform general global gather and
  scatter operations. Compared to SurjectiveComm the global_scatter_reduce function needs
  to perform an additional scatter as some elements of the source vector might be left empty
- * @tparam Index an integer thrust Vector (thrust::host_vector<int> or thrust::device_vector<int>) 
- * @tparam Vector a thrust Vector (thrust::host_vector<double> or thrust::device_vector<double>)
+ * @tparam Index an integer thrust Vector 
+ * @tparam Vector a thrust Vector 
  */
 template< class Index, class Vector>
 struct GeneralComm : public aCommunicator<Vector>
@@ -371,7 +371,7 @@ struct GeneralComm : public aCommunicator<Vector>
     GeneralComm( const thrust::host_vector<int>& localGatherMap, const thrust::host_vector<int>& pidGatherMap, MPI_Comm comm) {
         construct( localGatherMap, pidGatherMap, comm);
     }
-    ///@brief reconstruct from another type if non-empty; else same as default constructor
+    ///@brief reconstruct from another type; if src is empty same as default constructor
     template<class OtherIndex, class OtherVector>
     GeneralComm( const GeneralComm<OtherIndex, OtherVector>& src){
         if( src.size() > 0)
diff --git a/inc/dg/backend/mpi_communicator.h b/inc/dg/backend/mpi_communicator.h
index 3b9b63860..d9ae42abc 100644
--- a/inc/dg/backend/mpi_communicator.h
+++ b/inc/dg/backend/mpi_communicator.h
@@ -1,7 +1,21 @@
 #pragma once
 
+#include <mpi.h>
+
 namespace dg
 {
+///@cond
+template<class value_type>
+MPI_Datatype getMPIDataType(){ assert( false && "Type not supported!\n" ); return; }
+template<>
+MPI_Datatype getMPIDataType<double>(){ return MPI_DOUBLE;}
+template<>
+MPI_Datatype getMPIDataType<float>(){ return MPI_FLOAT;}
+template<>
+MPI_Datatype getMPIDataType<int>(){ return MPI_INT;} 
+template<>
+MPI_Datatype getMPIDataType<unsigned>(){ return MPI_UNSIGNED;} 
+///@endcond
 
 /**
  * @brief Struct that performs collective scatter and gather operations across processes
@@ -33,10 +47,11 @@ by a matrix. The gather matrix is just a
 (permutation) matrix of 1's and 0's with exactly one "1" in each line.
 In a "coo" formatted sparse matrix format the values array would consist only of "1"s, 
 row array is just the index and column array is the gather map.
-We uniquely define the corresponding scatter matrix as the transpose of the gather matrix. 
+We uniquely define the corresponding <b> scatter matrix </b> as the <b> transpose of the gather matrix</b>. 
 The scatter matrix can have zero, one or more "1"s in each line.
-\f[ w = G v \\
-    v = S w \f]
+\f[ w_1 = G v_1 \\
+    v_2 = S w_2 = G^\mathrm{T} w_2 \f]
+where \f$ v_1\f$ and \f$ v_2\f$ are data vectors and \f$ w_1\f$ and \f$ w_2\f$ are buffer vectors.
 
 This class performs these operations for the case that v and w are distributed across processes.
 We always assume that the source vector v is distributed equally among processes, i.e. 
@@ -49,13 +64,13 @@ In this case the buffer and the vector can swap their roles.
 @note Finally note that when v is filled with its indices, i.e. \f$ v[i] = i \f$, then
 the gather operation will reproduce the index map in the buffer w \f$ w[i] = \text{idx}[i]\f$ .
 
- * @tparam LocalContainer a container on a shared memory system
+ * @tparam LocalContainer a container on a shared memory system (must be default constructible)
  * @ingroup mpi_structures
  */
 template< class LocalContainer>
 struct aCommunicator
 {
-    typedef LocalContainer container_type; //!< typedef to derive container type
+    typedef LocalContainer container_type; //!< reveal local container type
 
     /**
      * @brief Allocate a buffer object of size size()
@@ -69,18 +84,22 @@ struct aCommunicator
 
     /**
      * @brief Globally (across processes) gather data into a buffer 
+     *
+     * This is the transpose operation of global_scatter_reduce()
      * @param values data; other processes collect data from this vector
-     * @param buffer object to hold the gathered data ( must be of size size())
+     * @param buffer on output holds the gathered data ( must be of size size())
      * @note if size()==0 nothing happens
      */
     void global_gather( const LocalContainer& values, LocalContainer& buffer)const
     {
         if( do_size() == 0 ) return;
-        do_global_gather( values, gather);
+        do_global_gather( values, buffer);
     }
 
     /**
      * @brief Globally (across processes) gather data into a buffer (memory allocating version)
+     *
+     * This is the transpose operation of global_scatter_reduce()
      * @param values data; other processes collect data from this vector
      * @return object that holds the gathered data
      * @note if size()==0 the default constructor of LocalContainer is called
@@ -94,8 +113,10 @@ struct aCommunicator
 
     /**
      * @brief Globally (across processes) scatter data accross processes and reduce on multiple indices
+     *
+     * This is the transpose operation of global_gather()
      * @param toScatter buffer vector; (has to be of size given by size())
-     * @param values contains values from other processes sent back to the origin 
+     * @param values on output contains values from other processes sent back to the origin 
      * @note if size()==0 nothing happens
      */
     void global_scatter_reduce( const LocalContainer& toScatter, LocalContainer& values) const{
@@ -144,4 +165,8 @@ struct aCommunicator
     virtual void do_global_scatter_reduce( const LocalContainer& toScatter, LocalContainer& values) const=0;
 };
 
+
+
+
+
 }//namespace dg
diff --git a/inc/dg/backend/mpi_vector.h b/inc/dg/backend/mpi_vector.h
index 2df718ab6..4972bddd8 100644
--- a/inc/dg/backend/mpi_vector.h
+++ b/inc/dg/backend/mpi_vector.h
@@ -308,15 +308,15 @@ void NearestNeighborComm<I,V>::sendrecv( V& sb1, V& sb2 , V& rb1, V& rb2) const
 #if THRUST_DEVICE_SYSTEM==THRUST_DEVICE_SYSTEM_CUDA
     cudaDeviceSynchronize(); //wait until device functions are finished before sending data
 #endif //THRUST_DEVICE_SYSTEM
-    MPI_Sendrecv(   thrust::raw_pointer_cast(sb1.data()), buffer_size(), MPI_DOUBLE,  //sender
+    MPI_Sendrecv(   thrust::raw_pointer_cast(sb1.data()), buffer_size(), getMPIDataType<typename VectorTraits<V>::value_type>(),  //sender
                     dest, 3,  //destination
-                    thrust::raw_pointer_cast(rb2.data()), buffer_size(), MPI_DOUBLE, //receiver
+                    thrust::raw_pointer_cast(rb2.data()), buffer_size(), getMPIDataType<typename VectorTraits<V>::value_type>(), //receiver
                     source, 3, //source
                     comm_, &status);
     MPI_Cart_shift( comm_, direction_, +1, &source, &dest);
-    MPI_Sendrecv(   thrust::raw_pointer_cast(sb2.data()), buffer_size(), MPI_DOUBLE,  //sender
+    MPI_Sendrecv(   thrust::raw_pointer_cast(sb2.data()), buffer_size(), getMPIDataType<typename VectorTraits<V>::value_type>(),  //sender
                     dest, 9,  //destination
-                    thrust::raw_pointer_cast(rb1.data()), buffer_size(), MPI_DOUBLE, //receiver
+                    thrust::raw_pointer_cast(rb1.data()), buffer_size(), getMPIDataType<typename VectorTraits<V>::value_type>(), //receiver
                     source, 9, //source
                     comm_, &status);
 }
-- 
GitLab


From 11932ad67fba41553c64525961871736d8955e1d Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Thu, 28 Sep 2017 12:04:28 +0200
Subject: [PATCH 321/453] first version of global idx to buffer idx

---
 inc/dg/backend/mpi_collective.h | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/inc/dg/backend/mpi_collective.h b/inc/dg/backend/mpi_collective.h
index 4155956cb..8873d79d6 100644
--- a/inc/dg/backend/mpi_collective.h
+++ b/inc/dg/backend/mpi_collective.h
@@ -438,4 +438,26 @@ struct GeneralComm : public aCommunicator<Vector>
     Buffer<Vector> store_;
     Index scatterMap_;
 };
+
+
+//given global indices -> make a sorted unique indices vector -> make a gather map into the unique vector
+void global2bufferIdx( const thrust::host_vector<int>& global_idx, thrust::host_vector<int>& idx_in_buffer, thrust::host_vector<int>& unique_global_idx)
+{
+    thrust::host_vector<int> copy(global_idx);
+    thrust::host_vector<int> index(copy);
+    thrust::sequence( index.begin(), index.end());
+    thrust::stable_sort_by_key( copy.begin(), copy.end(), index.begin());//note: this also sorts the pids
+    thrust::host_vector<int> ones( index.size(), 1);
+    thrust::host_vector<int> unique_global( index.size()), howmany( index.size());
+    thrust::pair<int*, int*> new_end;
+    new_end = thrust::reduce_by_key( copy.begin(), copy.end(), ones.begin(), unique_global.begin(), howmany.begin());
+    unique_global_idx.assign( unique_global.begin(), new_end.first);
+    thrust::host_vector<int> gather_map;
+    for( int i=0; i<unique_global_idx.size(); i++)
+        for( int j=0; j<howmany[i]; j++)
+            gather_map.append(i );
+    assert( gather_map.size() == global_idx.size());
+    idx_in_buffer.resize( global_idx.size());
+    thrust::scatter( gather_map.begin(), gather_map.end(), index.begin(), idx_in_buffer.begin());
+}
 }//namespace dg
-- 
GitLab


From 8aebb8f5aa3de9214f5019f41c337b384f6f3121 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Thu, 28 Sep 2017 13:41:26 +0200
Subject: [PATCH 322/453] updated mpi_base.h

---
 inc/dg/geometry/mpi_base.h | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/inc/dg/geometry/mpi_base.h b/inc/dg/geometry/mpi_base.h
index 491f8c6bf..15d7b36a9 100644
--- a/inc/dg/geometry/mpi_base.h
+++ b/inc/dg/geometry/mpi_base.h
@@ -166,7 +166,7 @@ struct CartesianMPIGrid3d : public aMPIGeometry3d
     }
 
     private:
-    MPI_Comm get_perp_comm( MPI_Comm src)
+    MPI_Comm get_perp_comm( MPI_Comm src) const
     {
         MPI_Comm planeComm;
         int remain_dims[] = {true,true,false}; //true true false
@@ -202,6 +202,13 @@ struct CylindricalMPIGrid3d: public aMPIGeometry3d
         return CartesianMPIGrid2d( global().x0(), global().x1(), global().y0(), global().y1(), global().n(), global().Nx(), global().Ny(), global().bcx(), global().bcy(), get_perp_comm( communicator() ));
     }
     private:
+    MPI_Comm get_perp_comm( MPI_Comm src) const
+    {
+        MPI_Comm planeComm;
+        int remain_dims[] = {true,true,false}; //true true false
+        MPI_Cart_sub( src, remain_dims, &planeComm);
+        return planeComm;
+    }
     virtual SparseTensor<host_vector > do_compute_metric()const{
         SparseTensor<host_vector> metric(1);
         host_vector R = dg::evaluate(dg::cooX3d, *this);
-- 
GitLab


From ff463157fe318ccf23cbca0232d649021f20da46 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Thu, 28 Sep 2017 14:27:12 +0200
Subject: [PATCH 323/453] rewrite polar project for new backend

---
 inc/geometries/hector.h          | 20 +++++++++---------
 inc/geometries/mpi_curvilinear.h | 10 ++++-----
 inc/geometries/polar.h           | 31 ++++++++++++++-------------
 src/polar/ns.h                   | 36 ++++++++++++++++----------------
 src/polar/polar.cu               | 10 ++++-----
 src/polar/polar_mpi.cu           | 13 ++++--------
 6 files changed, 58 insertions(+), 62 deletions(-)

diff --git a/inc/geometries/hector.h b/inc/geometries/hector.h
index 29ed2759e..4c86d1ed5 100644
--- a/inc/geometries/hector.h
+++ b/inc/geometries/hector.h
@@ -310,7 +310,7 @@ struct Hector : public aGenerator2d
      *
      * @return  orthogonal zeta, eta grid
      */
-    const dg::CurvilinearGrid2d& internal_grid() const {return g2d_;}
+    const dg::geo::CurvilinearGrid2d& internal_grid() const {return g2d_;}
     virtual Hector* clone() const{return new Hector(*this);}
     bool isConformal() const {return conformal_;}
     private:
@@ -362,9 +362,9 @@ struct Hector : public aGenerator2d
     {
         //first find u( \zeta, \eta)
         double eps = 1e10, eps_old = 2e10;
-        dg::CurvilinearGrid2d g2d_old = g2d_;
+        dg::geo::CurvilinearGrid2d g2d_old = g2d_;
         container adapt = dg::pullback(chi, g2d_old);
-        dg::Elliptic<dg::CurvilinearGrid2d, Matrix, container> ellipticD_old( g2d_old, dg::DIR, dg::PER, dg::not_normed, dg::centered);
+        dg::Elliptic<dg::geo::CurvilinearGrid2d, Matrix, container> ellipticD_old( g2d_old, dg::DIR, dg::PER, dg::not_normed, dg::centered);
         ellipticD_old.set_chi( adapt);
 
         container u_old = dg::evaluate( dg::zero, g2d_old), u(u_old);
@@ -374,9 +374,9 @@ struct Hector : public aGenerator2d
         while( (eps < eps_old||eps > 1e-7) && eps > eps_u)
         {
             eps = eps_old;
-            g2d_.multiplyCellNumber(2,2);
+            g2d_.multiplyCellNumbers(2,2);
             if(verbose) std::cout << "Nx "<<Nx<<" Ny "<<Ny<<std::flush;
-            dg::Elliptic<dg::CurvilinearGrid2d, Matrix, container> ellipticD( g2d_, dg::DIR, dg::PER, dg::not_normed, dg::centered);
+            dg::Elliptic<dg::geo::CurvilinearGrid2d, Matrix, container> ellipticD( g2d_, dg::DIR, dg::PER, dg::not_normed, dg::centered);
             adapt = dg::pullback(chi, g2d_);
             ellipticD.set_chi( adapt);
             lapu = dg::pullback( lapChiPsi, g2d_);
@@ -404,8 +404,8 @@ struct Hector : public aGenerator2d
         dg::geo::detail::LaplaceChiPsi lapChiPsi( psi, chi);
         //first find u( \zeta, \eta)
         double eps = 1e10, eps_old = 2e10;
-        dg::CurvilinearGrid2d g2d_old = g2d_;
-        dg::TensorElliptic<dg::CurvilinearGrid2d, Matrix, container> ellipticD_old( g2d_old, dg::DIR, dg::PER, dg::not_normed, dg::centered);
+        dg::geo::CurvilinearGrid2d g2d_old = g2d_;
+        dg::TensorElliptic<dg::geo::CurvilinearGrid2d, Matrix, container> ellipticD_old( g2d_old, dg::DIR, dg::PER, dg::not_normed, dg::centered);
         ellipticD_old.transform_and_set( chi.xx(), chi.xy(), chi.yy());
 
         container u_old = dg::evaluate( dg::zero, g2d_old), u(u_old);
@@ -415,9 +415,9 @@ struct Hector : public aGenerator2d
         while( (eps < eps_old||eps > 1e-7) && eps > eps_u)
         {
             eps = eps_old;
-            g2d_.multiplyCellNumber(2,2);
+            g2d_.multiplyCellNumbers(2,2);
             if(verbose)std::cout << "Nx "<<Nx<<" Ny "<<Ny<<std::flush;
-            dg::TensorElliptic<dg::CurvilinearGrid2d, Matrix, container> ellipticD( g2d_, dg::DIR, dg::PER, dg::not_normed, dg::centered);
+            dg::TensorElliptic<dg::geo::CurvilinearGrid2d, Matrix, container> ellipticD( g2d_, dg::DIR, dg::PER, dg::not_normed, dg::centered);
             ellipticD.transform_and_set( chi.xx(), chi.xy(), chi.yy() );
             lapu = dg::pullback( lapChiPsi, g2d_);
             const container vol2d = dg::create::weights( g2d_);
@@ -509,7 +509,7 @@ struct Hector : public aGenerator2d
     double c0_, lu_;
     thrust::host_vector<double> u_, ux_, uy_, vx_, vy_;
     thrust::host_vector<double> etaV_, zetaU_, etaU_;
-    dg::CurvilinearGrid2d g2d_;
+    dg::geo::CurvilinearGrid2d g2d_;
 
 };
 
diff --git a/inc/geometries/mpi_curvilinear.h b/inc/geometries/mpi_curvilinear.h
index df5e9ac1a..1be6001fa 100644
--- a/inc/geometries/mpi_curvilinear.h
+++ b/inc/geometries/mpi_curvilinear.h
@@ -34,7 +34,7 @@ struct CurvilinearMPIGrid2d : public dg::aMPIGeometry2d
         dg::aMPIGeometry2d( 0, generator.width(), 0., generator.height(), n, Nx, Ny, bcx, bcy, comm2d), handle_(generator)
     {
         //generate global 2d grid and then reduce to local 
-        dg::CurvilinearGrid2d g(generator, n, Nx, Ny);
+        CurvilinearGrid2d g(generator, n, Nx, Ny);
         divide_and_conquer(g);
     }
     ///explicit conversion of 3d product grid to the perpendicular grid
@@ -47,7 +47,7 @@ struct CurvilinearMPIGrid2d : public dg::aMPIGeometry2d
     virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny)
     {
         dg::aMPITopology2d::do_set(new_n, new_Nx, new_Ny);
-        dg::CurvilinearGrid2d g( handle_.get(), new_n, new_Nx, new_Ny);
+        CurvilinearGrid2d g( handle_.get(), new_n, new_Nx, new_Ny);
         divide_and_conquer(g);//distribute to processes
     }
     MPI_Comm get_perp_comm( MPI_Comm src)
@@ -57,7 +57,7 @@ struct CurvilinearMPIGrid2d : public dg::aMPIGeometry2d
         MPI_Cart_sub( src, remain_dims, &planeComm);
         return planeComm;
     }
-    void divide_and_conquer(const dg::CurvilinearGrid2d& g_)
+    void divide_and_conquer(const CurvilinearGrid2d& g_)
     {
         dg::SparseTensor<thrust::host_vector<double> > jacobian=g_.jacobian(); 
         dg::SparseTensor<thrust::host_vector<double> > metric=g_.metric(); 
@@ -94,7 +94,7 @@ struct CurvilinearMPIGrid2d : public dg::aMPIGeometry2d
  */
 struct CurvilinearProductMPIGrid3d : public dg::aMPIGeometry3d
 {
-    typedef dg::CurvilinearMPIGrid2d perpendicular_grid; //!< the two-dimensional grid
+    typedef dg::geo::CurvilinearMPIGrid2d perpendicular_grid; //!< the two-dimensional grid
     /// @opydoc hide_grid_parameters3d
     /// @param comm a three-dimensional Cartesian communicator
     /// @note the paramateres given in the constructor are global parameters 
@@ -193,7 +193,7 @@ struct CurvilinearProductMPIGrid3d : public dg::aMPIGeometry3d
     virtual std::vector<host_vector > do_compute_map()const{return map_;}
     dg::SparseTensor<host_vector > jac_;
     std::vector<host_vector > map_;
-    Handle<dg::aGenerator2d> handle_;
+    Handle<dg::geo::aGenerator2d> handle_;
 };
 ///@cond
 CurvilinearMPIGrid2d::CurvilinearMPIGrid2d( const CurvilinearProductMPIGrid3d& g):
diff --git a/inc/geometries/polar.h b/inc/geometries/polar.h
index 08a557fac..ec3b355c0 100644
--- a/inc/geometries/polar.h
+++ b/inc/geometries/polar.h
@@ -1,9 +1,10 @@
 #pragma once
+#include "generator.h"
 
 namespace dg {
 namespace geo {
 
-struct PolarGenerator
+struct PolarGenerator : public aGenerator2d
 {
     private:
         double r_min, r_max;
@@ -11,8 +12,10 @@ struct PolarGenerator
     public:
 
     PolarGenerator(double _r_min, double _r_max) : r_min(_r_min), r_max(_r_max) {}
+    virtual PolarGenerator* clone() const{return new PolarGenerator(*this); }
 
-    void operator()( 
+    private:
+    void do_generate( 
          const thrust::host_vector<double>& zeta1d, 
          const thrust::host_vector<double>& eta1d, 
          thrust::host_vector<double>& x, 
@@ -20,7 +23,7 @@ struct PolarGenerator
          thrust::host_vector<double>& zetaX, 
          thrust::host_vector<double>& zetaY, 
          thrust::host_vector<double>& etaX, 
-         thrust::host_vector<double>& etaY) {
+         thrust::host_vector<double>& etaY) const {
 
         int size_r   = zeta1d.size();
         int size_phi = eta1d.size();
@@ -47,14 +50,13 @@ struct PolarGenerator
 
     }
    
-    double width() const{return r_max-r_min;}
-    double height() const{return 2*M_PI;}
-    bool isOrthogonal() const{return true;}
-    bool isConformal()  const{return false;}
+    double do_width() const{return r_max-r_min;}
+    double do_height() const{return 2*M_PI;}
+    bool do_isOrthogonal() const{return true;}
 };
 
 
-struct LogPolarGenerator
+struct LogPolarGenerator : public aGenerator2d
 {
     private:
         double r_min, r_max;
@@ -62,8 +64,10 @@ struct LogPolarGenerator
     public:
 
     LogPolarGenerator(double _r_min, double _r_max) : r_min(_r_min), r_max(_r_max) {}
+    virtual LogPolarGenerator* clone() const{return new LogPolarGenerator(*this); }
 
-    void operator()(
+    private:
+    void do_generate(
          const thrust::host_vector<double>& zeta1d,
          const thrust::host_vector<double>& eta1d,
          thrust::host_vector<double>& x,
@@ -71,7 +75,7 @@ struct LogPolarGenerator
          thrust::host_vector<double>& zetaX,
          thrust::host_vector<double>& zetaY,
          thrust::host_vector<double>& etaX,
-         thrust::host_vector<double>& etaY) {
+         thrust::host_vector<double>& etaY) const {
 
         int size_r   = zeta1d.size();
         int size_phi = eta1d.size();
@@ -98,10 +102,9 @@ struct LogPolarGenerator
 
     }
 
-    double width() const{return log(r_max)-log(r_min);}
-    double height() const{return 2*M_PI;}
-    bool isOrthogonal() const{return true;}
-    bool isConformal()  const{return true;}
+    double do_width() const{return log(r_max)-log(r_min);}
+    double do_height() const{return 2*M_PI;}
+    bool do_isOrthogonal() const{return true;}
 };
 
 }
diff --git a/src/polar/ns.h b/src/polar/ns.h
index c92b01b0f..0bfc2b3aa 100644
--- a/src/polar/ns.h
+++ b/src/polar/ns.h
@@ -8,13 +8,13 @@
 #include "dg/elliptic.h"
 #include "dg/cg.h"
 
-namespace dg
+namespace polar
 {
 template< class Geometry, class Matrix, class container>
 struct Diffusion
 {
     Diffusion( const Geometry& g, double nu): nu_(nu),
-        w2d( dg::create::weights( g) ), v2d( dg::create::inv_weights(g) ) ,
+        w2d( dg::create::volume( g) ), v2d( dg::create::inv_volume(g) ) ,
         LaplacianM( g, dg::normed)
     { 
     }
@@ -24,6 +24,7 @@ struct Diffusion
         dg::blas1::scal( y, -nu_);
     }
     const container& weights(){return w2d;}
+    const container& inv_weights(){return v2d;}
     const container& precond(){return v2d;}
   private:
     double nu_;
@@ -32,14 +33,14 @@ struct Diffusion
 };
 
 template< class Geometry, class Matrix, class container >
-struct Shu 
+struct Explicit 
 {
     typedef container Vector;
 
-    Shu( const Geometry& grid, double eps);
+    Explicit( const Geometry& grid, double eps);
 
-    const Elliptic<Matrix, container, container>& lap() const { return laplaceM;}
-    ArakawaX<Geometry, Matrix, container>& arakawa() {return arakawa_;}
+    const dg::Elliptic<Matrix, container, container>& lap() const { return laplaceM;}
+    dg::ArakawaX<Geometry, Matrix, container>& arakawa() {return arakawa_;}
     /**
      * @brief Returns psi that belong to the last y in operator()
      *
@@ -47,30 +48,29 @@ struct Shu
      * @return psi is the potential
      */
     const container& potential( ) {return psi;}
-    void operator()( Vector& y, Vector& yp);
+    void operator()( const Vector& y, Vector& yp);
   private:
-    container psi, w2d, v2d;
-    Elliptic<Geometry, Matrix, container> laplaceM;
-    ArakawaX<Geometry, Matrix, container> arakawa_; 
-    Invert<container> invert;
+    container psi;
+    dg::Elliptic<Geometry, Matrix, container> laplaceM;
+    dg::ArakawaX<Geometry, Matrix, container> arakawa_; 
+    dg::Invert<container> invert;
 };
 
 template<class Geometry, class Matrix, class container>
-Shu< Geometry, Matrix, container>::Shu( const Geometry& g, double eps): 
-    psi( evaluate(dg::zero, g) ),
-    w2d( create::weights( g)), v2d( create::inv_weights(g)),  
-    laplaceM( g, not_normed),
+Explicit< Geometry, Matrix, container>::Explicit( const Geometry& g, double eps): 
+    psi( dg::evaluate(dg::zero, g) ),
+    laplaceM( g, dg::not_normed),
     arakawa_( g), 
     invert( psi, g.size(), eps)
 {
 }
 
 template<class Geometry, class Matrix, class container>
-void Shu<Geometry, Matrix, container>::operator()( Vector& y, Vector& yp)
+void Explicit<Geometry, Matrix, container>::operator()( const Vector& y, Vector& yp)
 {
-    invert( laplaceM, psi, y, w2d, v2d);
+    invert( laplaceM, psi, y);
     arakawa_( y, psi, yp); //A(y,psi)-> yp
 }
 
-}//namespace dg
+}//namespace polar
 
diff --git a/src/polar/polar.cu b/src/polar/polar.cu
index 2fee8c9e4..8dd053b55 100644
--- a/src/polar/polar.cu
+++ b/src/polar/polar.cu
@@ -26,8 +26,6 @@
 using namespace std;
 using namespace dg;
 
-#define Grid OrthogonalGrid2d<DVec>
-
 #ifdef LOG_POLAR
     typedef dg::geo::LogPolarGenerator Generator;
 #else
@@ -60,9 +58,9 @@ int main(int argc, char* argv[])
 
     //Grid2d grid( 0, p.lx, 0, p.ly, p.n, p.Nx, p.Ny, p.bc_x, p.bc_y);
     Generator generator(p.r_min, p.r_max); // Generator is defined by the compiler
-    Grid grid( generator, p.n, p.Nx, p.Ny, dg::DIR); // second coordiante is periodic by default
+    dg::geo::CurvilinearGrid2d grid( generator, p.n, p.Nx, p.Ny, dg::DIR, dg::PER);
 
-    DVec w2d( create::weights(grid));
+    DVec w2d( create::volume(grid));
 
 #ifdef OPENGL_WINDOW
     //create CUDA context that uses OpenGL textures in Glfw window
@@ -81,8 +79,8 @@ int main(int argc, char* argv[])
     DVec y0( omega ), y1( y0);
 
     //make solver and stepper
-    Shu<Grid, DMatrix, DVec> shu( grid, p.eps);
-    Diffusion<Grid, DMatrix, DVec> diffusion( grid, p.nu);
+    polar::Explicit<aGeometry2d, DMatrix, DVec> shu( grid, p.eps);
+    polar::Diffusion<aGeometry2d, DMatrix, DVec> diffusion( grid, p.nu);
     Karniadakis< DVec > ab( y0, y0.size(), p.eps_time);
 
     t.tic();
diff --git a/src/polar/polar_mpi.cu b/src/polar/polar_mpi.cu
index cc1eed20f..4908d368b 100644
--- a/src/polar/polar_mpi.cu
+++ b/src/polar/polar_mpi.cu
@@ -22,11 +22,6 @@
 
 using namespace std;
 using namespace dg;
-const unsigned k = 3;
-
-// local container for the grid
-#define Grid OrthogonalMPIGrid2d<DVec>
-
 
 #ifdef LOG_POLAR
     typedef dg::geo::LogPolarGenerator Generator;
@@ -95,9 +90,9 @@ int main(int argc, char* argv[])
 
     //Grid2d grid( 0, p.lx, 0, p.ly, p.n, p.Nx, p.Ny, p.bc_x, p.bc_y);
     Generator generator(p.r_min, p.r_max); // Generator is defined by the compiler
-    Grid grid( generator, p.n, p.Nx, p.Ny, dg::DIR, comm); // second coordiante is periodic by default
+    dg::geo::CurvilinearMPIGrid2d grid( generator, p.n, p.Nx, p.Ny, dg::DIR, dg::PER, comm); 
 
-    MDVec w2d( create::weights(grid));
+    MDVec w2d( create::volume(grid));
 
     dg::Lamb lamb( p.posX, p.posY, p.R, p.U);
     MHVec omega = evaluate ( lamb, grid);
@@ -109,8 +104,8 @@ int main(int argc, char* argv[])
     MDVec y0( omega ), y1( y0);
 
     //make solver and stepper
-    Shu<Grid, MDMatrix, MDVec> shu( grid, p.eps);
-    Diffusion<Grid, MDMatrix, MDVec> diffusion( grid, p.nu);
+    polar::Explicit<aMPIGeometry2d, MDMatrix, MDVec> shu( grid, p.eps);
+    polar::Diffusion<aMPIGeometry2d, MDMatrix, MDVec> diffusion( grid, p.nu);
     Karniadakis< MDVec > ab( y0, y0.size(), p.eps_time);
 
     t.tic();
-- 
GitLab


From ae2d15d003b13850ad7a3d1c897bd47e8bff7065 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Thu, 28 Sep 2017 16:04:01 +0200
Subject: [PATCH 324/453] corrected namespace geo in geometries programs

---
 inc/geometries/conformal_elliptic_b.cu    | 12 ++++++------
 inc/geometries/ds_b.cu                    |  2 +-
 inc/geometries/flux_t.cu                  | 11 +++++------
 inc/geometries/geometry_elliptic_b.cu     |  6 +++---
 inc/geometries/geometry_elliptic_mpib.cu  |  6 +++---
 inc/geometries/hector_t.cu                |  4 ++--
 inc/geometries/mpi_fieldaligned.h         |  2 +-
 inc/geometries/ribeiroX_t.cu              |  6 +++---
 inc/geometries/ribeiro_mpit.cu            |  4 ++--
 inc/geometries/ribeiro_t.cu               |  4 ++--
 inc/geometries/separatrix_orthogonal_t.cu |  4 ++--
 inc/geometries/simple_orthogonal_t.cu     |  4 ++--
 12 files changed, 32 insertions(+), 33 deletions(-)

diff --git a/inc/geometries/conformal_elliptic_b.cu b/inc/geometries/conformal_elliptic_b.cu
index 88c5c4afd..0e867e26d 100644
--- a/inc/geometries/conformal_elliptic_b.cu
+++ b/inc/geometries/conformal_elliptic_b.cu
@@ -108,7 +108,7 @@ int main(int argc, char**argv)
     dg::geo::SimpleOrthogonal generator0(c.get_psip(), psi_0, psi_1, gp.R_0, 0., 0);
     for( unsigned i=0; i<nIter; i++)
     {
-        dg::CurvilinearGrid2d g2d(generator0, n, Nx, Ny);
+        dg::geo::CurvilinearGrid2d g2d(generator0, n, Nx, Ny);
         compute_error_elliptic(c, g2d, psi_0, psi_1, eps);
         compute_cellsize(g2d);
         std::cout <<std::endl;
@@ -120,7 +120,7 @@ int main(int argc, char**argv)
     dg::geo::SimpleOrthogonal generator1(c.get_psip(), psi_0, psi_1, gp.R_0, 0., 1);
     for( unsigned i=0; i<nIter; i++)
     {
-        dg::CurvilinearGrid2d g2d(generator1, n, Nx, Ny);
+        dg::geo::CurvilinearGrid2d g2d(generator1, n, Nx, Ny);
         compute_error_elliptic(c, g2d, psi_0, psi_1, eps);
         compute_cellsize(g2d);
         std::cout <<std::endl;
@@ -132,7 +132,7 @@ int main(int argc, char**argv)
     dg::geo::Hector<dg::IHMatrix, dg::HMatrix, dg::HVec> hectorConf( c.get_psip(), psi_0, psi_1, gp.R_0, 0., nGrid,NxGrid,NyGrid, 1e-10, true);
     for( unsigned i=0; i<nIter; i++)
     {
-        dg::CurvilinearGrid2d g2d(hectorConf, n, Nx, Ny);
+        dg::geo::CurvilinearGrid2d g2d(hectorConf, n, Nx, Ny);
         compute_error_elliptic(c, g2d, psi_0, psi_1,eps);
         compute_cellsize(g2d);
         std::cout <<std::endl;
@@ -145,7 +145,7 @@ int main(int argc, char**argv)
     dg::geo::Hector<dg::IHMatrix, dg::HMatrix, dg::HVec> hectorMonitor( c.get_psip(), lc, psi_0, psi_1, gp.R_0, 0., nGrid,NxGrid,NyGrid, 1e-10, true);
     for( unsigned i=0; i<nIter; i++)
     {
-        dg::CurvilinearGrid2d g2d(hectorMonitor, n, Nx, Ny);
+        dg::geo::CurvilinearGrid2d g2d(hectorMonitor, n, Nx, Ny);
         compute_error_elliptic(c, g2d, psi_0, psi_1,eps);
         compute_cellsize(g2d);
         std::cout <<std::endl;
@@ -158,7 +158,7 @@ int main(int argc, char**argv)
     dg::geo::Hector<dg::IHMatrix, dg::HMatrix, dg::HVec> hectorAdapt( c.get_psip(), nc, psi_0, psi_1, gp.R_0, 0., nGrid,NxGrid,NyGrid, 1e-10, true);
     for( unsigned i=0; i<nIter; i++)
     {
-        dg::CurvilinearGrid2d g2d(hectorAdapt, n, Nx, Ny);
+        dg::geo::CurvilinearGrid2d g2d(hectorAdapt, n, Nx, Ny);
         compute_error_elliptic(c, g2d, psi_0, psi_1,eps);
         compute_cellsize(g2d);
         std::cout <<std::endl;
@@ -170,7 +170,7 @@ int main(int argc, char**argv)
     dg::geo::Ribeiro ribeiro( c.get_psip(), psi_0, psi_1, gp.R_0, 0.);
     for( unsigned i=0; i<nIter; i++)
     {
-        dg::CurvilinearGrid2d g2d(ribeiro, n, Nx, Ny);
+        dg::geo::CurvilinearGrid2d g2d(ribeiro, n, Nx, Ny);
         compute_error_elliptic(c, g2d, psi_0, psi_1,eps);
         compute_cellsize(g2d);
         std::cout <<std::endl;
diff --git a/inc/geometries/ds_b.cu b/inc/geometries/ds_b.cu
index 1bf614496..74b392dea 100644
--- a/inc/geometries/ds_b.cu
+++ b/inc/geometries/ds_b.cu
@@ -93,7 +93,7 @@ int main()
     std::cin >> mx >> my;
 
     dg::geo::FluxGenerator flux( mag.get_psip(), mag.get_ipol(), psi_0, psi_1, gp.R_0, 0., 1);
-    dg::CurvilinearProductGrid3d g3d(flux, n, Nx, Ny,Nz, dg::DIR);
+    dg::geo::CurvilinearProductGrid3d g3d(flux, n, Nx, Ny,Nz, dg::DIR);
     //dg::FieldAligned<dg::aGeometry3d, dg::IHMatrix, dg::HVec> fieldaligned( bhat, g3d, 1, 4, gp.rk4eps, dg::NoLimiter() ); 
     dg::DS<dg::aGeometry3d, dg::IHMatrix, dg::HMatrix, dg::HVec> ds( mag, g3d, dg::normed, dg::centered, false, true, mx, my);
 
diff --git a/inc/geometries/flux_t.cu b/inc/geometries/flux_t.cu
index 6ef2612a3..9f4b3e8e2 100644
--- a/inc/geometries/flux_t.cu
+++ b/inc/geometries/flux_t.cu
@@ -62,8 +62,8 @@ int main( int argc, char* argv[])
     }
     //write parameters from file into variables
     dg::geo::solovev::GeomParameters gp(js);
-    Psip psip( gp); 
-    std::cout << "Psi min "<<psip(gp.R_0, 0)<<"\n";
+    dg::geo::TokamakMagneticField c = dg::geo::createSolovevField( gp);
+    std::cout << "Psi min "<<c.psip()(gp.R_0, 0)<<"\n";
     std::cout << "Type psi_0 (-20) and psi_1 (-4)\n";
     double psi_0, psi_1;
     std::cin >> psi_0>> psi_1;
@@ -72,10 +72,9 @@ int main( int argc, char* argv[])
     //solovev::detail::Fpsi fpsi( gp, -10);
     std::cout << "Constructing flux grid ... \n";
     t.tic();
-    dg::geo::TokamakMagneticField c = dg::geo::createSolovevField( gp);
     dg::geo::FluxGenerator flux( c.get_psip(), c.get_ipol(), psi_0, psi_1, gp.R_0, 0., 1);
-    dg::CurvilinearProductGrid3d g3d(flux, n, Nx, Ny,Nz, dg::DIR);
-    dg::CurvilinearGrid2d g2d(flux, n, Nx,Ny, dg::NEU);
+    dg::geo::CurvilinearProductGrid3d g3d(flux, n, Nx, Ny,Nz, dg::DIR);
+    dg::geo::CurvilinearGrid2d g2d(flux, n, Nx,Ny, dg::NEU);
     dg::Grid2d g2d_periodic(g2d.x0(), g2d.x1(), g2d.y0(), g2d.y1(), g2d.n(), g2d.Nx(), g2d.Ny()+1); 
     t.toc();
     std::cout << "Construction took "<<t.diff()<<"s"<<std::endl;
@@ -100,7 +99,7 @@ int main( int argc, char* argv[])
         err = nc_def_var( ncid, names[i].data(), NC_DOUBLE, 2, dim2d, &varID[i]);
     ///////////////////////now fill variables///////////////////
 
-    thrust::host_vector<double> psi_p = dg::pullback( psip, g2d);
+    thrust::host_vector<double> psi_p = dg::pullback( c.psip(), g2d);
     //g.display();
     err = nc_put_var_double( ncid, varID[0], periodify(psi_p, g2d_periodic).data());
     dg::HVec temp0( g2d.size()), temp1(temp0);
diff --git a/inc/geometries/geometry_elliptic_b.cu b/inc/geometries/geometry_elliptic_b.cu
index 758b1e189..f9215a46e 100644
--- a/inc/geometries/geometry_elliptic_b.cu
+++ b/inc/geometries/geometry_elliptic_b.cu
@@ -44,9 +44,9 @@ int main(int argc, char**argv)
     std::cout << "Constructing grid ... \n";
     t.tic();
     dg::geo::SimpleOrthogonal generator( c.get_psip(), psi_0, psi_1, gp.R_0, 0., 1);
-    dg::CurvilinearProductGrid3d g3d( generator, n, Nx, Ny,Nz, dg::DIR);
-    dg::CurvilinearGrid2d g2d = g3d.perp_grid();
-    dg::Elliptic<dg::CurvilinearGrid2d, dg::DMatrix, dg::DVec> pol( g2d, dg::not_normed, dg::forward);
+    dg::geo::CurvilinearProductGrid3d g3d( generator, n, Nx, Ny,Nz, dg::DIR);
+    dg::geo::CurvilinearGrid2d g2d = g3d.perp_grid();
+    dg::Elliptic<dg::geo::CurvilinearGrid2d, dg::DMatrix, dg::DVec> pol( g2d, dg::not_normed, dg::forward);
     t.toc();
     std::cout << "Construction took "<<t.diff()<<"s\n";
     ///////////////////////////////////////////////////////////////////////////
diff --git a/inc/geometries/geometry_elliptic_mpib.cu b/inc/geometries/geometry_elliptic_mpib.cu
index f774b71cc..5b0e16146 100644
--- a/inc/geometries/geometry_elliptic_mpib.cu
+++ b/inc/geometries/geometry_elliptic_mpib.cu
@@ -51,9 +51,9 @@ int main(int argc, char**argv)
     dg::Timer t;
     t.tic();
     dg::geo::SimpleOrthogonal generator( c.get_psip(), psi_0, psi_1, gp.R_0, 0., 1);
-    dg::CurvilinearProductMPIGrid3d g3d( generator, n, Nx, Ny,Nz, dg::DIR, dg::PER, dg::PER, comm);
-    dg::CurvilinearMPIGrid2d g2d = g3d.perp_grid();
-    dg::Elliptic<dg::CurvilinearMPIGrid2d, dg::MDMatrix, dg::MDVec> pol( g2d, dg::not_normed, dg::forward);
+    dg::geo::CurvilinearProductMPIGrid3d g3d( generator, n, Nx, Ny,Nz, dg::DIR, dg::PER, dg::PER, comm);
+    dg::geo::CurvilinearMPIGrid2d g2d = g3d.perp_grid();
+    dg::Elliptic<dg::geo::CurvilinearMPIGrid2d, dg::MDMatrix, dg::MDVec> pol( g2d, dg::not_normed, dg::forward);
     t.toc();
     if(rank==0)std::cout << "Construction took "<<t.diff()<<"s\n";
     ///////////////////////////////////////////////////////////////////////////
diff --git a/inc/geometries/hector_t.cu b/inc/geometries/hector_t.cu
index 75831e16e..552e4474f 100644
--- a/inc/geometries/hector_t.cu
+++ b/inc/geometries/hector_t.cu
@@ -91,8 +91,8 @@ int main( int argc, char* argv[])
         hector = new Hector<dg::IDMatrix, dg::DMatrix, dg::DVec>( psip,lc, psi_0, psi_1, gp.R_0, 0., nGrid, NxGrid, NyGrid, epsHector, true);
     }
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-    dg::CurvilinearProductGrid3d g3d(*hector, n, Nx, Ny,Nz, dg::DIR);
-    dg::CurvilinearGrid2d g2d = g3d.perp_grid();
+    dg::geo::CurvilinearProductGrid3d g3d(*hector, n, Nx, Ny,Nz, dg::DIR);
+    dg::geo::CurvilinearGrid2d g2d = g3d.perp_grid();
 
     dg::Grid2d g2d_periodic(g2d.x0(), g2d.x1(), g2d.y0(), g2d.y1(), g2d.n(), g2d.Nx(), g2d.Ny()+1); 
     t.toc();
diff --git a/inc/geometries/mpi_fieldaligned.h b/inc/geometries/mpi_fieldaligned.h
index 3706effff..b97dbd464 100644
--- a/inc/geometries/mpi_fieldaligned.h
+++ b/inc/geometries/mpi_fieldaligned.h
@@ -55,7 +55,7 @@ aGeometry2d* clone_MPI3d_to_global_perp( const aMPIGeometry3d* grid_ptr)
 {
     const dg::CartesianMPIGrid3d* grid_cart = dynamic_cast<const dg::CartesianMPIGrid3d*>(grid_ptr);
     const dg::CylindricalMPIGrid3d* grid_cyl = dynamic_cast<const dg::CylindricalMPIGrid3d*>(grid_ptr);
-    const dg::CurvilinearProductMPIGrid3d*  grid_curvi = dynamic_cast<const dg::CurvilinearProductMPIGrid3d*>(grid_ptr);
+    const dg::geo::CurvilinearProductMPIGrid3d*  grid_curvi = dynamic_cast<const dg::geo::CurvilinearProductMPIGrid3d*>(grid_ptr);
     aGeometry2d* g2d_ptr;
     if( grid_cart) 
     {
diff --git a/inc/geometries/ribeiroX_t.cu b/inc/geometries/ribeiroX_t.cu
index f4ea9c1ae..4b50edcd9 100644
--- a/inc/geometries/ribeiroX_t.cu
+++ b/inc/geometries/ribeiroX_t.cu
@@ -23,7 +23,7 @@ using namespace dg::geo;
 //typedef dg::FieldAligned< solovev::ConformalXGrid3d<dg::DVec> , dg::IDMatrix, dg::DVec> DFA;
 double sine( double x) {return sin(x);}
 double cosine( double x) {return cos(x);}
-typedef dg::FieldAligned< dg::CurvilinearGridX3d<dg::DVec> , dg::IDMatrix, dg::DVec> DFA;
+typedef dg::FieldAligned< dg::geo::CurvilinearGridX3d<dg::DVec> , dg::IDMatrix, dg::DVec> DFA;
 
 thrust::host_vector<double> periodify( const thrust::host_vector<double>& in, const dg::GridX3d& g)
 {
@@ -110,8 +110,8 @@ int main( int argc, char* argv[])
 
     double R0 = gp.R_0, Z0 = 0;
     dg::geo::RibeiroX<solovev::Psip,solovev::PsipR,solovev::PsipZ,solovev::PsipRR, solovev::PsipRZ, solovev::PsipZZ> generator(psip, psipR, psipZ, psipRR, psipRZ, psipZZ, psi_0, fx_0, R_X,Z_X, R0, Z0);
-    dg::CurvilinearGridX3d<dg::DVec> g3d(generator, psi_0, fx_0, fy_0, n, Nx, Ny,Nz, dg::DIR, dg::NEU);
-    dg::CurvilinearGridX2d<dg::DVec> g2d = g3d.perp_grid();
+    dg::geo::CurvilinearGridX3d<dg::DVec> g3d(generator, psi_0, fx_0, fy_0, n, Nx, Ny,Nz, dg::DIR, dg::NEU);
+    dg::geo::CurvilinearGridX2d<dg::DVec> g2d = g3d.perp_grid();
     t.toc();
     dg::GridX3d g3d_periodic(g3d.x0(), g3d.x1(), g3d.y0(), g3d.y1(), g3d.z0(), g3d.z1(), g3d.fx(), g3d.fy(), g3d.n(), g3d.Nx(), g3d.Ny(), 2); 
     std::cout << "Construction took "<<t.diff()<<"s"<<std::endl;
diff --git a/inc/geometries/ribeiro_mpit.cu b/inc/geometries/ribeiro_mpit.cu
index e14029d05..9cf2738c4 100644
--- a/inc/geometries/ribeiro_mpit.cu
+++ b/inc/geometries/ribeiro_mpit.cu
@@ -64,8 +64,8 @@ int main( int argc, char* argv[])
     if(rank==0)std::cout << "Constructing grid ... \n";
     t.tic();
     dg::geo::Ribeiro ribeiro( psip, psi_0, psi_1, gp.R_0, 0., 1);
-    dg::CurvilinearProductMPIGrid3d g3d(ribeiro, n, Nx, Ny,Nz, dg::DIR,dg::PER, dg::PER,comm);
-    dg::CurvilinearMPIGrid2d g2d = g3d.perp_grid();
+    dg::geo::CurvilinearProductMPIGrid3d g3d(ribeiro, n, Nx, Ny,Nz, dg::DIR,dg::PER, dg::PER,comm);
+    dg::geo::CurvilinearMPIGrid2d g2d = g3d.perp_grid();
     t.toc();
     if(rank==0)std::cout << "Construction took "<<t.diff()<<"s"<<std::endl;
     int ncid;
diff --git a/inc/geometries/ribeiro_t.cu b/inc/geometries/ribeiro_t.cu
index d6b4bfe55..add55358f 100644
--- a/inc/geometries/ribeiro_t.cu
+++ b/inc/geometries/ribeiro_t.cu
@@ -72,8 +72,8 @@ int main( int argc, char* argv[])
     std::cout << "Constructing ribeiro grid ... \n";
     t.tic();
     dg::geo::Ribeiro ribeiro( psip, psi_0, psi_1, gp.R_0, 0., 1);
-    dg::CurvilinearProductGrid3d g3d(ribeiro, n, Nx, Ny,Nz, dg::DIR);
-    dg::CurvilinearGrid2d g2d = g3d.perp_grid();
+    dg::geo::CurvilinearProductGrid3d g3d(ribeiro, n, Nx, Ny,Nz, dg::DIR);
+    dg::geo::CurvilinearGrid2d g2d = g3d.perp_grid();
     dg::Grid2d g2d_periodic(g2d.x0(), g2d.x1(), g2d.y0(), g2d.y1(), g2d.n(), g2d.Nx(), g2d.Ny()+1); 
     t.toc();
     std::cout << "Construction took "<<t.diff()<<"s"<<std::endl;
diff --git a/inc/geometries/separatrix_orthogonal_t.cu b/inc/geometries/separatrix_orthogonal_t.cu
index 8ee070c75..30b3702d5 100644
--- a/inc/geometries/separatrix_orthogonal_t.cu
+++ b/inc/geometries/separatrix_orthogonal_t.cu
@@ -123,8 +123,8 @@ int main( int argc, char* argv[])
     dg::geo::SeparatrixOrthogonal generator(c.get_psip(), psi_0, R_X,Z_X, R0, Z0,0);
     //dg::geo::SimpleOrthogonalX generator(c.get_psip(), psi_0, R_X,Z_X, R0, Z0,0);
     dg::EquidistXRefinement equi(add_x, add_y, 1,1);
-    dg::CurvilinearRefinedProductGridX3d g3d(equi, generator, fx_0, fy_0, n, Nx, Ny,Nz, dg::DIR, dg::NEU);
-    dg::CurvilinearRefinedGridX2d g2d(equi, generator, fx_0, fy_0, n, Nx, Ny,dg::DIR, dg::NEU);
+    dg::geo::CurvilinearRefinedProductGridX3d g3d(equi, generator, fx_0, fy_0, n, Nx, Ny,Nz, dg::DIR, dg::NEU);
+    dg::geo::CurvilinearRefinedGridX2d g2d(equi, generator, fx_0, fy_0, n, Nx, Ny,dg::DIR, dg::NEU);
     t.toc();
     dg::GridX3d g3d_periodic(g3d.x0(), g3d.x1(), g3d.y0(), g3d.y1(), g3d.z0(), g3d.z1(), g3d.fx(), g3d.fy(), g3d.n(), g3d.Nx(), g3d.Ny(), 2); 
     std::cout << "Construction took "<<t.diff()<<"s"<<std::endl;
diff --git a/inc/geometries/simple_orthogonal_t.cu b/inc/geometries/simple_orthogonal_t.cu
index 35aaedc51..013298512 100644
--- a/inc/geometries/simple_orthogonal_t.cu
+++ b/inc/geometries/simple_orthogonal_t.cu
@@ -74,8 +74,8 @@ int main( int argc, char* argv[])
     t.tic();
 
     dg::geo::SimpleOrthogonal generator( psip, psi_0, psi_1, gp.R_0, 0., 1);
-    dg::CurvilinearProductGrid3d g3d(generator, n, Nx, Ny,Nz, dg::DIR);
-    dg::CurvilinearGrid2d g2d = g3d.perp_grid();
+    dg::geo::CurvilinearProductGrid3d g3d(generator, n, Nx, Ny,Nz, dg::DIR);
+    dg::geo::CurvilinearGrid2d g2d = g3d.perp_grid();
     dg::Grid2d g2d_periodic(g2d.x0(), g2d.x1(), g2d.y0(), g2d.y1(), g2d.n(), g2d.Nx(), g2d.Ny()+1); 
     t.toc();
     std::cout << "Construction took "<<t.diff()<<"s"<<std::endl;
-- 
GitLab


From 45a822ee964c135e496fb03c1b1536fc44d03afe Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Thu, 28 Sep 2017 16:33:25 +0200
Subject: [PATCH 325/453] messed up fieldaligned, what is with hp and hm?

---
 inc/geometries/fieldaligned.h | 49 +++++++++++++++++++++--------------
 1 file changed, 30 insertions(+), 19 deletions(-)

diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index 52c340e34..08730571e 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -159,6 +159,33 @@ void clip_to_boundary( thrust::host_vector<double>& x, const aTopology2d* grid)
     clip_to_boundary(x[0], x[1], grid);
 }
 
+//grid2d_ptr is global grid topolgy
+void interpolate_and_clip( const aTopolgy2d* grid2d_ptr, unsigned mx, unsigned my, 
+        const std::vector<thrust::host_vector<double> >& yp_coarse,
+        const std::vector<thrust::host_vector<double> >& ym_coarse,
+        std::vector<thrust::host_vector<double> >& yp_,
+        std::vector<thrust::host_vector<double> >& ym_
+        )
+{
+    dg::Grid2d g2dFine((dg::Grid2d(*grid2d_ptr)));//FINE GRID
+    g2dFine.multiplyCellNumbers((double)mx, (double)my);
+
+    dg::IHMatrix interpolate = dg::create::interpolation( g2dFine, *grid2d_ptr);  //INTERPOLATE TO FINE GRID
+    std::vector<thrust::host_vector<double> > yp( 3, dg::evaluate(dg::zero, g2dFine)), ym(yp); 
+    for( unsigned i=0; i<3; i++)
+    {
+        dg::blas2::symv( interpolate, yp_coarse[i], yp[i]);
+        dg::blas2::symv( interpolate, ym_coarse[i], ym[i]);
+    }
+    for( unsigned i=0; i<yp[0].size(); i++)
+    {
+        g2dFine.shift_topologic( yp[0][i], yp[1][i], yp[0][i], yp[1][i]);
+        g2dFine.shift_topologic( ym[0][i], ym[1][i], ym[0][i], ym[1][i]);
+        detail::clip_to_boundary( yp[0][i], yp[1][i], &g2dFine);
+        detail::clip_to_boundary( ym[0][i], ym[1][i], &g2dFine);
+    }
+    yp_=yp, ym_=ym;
+}
 
 /**
  * @brief Integrate a field line to find whether the result lies inside or outside of the box
@@ -292,6 +319,7 @@ void integrate_all_fieldlines2d( const dg::geo::BinaryVectorLvl0& vec, const aGe
 
 aGeometry2d* clone_3d_to_perp( const aGeometry3d* grid_ptr)
 {
+    //%%%%%%%%%%%downcast grid since we don't have a virtual function perp_grid%%%%%%%%%%%%%
     const dg::CartesianGrid3d* grid_cart = dynamic_cast<const dg::CartesianGrid3d*>(grid_ptr);
     const dg::CylindricalGrid3d* grid_cyl = dynamic_cast<const dg::CylindricalGrid3d*>(grid_ptr);
     const dg::geo::CurvilinearProductGrid3d*  grid_curvi = dynamic_cast<const dg::geo::CurvilinearProductGrid3d*>(grid_ptr);
@@ -478,7 +506,6 @@ FieldAligned<Geometry, IMatrix, container>::FieldAligned(const dg::geo::BinaryVe
     if( deltaPhi <=0) deltaPhi = grid.hz();
     else assert( grid.Nz() == 1 || grid.hz()==deltaPhi);
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-    //%%%%%%%%%%%downcast grid since we don't have a virtual function perp_grid%%%%%%%%%%%%%
     const aGeometry2d* grid2d_ptr = detail::clone_3d_to_perp(&grid);
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     //Resize vector to 2D grid size
@@ -489,30 +516,14 @@ FieldAligned<Geometry, IMatrix, container>::FieldAligned(const dg::geo::BinaryVe
     //%%%%%%%%%%%%%%%%%%%%%%%%%%Set starting points and integrate field lines%%%%%%%%%%%%%%
     std::cout << "Start fieldline integration!\n";
     dg::Timer t;
-    std::vector<thrust::host_vector<double> > yp_coarse( 3), ym_coarse(yp_coarse); 
+    std::vector<thrust::host_vector<double> > yp_coarse( 3), ym_coarse(yp_coarse), yp, ym; 
     t.tic();
     
     dg::aGeometry2d* g2dField_ptr = grid2d_ptr->clone();//INTEGRATE HIGH ORDER GRID
     g2dField_ptr->set( 7, g2dField_ptr->Nx(), g2dField_ptr->Ny());
     detail::integrate_all_fieldlines2d( vec, g2dField_ptr, yp_coarse, ym_coarse, deltaPhi, eps);
     delete g2dField_ptr;
-
-    dg::Grid2d g2dFine((dg::Grid2d(*grid2d_ptr)));//FINE GRID
-    g2dFine.multiplyCellNumbers((double)mx, (double)my);
-    IMatrix interpolate = dg::create::interpolation( g2dFine, *grid2d_ptr);  //INTERPOLATE TO FINE GRID
-    std::vector<thrust::host_vector<double> > yp( 3, dg::evaluate(dg::zero, g2dFine)), ym(yp); 
-    for( unsigned i=0; i<3; i++)
-    {
-        dg::blas2::symv( interpolate, yp_coarse[i], yp[i]);
-        dg::blas2::symv( interpolate, ym_coarse[i], ym[i]);
-    }
-    for( unsigned i=0; i<yp[0].size(); i++)
-    {
-        g2dFine.shift_topologic( yp[0][i], yp[1][i], yp[0][i], yp[1][i]);
-        g2dFine.shift_topologic( ym[0][i], ym[1][i], ym[0][i], ym[1][i]);
-        detail::clip_to_boundary( yp[0][i], yp[1][i], &g2dFine);
-        detail::clip_to_boundary( ym[0][i], ym[1][i], &g2dFine);
-    }
+    interpolate_and_clip( grid2d_ptr, mx, my, yp_coarse, ym_coarse, yp, ym);
     t.toc(); 
     std::cout << "Fieldline integration took "<<t.diff()<<"s\n";
     //%%%%%%%%%%%%%%%%%%Create interpolation and projection%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-- 
GitLab


From 72d5e32b5e8b010aa7804a901d18dcf3fdbd1ad2 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Fri, 29 Sep 2017 12:00:53 +0200
Subject: [PATCH 326/453] added global_geometry member function in geometry
 base classes

---
 inc/dg/cg.h                       |  2 +-
 inc/dg/geometry/mpi_base.h        | 27 ++++++++++++++++++++++++
 inc/geometries/fieldaligned.h     | 23 +++++++++++++--------
 inc/geometries/mpi_curvilinear.h  | 14 ++++++++++++-
 inc/geometries/mpi_fieldaligned.h | 34 ++++++++-----------------------
 5 files changed, 63 insertions(+), 37 deletions(-)

diff --git a/inc/dg/cg.h b/inc/dg/cg.h
index 1d6260716..7fd78e5d5 100644
--- a/inc/dg/cg.h
+++ b/inc/dg/cg.h
@@ -256,7 +256,7 @@ can be used to get initial guesses based on past solutions
  where the indices indicate the current (0) and past (negative) solutions.
 *
 * @copydoc hide_container
-* @ingroup misc
+* @ingroup invert
 */
 template<class container>
 struct Extrapolation
diff --git a/inc/dg/geometry/mpi_base.h b/inc/dg/geometry/mpi_base.h
index 15d7b36a9..533dc6a86 100644
--- a/inc/dg/geometry/mpi_base.h
+++ b/inc/dg/geometry/mpi_base.h
@@ -29,6 +29,8 @@ struct aMPIGeometry2d : public aMPITopology2d
     }
     ///Geometries are cloneable
     virtual aMPIGeometry2d* clone()const=0;
+    ///Construct the global non-MPI geometry
+    virtual aGeometry2d* global_geometry()const =0;
     ///allow deletion through base class pointer
     virtual ~aMPIGeometry2d(){}
     protected:
@@ -78,6 +80,8 @@ struct aMPIGeometry3d : public aMPITopology3d
     }
     ///Geometries are cloneable
     virtual aMPIGeometry3d* clone()const=0;
+    ///Construct the global non-MPI geometry
+    virtual aGeometry3d* global_geometry()const =0;
     ///allow deletion through base class pointer
     virtual ~aMPIGeometry3d(){}
     protected:
@@ -129,6 +133,13 @@ struct CartesianMPIGrid2d : public aMPIGeometry2d
     ///@param g existing grid object
     CartesianMPIGrid2d( const dg::MPIGrid2d& g): aMPIGeometry2d( g.global().x0(),g.global().x1(),g.global().y0(),g.global().y1(),g.global().n(),g.global().Nx(),g.global().Ny(),g.global().bcx(),g.global().bcy(),g.communicator()){}
     virtual CartesianMPIGrid2d* clone()const{return new CartesianMPIGrid2d(*this);}
+    virtual CartesianGrid2d* global_geometry()const{
+        return new CartesianGrid2d( 
+                global().x0(), global().x1(), 
+                global().y0(), global().y1(), 
+                global().n(), global().Nx(), global().Ny(),  
+                global().bcx(), global().bcy());
+    }
     private:
     virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny){
         aMPITopology2d::do_set(new_n,new_Nx,new_Ny);
@@ -155,6 +166,14 @@ struct CartesianMPIGrid3d : public aMPIGeometry3d
     ///@param g existing grid object
     CartesianMPIGrid3d( const dg::MPIGrid3d& g): aMPIGeometry3d( g.global().x0(),g.global().x1(),g.global().y0(),g.global().y1(),g.global().z0(),g.global().z1(),g.global().n(),g.global().Nx(),g.global().Ny(),g.global().Nz(),g.global().bcx(),g.global().bcy(),g.global().bcz(),g.communicator()){}
     virtual CartesianMPIGrid3d* clone()const{return new CartesianMPIGrid3d(*this);}
+    virtual CartesianGrid3d* global_geometry()const{
+        return new CartesianGrid3d( 
+                global().x0(), global().x1(), 
+                global().y0(), global().y1(), 
+                global().z0(), global().z1(), 
+                global().n(), global().Nx(), global().Ny(), global().Nz(), 
+                global().bcx(), global().bcy(), global().bcz());
+    }
     /*!
      * @brief The grid made up by the first two dimensions in space and process topology
      *
@@ -197,6 +216,14 @@ struct CylindricalMPIGrid3d: public aMPIGeometry3d
     CylindricalMPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, MPI_Comm comm):aMPIGeometry3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, bcx, bcy, dg::PER, comm){}
 
     virtual CylindricalMPIGrid3d* clone()const{return new CylindricalMPIGrid3d(*this);}
+    virtual CylindricalGrid3d* global_geometry()const{
+        return new CylindricalGrid3d( 
+                global().x0(), global().x1(), 
+                global().y0(), global().y1(), 
+                global().z0(), global().z1(), 
+                global().n(), global().Nx(), global().Ny(), global().Nz(), 
+                global().bcx(), global().bcy(), global().bcz());
+    }
     ///@copydoc CartesianMPIGrid3d::perp_grid()const
     CartesianMPIGrid2d perp_grid()const{ 
         return CartesianMPIGrid2d( global().x0(), global().x1(), global().y0(), global().y1(), global().n(), global().Nx(), global().Ny(), global().bcx(), global().bcy(), get_perp_comm( communicator() ));
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index 08730571e..14cedaf59 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -8,6 +8,7 @@
 #include "dg/backend/interpolation.cuh"
 #include "dg/backend/projection.cuh"
 #include "dg/backend/functions.h"
+#include "dg/backend/typedefs.cuh"
 
 #include "dg/geometry/geometry.h"
 #include "dg/functors.h"
@@ -160,16 +161,13 @@ void clip_to_boundary( thrust::host_vector<double>& x, const aTopology2d* grid)
 }
 
 //grid2d_ptr is global grid topolgy
-void interpolate_and_clip( const aTopolgy2d* grid2d_ptr, unsigned mx, unsigned my, 
+void interpolate_and_clip( const aTopolgy2d* grid2d_ptr, const aTopology2d* g2dFine, 
         const std::vector<thrust::host_vector<double> >& yp_coarse,
         const std::vector<thrust::host_vector<double> >& ym_coarse,
         std::vector<thrust::host_vector<double> >& yp_,
         std::vector<thrust::host_vector<double> >& ym_
         )
 {
-    dg::Grid2d g2dFine((dg::Grid2d(*grid2d_ptr)));//FINE GRID
-    g2dFine.multiplyCellNumbers((double)mx, (double)my);
-
     dg::IHMatrix interpolate = dg::create::interpolation( g2dFine, *grid2d_ptr);  //INTERPOLATE TO FINE GRID
     std::vector<thrust::host_vector<double> > yp( 3, dg::evaluate(dg::zero, g2dFine)), ym(yp); 
     for( unsigned i=0; i<3; i++)
@@ -483,7 +481,7 @@ struct FieldAligned
     void eMinus(enum whichMatrix which, const container& in, container& out);
     typedef cusp::array1d_view< typename container::iterator> View;
     typedef cusp::array1d_view< typename container::const_iterator> cView;
-    IMatrix plus, minus, plusT, minusT; //interpolation matrices
+    IMatrix m_plus, m_minus, m_plusT, m_minusT; //interpolation matrices
     container hz_, hp_,hm_, ghostM, ghostP;
     unsigned Nz_, perp_size_;
     dg::bc bcz_;
@@ -523,14 +521,17 @@ FieldAligned<Geometry, IMatrix, container>::FieldAligned(const dg::geo::BinaryVe
     g2dField_ptr->set( 7, g2dField_ptr->Nx(), g2dField_ptr->Ny());
     detail::integrate_all_fieldlines2d( vec, g2dField_ptr, yp_coarse, ym_coarse, deltaPhi, eps);
     delete g2dField_ptr;
-    interpolate_and_clip( grid2d_ptr, mx, my, yp_coarse, ym_coarse, yp, ym);
+
+    dg::Grid2d g2dFine((dg::Grid2d(*grid2d_ptr)));//FINE GRID
+    g2dFine.multiplyCellNumbers((double)mx, (double)my);
+    interpolate_and_clip( grid2d_ptr, g2dFine, yp_coarse, ym_coarse, yp, ym);
     t.toc(); 
     std::cout << "Fieldline integration took "<<t.diff()<<"s\n";
     //%%%%%%%%%%%%%%%%%%Create interpolation and projection%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     t.tic();
-    IMatrix plusFine  = dg::create::interpolation( yp[0], yp[1], *grid2d_ptr, globalbcx, globalbcy);
-    IMatrix minusFine = dg::create::interpolation( ym[0], ym[1], *grid2d_ptr, globalbcx, globalbcy);
-    IMatrix projection = dg::create::projection( *grid2d_ptr, g2dFine);
+    dg::IHMatrix plusFine  = dg::create::interpolation( yp[0], yp[1], *grid2d_ptr, globalbcx, globalbcy), plus, plusT;
+    dg::IHMatrix minusFine = dg::create::interpolation( ym[0], ym[1], *grid2d_ptr, globalbcx, globalbcy), minus, minusT;
+    dg::IHMatrix projection = dg::create::projection( *grid2d_ptr, g2dFine);
     t.toc();
     std::cout <<"Creation of interpolation/projection took "<<t.diff()<<"s\n";
     t.tic();
@@ -541,6 +542,10 @@ FieldAligned<Geometry, IMatrix, container>::FieldAligned(const dg::geo::BinaryVe
     //%Transposed matrices work only for csr_matrix due to bad matrix form for ell_matrix!!!
     cusp::transpose( plus, plusT);
     cusp::transpose( minus, minusT);     
+    dg::blas2::transfer( plus, m_plus);
+    dg::blas2::transfer( plusT, m_plusT);
+    dg::blas2::transfer( minus, m_minus);
+    dg::blas2::transfer( minusT, m_minusT);
     //%%%%%%%%%%%%%%%%%%%%%%%project h and copy into h vectors%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     thrust::host_vector<double> hp( perp_size_), hm(hp);
     dg::blas2::symv( projection, yp[2], hp);
diff --git a/inc/geometries/mpi_curvilinear.h b/inc/geometries/mpi_curvilinear.h
index 1be6001fa..fa1d73540 100644
--- a/inc/geometries/mpi_curvilinear.h
+++ b/inc/geometries/mpi_curvilinear.h
@@ -43,6 +43,12 @@ struct CurvilinearMPIGrid2d : public dg::aMPIGeometry2d
     ///read access to the generator 
     const aGenerator2d& generator() const{return handle_.get();}
     virtual CurvilinearMPIGrid2d* clone()const{return new CurvilinearMPIGrid2d(*this);}
+    virtual CurvilinearGrid2d* global_geometry()const{
+        return new CurvilinearGrid2d( 
+                handle_.get(),
+                global().n(), global().Nx(), global().Ny(),
+                global().bcx(), global().bcy());
+    }
     private:
     virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny)
     {
@@ -119,6 +125,12 @@ struct CurvilinearProductMPIGrid3d : public dg::aMPIGeometry3d
     ///read access to the generator
     const aGenerator2d& generator() const{return handle_.get();}
     virtual CurvilinearProductMPIGrid3d* clone()const{return new CurvilinearProductMPIGrid3d(*this);}
+    virtual CurvilinearProductGrid3d* global_geometry()const{
+        return new CurvilinearProductGrid3d( 
+                handle_.get(),
+                global().n(), global().Nx(), global().Ny(), global().Nz(), 
+                global().bcx(), global().bcy(), global().bcz());
+    }
     private:
     MPI_Comm get_perp_comm( MPI_Comm src)
     {
@@ -130,7 +142,7 @@ struct CurvilinearProductMPIGrid3d : public dg::aMPIGeometry3d
     virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz)
     {
         dg::aMPITopology3d::do_set(new_n, new_Nx, new_Ny, new_Nz);
-        if( !( new_n == n() && new_Nx == Nx() && new_Ny == Ny() ) )
+        if( !( new_n == n() && new_Nx == global().Nx() && new_Ny == global().Ny() ) )
         {
             CurvilinearMPIGrid2d g(handle_.get(),new_n,new_Nx,new_Ny, this->bcx(), this->bcy(), get_perp_comm(communicator()));
             constructPerp( g);
diff --git a/inc/geometries/mpi_fieldaligned.h b/inc/geometries/mpi_fieldaligned.h
index b97dbd464..f9cf4d2b5 100644
--- a/inc/geometries/mpi_fieldaligned.h
+++ b/inc/geometries/mpi_fieldaligned.h
@@ -53,6 +53,7 @@ void sendBackward( InputIterator begin, InputIterator end, OutputIterator result
 
 aGeometry2d* clone_MPI3d_to_global_perp( const aMPIGeometry3d* grid_ptr)
 {
+    //%%%%%%%%%%%downcast grid since we don't have a virtual function perp_grid%%%%%%%%%%%%%
     const dg::CartesianMPIGrid3d* grid_cart = dynamic_cast<const dg::CartesianMPIGrid3d*>(grid_ptr);
     const dg::CylindricalMPIGrid3d* grid_cyl = dynamic_cast<const dg::CylindricalMPIGrid3d*>(grid_ptr);
     const dg::geo::CurvilinearProductMPIGrid3d*  grid_curvi = dynamic_cast<const dg::geo::CurvilinearProductMPIGrid3d*>(grid_ptr);
@@ -146,8 +147,8 @@ struct FieldAligned< Geometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vec
     LocalContainer limiter_;
     std::vector<LocalContainer> tempXYplus_, tempXYminus_, temp_; 
     CommunicatorXY commXYplus_, commXYminus_;
-    LocalIMatrix plus, minus; //interpolation matrices
-    LocalIMatrix plusT, minusT; //interpolation matrices
+    LocalIMatrix m_plus, m_minus; //interpolation matrices
+    LocalIMatrix m_plusT, m_minusT; //interpolation matrices
 };
 //////////////////////////////////////DEFINITIONS/////////////////////////////////////
 template<class MPIGeometry, class LocalIMatrix, class CommunicatorXY, class LocalContainer>
@@ -161,8 +162,7 @@ FieldAligned<MPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vector<L
     if( deltaPhi <=0) deltaPhi = grid.hz();
     else assert( grid.Nz() == 1 || grid.hz()==deltaPhi);
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-    //%%%%%%%%%%%downcast grid since we don't have a virtual function perp_grid%%%%%%%%%%%%%
-    const aGeometry2d* g2d_ptr = detail::clone_MPI3d_to_global_perp(&grid);
+    const aGeometry2d* grid2d_ptr = detail::clone_MPI3d_to_global_perp(&grid);
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     unsigned localsize = grid2d_ptr->size();
     limiter_ = dg::evaluate( limit, grid2d_ptr->local());
@@ -172,21 +172,13 @@ FieldAligned<MPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vector<L
     std::vector<thrust::host_vector<double> > yp_coarse( 3), ym_coarse(yp_coarse); 
     
     dg::aGeometry2d* g2dField_ptr = grid2d_ptr->clone();//INTEGRATE HIGH ORDER GRID
-    g2dField_ptr->set( 7, g2dField_ptr->Nx(), g2dField_ptr->Ny());
+    g2dField_ptr->set( 7, g2dField_ptr->global().Nx(), g2dField_ptr->global().Ny());
     detail::integrate_all_fieldlines2d( vec, g2dField_ptr, yp_coarse, ym_coarse, deltaPhi, eps);
     delete g2dField_ptr;
 
-    //determine pid of result 
-    thrust::host_vector<int> pids( localsize);
-    for( unsigned i=0; i<localsize; i++)
-    {
-        pids[i]  = grid2d_ptr.pidOf( yp[0][i], yp[1][i]);
-        if( pids[i]  == -1)
-        {
-            std::cerr << "ERROR: PID NOT FOUND!\n";
-            return;
-        }
-    }
+    dg::Grid2d g2dFine((dg::Grid2d(*grid2d_ptr)));//FINE GRID
+    g2dFine.multiplyCellNumbers((double)mx, (double)my);
+    interpolate_and_clip( grid2d_ptr, g2dFine, yp_coarse, ym_coarse, yp, ym);
 
     CommunicatorXY cp( pids, grid2d_ptr.communicator());
     commXYplus_ = cp;
@@ -198,16 +190,6 @@ FieldAligned<MPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vector<L
     plus = dg::create::interpolation( pX, pY, grid2d_ptr.local(), globalbcz); //inner points hopefully never lie exactly on local boundary
     cusp::transpose( plus, plusT);
 
-    //do the same for the minus z-plane
-    for( unsigned i=0; i<pids.size(); i++)
-    {
-        pids[i]  = grid2d_ptr.pidOf( ym[0][i], ym[1][i]);
-        if( pids[i] == -1)
-        {
-            std::cerr << "ERROR: PID NOT FOUND!\n";
-            return;
-        }
-    }
     CommunicatorXY cm( pids, grid2d_ptr.communicator());
     commXYminus_ = cm;
     dg::blas1::transfer( cm.global_gather( ym[0]), pX);
-- 
GitLab


From e0ab61f9b43889348609b49e140b7984c6bd0e79 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Fri, 29 Sep 2017 14:16:13 +0200
Subject: [PATCH 327/453] work on unificating fieldaligned and mpi_fieldaligned

---
 inc/geometries/fieldaligned.h     | 48 ++++++++++-----------
 inc/geometries/mpi_fieldaligned.h | 71 +++++++++++++++++++++----------
 2 files changed, 72 insertions(+), 47 deletions(-)

diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index 14cedaf59..7d77dd06f 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -160,15 +160,13 @@ void clip_to_boundary( thrust::host_vector<double>& x, const aTopology2d* grid)
     clip_to_boundary(x[0], x[1], grid);
 }
 
-//grid2d_ptr is global grid topolgy
-void interpolate_and_clip( const aTopolgy2d* grid2d_ptr, const aTopology2d* g2dFine, 
+void interpolate_and_clip( const dg::IHMatrix& interpolate, const aTopolgy2d* boundary_ptr,
         const std::vector<thrust::host_vector<double> >& yp_coarse,
         const std::vector<thrust::host_vector<double> >& ym_coarse,
         std::vector<thrust::host_vector<double> >& yp_,
         std::vector<thrust::host_vector<double> >& ym_
         )
 {
-    dg::IHMatrix interpolate = dg::create::interpolation( g2dFine, *grid2d_ptr);  //INTERPOLATE TO FINE GRID
     std::vector<thrust::host_vector<double> > yp( 3, dg::evaluate(dg::zero, g2dFine)), ym(yp); 
     for( unsigned i=0; i<3; i++)
     {
@@ -177,10 +175,10 @@ void interpolate_and_clip( const aTopolgy2d* grid2d_ptr, const aTopology2d* g2dF
     }
     for( unsigned i=0; i<yp[0].size(); i++)
     {
-        g2dFine.shift_topologic( yp[0][i], yp[1][i], yp[0][i], yp[1][i]);
-        g2dFine.shift_topologic( ym[0][i], ym[1][i], ym[0][i], ym[1][i]);
-        detail::clip_to_boundary( yp[0][i], yp[1][i], &g2dFine);
-        detail::clip_to_boundary( ym[0][i], ym[1][i], &g2dFine);
+        boundary_ptr->shift_topologic( yp[0][i], yp[1][i], yp[0][i], yp[1][i]);
+        boundary_ptr->shift_topologic( ym[0][i], ym[1][i], ym[0][i], ym[1][i]);
+        detail::clip_to_boundary( yp[0][i], yp[1][i], boundary_ptr);
+        detail::clip_to_boundary( ym[0][i], ym[1][i], boundary_ptr);
     }
     yp_=yp, ym_=ym;
 }
@@ -188,9 +186,9 @@ void interpolate_and_clip( const aTopolgy2d* grid2d_ptr, const aTopology2d* g2dF
 /**
  * @brief Integrate a field line to find whether the result lies inside or outside of the box
  * @tparam Field Must be usable in the integrateRK() functions
- * @tparam Grid must provide 2d contains function
+ * @tparam Topology must provide 2d contains function
  */
-template < class Field, class Grid>
+template < class Field, class Topology>
 struct BoxIntegrator
 {
     /**
@@ -200,7 +198,7 @@ struct BoxIntegrator
      * @param g The 2d or 3d grid
      * @param eps the accuracy of the runge kutta integrator
      */
-    BoxIntegrator( const Field& field, const Grid& g, double eps): field_(field), g_(g), coords_(3), coordsp_(3), eps_(eps) {}
+    BoxIntegrator( const Field& field, const Topology& g, double eps): field_(field), g_(g), coords_(3), coordsp_(3), eps_(eps) {}
     /**
      * @brief Set the starting coordinates for next field line integration
      * @param coords the new coords (must have size = 3)
@@ -220,7 +218,7 @@ struct BoxIntegrator
     }
     private:
     const Field& field_;
-    const Grid& g_;
+    const Topology& g_;
     thrust::host_vector<double> coords_, coordsp_;
     double eps_;
 };
@@ -229,7 +227,7 @@ struct BoxIntegrator
  * @brief Integrate one field line in a given box, Result is guaranteed to lie inside the box modulo periodic boundary conditions
  *
  * @tparam Field Must be usable in the integrateRK function
- * @tparam Grid must provide 2d contains function
+ * @tparam Topology must provide 2d contains and shift_topologic function
  * @param field The field to use
  * @param grid instance of the Grid class 
  * @param coords0 The initial condition
@@ -237,8 +235,8 @@ struct BoxIntegrator
  * @param phi1 The angle (read/write) contains maximum phi on input and resulting phi on output
  * @param eps error
  */
-template< class Field, class Grid>
-void boxintegrator( const Field& field, const Grid& grid, 
+template< class Field, class Topology>
+void boxintegrator( const Field& field, const Topology& grid, 
         const thrust::host_vector<double>& coords0, 
         thrust::host_vector<double>& coords1, 
         double& phi1, double eps)
@@ -253,7 +251,7 @@ void boxintegrator( const Field& field, const Grid& grid,
         std::cerr << "point "<<coords1[0]<<" "<<coords1[1]<<" is somewhere else!\n";
 #endif //DG_DEBUG
         double deltaPhi = phi1;
-        BoxIntegrator<Field, Grid> boxy( field, grid, eps);//stores references to field and grid
+        BoxIntegrator<Field, Topology> boxy( field, grid, eps);//stores references to field and grid
         boxy.set_coords( coords0); //nimm alte koordinaten
         if( phi1 > 0)
         {
@@ -279,11 +277,13 @@ void boxintegrator( const Field& field, const Grid& grid,
 }
 
 //used in constructor of FieldAligned
-void integrate_all_fieldlines2d( const dg::geo::BinaryVectorLvl0& vec, const aGeometry2d* g2dField_ptr, std::vector<thrust::host_vector<double> >& yp_result, std::vector<thrust::host_vector<double> >& ym_result , double deltaPhi, double eps)
+void integrate_all_fieldlines2d( const dg::geo::BinaryVectorLvl0& vec, const aGeometry2d* g2dField_ptr, const aTopology2d* evaluate_ptr, std::vector<thrust::host_vector<double> >& yp_result, std::vector<thrust::host_vector<double> >& ym_result , double deltaPhi, double eps)
 {
-    std::vector<thrust::host_vector<double> > y( 3, dg::evaluate( dg::cooX2d, *g2dField_ptr)); //x
-    y[1] = dg::evaluate( dg::cooY2d, *g2dField_ptr); //y
-    y[2] = dg::evaluate( dg::zero, *g2dField_ptr); //s
+    //g2dField contains the global geometry 
+    //evaluate_ptr contains the points to actually integrate
+    std::vector<thrust::host_vector<double> > y( 3, dg::evaluate( dg::cooX2d, *evaluate_ptr)); //x
+    y[1] = dg::evaluate( dg::cooY2d, *evaluate_ptr); //y
+    y[2] = dg::evaluate( dg::zero, *evaluate_ptr); //s
     std::vector<thrust::host_vector<double> > yp( 3, y[0]), ym(yp); 
     //construct field on high polynomial grid, then integrate it
     Timer t;
@@ -312,7 +312,6 @@ void integrate_all_fieldlines2d( const dg::geo::BinaryVectorLvl0& vec, const aGe
     }
     yp_result=yp;
     ym_result=ym;
-    delete g2dField_ptr;
 }
 
 aGeometry2d* clone_3d_to_perp( const aGeometry3d* grid_ptr)
@@ -508,7 +507,7 @@ FieldAligned<Geometry, IMatrix, container>::FieldAligned(const dg::geo::BinaryVe
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     //Resize vector to 2D grid size
     perp_size_ = grid2d_ptr->size();
-    limiter_ = dg::evaluate( limit, *grid2d_ptr);
+    limiter_ = dg::pullback( limit, *grid2d_ptr);
     right_ = left_ = dg::evaluate( zero, *grid2d_ptr);
     ghostM.resize( perp_size_); ghostP.resize( perp_size_);
     //%%%%%%%%%%%%%%%%%%%%%%%%%%Set starting points and integrate field lines%%%%%%%%%%%%%%
@@ -519,12 +518,13 @@ FieldAligned<Geometry, IMatrix, container>::FieldAligned(const dg::geo::BinaryVe
     
     dg::aGeometry2d* g2dField_ptr = grid2d_ptr->clone();//INTEGRATE HIGH ORDER GRID
     g2dField_ptr->set( 7, g2dField_ptr->Nx(), g2dField_ptr->Ny());
-    detail::integrate_all_fieldlines2d( vec, g2dField_ptr, yp_coarse, ym_coarse, deltaPhi, eps);
-    delete g2dField_ptr;
+    detail::integrate_all_fieldlines2d( vec, g2dField_ptr, g2dField_ptr, yp_coarse, ym_coarse, deltaPhi, eps);
 
     dg::Grid2d g2dFine((dg::Grid2d(*grid2d_ptr)));//FINE GRID
     g2dFine.multiplyCellNumbers((double)mx, (double)my);
-    interpolate_and_clip( grid2d_ptr, g2dFine, yp_coarse, ym_coarse, yp, ym);
+    dg::IHMatrix interpolate = dg::create::interpolation( g2dFine, *g2dField_ptr);  //INTERPOLATE TO FINE GRID
+    interpolate_and_clip( interpolate, grid2d_ptr, yp_coarse, ym_coarse, yp, ym);
+    delete g2dField_ptr;
     t.toc(); 
     std::cout << "Fieldline integration took "<<t.diff()<<"s\n";
     //%%%%%%%%%%%%%%%%%%Create interpolation and projection%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
diff --git a/inc/geometries/mpi_fieldaligned.h b/inc/geometries/mpi_fieldaligned.h
index f9cf4d2b5..c5ccb6f20 100644
--- a/inc/geometries/mpi_fieldaligned.h
+++ b/inc/geometries/mpi_fieldaligned.h
@@ -51,30 +51,27 @@ void sendBackward( InputIterator begin, InputIterator end, OutputIterator result
                     comm, &status);
 }
 
-aGeometry2d* clone_MPI3d_to_global_perp( const aMPIGeometry3d* grid_ptr)
+aMPIGeometry2d* clone_MPI3d_to_perp( const aMPIGeometry3d* grid_ptr)
 {
     //%%%%%%%%%%%downcast grid since we don't have a virtual function perp_grid%%%%%%%%%%%%%
     const dg::CartesianMPIGrid3d* grid_cart = dynamic_cast<const dg::CartesianMPIGrid3d*>(grid_ptr);
     const dg::CylindricalMPIGrid3d* grid_cyl = dynamic_cast<const dg::CylindricalMPIGrid3d*>(grid_ptr);
     const dg::geo::CurvilinearProductMPIGrid3d*  grid_curvi = dynamic_cast<const dg::geo::CurvilinearProductMPIGrid3d*>(grid_ptr);
-    aGeometry2d* g2d_ptr;
+    aMPIGeometry2d* g2d_ptr;
     if( grid_cart) 
     {
         dg::CartesianMPIGrid2d cart = grid_cart->perp_grid();
-        dg::CartesianGrid2d global_cart( cart.global());
-        g2d_ptr = global_cart.clone();
+        g2d_ptr = cart.clone();
     }
     else if( grid_cyl) 
     {
         dg::CartesianMPIGrid2d cart = grid_cyl->perp_grid();
-        dg::CartesianGrid2d global_cart( cart.global());
-        g2d_ptr = global_cart.clone();
+        g2d_ptr = cart.clone();
     }
     else if( grid_curvi) 
     {
         dg::geo::CurvilinearMPIGrid2d curv = grid_curvi->perp_grid();
-        dg::geo::CurvilinearGrid2d global_curv( curv.generator(), curv.global().n(), curv.global().Nx(), curv.global().Ny(), curv.bcx(), curv.bcy());
-        g2d_ptr = global_curv.clone();
+        g2d_ptr = curv.clone();
     }
     else
         throw dg::Error( dg::Message(_ping_)<<"Grid class not recognized!");
@@ -162,23 +159,59 @@ FieldAligned<MPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vector<L
     if( deltaPhi <=0) deltaPhi = grid.hz();
     else assert( grid.Nz() == 1 || grid.hz()==deltaPhi);
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-    const aGeometry2d* grid2d_ptr = detail::clone_MPI3d_to_global_perp(&grid);
+    const aMPIGeometry2d* grid2d_ptr = detail::clone_MPI3d_to_global_perp(&grid);
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     unsigned localsize = grid2d_ptr->size();
-    limiter_ = dg::evaluate( limit, grid2d_ptr->local());
+    MPI_Vector<dg::HVec> temp = dg::pullback( limit, *grid2d_ptr);
+    dg::blas1::transfer( temp.data(), limiter_);
     right_ = left_ = dg::evaluate( zero, grid2d_ptr->local());
     ghostM.resize( localsize); ghostP.resize( localsize);
     //%%%%%%%%%%%%%%%%%%%%%%%%%%Set starting points and integrate field lines%%%%%%%%%%%%%%
     std::vector<thrust::host_vector<double> > yp_coarse( 3), ym_coarse(yp_coarse); 
     
-    dg::aGeometry2d* g2dField_ptr = grid2d_ptr->clone();//INTEGRATE HIGH ORDER GRID
+    dg::aMPIGeometry2d* g2dField_ptr = grid2d_ptr->clone();//INTEGRATE HIGH ORDER GRID
     g2dField_ptr->set( 7, g2dField_ptr->global().Nx(), g2dField_ptr->global().Ny());
-    detail::integrate_all_fieldlines2d( vec, g2dField_ptr, yp_coarse, ym_coarse, deltaPhi, eps);
-    delete g2dField_ptr;
+    dg::aGeometry2d* global_g2dField_ptr = g2dField_ptr->global_grid();
+    dg::aGrid2d local_g2dField = g2dField_ptr->local();
+    detail::integrate_all_fieldlines2d( vec, global_g2dField_ptr, &local_g2dField, yp_coarse, ym_coarse, deltaPhi, eps);
 
-    dg::Grid2d g2dFine((dg::Grid2d(*grid2d_ptr)));//FINE GRID
+    dg::MPIGrid2d g2dFine((dg::MPIGrid2d(*grid2d_ptr)));//FINE GRID
     g2dFine.multiplyCellNumbers((double)mx, (double)my);
-    interpolate_and_clip( grid2d_ptr, g2dFine, yp_coarse, ym_coarse, yp, ym);
+    dg::IHMatrix interpolate = dg::create::interpolation( g2dFine.local(), local_g2dField);  //INTERPOLATE TO FINE GRID
+    interpolate_and_clip( interpolate, global_g2dField_ptr, yp_coarse, ym_coarse, yp, ym);
+    delete g2dField_ptr;
+    delete global_g2dField_ptr;
+    //%%%%%%%%%%%%%%%%%%Create interpolation and projection%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+    t.tic();
+    dg::IHMatrix plusFine  = dg::create::interpolation( yp[0], yp[1], grid2d_ptr->global(), globalbcx, globalbcy), plus, plusT;
+    dg::IHMatrix minusFine = dg::create::interpolation( ym[0], ym[1], grid2d_ptr->global(), globalbcx, globalbcy), minus, minusT;
+    dg::IHMatrix projection = dg::create::projection( grid2d_ptr->local(), g2dFine->local());
+    t.toc();
+    std::cout <<"Creation of interpolation/projection took "<<t.diff()<<"s\n";
+    t.tic();
+    cusp::multiply( projection, plusFine, plus);
+    cusp::multiply( projection, minusFine, minus);
+    t.toc();
+    std::cout<< "Multiplication of P*I took: "<<t.diff()<<"s\n";
+    //%Transposed matrices work only for csr_matrix due to bad matrix form for ell_matrix!!!
+    cusp::transpose( plus, plusT);
+    cusp::transpose( minus, minusT);     
+    dg::blas2::transfer( plus, m_plus);
+    dg::blas2::transfer( plusT, m_plusT);
+    dg::blas2::transfer( minus, m_minus);
+    dg::blas2::transfer( minusT, m_minusT);
+    //%%%%%%%%%%%%%%%%%%%%%%%project h and copy into h vectors%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+    thrust::host_vector<double> hp( perp_size_), hm(hp);
+    dg::blas2::symv( projection, yp[2], hp);
+    dg::blas2::symv( projection, ym[2], hm);
+    for( unsigned i=0; i<grid.Nz(); i++)
+    {
+        thrust::copy( hp.begin(), hp.end(), hp_.data().begin() + i*localsize_);
+        thrust::copy( hm.begin(), hm.end(), hm_.data().begin() + i*localsize_);
+    }
+    dg::blas1::scal( hm_, -1.);
+    dg::blas1::axpby(  1., hp_, +1., hm_, hz_);
+    delete grid2d_ptr;
 
     CommunicatorXY cp( pids, grid2d_ptr.communicator());
     commXYplus_ = cp;
@@ -196,14 +229,6 @@ FieldAligned<MPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vector<L
     dg::blas1::transfer( cm.global_gather( ym[1]), pY);
     minus = dg::create::interpolation( pX, pY, grid2d_ptr.local(), globalbcz); //inner points hopefully never lie exactly on local boundary
     cusp::transpose( minus, minusT);
-    //copy to device
-    for( unsigned i=0; i<g_.Nz(); i++)
-    {
-        thrust::copy( yp[2].begin(), yp[2].end(), hp_.data().begin() + i*localsize);
-        thrust::copy( ym[2].begin(), ym[2].end(), hm_.data().begin() + i*localsize);
-    }
-    dg::blas1::scal( hm_, -1.);
-    dg::blas1::axpby(  1., hp_, +1., hm_, hz_);
     for( unsigned i=0; i<g_.Nz(); i++)
     {
         tempXYplus_[i].resize( commXYplus_.size());
-- 
GitLab


From 66b8b01f63b820af9cec9bd37e850ed1b68ea191 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Fri, 29 Sep 2017 15:59:20 +0200
Subject: [PATCH 328/453] start to make aProductGeometry3d

---
 inc/dg/geometry/base_geometry.h   | 52 ++++++++++++++++++++-----------
 inc/geometries/mpi_fieldaligned.h |  4 +--
 2 files changed, 34 insertions(+), 22 deletions(-)

diff --git a/inc/dg/geometry/base_geometry.h b/inc/dg/geometry/base_geometry.h
index 4973fad79..775697f1d 100644
--- a/inc/dg/geometry/base_geometry.h
+++ b/inc/dg/geometry/base_geometry.h
@@ -9,9 +9,7 @@ namespace dg
 ///@addtogroup basicgeometry
 ///@{
 
-/**
- * @brief This is the abstract interface class for a two-dimensional Geometry
- */
+///@brief This is the abstract interface class for a two-dimensional Geometry
 struct aGeometry2d : public aTopology2d
 {
     /**
@@ -86,9 +84,7 @@ struct aGeometry2d : public aTopology2d
 
 };
 
-/**
- * @brief This is the abstract interface class for a three-dimensional Geometry
- */
+///@brief This is the abstract interface class for a three-dimensional Geometry
 struct aGeometry3d : public aTopology3d
 {
     /**
@@ -167,6 +163,30 @@ struct aGeometry3d : public aTopology3d
     }
 };
 
+///@brief a 3d product space Geometry
+struct aProductGeometry3d : public aGeometry3d
+{
+    /*!
+     * @brief The grid made up by the first two dimensions
+     *
+     * This is possible because the 3d grid is a product grid of a 2d perpendicular grid and a 1d parallel grid
+     * @return A newly constructed perpendicular grid
+     */
+    aGeometry2d* perp_grid()const{
+        return do_perp_grid();
+    }
+    //default copy, assignment are public but cannot directly be called because of pure virtual functions; 
+    //default deletion is public and virtual
+    protected:
+    /*!
+     * @copydoc aTopology3d::aTopology3d()
+     * @note the default coordinate map will be the identity 
+     */
+    aProductGeometry3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz): aGeometry3d(x0,x1,y0,y1,z0,z1,n,Nx,Ny,Nz,bcx,bcy,bcz){}
+    private:
+    virtual aGeometry2d* do_perp_grid()const=0;
+};
+
 ///@}
 
 ///@addtogroup geometry
@@ -194,26 +214,20 @@ struct CartesianGrid2d: public dg::aGeometry2d
 /**
  * @brief three-dimensional Grid with Cartesian metric
  */
-struct CartesianGrid3d: public dg::aGeometry3d
+struct CartesianGrid3d: public dg::aProductGeometry3d
 {
     typedef CartesianGrid2d perpendicular_grid;
     ///@copydoc hide_grid_parameters3d
     ///@copydoc hide_bc_parameters3d
-    CartesianGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = PER, bc bcy = PER, bc bcz = PER): dg::aGeometry3d(x0,x1,y0,y1,z0,z1,n,Nx,Ny,Nz,bcx,bcy,bcz){}
+    CartesianGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = PER, bc bcy = PER, bc bcz = PER): dg::aProductGeometry3d(x0,x1,y0,y1,z0,z1,n,Nx,Ny,Nz,bcx,bcy,bcz){}
     /**
      * @brief Implicit type conversion from Grid3d
      * @param g existing grid object
      */
-    CartesianGrid3d( const dg::Grid3d& g):dg::aGeometry3d(g.x0(), g.x1(), g.y0(), g.y1(), g.z0(), g.z1(),g.n(),g.Nx(),g.Ny(),g.Nz(),g.bcx(),g.bcy(),g.bcz()){}
+    CartesianGrid3d( const dg::Grid3d& g):dg::aProductGeometry3d(g.x0(), g.x1(), g.y0(), g.y1(), g.z0(), g.z1(),g.n(),g.Nx(),g.Ny(),g.Nz(),g.bcx(),g.bcy(),g.bcz()){}
     virtual CartesianGrid3d* clone()const{return new CartesianGrid3d(*this);}
-    /*!
-     * @brief The grid made up by the first two dimensions
-     *
-     * This is possible because the 3d grid is a product grid of a 2d perpendicular grid and a 1d parallel grid
-     * @return A newly constructed perpendicular grid
-     */
-    CartesianGrid2d perp_grid() const{ return CartesianGrid2d(x0(),x1(),y0(),y1(),n(),Nx(),Ny(),bcx(),bcy());}
     private:
+    CartesianGrid2d* do_perp_grid() const{ return new CartesianGrid2d(x0(),x1(),y0(),y1(),n(),Nx(),Ny(),bcx(),bcy());}
     virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz){
         aTopology3d::do_set(new_n,new_Nx,new_Ny,new_Nz);
     }
@@ -222,17 +236,17 @@ struct CartesianGrid3d: public dg::aGeometry3d
 /**
  * @brief three-dimensional Grid with Cylindrical metric
  */
-struct CylindricalGrid3d: public dg::aGeometry3d
+struct CylindricalGrid3d: public dg::aProductGeometry3d
 {
     typedef CartesianGrid2d perpendicular_grid;
     ///@copydoc hide_grid_parameters3d
     ///@copydoc hide_bc_parameters3d
     ///@note x corresponds to R, y to Z and z to phi, the volume element is R
-    CylindricalGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = PER, bc bcy = PER, bc bcz = PER): dg::aGeometry3d(x0,x1,y0,y1,z0,z1,n,Nx,Ny,Nz,bcx,bcy,bcz){}
+    CylindricalGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = PER, bc bcy = PER, bc bcz = PER): dg::aProductGeometry3d(x0,x1,y0,y1,z0,z1,n,Nx,Ny,Nz,bcx,bcy,bcz){}
     virtual CylindricalGrid3d* clone()const{return new CylindricalGrid3d(*this);}
     ///@copydoc  CartesianGrid3d::perp_grid()const
-    CartesianGrid2d perp_grid() const{ return CartesianGrid2d(x0(),x1(),y0(),y1(),n(),Nx(),Ny(),bcx(),bcy());}
     private:
+    CartesianGrid2d* do_perp_grid() const{ return new CartesianGrid2d(x0(),x1(),y0(),y1(),n(),Nx(),Ny(),bcx(),bcy());}
     virtual SparseTensor<thrust::host_vector<double> > do_compute_metric()const{
         SparseTensor<thrust::host_vector<double> > metric(1);
         thrust::host_vector<double> R = dg::evaluate(dg::cooX3d, *this);
diff --git a/inc/geometries/mpi_fieldaligned.h b/inc/geometries/mpi_fieldaligned.h
index c5ccb6f20..88999d96e 100644
--- a/inc/geometries/mpi_fieldaligned.h
+++ b/inc/geometries/mpi_fieldaligned.h
@@ -563,9 +563,7 @@ void FieldAligned<G,RowDistMat<M,C>,MPI_Vector<container> >::einsPlusT( const MP
             //first exchange data in XY
             thrust::copy( in.cbegin() + i0*size2d, in.cbegin() + (i0+1)*size2d, temp_[i0].begin());
             tempXYplus_[i0] = commXYplus_.global_gather( temp_[i0]);
-            cView inV( tempXYplus_[i0].cbegin(), tempXYplus_[i0].cend() );
-            View tempV( temp_[i0].begin(), temp_[i0].end() );
-            cusp::multiply( plusT, inV, tempV);
+            dg::blas2::symv( plusT, tempXYplus_[i0], temp_[i0]);
         }
     }
     else //directly compute in temp_
-- 
GitLab


From c28c49e8edfbf71698c9c82fda2309cfc315e3e9 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sat, 30 Sep 2017 16:03:15 +0200
Subject: [PATCH 329/453] added mpi Product Geometry3d

---
 inc/dg/geometry/base_geometry.h  | 11 ++++--
 inc/dg/geometry/mpi_base.h       | 62 +++++++++++++++++++++-----------
 inc/geometries/mpi_curvilinear.h |  8 +----
 3 files changed, 52 insertions(+), 29 deletions(-)

diff --git a/inc/dg/geometry/base_geometry.h b/inc/dg/geometry/base_geometry.h
index 775697f1d..cc3cce7cd 100644
--- a/inc/dg/geometry/base_geometry.h
+++ b/inc/dg/geometry/base_geometry.h
@@ -175,9 +175,16 @@ struct aProductGeometry3d : public aGeometry3d
     aGeometry2d* perp_grid()const{
         return do_perp_grid();
     }
-    //default copy, assignment are public but cannot directly be called because of pure virtual functions; 
-    //default deletion is public and virtual
+    ///allow deletion through base class pointer
+    virtual ~aProductGeometry3d(){}
     protected:
+    ///@copydoc aTopology3d::aTopology3d(const aTopology3d&)
+    aProductGeometry3d( const aProductGeometry3d& src):aGeometry3d(src){}
+    ///@copydoc aTopology3d::operator=(const aTopology3d&)
+    aProductGeometry3d& operator=( const aProductGeometry3d& src){
+        aGeometry3d::operator=(src);
+        return *this;
+    }
     /*!
      * @copydoc aTopology3d::aTopology3d()
      * @note the default coordinate map will be the identity 
diff --git a/inc/dg/geometry/mpi_base.h b/inc/dg/geometry/mpi_base.h
index 533dc6a86..a95fb2a3d 100644
--- a/inc/dg/geometry/mpi_base.h
+++ b/inc/dg/geometry/mpi_base.h
@@ -111,6 +111,35 @@ struct aMPIGeometry3d : public aMPITopology3d
     }
 };
 
+///@brief a 3d product space MPI Geometry
+struct aProductMPIGeometry3d : public aMPIGeometry3d
+{
+    /*!
+     * @brief The grid made up by the first two dimensions
+     *
+     * This is possible because the 3d grid is a product grid of a 2d perpendicular grid and a 1d parallel grid
+     * @return A newly constructed perpendicular grid
+     */
+    aMPIGeometry2d* perp_grid()const{
+        return do_perp_grid();
+    }
+    ///allow deletion through base class pointer
+    virtual ~aProductMPIGeometry3d(){}
+    protected:
+    ///@copydoc aMPITopology3d::aMPITopology3d()
+    aProductMPIGeometry3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz, MPI_Comm comm):
+        aMPIGeometry3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, bcx, bcy, bcz, comm){}
+    ///@copydoc aMPITopology3d::aMPITopology3d(const aMPITopology3d&)
+    aProductMPIGeometry3d( const aProductMPIGeometry3d& src):aMPIGeometry3d(src){}
+    ///@copydoc aMPITopology3d::operator=(const aMPITopology3d&)
+    aProductMPIGeometry3d& operator=( const aProductMPIGeometry3d& src){
+        aMPIGeometry3d::operator=(src);
+        return *this;
+    }
+    private:
+    virtual aMPIGeometry2d* do_perp_grid()const=0;
+};
+
 ///@}
 
 ///@addtogroup geometry
@@ -150,21 +179,21 @@ struct CartesianMPIGrid2d : public aMPIGeometry2d
 /**
  * @brief The mpi version of CartesianGrid3d
  */
-struct CartesianMPIGrid3d : public aMPIGeometry3d
+struct CartesianMPIGrid3d : public aProductMPIGeometry3d
 {
     typedef CartesianMPIGrid2d perpendicular_grid;
     ///@copydoc hide_grid_parameters3d
     ///@copydoc hide_comm_parameters3d
-    CartesianMPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, MPI_Comm comm): aMPIGeometry3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, dg::PER,dg::PER,dg::PER, comm){}
+    CartesianMPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, MPI_Comm comm): aProductMPIGeometry3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, dg::PER,dg::PER,dg::PER, comm){}
 
     ///@copydoc hide_grid_parameters3d
     ///@copydoc hide_bc_parameters3d
     ///@copydoc hide_comm_parameters3d
-    CartesianMPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz, MPI_Comm comm):aMPIGeometry3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, bcx, bcy, bcz, comm){}
+    CartesianMPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz, MPI_Comm comm):aProductMPIGeometry3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, bcx, bcy, bcz, comm){}
 
     ///@brief Implicit type conversion from MPIGrid3d
     ///@param g existing grid object
-    CartesianMPIGrid3d( const dg::MPIGrid3d& g): aMPIGeometry3d( g.global().x0(),g.global().x1(),g.global().y0(),g.global().y1(),g.global().z0(),g.global().z1(),g.global().n(),g.global().Nx(),g.global().Ny(),g.global().Nz(),g.global().bcx(),g.global().bcy(),g.global().bcz(),g.communicator()){}
+    CartesianMPIGrid3d( const dg::MPIGrid3d& g): aProductMPIGeometry3d( g.global().x0(),g.global().x1(),g.global().y0(),g.global().y1(),g.global().z0(),g.global().z1(),g.global().n(),g.global().Nx(),g.global().Ny(),g.global().Nz(),g.global().bcx(),g.global().bcy(),g.global().bcz(),g.communicator()){}
     virtual CartesianMPIGrid3d* clone()const{return new CartesianMPIGrid3d(*this);}
     virtual CartesianGrid3d* global_geometry()const{
         return new CartesianGrid3d( 
@@ -174,15 +203,6 @@ struct CartesianMPIGrid3d : public aMPIGeometry3d
                 global().n(), global().Nx(), global().Ny(), global().Nz(), 
                 global().bcx(), global().bcy(), global().bcz());
     }
-    /*!
-     * @brief The grid made up by the first two dimensions in space and process topology
-     *
-     * This is possible because the 3d grid is a product grid of a 2d perpendicular grid and a 1d parallel grid
-     * @return A newly constructed perpendicular grid with the perpendicular communicator
-     */
-    CartesianMPIGrid2d perp_grid()const{ 
-        return CartesianMPIGrid2d( global().x0(), global().x1(), global().y0(), global().y1(), global().n(), global().Nx(), global().Ny(), global().bcx(), global().bcy(), get_perp_comm( communicator() ));
-    }
 
     private:
     MPI_Comm get_perp_comm( MPI_Comm src) const
@@ -192,6 +212,9 @@ struct CartesianMPIGrid3d : public aMPIGeometry3d
         MPI_Cart_sub( src, remain_dims, &planeComm);
         return planeComm;
     }
+    virtual CartesianMPIGrid2d* do_perp_grid()const{ 
+        return new CartesianMPIGrid2d( global().x0(), global().x1(), global().y0(), global().y1(), global().n(), global().Nx(), global().Ny(), global().bcx(), global().bcy(), get_perp_comm( communicator() ));
+    }
     virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz){
         aMPITopology3d::do_set(new_n,new_Nx,new_Ny,new_Nz);
     }
@@ -200,20 +223,20 @@ struct CartesianMPIGrid3d : public aMPIGeometry3d
 /**
  * @brief the mpi version of CylindricalGrid3d
  */
-struct CylindricalMPIGrid3d: public aMPIGeometry3d
+struct CylindricalMPIGrid3d: public aProductMPIGeometry3d
 {
     typedef CartesianMPIGrid2d perpendicular_grid;
     ///@copydoc hide_grid_parameters3d
     ///@copydoc hide_bc_parameters3d
     ///@copydoc hide_comm_parameters3d
     ///@note x corresponds to R, y to Z and z to phi, the volume element is R
-    CylindricalMPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz, MPI_Comm comm):aMPIGeometry3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, bcx, bcy, bcz, comm){}
+    CylindricalMPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz, MPI_Comm comm):aProductMPIGeometry3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, bcx, bcy, bcz, comm){}
     ///@copydoc hide_grid_parameters3d
     ///@copydoc hide_bc_parameters2d
     ///@note bcz is dg::PER
     ///@copydoc hide_comm_parameters3d
     ///@note x corresponds to R, y to Z and z to phi, the volume element is R
-    CylindricalMPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, MPI_Comm comm):aMPIGeometry3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, bcx, bcy, dg::PER, comm){}
+    CylindricalMPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, MPI_Comm comm):aProductMPIGeometry3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, bcx, bcy, dg::PER, comm){}
 
     virtual CylindricalMPIGrid3d* clone()const{return new CylindricalMPIGrid3d(*this);}
     virtual CylindricalGrid3d* global_geometry()const{
@@ -224,11 +247,10 @@ struct CylindricalMPIGrid3d: public aMPIGeometry3d
                 global().n(), global().Nx(), global().Ny(), global().Nz(), 
                 global().bcx(), global().bcy(), global().bcz());
     }
-    ///@copydoc CartesianMPIGrid3d::perp_grid()const
-    CartesianMPIGrid2d perp_grid()const{ 
-        return CartesianMPIGrid2d( global().x0(), global().x1(), global().y0(), global().y1(), global().n(), global().Nx(), global().Ny(), global().bcx(), global().bcy(), get_perp_comm( communicator() ));
-    }
     private:
+    virtual CartesianMPIGrid2d* do_perp_grid()const{ 
+        return new CartesianMPIGrid2d( global().x0(), global().x1(), global().y0(), global().y1(), global().n(), global().Nx(), global().Ny(), global().bcx(), global().bcy(), get_perp_comm( communicator() ));
+    }
     MPI_Comm get_perp_comm( MPI_Comm src) const
     {
         MPI_Comm planeComm;
diff --git a/inc/geometries/mpi_curvilinear.h b/inc/geometries/mpi_curvilinear.h
index fa1d73540..20817cbfb 100644
--- a/inc/geometries/mpi_curvilinear.h
+++ b/inc/geometries/mpi_curvilinear.h
@@ -114,13 +114,6 @@ struct CurvilinearProductMPIGrid3d : public dg::aMPIGeometry3d
         constructParallel(this->Nz());
     }
 
-    /*!
-     * @brief The grid made up by the first two dimensions in space and process topology
-     *
-     * This is possible because the 3d grid is a product grid of a 2d perpendicular grid and a 1d parallel grid
-     * @return A newly constructed perpendicular grid with the perpendicular communicator
-     */
-    perpendicular_grid perp_grid() const { return perpendicular_grid(*this);}
 
     ///read access to the generator
     const aGenerator2d& generator() const{return handle_.get();}
@@ -132,6 +125,7 @@ struct CurvilinearProductMPIGrid3d : public dg::aMPIGeometry3d
                 global().bcx(), global().bcy(), global().bcz());
     }
     private:
+    virtual perpendicular_grid* do_perp_grid() const { return new perpendicular_grid(*this);}
     MPI_Comm get_perp_comm( MPI_Comm src)
     {
         MPI_Comm planeComm;
-- 
GitLab


From b6a33ec98353ef05022ea99cfbb8dbeb5703493b Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sat, 30 Sep 2017 16:11:47 +0200
Subject: [PATCH 330/453] excluded ell_interpolation from documentation and
 added ProductGrid3d to derive from ProductGeometry

---
 inc/dg/Doxyfile                  |  1 +
 inc/doc/Makefile                 | 12 ++++++++++--
 inc/geometries/curvilinear.h     |  4 ++--
 inc/geometries/mpi_curvilinear.h |  4 ++--
 4 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/inc/dg/Doxyfile b/inc/dg/Doxyfile
index 11667eb36..3c809011f 100644
--- a/inc/dg/Doxyfile
+++ b/inc/dg/Doxyfile
@@ -831,6 +831,7 @@ EXCLUDE                = ../dg/backend/creation.cuh \
                          ../dg/backend/sparseblockmat_omp_kernels.h \
                          ../dg/backend/transpose.h \
                          ../dg/backend/average.h \
+                         ../dg/backend/ell_interpolation.cuh \
 
 # The EXCLUDE_SYMLINKS tag can be used to select whether or not files or
 # directories that are symbolic links (a Unix file system feature) are excluded
diff --git a/inc/doc/Makefile b/inc/doc/Makefile
index 6eee17e96..d25c382fb 100644
--- a/inc/doc/Makefile
+++ b/inc/doc/Makefile
@@ -6,7 +6,7 @@
 
 all: doc
 
-.PHONY: clean doc dg.tag dg file geometries
+.PHONY: clean doc dg.tag geometries.tag dg file geometries
 
 dg.tag:
 	(cat ../dg/Doxyfile; \
@@ -15,6 +15,13 @@ dg.tag:
 	echo "GENERATE_HTML = NO"; \
 	echo "GENERATE_TAGFILE = ./dg.tag" ) | doxygen - ;
 
+geometries.tag:
+	(cat ../geometries/Doxyfile; \
+	echo "INPUT = ../geometries/"; \
+	echo "OUTPUT_DIRECTORY = ./geometries"; \
+	echo "GENERATE_HTML = NO"; \
+	echo "GENERATE_TAGFILE = ./geometries.tag" ) | doxygen - ;
+
 dg:
 	(cat ../dg/Doxyfile; \
 	echo "INPUT = ../dg/"; \
@@ -22,7 +29,8 @@ dg:
 	echo "HTML_HEADER = header.html"; \
     echo "EXTERNAL_GROUPS=NO" ;\
     echo "EXTERNAL_PAGES=NO" ;\
-	echo ) | doxygen - ; 
+	echo "TAGFILES = geometries.tag=../../geometries/html") | doxygen - ; 
+	#echo ) | doxygen - ; 
 
 geometries: dg.tag
 	(cat ../geometries/Doxyfile; \
diff --git a/inc/geometries/curvilinear.h b/inc/geometries/curvilinear.h
index e49e877df..04b78ab04 100644
--- a/inc/geometries/curvilinear.h
+++ b/inc/geometries/curvilinear.h
@@ -76,13 +76,13 @@ void square( const dg::SparseTensor<thrust::host_vector<double> >& jac, const th
  * 
  * The base coordinate system is the cylindrical coordinate system R,Z,phi
  */
-struct CurvilinearProductGrid3d : public dg::aGeometry3d
+struct CurvilinearProductGrid3d : public dg::aProductGeometry3d
 {
     typedef CurvilinearGrid2d perpendicular_grid;
 
     ///@copydoc hide_grid_parameters3d
     CurvilinearProductGrid3d( const aGenerator2d& generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx=dg::DIR, bc bcy=dg::PER, bc bcz=dg::PER):
-        dg::aGeometry3d( 0, generator.width(), 0., generator.height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, bcy, bcz)
+        dg::aProductGeometry3d( 0, generator.width(), 0., generator.height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, bcy, bcz)
     { 
         map_.resize(3);
         handle_ = generator;
diff --git a/inc/geometries/mpi_curvilinear.h b/inc/geometries/mpi_curvilinear.h
index 20817cbfb..38c0d88a8 100644
--- a/inc/geometries/mpi_curvilinear.h
+++ b/inc/geometries/mpi_curvilinear.h
@@ -98,14 +98,14 @@ struct CurvilinearMPIGrid2d : public dg::aMPIGeometry2d
 /**
  * This is s 2x1 curvilinear product space MPI grid
  */
-struct CurvilinearProductMPIGrid3d : public dg::aMPIGeometry3d
+struct CurvilinearProductMPIGrid3d : public dg::aProductMPIGeometry3d
 {
     typedef dg::geo::CurvilinearMPIGrid2d perpendicular_grid; //!< the two-dimensional grid
     /// @opydoc hide_grid_parameters3d
     /// @param comm a three-dimensional Cartesian communicator
     /// @note the paramateres given in the constructor are global parameters 
     CurvilinearProductMPIGrid3d( const aGenerator2d& generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz, MPI_Comm comm): 
-        dg::aMPIGeometry3d( 0, generator.width(), 0., generator.height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, bcy, bcz, comm),
+        dg::aProductMPIGeometry3d( 0, generator.width(), 0., generator.height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, bcy, bcz, comm),
         handle_( generator)
     {
         map_.resize(3);
-- 
GitLab


From 1f0ed2f465a4374eec1251dfb72c5780c6583857 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sat, 30 Sep 2017 17:46:06 +0200
Subject: [PATCH 331/453] added split and join functions

---
 inc/dg/backend/split_and_join.h | 82 +++++++++++++++++++++++++++++++++
 inc/doc/Makefile                |  2 +-
 2 files changed, 83 insertions(+), 1 deletion(-)
 create mode 100644 inc/dg/backend/split_and_join.h

diff --git a/inc/dg/backend/split_and_join.h b/inc/dg/backend/split_and_join.h
new file mode 100644
index 000000000..1a769e9a0
--- /dev/null
+++ b/inc/dg/backend/split_and_join.h
@@ -0,0 +1,82 @@
+#pragma once
+#include <thrust/host_vector.h>
+#include <thrust/device_vector.h>
+#include "grid.h"
+#ifdef MPI_VERSION
+#include "mpi_vector.h"
+#include "mpi_grid.h"
+#endif //MPI_VERSION
+namespace dg
+{
+///@ingroup scatter
+///@{
+/** @brief  Split a vector into planes 
+*
+* @tparam thrust_vector either thrust::host_vector or thrust::device_vector
+* @param in contiguous 3d vector (must be of size grid.size())
+* @param out contains grid.Nz() 2d vectors of 2d size on output (gets resized if necessary)
+* @param grid provide dimensions in 3rd and first two dimensions
+*/
+template<class thrust_vector>
+void split( const thrust_vector& in, std::vector<thrust_vector>& out, const aTopology3d* grid)
+{
+    unsigned size2d=grid->n()*grid->n()*grid->Nx()*grid->Ny();
+    out.resize( grid.Nz());
+    for(unsigned i=0; i<grid->Nz(); i++)
+        out[i].assign( in.begin() + i*size2d, in.begin()+(i+1)*size2d);
+}
+#ifdef MPI_VERSION
+///@brief MPI Version of split
+///@copydetails dg::split()
+///@note every plane in out gets its own 2d Cartesian communicator
+template <class thrust_vector>
+void split( const MPI_Vector<thrust_vector>& in, std::vector<MPI_Vector<thrust_vector> >& out, const aMPITopology3d* grid)
+{
+    assert( in.communicator() == grid.communicator());
+    MPI_Comm planeComm;
+    int remain_dims[] = {true,true,false}; //true true false
+    MPI_Cart_sub( in.communicator(), remain_dims, &planeComm);
+    //local size2d
+    unsigned size2d=grid->n()*grid->n()*grid->Nx()*grid->Ny();
+    out.resize( grid.Nz());
+    for(unsigned i=0; i<grid->Nz(); i++)
+    {
+        out[i].data().assign( in.data().begin() + i*size2d, in.data().begin()+(i+1)*size2d);
+        out[i].communicator() = planeComm;
+    }
+}
+#endif //MPI_VERSION
+/**
+* @brief Revert split operation
+*
+* @tparam thrust_vector either thrust::host_vector or thrust::device_vector
+* @param in grid.Nz() 2d vectors of 2d size 
+* @param out contiguous 3d vector (gets resized if necessary) 
+* @param grid provide dimensions in 3rd and first two dimensions
+* @note split followed by join restores the original vector
+*/
+template<class thrust_vector>
+void join( const std::vector<thrust_vector>& in, thrust_vector& out, const aTopology3d* grid)
+{
+    unsigned size2d=grid->n()*grid->n()*grid->Nx()*grid->Ny();
+    out.resize( size2d*grid.Nz());
+    for(unsigned i=0; i<grid->Nz(); i++)
+        thrust::copy( in[i].begin(), in[i].end(), out.begin()+i*size2d);
+}
+
+#ifdef MPI_VERSION
+///@brief MPI Version of join
+///@copydetail dg::join()
+template<class thrust_vector>
+void join( const std::vector<MPI_Vector<thrust_vector> >& in, MPI_Vector<thrust_vector >& out, const aMPITopology3d* grid)
+{
+    unsigned size2d=grid->n()*grid->n()*grid->Nx()*grid->Ny();
+    out.data().resize( size2d*grid.Nz());
+    out.communicator() = grid.communicator();
+    for(unsigned i=0; i<grid->Nz(); i++)
+        thrust::copy( in[i].data().begin(), in[i].data().end(), out.data().begin()+i*size2d);
+}
+
+#endif //MPI_VERSION
+///@}
+}//namespace dg
diff --git a/inc/doc/Makefile b/inc/doc/Makefile
index d25c382fb..590c85315 100644
--- a/inc/doc/Makefile
+++ b/inc/doc/Makefile
@@ -22,7 +22,7 @@ geometries.tag:
 	echo "GENERATE_HTML = NO"; \
 	echo "GENERATE_TAGFILE = ./geometries.tag" ) | doxygen - ;
 
-dg:
+dg: geometries.tag
 	(cat ../dg/Doxyfile; \
 	echo "INPUT = ../dg/"; \
 	echo "OUTPUT_DIRECTORY = ./dg"; \
-- 
GitLab


From 602e39e144b056c04c23a5e54ca683b7a766b449 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sat, 30 Sep 2017 22:42:06 +0200
Subject: [PATCH 332/453] added split and join functions to blas_b and updated
 blas_mpib

---
 inc/dg/backend/split_and_join.h |  24 ++++----
 inc/dg/blas_b.cu                |  16 ++++-
 inc/dg/blas_mpib.cu             | 104 +++++++++++++++++++++++---------
 3 files changed, 101 insertions(+), 43 deletions(-)

diff --git a/inc/dg/backend/split_and_join.h b/inc/dg/backend/split_and_join.h
index 1a769e9a0..ba4dc8f5b 100644
--- a/inc/dg/backend/split_and_join.h
+++ b/inc/dg/backend/split_and_join.h
@@ -18,11 +18,11 @@ namespace dg
 * @param grid provide dimensions in 3rd and first two dimensions
 */
 template<class thrust_vector>
-void split( const thrust_vector& in, std::vector<thrust_vector>& out, const aTopology3d* grid)
+void split( const thrust_vector& in, std::vector<thrust_vector>& out, const aTopology3d& grid)
 {
-    unsigned size2d=grid->n()*grid->n()*grid->Nx()*grid->Ny();
+    unsigned size2d=grid.n()*grid.n()*grid.Nx()*grid.Ny();
     out.resize( grid.Nz());
-    for(unsigned i=0; i<grid->Nz(); i++)
+    for(unsigned i=0; i<grid.Nz(); i++)
         out[i].assign( in.begin() + i*size2d, in.begin()+(i+1)*size2d);
 }
 #ifdef MPI_VERSION
@@ -30,16 +30,16 @@ void split( const thrust_vector& in, std::vector<thrust_vector>& out, const aTop
 ///@copydetails dg::split()
 ///@note every plane in out gets its own 2d Cartesian communicator
 template <class thrust_vector>
-void split( const MPI_Vector<thrust_vector>& in, std::vector<MPI_Vector<thrust_vector> >& out, const aMPITopology3d* grid)
+void split( const MPI_Vector<thrust_vector>& in, std::vector<MPI_Vector<thrust_vector> >& out, const aMPITopology3d& grid)
 {
     assert( in.communicator() == grid.communicator());
     MPI_Comm planeComm;
     int remain_dims[] = {true,true,false}; //true true false
     MPI_Cart_sub( in.communicator(), remain_dims, &planeComm);
     //local size2d
-    unsigned size2d=grid->n()*grid->n()*grid->Nx()*grid->Ny();
+    unsigned size2d=grid.n()*grid.n()*grid.Nx()*grid.Ny();
     out.resize( grid.Nz());
-    for(unsigned i=0; i<grid->Nz(); i++)
+    for(unsigned i=0; i<grid.Nz(); i++)
     {
         out[i].data().assign( in.data().begin() + i*size2d, in.data().begin()+(i+1)*size2d);
         out[i].communicator() = planeComm;
@@ -56,11 +56,11 @@ void split( const MPI_Vector<thrust_vector>& in, std::vector<MPI_Vector<thrust_v
 * @note split followed by join restores the original vector
 */
 template<class thrust_vector>
-void join( const std::vector<thrust_vector>& in, thrust_vector& out, const aTopology3d* grid)
+void join( const std::vector<thrust_vector>& in, thrust_vector& out, const aTopology3d& grid)
 {
-    unsigned size2d=grid->n()*grid->n()*grid->Nx()*grid->Ny();
+    unsigned size2d=grid.n()*grid.n()*grid.Nx()*grid.Ny();
     out.resize( size2d*grid.Nz());
-    for(unsigned i=0; i<grid->Nz(); i++)
+    for(unsigned i=0; i<grid.Nz(); i++)
         thrust::copy( in[i].begin(), in[i].end(), out.begin()+i*size2d);
 }
 
@@ -68,12 +68,12 @@ void join( const std::vector<thrust_vector>& in, thrust_vector& out, const aTopo
 ///@brief MPI Version of join
 ///@copydetail dg::join()
 template<class thrust_vector>
-void join( const std::vector<MPI_Vector<thrust_vector> >& in, MPI_Vector<thrust_vector >& out, const aMPITopology3d* grid)
+void join( const std::vector<MPI_Vector<thrust_vector> >& in, MPI_Vector<thrust_vector >& out, const aMPITopology3d& grid)
 {
-    unsigned size2d=grid->n()*grid->n()*grid->Nx()*grid->Ny();
+    unsigned size2d=grid.n()*grid.n()*grid.Nx()*grid.Ny();
     out.data().resize( size2d*grid.Nz());
     out.communicator() = grid.communicator();
-    for(unsigned i=0; i<grid->Nz(); i++)
+    for(unsigned i=0; i<grid.Nz(); i++)
         thrust::copy( in[i].data().begin(), in[i].data().end(), out.data().begin()+i*size2d);
 }
 
diff --git a/inc/dg/blas_b.cu b/inc/dg/blas_b.cu
index f8d5398d2..671fffa5a 100644
--- a/inc/dg/blas_b.cu
+++ b/inc/dg/blas_b.cu
@@ -9,6 +9,7 @@
 #include "backend/derivatives.h"
 #include "backend/evaluation.cuh"
 #include "backend/fast_interpolation.h"
+#include "backend/split_and_join.h"
 
 const double lx = 2.*M_PI;
 const double ly = 2.*M_PI;
@@ -50,7 +51,7 @@ int main()
     dg::blas2::transfer(dg::create::fast_interpolation( grid_half, 2,2), inter);
     dg::blas2::transfer(dg::create::fast_projection( grid, 2,2), project);
     std::cout << "Done...\n";
-    int multi=200;
+    int multi=100;
     t.tic();
     value_type norm=0;
     for( int i=0; i<multi; i++)
@@ -104,8 +105,17 @@ int main()
         dg::blas2::gemv( project, x, x_half);
     t.toc();
     std::cout<<"Projection full to half grid     "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
-    
-
+    t.tic();
+    std::vector<Vector> y_split;
+    for( int i=0; i<multi; i++)
+        dg::split( y, y_split, grid);
+    t.toc();
+    std::cout<<"Split                  took      "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
+    t.tic();
+    for( int i=0; i<multi; i++)
+        dg::join( y_split, x, grid);
+    t.toc();
+    std::cout<<"                  Join took      "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
     t.tic();
     for( int i=0; i<multi; i++)
         dg::blas1::axpby( 1., y, -1., x);
diff --git a/inc/dg/blas_mpib.cu b/inc/dg/blas_mpib.cu
index 30c4323cf..bb3f79851 100644
--- a/inc/dg/blas_mpib.cu
+++ b/inc/dg/blas_mpib.cu
@@ -9,10 +9,12 @@
 #include "backend/mpi_evaluation.h"
 #include "backend/mpi_derivatives.h"
 #include "backend/mpi_init.h"
+#include "backend/fast_interpolation.h"
+#include "backend/split_and_join.h"
 
 const double lx = 2.*M_PI;
 const double ly = 2.*M_PI;
-double function(double x, double y){ return sin(y)*sin(x);}
+double function(double x, double y, double z){ return sin(y)*sin(x);}
 
 typedef dg::MDVec Vector;
 typedef dg::MDMatrix Matrix;
@@ -20,17 +22,17 @@ typedef dg::MDMatrix Matrix;
 int main( int argc, char* argv[])
 {
     MPI_Init(&argc, &argv);
-    unsigned n, Nx, Ny; 
+    unsigned n, Nx, Ny, Nz; 
     MPI_Comm comm;
-    dg::mpi_init2d( dg::PER, dg::PER, n, Nx, Ny, comm);
+    dg::mpi_init3d( dg::PER, dg::PER, dg::PER, n, Nx, Ny, Nz, comm);
     int rank;
     MPI_Comm_rank( MPI_COMM_WORLD, &rank);
 
-    dg::MPIGrid2d grid( 0., lx, 0, ly, n, Nx, Ny, comm);
+    dg::MPIGrid3d grid( 0., lx, 0, ly,0., ly, n, Nx, Ny, Nz, comm);
+    dg::MPIGrid3d grid_half = grid; grid_half.multiplyCellNumbers(0.5, 0.5);
     Vector w2d;
     dg::blas1::transfer( dg::create::weights(grid), w2d);
     if(rank==0)std::cout<<"Evaluate a function on the grid\n";
-    if(rank==0)std::cout<<"Evaluate a function on the grid\n";
     dg::Timer t;
     t.tic();
     Vector x;
@@ -40,73 +42,119 @@ int main( int argc, char* argv[])
     if(rank==0)std::cout << "Sizeof value type is "<<sizeof(double)<<"\n";
     double gbytes=(double)grid.global().size()*sizeof(double)/1e9;
     if(rank==0)std::cout << "Sizeof vectors is "<<gbytes<<" GB\n";
+    dg::MultiMatrix<Matrix, Vector> inter, project; 
+    dg::blas2::transfer(dg::create::fast_interpolation( grid_half, 2,2), inter);
+    dg::blas2::transfer(dg::create::fast_projection( grid, 2,2), project);
+
+    int multi=100;
     t.tic();
     double norm=0;
-    for( unsigned i=0; i<20; i++)
+    for( int i=0; i<multi; i++)
         norm += dg::blas1::dot( w2d, x);
     t.toc();
-    if(rank==0)std::cout<<"DOT took                         " <<t.diff()/20.<<"s\t"<<gbytes*20/t.diff()<<"GB/s\n";
-    Vector y(x);
+    if(rank==0)std::cout<<"DOT took                         " <<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
+    Vector y(x), z(x), u(x), v(x);
     Matrix M;
     dg::blas2::transfer(dg::create::dx( grid, dg::centered), M);
     t.tic();
-    for( int i=0; i<20; i++)
+    for( int i=0; i<multi; i++)
         dg::blas2::symv( M, x, y);
     t.toc();
-    if(rank==0)std::cout<<"centered x derivative took       "<<t.diff()/20.<<"s\t"<<gbytes*20/t.diff()<<"GB/s\n";
+    if(rank==0)std::cout<<"centered x derivative took       "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
 
     dg::blas2::transfer(dg::create::dx( grid, dg::forward), M);
     t.tic();
-    for( int i=0; i<20; i++)
+    for( int i=0; i<multi; i++)
         dg::blas2::symv( M, x, y);
     t.toc();
-    if(rank==0)std::cout<<"forward x derivative took        "<<t.diff()/20<<"s\t"<<gbytes*20/t.diff()<<"GB/s\n";
+    if(rank==0)std::cout<<"forward x derivative took        "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
 
     dg::blas2::transfer(dg::create::dy( grid, dg::forward), M);
     t.tic();
-    for( int i=0; i<20; i++)
+    for( int i=0; i<multi; i++)
         dg::blas2::symv( M, x, y);
     t.toc();
-    if(rank==0)std::cout<<"forward y derivative took        "<<t.diff()/20<<"s\t"<<gbytes*20/t.diff()<<"GB/s\n";
+    if(rank==0)std::cout<<"forward y derivative took        "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
 
     dg::blas2::transfer(dg::create::dy( grid, dg::centered), M);
     t.tic();
-    for( int i=0; i<20; i++)
+    for( int i=0; i<multi; i++)
         dg::blas2::symv( M, x, y);
     t.toc();
-    if(rank==0)std::cout<<"centered y derivative took       "<<t.diff()/20<<"s\t"<<gbytes*20/t.diff()<<"GB/s\n";
+    if(rank==0)std::cout<<"centered y derivative took       "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
 
     dg::blas2::transfer(dg::create::jumpX( grid), M);
     t.tic();
-    for( int i=0; i<20; i++)
+    for( int i=0; i<multi; i++)
         dg::blas2::symv( M, x, y);
     t.toc();
-    if(rank==0)std::cout<<"jump X took                      "<<t.diff()/20<<"s\t"<<gbytes*20/t.diff()<<"GB/s\n";
+    if(rank==0)std::cout<<"jump X took                      "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
+    Vector x_half = dg::evaluate( dg::zero, grid_half);
+    t.tic();
+    for( int i=0; i<multi; i++)
+        dg::blas2::gemv( inter, x_half, x);
+    t.toc();
+    if(rank==0)std::cout<<"Interpolation half to full grid  "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
+    t.tic();
+    for( int i=0; i<multi; i++)
+        dg::blas2::gemv( project, x, x_half);
+    t.toc();
+    if(rank==0)std::cout<<"Projection full to half grid     "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
+    std::vector<Vector> y_split;
+    t.tic();
+    for( int i=0; i<multi; i++)
+        dg::split( y, y_split, grid);
+    t.toc();
+    if(rank==0)std::cout<<"Split                  took      "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
+    t.tic();
+    for( int i=0; i<multi; i++)
+        dg::join( y_split, x, grid);
+    t.toc();
+    if(rank==0)std::cout<<"                  Join took      "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
 
     t.tic();
-    for( int i=0; i<20; i++)
+    for( int i=0; i<multi; i++)
         dg::blas1::axpby( 1., y, -1., x);
     t.toc();
-    if(rank==0)std::cout<<"AXPBY took                       "<<t.diff()/20<<"s\t"<<gbytes*20/t.diff()<<"GB/s\n";
+    if(rank==0)std::cout<<"AXPBY took                       "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
+    t.tic();
+    for( int i=0; i<multi; i++)
+        dg::blas1::axpbypgz( 1., x, -1., y, 2., z);
+    t.toc();
+    if(rank==0)std::cout<<"AXPBYPGZ (1*x-1*y+2*z=z)         "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
+    t.tic();
+    for( int i=0; i<multi; i++)
+        dg::blas1::axpbypgz( 1., x, -1., y, 3., x);
+    t.toc();
+    if(rank==0)std::cout<<"AXPBYPGZ (1*x-1.*y+3*x=x)        "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
 
     t.tic();
-    for( int i=0; i<20; i++)
-        dg::blas1::pointwiseDot( y, x, x);
+    for( int i=0; i<multi; i++)
+        dg::blas1::pointwiseDot(  y, x, x);
     t.toc();
-    if(rank==0)std::cout<<"pointwiseDot took                "<<t.diff()/20<<"s\t" <<gbytes*20/t.diff()<<"GB/s\n";
+    if(rank==0)std::cout<<"pointwiseDot (yx=x)              "<<t.diff()/multi<<"s\t" <<gbytes*multi/t.diff()<<"GB/s\n";
     t.tic();
-    for( int i=0; i<20; i++)
+    for( int i=0; i<multi; i++)
+        dg::blas1::pointwiseDot( 1., y, x, 2.,u,v,0.,  z);
+    t.toc();
+    if(rank==0)std::cout<<"pointwiseDot (1*yx+2*uv=z)       "<<t.diff()/multi<<"s\t" <<gbytes*multi/t.diff()<<"GB/s\n";
+    t.tic();
+    for( int i=0; i<multi; i++)
+        dg::blas1::pointwiseDot( 1., y, x, 2.,u,v,0.,  v);
+    t.toc();
+    if(rank==0)std::cout<<"pointwiseDot (1*yx+2*uv=v)       "<<t.diff()/multi<<"s\t" <<gbytes*multi/t.diff()<<"GB/s\n";
+    t.tic();
+    for( int i=0; i<multi; i++)
         norm += dg::blas2::dot( w2d, y);
     t.toc();
-    if(rank==0)std::cout<<"DOT(w,y) took                    " <<t.diff()/20.<<"s\t"<<gbytes*20/t.diff()<<"GB/s\n";
-
+    if(rank==0)std::cout<<"DOT(w,y) took                    " <<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
     t.tic();
-    for( int i=0; i<20; i++)
+    for( int i=0; i<multi; i++)
     {
         norm += dg::blas2::dot( x, w2d, y);
     }
     t.toc();
-    if(rank==0)std::cout<<"DOT(x,w,y) took                  " <<t.diff()/20<<"s\t"<<gbytes*20/t.diff()<<"GB/s\n";
+    if(rank==0)std::cout<<"DOT(x,w,y) took                  " <<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
     if(rank==0)std::cout<<norm<<std::endl;
 
     MPI_Finalize();
-- 
GitLab


From dcc9afce5c57c985a43878671bffa7a6dfb3f0c4 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 2 Oct 2017 11:26:28 +0200
Subject: [PATCH 333/453] created dg::expand function

---
 inc/dg/backend/mpi_evaluation.h |  6 +--
 inc/dg/backend/split_and_join.h | 33 +++++++++++++-
 inc/dg/backend/xspacelib.cuh    | 77 +++++----------------------------
 inc/dg/blas_b.cu                | 12 -----
 inc/dg/blas_mpib.cu             | 13 ------
 inc/geometries/fieldaligned.h   |  1 +
 6 files changed, 44 insertions(+), 98 deletions(-)

diff --git a/inc/dg/backend/mpi_evaluation.h b/inc/dg/backend/mpi_evaluation.h
index 80eb14166..f15a4bd1c 100644
--- a/inc/dg/backend/mpi_evaluation.h
+++ b/inc/dg/backend/mpi_evaluation.h
@@ -97,11 +97,7 @@ MPI_Vector<thrust::host_vector<double> > global2local( const thrust::host_vector
     return MPI_Vector<thrust::host_vector<double> >(temp, g.communicator()); 
 }
 /**
- * @brief Take the relevant local part of a global vector
- *
- * @param global a vector the size of the global grid 
- * @param g the assumed topology
- * @return an MPI_Vector that is the distributed version of the global vector
+ * @copydoc global2local
  * @ingroup scatter
  */
 MPI_Vector<thrust::host_vector<double> > global2local( const thrust::host_vector<double>& global, const aMPITopology2d& g)
diff --git a/inc/dg/backend/split_and_join.h b/inc/dg/backend/split_and_join.h
index ba4dc8f5b..4d68c554e 100644
--- a/inc/dg/backend/split_and_join.h
+++ b/inc/dg/backend/split_and_join.h
@@ -66,7 +66,7 @@ void join( const std::vector<thrust_vector>& in, thrust_vector& out, const aTopo
 
 #ifdef MPI_VERSION
 ///@brief MPI Version of join
-///@copydetail dg::join()
+///@copydetails dg::join()
 template<class thrust_vector>
 void join( const std::vector<MPI_Vector<thrust_vector> >& in, MPI_Vector<thrust_vector >& out, const aMPITopology3d& grid)
 {
@@ -76,7 +76,38 @@ void join( const std::vector<MPI_Vector<thrust_vector> >& in, MPI_Vector<thrust_
     for(unsigned i=0; i<grid.Nz(); i++)
         thrust::copy( in[i].data().begin(), in[i].data().end(), out.data().begin()+i*size2d);
 }
+#endif //MPI_VERSION
 
+/**
+* @brief Expand 2d to 3d vector
+* @tparam thrust_vector1 either thrust::host_vector or thrust::device_vector
+* @tparam thrust_vector2 either thrust::host_vector or thrust::device_vector
+* @param in 2d vector of 2d size 
+* @param out contiguous 3d vector (gets resized if necessary) 
+* @param grid provide dimensions in 3rd and first two dimensions
+*/
+template<class thrust_vector1, class thrust_vector2>
+void expand( const thrust_vector1& in, thrust_vector2& out, const aTopology3d& grid)
+{
+    unsigned perp_size = grid.n()*grid.n()*grid.Nx()*grid.Ny();
+    out.resize( grid.size());
+    for( unsigned i=0; i<grid.Nz(); i++)
+        thrust::copy( in.begin(), in.end(), out.begin() + i*perp_size);
+}
+#ifdef MPI_VERSION
+///@brief MPI Version of expand
+///@copydetails dg::expand()
+template<class thrust_vector1, class thrust_vector2>
+void expand( const MPI_Vector<thrust_vector1>& in, MPI_Vector<thrust_vector2>& out, const aMPITopology3d& grid)
+{
+    unsigned perp_size = grid.n()*grid.n()*grid.Nx()*grid.Ny();
+    out.data().resize( grid.size());
+    out.communicator() = grid.communicator();
+    for( unsigned i=0; i<grid.Nz(); i++)
+        thrust::copy( in.data().begin(), in.data().end(), out.data().begin() + i*perp_size);
+}
 #endif //MPI_VERSION
+
+
 ///@}
 }//namespace dg
diff --git a/inc/dg/backend/xspacelib.cuh b/inc/dg/backend/xspacelib.cuh
index 26bc70624..71c1dc366 100644
--- a/inc/dg/backend/xspacelib.cuh
+++ b/inc/dg/backend/xspacelib.cuh
@@ -26,57 +26,6 @@ namespace create{
 ///@addtogroup scatter
 ///@{
 
-/**
- * @brief Create a permutation matrix from a permutation map
- *
- * A permutation can be done in two ways. Either you name to every index in a vector
- * an index where this place should go to ( scatter) or you name to every index the 
- * index of the position that comes to this place (gather). A Scatter is the
- * inverse of a Gather operation with the same index-map. 
- * When transformed to a
- * permutation matrix scatter is the inverse ( = transpose) of gather. (Permutation
- * matrices are orthogonal and sparse)
- * @param map index map
- *
- * @return Permutation matrix
- */
-cusp::coo_matrix<int, double, cusp::host_memory> gather( const thrust::host_vector<int>& map)
-{
-    typedef cusp::coo_matrix<int, double, cusp::host_memory> Matrix;
-    Matrix p( map.size(), map.size(), map.size());
-    cusp::array1d<int, cusp::host_memory> rows( thrust::make_counting_iterator<int>(0), thrust::make_counting_iterator<int>(map.size()));
-    cusp::array1d<int, cusp::host_memory> cols( map.begin(), map.end());
-    cusp::array1d<double, cusp::host_memory> values(map.size(), 1.);
-    p.row_indices = rows;
-    p.column_indices = cols;
-    p.values = values;
-    p.sort_by_row_and_column(); //important!!
-    return p;
-}
-
-/**
- * @brief Create a permutation matrix from a permutation map
- *
- * A permutation can be done in two ways. Either you name to every index in a vector
- * an index where this place should go to ( scatter) or you name to every index the 
- * index of the position that comes to this place (gather). A Scatter is the
- * inverse of a Gather operation with the same index-map. 
- * When transformed to a
- * permutation matrix scatter is the inverse ( = transpose) of gather. (Permutation
- * matrices are orthogonal and sparse)
- * @param map index map
- *
- * @return Permutation matrix
- */
-cusp::coo_matrix<int, double, cusp::host_memory> scatter( const thrust::host_vector<int>& map)
-{
-    typedef cusp::coo_matrix<int, double, cusp::host_memory> Matrix;
-    Matrix p = gather( map);
-    p.row_indices.swap( p.column_indices);
-    p.sort_by_row_and_column(); //important!!
-    return p;
-}
-
 /**
  * @brief make a matrix that transforms values to an equidistant grid ready for visualisation
  *
@@ -84,9 +33,9 @@ cusp::coo_matrix<int, double, cusp::host_memory> scatter( const thrust::host_vec
  * @param g The grid on which to operate 
  *
  * @return transformation matrix
- * @note this matrix has ~n^4 N^2 entries and is not sorted
+ * @note this matrix has ~n^4 N^2 entries 
  */
-cusp::coo_matrix<int, double, cusp::host_memory> backscatter( const aTopology2d& g)
+dg::IHMatrix backscatter( const aTopology2d& g)
 {
     typedef cusp::coo_matrix<int, double, cusp::host_memory> Matrix;
     //create equidistant backward transformation
@@ -104,26 +53,20 @@ cusp::coo_matrix<int, double, cusp::host_memory> backscatter( const aTopology2d&
     //cusp::multiply( p, backward, scatter);
     //choose vector layout
     //return scatter;
-    return backward; 
+    return (dg::IHMatrix)backward; 
 
 }
-/**
- * @brief make a matrix that transforms values to an equidistant grid ready for visualisation
- *
- * Useful if you want to visualize a dg-formatted vector.
- * @param g The 3d grid on which to operate 
- *
- * @return transformation matrix
- * @note this matrix has ~n^4 N^2 entries and is not sorted
- */
-cusp::coo_matrix<int, double, cusp::host_memory> backscatter( const aTopology3d& g)
+
+///@copydoc backscatter(const aTopology2d&)
+dg::IHMatrix backscatter( const aTopology3d& g)
 {
     Grid2d g2d( g.x0(), g.x1(), g.y0(), g.y1(), g.n(), g.Nx(), g.Ny(), g.bcx(), g.bcy());
     cusp::coo_matrix<int,double, cusp::host_memory> back2d = backscatter( g2d);
-    return dgtensor<double>( 1, tensorproduct<double>( g.Nz(), delta(1)), back2d);
+    return (dg::IHMatrix)dgtensor<double>( 1, tensorproduct<double>( g.Nz(), delta(1)), back2d);
 }
+///@}
 
-
+///@cond
 /**
  * @brief Index map for scatter operation on dg - formatted vectors
  *
@@ -172,8 +115,8 @@ thrust::host_vector<int> contiguousLineNumbers( unsigned rows, unsigned cols)
     }
     return map;
 }
+///@endcond
 
-///@}
 
 } //namespace create
 }//namespace dg
diff --git a/inc/dg/blas_b.cu b/inc/dg/blas_b.cu
index 671fffa5a..70780b25c 100644
--- a/inc/dg/blas_b.cu
+++ b/inc/dg/blas_b.cu
@@ -9,7 +9,6 @@
 #include "backend/derivatives.h"
 #include "backend/evaluation.cuh"
 #include "backend/fast_interpolation.h"
-#include "backend/split_and_join.h"
 
 const double lx = 2.*M_PI;
 const double ly = 2.*M_PI;
@@ -106,17 +105,6 @@ int main()
     t.toc();
     std::cout<<"Projection full to half grid     "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
     t.tic();
-    std::vector<Vector> y_split;
-    for( int i=0; i<multi; i++)
-        dg::split( y, y_split, grid);
-    t.toc();
-    std::cout<<"Split                  took      "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
-    t.tic();
-    for( int i=0; i<multi; i++)
-        dg::join( y_split, x, grid);
-    t.toc();
-    std::cout<<"                  Join took      "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
-    t.tic();
     for( int i=0; i<multi; i++)
         dg::blas1::axpby( 1., y, -1., x);
     t.toc();
diff --git a/inc/dg/blas_mpib.cu b/inc/dg/blas_mpib.cu
index bb3f79851..41ee9bc6f 100644
--- a/inc/dg/blas_mpib.cu
+++ b/inc/dg/blas_mpib.cu
@@ -10,7 +10,6 @@
 #include "backend/mpi_derivatives.h"
 #include "backend/mpi_init.h"
 #include "backend/fast_interpolation.h"
-#include "backend/split_and_join.h"
 
 const double lx = 2.*M_PI;
 const double ly = 2.*M_PI;
@@ -100,18 +99,6 @@ int main( int argc, char* argv[])
         dg::blas2::gemv( project, x, x_half);
     t.toc();
     if(rank==0)std::cout<<"Projection full to half grid     "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
-    std::vector<Vector> y_split;
-    t.tic();
-    for( int i=0; i<multi; i++)
-        dg::split( y, y_split, grid);
-    t.toc();
-    if(rank==0)std::cout<<"Split                  took      "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
-    t.tic();
-    for( int i=0; i<multi; i++)
-        dg::join( y_split, x, grid);
-    t.toc();
-    if(rank==0)std::cout<<"                  Join took      "<<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
-
     t.tic();
     for( int i=0; i<multi; i++)
         dg::blas1::axpby( 1., y, -1., x);
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index 7d77dd06f..5f9849ebf 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -9,6 +9,7 @@
 #include "dg/backend/projection.cuh"
 #include "dg/backend/functions.h"
 #include "dg/backend/typedefs.cuh"
+#include "dg/backend/split_and_join.h"
 
 #include "dg/geometry/geometry.h"
 #include "dg/functors.h"
-- 
GitLab


From 41ea7bf7d27df37412ecc1dd1bda94167753efca Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 2 Oct 2017 11:57:20 +0200
Subject: [PATCH 334/453] created parallel member in tensor.h

---
 inc/dg/geometry/tensor.h    | 20 +++++++++++++++++---
 inc/dg/geometry/tensor_t.cu |  2 ++
 2 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/inc/dg/geometry/tensor.h b/inc/dg/geometry/tensor.h
index edd7701df..87f281037 100644
--- a/inc/dg/geometry/tensor.h
+++ b/inc/dg/geometry/tensor.h
@@ -73,7 +73,7 @@ struct SparseElement
 * This class enables shared access to stored Ts 
 * or not store them at all since the storage of (and computation with) a T is expensive.
 
-* This class contains both a (dense) matrix of integers.
+* This class contains a (dense) 3x3 matrix of integers.
 * If positive or zero, the integer represents a gather index into the stored array of Ts, 
 if negative the value of the T is assumed to be 1, except for the off-diagonal entries
     in the matrix where it is assumed to be 0.
@@ -169,7 +169,7 @@ struct SparseTensor
     //if you're looking for this function: YOU DON'T NEED IT!!ALIASING
     //T& value(size_t i, size_t j);
     /**
-     * @brief Return the T at given position, create one if there isn't one already
+     * @brief Return the T at given position, default construct one if there isn't one already
      * @param i index into the values array
      * @return  always returns a T 
      */
@@ -251,9 +251,11 @@ struct SparseTensor
      
      ///construct an empty Tensor
      SparseTensor empty()const{return SparseTensor();}
-     ///copy and erase all values in the third dimension
+     ///erase all values in the third dimension
      ///@note calls clear_unused_values() to get rid of the elements
      SparseTensor perp()const;
+     ///erase all values in the first two dimensions (leave only (2,2))
+     SparseTensor parallel()const;
 
     /**
      * @brief Return the transpose of the currrent tensor
@@ -476,6 +478,18 @@ SparseTensor<container> SparseTensor<container>::perp() const
     t.clear_unused_values();
     return t;
 }
+template<class container>
+SparseTensor<container> SparseTensor<container>::parallel() const
+{
+    SparseTensor<container> t;
+    if( isEmpty()) return t;
+    if( isSet( 2,2) ) 
+    {
+        t.mat_idx_(2,2) = 0;
+        t.values_.assign(1,values_[idx(2,2)] );
+    }
+    return t;
+}
 
 template<class container>
 void SparseTensor<container>::clear_unused_values()
diff --git a/inc/dg/geometry/tensor_t.cu b/inc/dg/geometry/tensor_t.cu
index ba6f79846..722e38777 100644
--- a/inc/dg/geometry/tensor_t.cu
+++ b/inc/dg/geometry/tensor_t.cu
@@ -68,6 +68,8 @@ int main()
     if(!sparse3d.isDense())print( sparse3d);
     std::cout << "perp \n";
     if( sparse3d.perp().isPerp()) print( sparse3d.perp());
+    std::cout << "parallel \n";
+    if( sparse3d.parallel().isDiagonal()) print( sparse3d.parallel());
     std::cout << "transpose \n";
     print( sparse3d.transpose());
 
-- 
GitLab


From 2ad995fac0ba37f5780db2932a028656e36d8b1f Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 2 Oct 2017 15:18:57 +0200
Subject: [PATCH 335/453] use MPI_Comm_compare instead of directly comparing
 communicators

---
 inc/dg/backend/mpi_matrix.h      | 28 +++++++++++++-----
 inc/dg/backend/mpi_precon_blas.h |  7 +++--
 inc/dg/backend/mpi_vector_blas.h | 12 ++++++--
 inc/dg/backend/split_and_join.h  |  6 ++--
 inc/dg/elliptic_b.cu             | 45 ++++++++++++++++++++++++++--
 inc/dg/elliptic_mpib.cu          | 51 ++++++++++++++++++++++++++++----
 inc/dg/geometry/base_geometry.h  |  3 +-
 inc/dg/geometry/mpi_base.h       |  2 +-
 inc/dg/geometry/multiply.h       |  2 +-
 9 files changed, 131 insertions(+), 25 deletions(-)

diff --git a/inc/dg/backend/mpi_matrix.h b/inc/dg/backend/mpi_matrix.h
index bc0684f0b..34f550872 100644
--- a/inc/dg/backend/mpi_matrix.h
+++ b/inc/dg/backend/mpi_matrix.h
@@ -112,8 +112,11 @@ struct RowColDistMat
             return;
 
         }
-        assert( x.communicator() == y.communicator());
-        assert( x.communicator() == m_c.get().communicator());
+        int result;
+        MPI_Comm_compare( x.communicator(), y.communicator(), &result);
+        assert( result == MPI_IDENT);
+        MPI_Comm_compare( x.communicator(), m_c.get().communicator(), &result);
+        assert( result == MPI_IDENT);
         //1. compute inner points
         dg::blas2::detail::doSymv( alpha, m_i, x.data(), beta, y.data(), 
                        typename dg::MatrixTraits<LocalMatrixInner>::matrix_category(), 
@@ -148,8 +151,11 @@ struct RowColDistMat
             return;
 
         }
-        assert( x.communicator() == y.communicator());
-        assert( x.communicator() == m_c.get().communicator());
+        int result;
+        MPI_Comm_compare( x.communicator(), y.communicator(), &result);
+        assert( result == MPI_IDENT);
+        MPI_Comm_compare( x.communicator(), m_c.get().communicator(), &result);
+        assert( result == MPI_IDENT);
         //1. compute inner points
         dg::blas2::detail::doSymv( m_i, x.data(), y.data(), 
                        typename dg::MatrixTraits<LocalMatrixInner>::matrix_category(), 
@@ -245,8 +251,11 @@ struct MPIDistMat
             return;
 
         }
-        assert( x.communicator() == y.communicator());
-        assert( x.communicator() == m_c.communicator());
+        int result;
+        MPI_Comm_compare( x.communicator(), y.communicator(), &result);
+        assert( result == MPI_IDENT);
+        MPI_Comm_compare( x.communicator(), m_c.get().communicator(), &result);
+        assert( result == MPI_IDENT);
         if( m_dist == row_dist){
             m_c.global_gather( x.data(), m_buffer.data());
             dg::blas2::detail::doSymv( alpha, m_m, m_buffer.data(), beta, y.data(), 
@@ -272,8 +281,11 @@ struct MPIDistMat
             return;
 
         }
-        assert( x.communicator() == y.communicator());
-        assert( x.communicator() == m_c.communicator());
+        int result;
+        MPI_Comm_compare( x.communicator(), y.communicator(), &result);
+        assert( result == MPI_IDENT);
+        MPI_Comm_compare( x.communicator(), m_c.get().communicator(), &result);
+        assert( result == MPI_IDENT);
         if( m_dist == row_dist){
             m_c.get().global_gather( x.data(), m_buffer.data());
             dg::blas2::detail::doSymv( m_m, m_buffer.data(), y.data(), 
diff --git a/inc/dg/backend/mpi_precon_blas.h b/inc/dg/backend/mpi_precon_blas.h
index d5b3e39ce..267d796e2 100644
--- a/inc/dg/backend/mpi_precon_blas.h
+++ b/inc/dg/backend/mpi_precon_blas.h
@@ -14,8 +14,11 @@ template< class Precon, class Vector>
 inline typename MatrixTraits<Precon>::value_type doDot( const Vector& x, const Precon& P, const Vector& y, MPIPreconTag, MPIVectorTag)
 {
 #ifdef DG_DEBUG
-    assert( x.communicator() == y.communicator());
-    assert( x.communicator() == P.communicator());
+    int result;
+    MPI_Comm_compare( x.communicator(), y.communicator(), &result);
+    assert( result == MPI_IDENT);
+    MPI_Comm_compare( x.communicator(), P.communicator(), &result);
+    assert( result == MPI_IDENT);
 #endif //DG_DEBUG
     //computation
     typename MatrixTraits<Precon>::value_type temp= doDot(x.data(), P.data(), y.data(), ThrustMatrixTag(), ThrustVectorTag());
diff --git a/inc/dg/backend/mpi_vector_blas.h b/inc/dg/backend/mpi_vector_blas.h
index 3cde835d9..589c1d709 100644
--- a/inc/dg/backend/mpi_vector_blas.h
+++ b/inc/dg/backend/mpi_vector_blas.h
@@ -17,7 +17,9 @@ namespace detail{
 template< class Vector1, class Vector2>
 void doTransfer( const Vector1& in, Vector2& out, MPIVectorTag, MPIVectorTag)
 {
-    out.communicator() = in.communicator();
+    int result;
+    MPI_Comm_compare( in.communicator(), out.communicator(), &result);
+    assert( result == MPI_IDENT);
     //local computation 
     typedef typename Vector1::container_type container1;
     typedef typename Vector2::container_type container2;
@@ -28,7 +30,9 @@ template< class Vector>
 typename VectorTraits<Vector>::value_type doDot( const Vector& x, const Vector& y, MPIVectorTag)
 {
 #ifdef DG_DEBUG
-    assert( x.communicator() == y.communicator());
+    int result;
+    MPI_Comm_compare( x.communicator(), y.communicator(), &result);
+    assert( result == MPI_IDENT);
 #endif //DG_DEBUG
     typedef typename Vector::container_type container;
     
@@ -65,7 +69,9 @@ inline void doTransform(  const Vector& x, Vector& y,
                           MPIVectorTag)
 {
 #ifdef DG_DEBUG
-    assert( x.communicator() == y.communicator());
+    int result;
+    MPI_Comm_compare( x.communicator(), y.communicator(), &result);
+    assert( result == MPI_IDENT);
 #endif //DG_DEBUG
     typedef typename Vector::container_type container;
     doTransform( x.data(), y.data(), op, typename VectorTraits<container>::vector_category());
diff --git a/inc/dg/backend/split_and_join.h b/inc/dg/backend/split_and_join.h
index 4d68c554e..4fa9f6ef2 100644
--- a/inc/dg/backend/split_and_join.h
+++ b/inc/dg/backend/split_and_join.h
@@ -32,9 +32,11 @@ void split( const thrust_vector& in, std::vector<thrust_vector>& out, const aTop
 template <class thrust_vector>
 void split( const MPI_Vector<thrust_vector>& in, std::vector<MPI_Vector<thrust_vector> >& out, const aMPITopology3d& grid)
 {
-    assert( in.communicator() == grid.communicator());
+    int result;
+    MPI_Comm_compare( x.communicator(), grid.communicator(), &result);
+    assert( result == MPI_IDENT);
     MPI_Comm planeComm;
-    int remain_dims[] = {true,true,false}; //true true false
+    int remain_dims[] = {true,true,false}; 
     MPI_Cart_sub( in.communicator(), remain_dims, &planeComm);
     //local size2d
     unsigned size2d=grid.n()*grid.n()*grid.Nx()*grid.Ny();
diff --git a/inc/dg/elliptic_b.cu b/inc/dg/elliptic_b.cu
index 746ba0507..2c81f1fa9 100644
--- a/inc/dg/elliptic_b.cu
+++ b/inc/dg/elliptic_b.cu
@@ -11,6 +11,7 @@
 #include "backend/derivatives.h"
 #include "backend/typedefs.cuh"
 #include "backend/cusp_thrust_backend.h"
+#include "backend/split_and_join.h"
 
 #include "cg.h"
 #include "elliptic.h"
@@ -61,10 +62,12 @@ int main()
     
     std::cout << "For a precision of "<< eps<<" ..."<<std::endl;
     x = dg::evaluate( initial, grid);
+    unsigned num;
     t.tic();
-    std::cout << "Number of pcg iterations "<< pcg( laplace, x, b, v3d, eps)<<std::endl;
+    num = pcg( laplace, x, b, v3d, eps);
     t.toc();
-    std::cout << "... on the device took "<< t.diff()<<"s\n";
+    std::cout << "Number of pcg iterations "<<num<<std::endl;
+    std::cout << "... on the device took   "<< t.diff()<<"s\n";
     dg::DVec  error(  solution);
     dg::blas1::axpby( 1., x,-1., error);
 
@@ -76,6 +79,44 @@ int main()
     normerr = dg::blas2::dot( w3d, error); 
     norm = dg::blas2::dot( w3d, deriv);
     std::cout << "L2 Norm of relative error in derivative is: " <<sqrt( normerr/norm)<<std::endl;
+    
+    std::cout << "TEST SPLIT SOLUTION\n";
+    x = dg::evaluate( initial, grid);
+    b = dg::evaluate ( laplace_fct, grid);
+    //create grid and perp and parallel volume
+    dg::Handle<dg::aGeometry2d> grid_perp = grid.perp_grid();
+    dg::DVec v2d = dg::create::inv_volume( grid_perp.get());
+    dg::DVec w2d = dg::create::volume( grid_perp.get());
+    dg::SparseElement<dg::DVec> g_parallel = dg::tensor::volume( grid.metric().parallel());
+    dg::DVec chi = dg::evaluate( dg::one, grid);
+    dg::tensor::pointwiseDot( chi, g_parallel, chi);
+    //create split Laplacian
+    std::vector< dg::Elliptic<dg::aGeometry2d, dg::DMatrix, dg::DVec> > laplace_split( 
+            grid.Nz(), dg::Elliptic<dg::aGeometry2d, dg::DMatrix, dg::DVec>(grid_perp.get(), dg::not_normed, dg::centered));
+    // create split  vectors and solve
+    std::vector<dg::DVec> b_split, x_split, chi_split;
+    pcg.construct( w2d, w2d.size());
+    std::vector<unsigned>  number(grid.Nz());
+    t.tic();
+    dg::tensor::pointwiseDot( b, g_parallel, b);
+    dg::split( b, b_split, grid);
+    dg::split( chi, chi_split, grid);
+    dg::split( x, x_split, grid);
+    for( unsigned i=0; i<grid.Nz(); i++)
+    {
+        laplace_split[i].set_chi( chi_split[i]);
+        dg::blas1::pointwiseDot( b_split[i], w2d, b_split[i]);
+        number[i] = pcg( laplace_split[i], x_split[i], b_split[i], v2d, eps);
+    }
+    dg::join( x_split, x, grid);
+    t.toc();
+    std::cout << "Number of iterations in split     "<< number[0]<<"\n";
+    std::cout << "Split solution on the device took "<< t.diff()<<"s\n";
+    dg::blas1::axpby( 1., x,-1., solution, error);
+    normerr = dg::blas2::dot( w3d, error);
+    norm = dg::blas2::dot( w3d, solution);
+    std::cout << "L2 Norm of relative error is:     " <<sqrt( normerr/norm)<<std::endl;
+
     //both function and derivative converge with order P 
 
     return 0;
diff --git a/inc/dg/elliptic_mpib.cu b/inc/dg/elliptic_mpib.cu
index 3b0f07fb9..b1db4822a 100644
--- a/inc/dg/elliptic_mpib.cu
+++ b/inc/dg/elliptic_mpib.cu
@@ -8,6 +8,7 @@
 
 #include "backend/timer.cuh"
 #include "backend/mpi_init.h"
+#include "backend/split_and_join.h"
 
 
 const double R_0 = 1000;
@@ -18,6 +19,7 @@ double fct( double x, double y, double z){ return sin(x-R_0)*sin(y);}
 double derivative( double x, double y, double z){return cos(x-R_0)*sin(y);}
 double laplace_fct( double x, double y, double z) { return -1./x*cos(x-R_0)*sin(y) + 2.*sin(y)*sin(x-R_0);}
 dg::bc bcx = dg::DIR;
+dg::bc bcy = dg::PER;
 double initial( double x, double y, double z) {return sin(0);}
 
 
@@ -26,9 +28,9 @@ int main( int argc, char* argv[])
     MPI_Init(&argc, &argv);
     unsigned n, Nx, Ny, Nz; 
     MPI_Comm comm;
-    dg::mpi_init3d( bcx, dg::PER, dg::PER, n, Nx, Ny, Nz, comm);
+    dg::mpi_init3d( bcx, bcy, dg::PER, n, Nx, Ny, Nz, comm);
 
-    dg::CylindricalMPIGrid3d grid( R_0, R_0+lx, 0, ly, 0,lz, n, Nx, Ny,Nz, bcx, dg::PER, dg::PER, comm);
+    dg::CylindricalMPIGrid3d grid( R_0, R_0+lx, 0, ly, 0,lz, n, Nx, Ny,Nz, bcx, bcy, dg::PER, comm);
     const dg::MDVec w3d = dg::create::volume( grid);
     const dg::MDVec v3d = dg::create::inv_volume( grid);
     int rank;
@@ -60,10 +62,10 @@ int main( int argc, char* argv[])
     
     if(rank==0)std::cout << "For a precision of "<< eps<<" ..."<<std::endl;
     t.tic();
-    unsigned number = pcg( laplace,x,b,v3d,eps);
-    if(rank==0)std::cout << "Number of pcg iterations "<< number<<std::endl;
+    unsigned num = pcg( laplace,x,b,v3d,eps);
+    if(rank==0)std::cout << "Number of pcg iterations "<< num<<std::endl;
     t.toc();
-    if(rank==0)std::cout << " took "<< t.diff()<<"s\n";
+    if(rank==0)std::cout << "... took                 "<< t.diff()<<"s\n";
     dg::MDVec  error(  solution);
     dg::blas1::axpby( 1., x,-1., error);
 
@@ -75,6 +77,45 @@ int main( int argc, char* argv[])
     normerr = dg::blas2::dot( w3d, error); 
     norm = dg::blas2::dot( w3d, deriv);
     if(rank==0)std::cout << "L2 Norm of relative error in derivative is: " <<sqrt( normerr/norm)<<std::endl;
+
+    if(rank==0)std::cout << "TEST SPLIT SOLUTION\n";
+    x = dg::evaluate( initial, grid);
+    b = dg::evaluate ( laplace_fct, grid);
+    //create grid and perp and parallel volume
+    dg::Handle<dg::aMPIGeometry2d> grid_perp = grid.perp_grid();
+    dg::MDVec v2d = dg::create::inv_volume( grid_perp.get());
+    dg::MDVec w2d = dg::create::volume( grid_perp.get());
+    dg::SparseElement<dg::MDVec> g_parallel = dg::tensor::volume( grid.metric().parallel());
+    dg::MDVec chi = dg::evaluate( dg::one, grid);
+    dg::tensor::pointwiseDot( chi, g_parallel, chi);
+    //create split Laplacian
+    std::vector< dg::Elliptic<dg::aMPIGeometry2d, dg::MDMatrix, dg::MDVec> > laplace_split( 
+            grid.Nz(), dg::Elliptic<dg::aMPIGeometry2d, dg::MDMatrix, dg::MDVec>(grid_perp.get(), dg::not_normed, dg::centered));
+    // create split  vectors and solve
+    std::vector<dg::MDVec> b_split, x_split, chi_split;
+    pcg.construct( w2d, w2d.size());
+    std::vector<unsigned>  number(grid.Nz());
+    t.tic();
+    dg::tensor::pointwiseDot( b, g_parallel, b);
+    dg::split( b, b_split, grid);
+    dg::split( chi, chi_split, grid);
+    dg::split( x, x_split, grid);
+    for( unsigned i=0; i<grid.Nz(); i++)
+        if(rank==0)std::cout << " "<<x_split[i].communicator()<<" "<<b_split[i].communicator()<<" "<<v2d.communicator()<<std::endl;
+    for( unsigned i=0; i<grid.Nz(); i++)
+    {
+        laplace_split[i].set_chi( chi_split[i]);
+        dg::blas1::pointwiseDot( b_split[i], w2d, b_split[i]);
+        number[i] = pcg( laplace_split[i], x_split[i], b_split[i], v2d, eps);
+    }
+    dg::join( x_split, x, grid);
+    t.toc();
+    if(rank==0)std::cout << "Number of iterations in split     "<< number[0]<<"\n";
+    if(rank==0)std::cout << "Split solution on the device took "<< t.diff()<<"s\n";
+    dg::blas1::axpby( 1., x,-1., solution, error);
+    normerr = dg::blas2::dot( w3d, error);
+    norm = dg::blas2::dot( w3d, solution);
+    if(rank==0)std::cout << "L2 Norm of relative error is:     " <<sqrt( normerr/norm)<<std::endl;
     //both function and derivative converge with order P 
 
     MPI_Finalize();
diff --git a/inc/dg/geometry/base_geometry.h b/inc/dg/geometry/base_geometry.h
index cc3cce7cd..a539f1d96 100644
--- a/inc/dg/geometry/base_geometry.h
+++ b/inc/dg/geometry/base_geometry.h
@@ -33,6 +33,7 @@ struct aGeometry2d : public aTopology2d
     g = \begin{pmatrix} g^{xx}(x,y) & g^{xy}(x,y) \\  & g^{yy}(x,y) \end{pmatrix}
     \f]
     * @return symmetric tensor
+    * @note use the dg::tensor functions to compute the volume element from here
     */
     SparseTensor<thrust::host_vector<double> > metric()const { 
         return do_compute_metric(); 
@@ -112,6 +113,7 @@ struct aGeometry3d : public aTopology3d
       & & g^{zz}(x,y,z)\end{pmatrix}
     \f]
     * @return symmetric tensor
+    * @note use the dg::tensor functions to compute the volume element from here
     */
     SparseTensor<thrust::host_vector<double> > metric()const { 
         return do_compute_metric(); 
@@ -251,7 +253,6 @@ struct CylindricalGrid3d: public dg::aProductGeometry3d
     ///@note x corresponds to R, y to Z and z to phi, the volume element is R
     CylindricalGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx = PER, bc bcy = PER, bc bcz = PER): dg::aProductGeometry3d(x0,x1,y0,y1,z0,z1,n,Nx,Ny,Nz,bcx,bcy,bcz){}
     virtual CylindricalGrid3d* clone()const{return new CylindricalGrid3d(*this);}
-    ///@copydoc  CartesianGrid3d::perp_grid()const
     private:
     CartesianGrid2d* do_perp_grid() const{ return new CartesianGrid2d(x0(),x1(),y0(),y1(),n(),Nx(),Ny(),bcx(),bcy());}
     virtual SparseTensor<thrust::host_vector<double> > do_compute_metric()const{
diff --git a/inc/dg/geometry/mpi_base.h b/inc/dg/geometry/mpi_base.h
index a95fb2a3d..d89d78a1d 100644
--- a/inc/dg/geometry/mpi_base.h
+++ b/inc/dg/geometry/mpi_base.h
@@ -254,7 +254,7 @@ struct CylindricalMPIGrid3d: public aProductMPIGeometry3d
     MPI_Comm get_perp_comm( MPI_Comm src) const
     {
         MPI_Comm planeComm;
-        int remain_dims[] = {true,true,false}; //true true false
+        int remain_dims[] = {true,true,false}; 
         MPI_Cart_sub( src, remain_dims, &planeComm);
         return planeComm;
     }
diff --git a/inc/dg/geometry/multiply.h b/inc/dg/geometry/multiply.h
index 4e175f647..d31caf562 100644
--- a/inc/dg/geometry/multiply.h
+++ b/inc/dg/geometry/multiply.h
@@ -229,7 +229,7 @@ SparseElement<container> determinant( const SparseTensor<container>& t)
  *
  * This is a convenience function that is the same as
  * @code
-    SparseElement<container> volume=determinant(g);
+    SparseElement<container> volume=determinant(t);
     invert(volume);
     sqrt(volume);
     @endcode
-- 
GitLab


From d3c2b7137bb17c4cd5b02050e4259f054a1fdda0 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 2 Oct 2017 16:30:06 +0200
Subject: [PATCH 336/453] implemented split solution method in elliptic_b and
 elliptic_mpib

---
 inc/dg/backend/mpi_matrix.h      | 16 ++++++++--------
 inc/dg/backend/mpi_precon_blas.h |  4 ++--
 inc/dg/backend/mpi_vector_blas.h | 10 ++++------
 inc/dg/backend/split_and_join.h  |  7 ++++---
 inc/dg/elliptic_mpib.cu          |  2 --
 5 files changed, 18 insertions(+), 21 deletions(-)

diff --git a/inc/dg/backend/mpi_matrix.h b/inc/dg/backend/mpi_matrix.h
index 34f550872..bc2f91e9c 100644
--- a/inc/dg/backend/mpi_matrix.h
+++ b/inc/dg/backend/mpi_matrix.h
@@ -114,9 +114,9 @@ struct RowColDistMat
         }
         int result;
         MPI_Comm_compare( x.communicator(), y.communicator(), &result);
-        assert( result == MPI_IDENT);
+        assert( result == MPI_CONGRUENT || result == MPI_IDENT);
         MPI_Comm_compare( x.communicator(), m_c.get().communicator(), &result);
-        assert( result == MPI_IDENT);
+        assert( result == MPI_CONGRUENT || result == MPI_IDENT);
         //1. compute inner points
         dg::blas2::detail::doSymv( alpha, m_i, x.data(), beta, y.data(), 
                        typename dg::MatrixTraits<LocalMatrixInner>::matrix_category(), 
@@ -153,9 +153,9 @@ struct RowColDistMat
         }
         int result;
         MPI_Comm_compare( x.communicator(), y.communicator(), &result);
-        assert( result == MPI_IDENT);
+        assert( result == MPI_CONGRUENT || result == MPI_IDENT);
         MPI_Comm_compare( x.communicator(), m_c.get().communicator(), &result);
-        assert( result == MPI_IDENT);
+        assert( result == MPI_CONGRUENT || result == MPI_IDENT);
         //1. compute inner points
         dg::blas2::detail::doSymv( m_i, x.data(), y.data(), 
                        typename dg::MatrixTraits<LocalMatrixInner>::matrix_category(), 
@@ -253,9 +253,9 @@ struct MPIDistMat
         }
         int result;
         MPI_Comm_compare( x.communicator(), y.communicator(), &result);
-        assert( result == MPI_IDENT);
+        assert( result == MPI_CONGRUENT);
         MPI_Comm_compare( x.communicator(), m_c.get().communicator(), &result);
-        assert( result == MPI_IDENT);
+        assert( result == MPI_CONGRUENT);
         if( m_dist == row_dist){
             m_c.global_gather( x.data(), m_buffer.data());
             dg::blas2::detail::doSymv( alpha, m_m, m_buffer.data(), beta, y.data(), 
@@ -283,9 +283,9 @@ struct MPIDistMat
         }
         int result;
         MPI_Comm_compare( x.communicator(), y.communicator(), &result);
-        assert( result == MPI_IDENT);
+        assert( result == MPI_CONGRUENT);
         MPI_Comm_compare( x.communicator(), m_c.get().communicator(), &result);
-        assert( result == MPI_IDENT);
+        assert( result == MPI_CONGRUENT);
         if( m_dist == row_dist){
             m_c.get().global_gather( x.data(), m_buffer.data());
             dg::blas2::detail::doSymv( m_m, m_buffer.data(), y.data(), 
diff --git a/inc/dg/backend/mpi_precon_blas.h b/inc/dg/backend/mpi_precon_blas.h
index 267d796e2..090da543d 100644
--- a/inc/dg/backend/mpi_precon_blas.h
+++ b/inc/dg/backend/mpi_precon_blas.h
@@ -16,9 +16,9 @@ inline typename MatrixTraits<Precon>::value_type doDot( const Vector& x, const P
 #ifdef DG_DEBUG
     int result;
     MPI_Comm_compare( x.communicator(), y.communicator(), &result);
-    assert( result == MPI_IDENT);
+    assert( result == MPI_CONGRUENT || result == MPI_IDENT);
     MPI_Comm_compare( x.communicator(), P.communicator(), &result);
-    assert( result == MPI_IDENT);
+    assert( result == MPI_CONGRUENT || result == MPI_IDENT);
 #endif //DG_DEBUG
     //computation
     typename MatrixTraits<Precon>::value_type temp= doDot(x.data(), P.data(), y.data(), ThrustMatrixTag(), ThrustVectorTag());
diff --git a/inc/dg/backend/mpi_vector_blas.h b/inc/dg/backend/mpi_vector_blas.h
index 589c1d709..1592ea35b 100644
--- a/inc/dg/backend/mpi_vector_blas.h
+++ b/inc/dg/backend/mpi_vector_blas.h
@@ -17,13 +17,11 @@ namespace detail{
 template< class Vector1, class Vector2>
 void doTransfer( const Vector1& in, Vector2& out, MPIVectorTag, MPIVectorTag)
 {
-    int result;
-    MPI_Comm_compare( in.communicator(), out.communicator(), &result);
-    assert( result == MPI_IDENT);
-    //local computation 
+    out.communicator() = in.communicator();
     typedef typename Vector1::container_type container1;
     typedef typename Vector2::container_type container2;
     doTransfer( in.data(), out.data(), typename VectorTraits<container1>::vector_category(), typename VectorTraits<container2>::vector_category());
+
 }
 
 template< class Vector>
@@ -32,7 +30,7 @@ typename VectorTraits<Vector>::value_type doDot( const Vector& x, const Vector&
 #ifdef DG_DEBUG
     int result;
     MPI_Comm_compare( x.communicator(), y.communicator(), &result);
-    assert( result == MPI_IDENT);
+    assert( result == MPI_CONGRUENT || result == MPI_IDENT); 
 #endif //DG_DEBUG
     typedef typename Vector::container_type container;
     
@@ -71,7 +69,7 @@ inline void doTransform(  const Vector& x, Vector& y,
 #ifdef DG_DEBUG
     int result;
     MPI_Comm_compare( x.communicator(), y.communicator(), &result);
-    assert( result == MPI_IDENT);
+    assert( result == MPI_CONGRUENT || result == MPI_IDENT); 
 #endif //DG_DEBUG
     typedef typename Vector::container_type container;
     doTransform( x.data(), y.data(), op, typename VectorTraits<container>::vector_category());
diff --git a/inc/dg/backend/split_and_join.h b/inc/dg/backend/split_and_join.h
index 4fa9f6ef2..3d76c772a 100644
--- a/inc/dg/backend/split_and_join.h
+++ b/inc/dg/backend/split_and_join.h
@@ -28,13 +28,14 @@ void split( const thrust_vector& in, std::vector<thrust_vector>& out, const aTop
 #ifdef MPI_VERSION
 ///@brief MPI Version of split
 ///@copydetails dg::split()
-///@note every plane in out gets its own 2d Cartesian communicator
+///@note every plane in out holds a 2d Cartesian MPI_Communicator 
+///@note two seperately split vectors have congruent (not identical) MPI_Communicators (Note here the MPI concept of congruent vs. identical communicators)
 template <class thrust_vector>
 void split( const MPI_Vector<thrust_vector>& in, std::vector<MPI_Vector<thrust_vector> >& out, const aMPITopology3d& grid)
 {
     int result;
-    MPI_Comm_compare( x.communicator(), grid.communicator(), &result);
-    assert( result == MPI_IDENT);
+    MPI_Comm_compare( in.communicator(), grid.communicator(), &result);
+    assert( result == MPI_CONGRUENT || result == MPI_IDENT);
     MPI_Comm planeComm;
     int remain_dims[] = {true,true,false}; 
     MPI_Cart_sub( in.communicator(), remain_dims, &planeComm);
diff --git a/inc/dg/elliptic_mpib.cu b/inc/dg/elliptic_mpib.cu
index b1db4822a..7cdb03fa9 100644
--- a/inc/dg/elliptic_mpib.cu
+++ b/inc/dg/elliptic_mpib.cu
@@ -100,8 +100,6 @@ int main( int argc, char* argv[])
     dg::split( b, b_split, grid);
     dg::split( chi, chi_split, grid);
     dg::split( x, x_split, grid);
-    for( unsigned i=0; i<grid.Nz(); i++)
-        if(rank==0)std::cout << " "<<x_split[i].communicator()<<" "<<b_split[i].communicator()<<" "<<v2d.communicator()<<std::endl;
     for( unsigned i=0; i<grid.Nz(); i++)
     {
         laplace_split[i].set_chi( chi_split[i]);
-- 
GitLab


From e642885f7f026ee0b6c2589810eddf24c54d6c24 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Tue, 3 Oct 2017 17:44:57 +0200
Subject: [PATCH 337/453] about to change ds.h and changed jump terms to be
 self-adjoint in elliptic.h

---
 inc/dg/elliptic.h             |  41 +++--
 inc/geometries/ds.h           | 290 +++++++++++++++++-----------------
 inc/geometries/fieldaligned.h |   4 +-
 3 files changed, 170 insertions(+), 165 deletions(-)

diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index 23341af78..4a7b5f343 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -24,19 +24,21 @@ namespace dg
  * @ingroup matrixoperators
  *
  * The term discretized is \f[ -\nabla \cdot ( \chi \nabla_\perp ) \f]
- * where \f$ \nabla_\perp \f$ is the perpendicular gradient. In general 
- * coordinates that means 
+ * where \f$ \nabla_\perp \f$ is the perpendicular gradient and \f$\chi\f$ is a spatially dependent function. 
+ * In general coordinates that means 
  * \f[ -\frac{1}{\sqrt{g}}\left( 
  * \partial_x\left(\sqrt{g}\chi \left(g^{xx}\partial_x + g^{xy}\partial_y \right)\right) 
  + \partial_y\left(\sqrt{g}\chi \left(g^{yx}\partial_x + g^{yy}\partial_y \right)\right) \right)\f]
- is discretized. Note that the discontinuous Galerkin discretization adds so-called
- jump terms via a jump matrix 
- \f[ D^\dagger_x \chi D_x + \alpha J \f]
- where \f$\alpha\f$  is a scale factor ( = jfactor). Usually the default \f$ \alpha=1 \f$ is a good choice.
+ is discretized. Note that the local discontinuous Galerkin discretization adds so-called
+ jump terms 
+ \f[ D^\dagger \chi D + \alpha J \f]
+ where \f$\alpha\f$  is a scale factor ( = jfactor), \f$ D \f$ contains the discretizations of the above derivatives, and \f$ J\f$ is a self-adjoint matrix. 
+ (The symmetric part of \f$J\f$ is added @b before the volume element is divided). The adjoint of a matrix is defined with respect to the volume element including dG weights.
+ Usually the default \f$ \alpha=1 \f$ is a good choice.
  However, in some cases, e.g. when \f$ \chi \f$ exhibits very large variations
  \f$ \alpha=0.1\f$ or \f$ \alpha=0.01\f$ might be better values. 
  In a time dependent problem the value of \f$\alpha\f$ determines the 
- numerical diffusion, i.e. for low values numerical oscillations may appear. 
+ numerical diffusion, i.e. for too low values numerical oscillations may appear. 
  Also note that a forward discretization has more diffusion than a centered discretization.
 
  * @copydoc hide_geometry_matrix_container
@@ -44,9 +46,8 @@ namespace dg
  * and thus in a conjugate gradient solver. 
  * @note The constructors initialize \f$ \chi=1\f$ so that a negative laplacian operator
  * results
- * @note The inverse dG weights make a good general purpose preconditioner, but 
- * the inverse of \f$ \chi\f$ should also seriously be considered
- * @note the jump term \f$ \alpha J\f$  adds artificial numerical diffusion
+ * @note The inverse of \f$ \chi\f$ makes a good general purpose preconditioner
+ * @note the jump term \f$ \alpha J\f$  adds artificial numerical diffusion as discussed above
  * @attention Pay attention to the negative sign 
  */
 template <class Geometry, class Matrix, class container>
@@ -184,15 +185,14 @@ class Elliptic
         //now take divergence
         dg::blas2::symv( lefty, tempy, y);  
         dg::blas2::symv( -1., leftx, gradx, -1., y);  
-        if( no_ == normed)
-            dg::tensor::pointwiseDivide( y, vol_, y);
 
         //add jump terms
         dg::blas2::symv( jfactor_, jumpX, x, 1., y);
         dg::blas2::symv( jfactor_, jumpY, x, 1., y);
+        if( no_ == normed)
+            dg::tensor::pointwiseDivide( y, vol_, y);
         if( no_ == not_normed)//multiply weights without volume
             dg::blas2::symv( weights_wo_vol, y, y);
-
     }
 
     private:
@@ -337,6 +337,8 @@ struct GeneralElliptic
         zchi = chi[2];
     }
 
+    ///@copydoc Elliptic::weights()
+    const container& weights()const {return weights_;}
     ///@copydoc Elliptic::inv_weights()
     const container& inv_weights()const {return inv_weights_;}
     /**
@@ -370,15 +372,12 @@ struct GeneralElliptic
         dg::blas1::pointwiseDot( zchi, temp0, temp1); 
         dg::blas2::gemv( -1., leftz, temp1, 1., y); 
 
-        if( no_==normed) 
-            dg::tensor::pointwiseDivide( temp0, vol_, temp0);
-        
         dg::blas2::symv( +1., jumpX, x, 1., y);
         dg::blas2::symv( +1., jumpY, x, 1., y);
-        if( no_==not_normed)//multiply weights w/o volume
+        dg::tensor::pointwiseDivide( y, vol_, y);
+        if( no_==not_normed)//multiply weights 
         {
-            dg::tensor::pointwiseDivide( y, vol_, y);
-            dg::blas1::pointwiseDivide( y, inv_weights_, y);
+            dg::blas1::pointwiseDot( y, weights_, y);
         }
     }
     private:
@@ -620,12 +619,12 @@ struct TensorElliptic
         //now take divergence
         dg::blas2::gemv( -1., leftx, gradx_, 0., y);  
         dg::blas2::gemv( -1., lefty, tempy_, 1., y);  
-        if( no_ == normed)
-            dg::tensor::pointwiseDivide( y, vol_,y);
 
         //add jump terms
         dg::blas2::symv( +1., jumpX, x, 1., y);
         dg::blas2::symv( +1., jumpY, x, 1., y);
+        if( no_ == normed)
+            dg::tensor::pointwiseDivide( y, vol_,y);
         if( no_ == not_normed)//multiply weights without volume
             dg::blas2::symv( weights_wo_vol, y, y);
     }
diff --git a/inc/geometries/ds.h b/inc/geometries/ds.h
index 48bd02ede..aec57660a 100644
--- a/inc/geometries/ds.h
+++ b/inc/geometries/ds.h
@@ -3,6 +3,7 @@
 #include "dg/blas.h"
 #include "dg/geometry/geometry.h"
 #include "dg/backend/derivatives.h"
+#include "backend/split_and_join.h"
 #include "fieldaligned.h"
 #ifdef MPI_VERSION
 #include "backend/mpi_derivatives.h"
@@ -49,11 +50,15 @@ struct DS
     /**
     * @brief Apply the forward derivative on a 3d vector
     *
-    * forward derivative \f$ \frac{1}{h_z^+}(f_{i+1} - f_{i})\f$
+    * forward derivative \f$ g_i = \alpha \frac{1}{h_z^+}(f_{i+1} - f_{i}) + \beta g_i\f$
     * @param f The vector to derive
     * @param dsf contains result on output (write only)
     */
-    void forward( const container& f, container& dsf);
+    void forward( double alpha, const container& f, double beta, container& dsf){
+        dg::split( f, m_f);
+        do_forward( alpha, m_f, beta, m_dsf);
+        dg::join( m_dsf, dsf);
+    }
     /**
     * @brief Apply the backward derivative on a 3d vector
     *
@@ -61,7 +66,11 @@ struct DS
     * @param f The vector to derive
     * @param dsf contains result on output (write only)
     */
-    void backward( const container& f, container& dsf);
+    void backward( double alpha, const container& f, double beta, container& dsf){
+        dg::split( f, m_f);
+        do_backward( alpha, m_f, beta, m_dsf);
+        dg::join( m_dsf, dsf);
+    }
     /**
     * @brief Apply the centered derivative on a 3d vector
     *
@@ -69,7 +78,11 @@ struct DS
     * @param f The vector to derive
     * @param dsf contains result on output (write only)
     */
-    void centered( const container& f, container& dsf);
+    void centered( double alpha, const container& f, double beta, container& dsf){
+        dg::split( f, m_f);
+        do_centered( alpha, m_f, beta, m_dsf);
+        dg::join( m_dsf, dsf);
+    }
 
     /**
     * @brief Apply the negative adjoint derivative on a 3d vector
@@ -77,21 +90,33 @@ struct DS
     * @param f The vector to derive
     * @param dsf contains result on output (write only)
     */
-    void forwardAdj( const container& f, container& dsf);
+    void forwardAdj( double alpha, const container& f, double beta, container& dsf){
+        dg::split( m_temp, m_f);
+        do_forwardAdj( alpha, m_f, beta, m_dsf, dg::normed);
+        dg::join( m_dsf, dsf);
+    }
     /**
     * @brief Apply the negative adjoint derivative on a 3d vector
     *
     * @param f The vector to derive
     * @param dsf contains result on output (write only)
     */
-    void backwardAdj( const container& f, container& dsf);
+    void backwardAdj( double alpha, const container& f, double beta, container& dsf){
+        dg::split( f, m_f);
+        do_backwardAdj( alpha, m_f, beta, m_dsf, dg::normed);
+        dg::join( m_dsf, dsf);
+    }
     /**
     * @brief Apply the negative adjoint derivative on a 3d vector
     *
     * @param f The vector to derive
     * @param dsf contains result on output (write only)
     */
-    void centeredAdjDir( const container& f, container& dsf);
+    void centeredAdj(double alpha, const container& f, double beta, container& dsf){
+        dg::split( f, m_f);
+        do_centeredAdj( alpha, m_f, beta, m_dsf, dg::normed);
+        dg::join( m_dsf, dsf);
+    }
     /**
     * @brief Apply the negative adjoint derivative on a 3d vector with the direct method
     *
@@ -99,21 +124,36 @@ struct DS
     * @param f The vector to derive
     * @param dsf contains result on output (write only)
     */
-    void forwardAdjDir( const container& f, container& dsf);
+    void forwardAdjDir( double alpha, const container& f, double beta, container& dsf)
+    {
+        dg::blas1::pointwiseDivide( f, m_B, temp);
+        forward( temp, temp);
+        dg::blas1::pointwiseDot( alpha, temp, m_B, beta, dsf);
+    }
     /**
     * @brief Apply the negative adjoint derivative on a 3d vector with the direct method
     *
     * @param f The vector to derive
     * @param dsf contains result on output (write only)
     */
-    void backwardAdjDir( const container& f, container& dsf);
+    void backwardAdjDir( double alpha, const container& f, double beta, container& dsf)
+    {
+        dg::blas1::pointwiseDivide( f, m_B, temp);
+        backward( temp, temp);
+        dg::blas1::pointwiseDot( alpha, temp, m_B, beta, dsf);
+    }
     /**
     * @brief Apply the negative adjoint derivative on a 3d vector with the direct method
     *
     * @param f The vector to derive
     * @param dsf contains result on output (write only)
     */
-    void centeredAdj( const container& f, container& dsf);
+    void centeredAdjDir( double alpha, const container& f, double beta, container& dsf)
+    {
+        dg::blas1::pointwiseDivide( f, m_B, temp);
+        backward( temp, temp);
+        dg::blas1::pointwiseDot( alpha, temp, m_B, beta, dsf);
+    }
 
     /**
     * @brief compute parallel derivative
@@ -133,7 +173,10 @@ struct DS
      * @param dsTdsf contains result on output (write only)
      * @note if apply_jumpX is false then no jumpy terms will be added in the x-direction
      */
-    void symv( const container& f, container& dsTdsf);
+    void symv( const container& f, container& dsTdsf){ symv( 1., f, 0., dsTdsf);}
+    void symv( double alpha, const container& f, double beta, container& dsTdsf){
+        do_symv( alpha, f, beta, dsTdsf);
+    }
 
     /**
     * @brief Set boundary conditions in the limiter region
@@ -183,6 +226,7 @@ struct DS
      * @return weights
      */
     const container& weights()const {return vol3d;}
+    const container& inv_weights()const {return inv3d;}
     /**
      * @brief Returns the preconditioner to use in conjugate gradient
      *
@@ -199,17 +243,25 @@ struct DS
     */
     const FieldAligned<Geometry, IMatrix, container>& fieldaligned() const{return f_;}
     private:
-    FieldAligned<Geometry,IMatrix,container> f_;
-    Matrix jumpX, jumpY;
-    container tempP, temp0, tempM;
-    container f, dsf;
-    container vol3d, inv3d;
-    container invB;
+    void do_forward(double alpha, const std::vector<container>& f, double beta, std::vector<container>& dsf);
+    void do_backward(double alpha, const std::vector<container>& f, double beta, std::vector<container>& dsf);
+    void do_centered(double alpha, const std::vector<container>& f, double beta, std::vector<container>& dsf);
+    void do_forwardAdj(double alpha, const std::vector<container>& f, double beta, std::vector<container>& dsf, dg::norm no);
+    void do_backwardAdj(double alpha, const std::vector<container>& f, double beta, std::vector<container>& dsf, dg::norm no);
+    void do_centeredAdj(double alpha, const std::vector<container>& f, double beta, std::vector<container>& dsf, dg::norm no);
+    void do_symv(double alpha, const std::vector<container>& f, double beta, std::vector<container>& dsf);
+
+    FieldAligned<Geometry,IMatrix,container> m_fa;
+    Matrix m_jumpX, m_jumpY;
+    container m_temp;
+    std::vector<container> m_tempP, m_temp0, m_tempM;
+    std::vector<container> m_f, m_dsf;
+    std::vector<container> m_vol3d, m_inv3d;
+    container m_B;
     //container R_;
-    dg::norm no_;
-    dg::direction dir_;
-    bool apply_jumpX_;
-    dg::SparseElement<container> volume_;
+    dg::norm m_no;
+    dg::direction m_dir;
+    bool m_apply_jumpX, m_apply_jumpY;
 };
 
 ///@cond
@@ -217,165 +269,119 @@ struct DS
 
 template<class Geometry, class I, class M, class container>
 DS<Geometry, I, M,container>::DS(const dg::geo::TokamakMagneticField& mag, const Geometry& grid, dg::norm no, dg::direction dir, bool jumpX, bool jumpY, unsigned mx, unsigned my):
-        f_( dg::geo::BinaryVectorLvl0( dg::geo::BHatR(mag), dg::geo::BHatZ(mag), dg::geo::BHatP(mag)), grid, mx, my, 1e-5, FullLimiter(), grid.bcx(), grid.bcy()),
+        m_fa( dg::geo::BinaryVectorLvl0( dg::geo::BHatR(mag), dg::geo::BHatZ(mag), dg::geo::BHatP(mag)), grid, mx, my, 1e-5, FullLimiter(), grid.bcx(), grid.bcy()),
         jumpX( dg::create::jumpX( grid)),
         jumpY( dg::create::jumpY( grid)),
-        tempP( dg::evaluate( dg::zero, grid)), temp0( tempP), tempM( tempP), 
-        f(tempP), dsf(tempP),
-        vol3d( dg::create::volume( grid)), inv3d( dg::create::inv_volume( grid)),
-        invB(dg::pullback(dg::geo::InvB(mag), grid)), 
-        no_(no), dir_(dir), apply_jumpX_(jumpX)
+        m_no(no), m_dir(dir), m_apply_jumpX(jumpX), m_apply_jumpY(jumpY)
 {
-    volume_ = dg::tensor::volume(grid.metric());
+    dg::blas1::transfer( dg::pullback( dg::geo::Bmodule(mag), grid), m_B);
+    m_temp = m_B;
+    dg::blas1::transfer( dg::create::volume(     grid), m_temp); 
+    dg::split( m_temp, m_vol3d);
+    dg::blas1::transfer( dg::create::inv_volume( grid), m_temp); 
+    dg::split( m_temp, m_inv3d);
+    dg::split( m_temp, m_tempP);
+    dg::split( m_temp, m_temp0);
+    dg::split( m_temp, m_tempM);
+    dg::split( m_temp, m_f);
+    dg::split( m_temp, m_dsf);
 }
 
 template<class G, class I, class M, class container>
 inline void DS<G,I,M,container>::operator()( const container& f, container& dsf) { 
     if( dir_ == dg::centered)
-        return centered( f, dsf);
+        return centered( 1., f, 0., dsf);
     else if( dir_ == dg::forward)
-        return forward( f, dsf);
+        return forward( 1., f, 0., dsf);
     else
-        return backward( f, dsf);
+        return backward( 1., f, 0., dsf);
 }
 
-
 template<class G, class I, class M, class container>
-void DS<G, I,M,container>::centered( const container& f, container& dsf)
+void DS<G,I,M,container>::do_forward( double alpha, const std::vector<container>& f, double beta, std::vector<container>& dsf)
 {
-    //direct discretisation
-    assert( &f != &dsf);
-    f_(einsPlus, f, tempP);
-    f_(einsMinus, f, tempM);
-    dg::blas1::axpby( 1., tempP, -1., tempM);
-    dg::blas1::pointwiseDivide( tempM, f_.hz(), dsf);
-}
-
-template<class G, class I, class M, class container>
-void DS<G, I,M,container>::centeredAdj( const container& f, container& dsf)
-{               
-    //adjoint discretisation
-    assert( &f != &dsf);    
-    dg::blas1::pointwiseDot( vol3d, f, dsf);
-    dg::blas1::pointwiseDivide( dsf, f_.hz(), dsf);
-    f_(einsPlusT, dsf, tempP);
-    f_(einsMinusT, dsf, tempM);
-    dg::blas1::axpby( 1., tempM, -1., tempP, dsf);        
-    dg::blas1::pointwiseDot( inv3d, dsf, dsf); 
-}
-
-template<class G, class I, class M, class container>
-void DS<G,I,M,container>::centeredAdjDir( const container& f, container& dsf)
-{       
-//     Direct discretisation
-    assert( &f != &dsf);    
-    dg::blas1::pointwiseDot( f, invB, dsf);
-    f_(einsPlus, dsf, tempP);
-    f_(einsMinus, dsf, tempM);
-    dg::blas1::axpby( 1., tempP, -1., tempM);
-    dg::blas1::pointwiseDivide( tempM, f_.hz(), dsf);        
-    dg::blas1::pointwiseDivide( dsf, invB, dsf);
+    //direct
+    m_fa(einsPlus, f, m_tempP);
+    dg::blas1::axpby( 1., m_tempP, -1., f, m_tempP);
+    dg::blas1::pointwiseDot( alpha, m_tempP, m_fa.hp_inv(), beta, dsf);
 }
-
-template<class G, class I, class M, class container>
-void DS<G,I,M,container>::forward( const container& f, container& dsf)
+template<class G,class I, class M, class container>
+void DS<G,I,M,container>::do_backward( double alpha, const std::vector<container>& f, double beta, std::vector<container>& dsf)
 {
     //direct
-    assert( &f != &dsf);
-    f_(einsPlus, f, tempP);
-    dg::blas1::axpby( 1., tempP, -1., f, tempP);
-    dg::blas1::pointwiseDivide( tempP, f_.hp(), dsf);
+    m_fa(einsMinus, f, m_tempM);
+    dg::blas1::axpby( 1., m_tempM, -1., f, m_tempM);
+    dg::blas1::pointwiseDot( alpha, m_tempM, m_fa.hp_inv(), beta, dsf);
 }
-
-template<class G, class I, class M, class container>
-void DS<G,I,M,container>::forwardAdj( const container& f, container& dsf)
-{    
-    //adjoint discretisation
-    assert( &f != &dsf);
-    dg::blas1::pointwiseDot( vol3d, f, dsf);
-    dg::blas1::pointwiseDivide( dsf, f_.hp(), dsf);
-    f_(einsPlusT, dsf, tempP);
-    dg::blas1::axpby( -1., tempP, 1., dsf, dsf);
-    dg::blas1::pointwiseDot( inv3d, dsf, dsf); 
-}
-
 template<class G, class I, class M, class container>
-void DS<G,I,M,container>::forwardAdjDir( const container& f, container& dsf)
+void DS<G, I,M,container>::do_centered( double alpha, const std::vector<container>& f, double beta, std::vector<container>& dsf)
 {
     //direct discretisation
-    assert( &f != &dsf);
-    dg::blas1::pointwiseDot( f, invB, dsf);
-    f_(einsMinus, dsf, tempP);
-    dg::blas1::axpby( -1., tempP, 1., dsf, dsf);
-    dg::blas1::pointwiseDivide( dsf, f_.hm(), dsf);        
-    dg::blas1::pointwiseDivide( dsf, invB, dsf);
+    m_fa(einsPlus, f, m_tempP);
+    m_fa(einsMinus, f, m_tempM);
+    dg::blas1::axpby( 1., m_tempP, -1., m_tempM);
+    dg::blas1::pointwiseDot( alpha, m_tempM, m_fa.hz_inv(), beta, dsf);
 }
-
-template<class G,class I, class M, class container>
-void DS<G,I,M,container>::backward( const container& f, container& dsf)
-{
-    //direct
-    assert( &f != &dsf);
-    f_(einsMinus, f, tempM);
-    dg::blas1::axpby( 1., tempM, -1., f, tempM);
-    dg::blas1::pointwiseDivide( tempM, f_.hm(), dsf);
+template<class G, class I, class M, class container>
+void DS<G,I,M,container>::do_forwardAdj( double alpha, std::vector<container>& f, double beta, std::vector<container>& dsf, dg::norm no)
+{    
+    //adjoint discretisation
+    dg::blas1::pointwiseDot( m_vol3d, f, f);
+    dg::blas1::pointwiseDot( f, m_fa.hp_inv(), f);
+    m_fa(einsPlusT, f, m_tempP);
+    dg::blas1::axpby( -1., m_tempP, 1., f, f);
+    if(no == dg::normed) 
+        dg::blas1::pointwiseDot( alpha, m_inv3d, f, beta, dsf); 
 }
-
 template<class G,class I, class M, class container>
-void DS<G,I,M,container>::backwardAdj( const container& f, container& dsf)
+void DS<G,I,M,container>::do_backwardAdj( double alpha, std::vector<container>& f, double beta, std::vector<container>& dsf, dg::norm no)
 {    
     //adjoint discretisation
-    assert( &f != &dsf);
-    dg::blas1::pointwiseDot( vol3d, f, dsf);
-    dg::blas1::pointwiseDivide( dsf, f_.hm(), dsf);
-    f_(einsMinusT, dsf, tempM);
-    dg::blas1::axpby( -1., tempM, 1., dsf, dsf);
-    dg::blas1::pointwiseDot( inv3d, dsf, dsf); 
+    dg::blas1::pointwiseDot( m_vol3d, f, f);
+    dg::blas1::pointwiseDot( f, m_fa.hm_inv(), f);
+    m_fa(einsMinusT, f, m_tempM);
+    dg::blas1::axpby( -1., m_tempM, 1., f, f);
+    if(no == dg::normed) 
+        dg::blas1::pointwiseDot( alpha, m_inv3d, f, beta, dsf); 
 }
-
-template<class G,class I, class M, class container>
-void DS<G,I,M,container>::backwardAdjDir( const container& f, container& dsf)
-{
-    //direct
-    assert( &f != &dsf);
-    dg::blas1::pointwiseDot( f, invB, dsf);
-    f_(einsPlus, dsf, tempM);
-    dg::blas1::axpby( -1., tempM, 1., dsf, dsf);
-    dg::blas1::pointwiseDivide( dsf, f_.hp(), dsf);        
-    dg::blas1::pointwiseDivide( dsf, invB, dsf);
+template<class G, class I, class M, class container>
+void DS<G, I,M,container>::do_centeredAdj( double alpha, std::vector<container>& f, double beta, std::vector<container>& dsf, dg::norm no)
+{               
+    //adjoint discretisation
+    dg::blas1::pointwiseDot( m_vol3d, f, f);
+    dg::blas1::pointwiseDot( f, m_fa.hz_inv(), f);
+    m_fa(einsPlusT,  f, m_tempP);
+    m_fa(einsMinusT, f, m_tempM);
+    dg::blas1::axpby( 1., m_tempP, -1., m_tempM);
+    if(no == dg::normed) 
+        dg::blas1::pointwiseDot( alpha, m_inv3d, m_tempM, beta, dsf); 
 }
 
 template<class G,class I, class M, class container>
-void DS<G,I,M,container>::symv( const container& f, container& dsTdsf)
+void DS<G,I,M,container>::do_symv( double alpha, const container& f, double beta, container& dsTdsf)
 {
-    if(dir_ == dg::centered)
+    dg::split( f, m_f);
+    if(m_dir == dg::centered)
     {
-        centered( f, tempP);
-        centeredAdj( tempP, dsTdsf);
+        do_centered( m_f, m_tempP);
+        do_centeredAdj( m_tempP, m_dsf);
     }
     else 
     {
-        forward( f, tempP);
-        forwardAdj( tempP, dsTdsf);
-        backward( f, tempM);
-        backwardAdj( tempM, temp0);
-        dg::blas1::axpby(0.5,temp0,0.5,dsTdsf,dsTdsf);
-    }
-//     add jump term 
-
-    if(apply_jumpX_)
-    {
-        dg::blas2::symv( jumpX, f, temp0);
-        dg::tensor::pointwiseDivide( temp0, volume_, temp0);
-        dg::blas1::axpby( -1., temp0, 1., dsTdsf, dsTdsf);
+        do_forward( m_f, m_tempP);
+        do_forwardAdj( m_tempP, m_dsf);
+        do_backward( m_f, m_tempM);
+        do_backwardAdj( m_tempM, m_temp0);
+        dg::blas1::axpby(0.5,m_temp0,0.5,m_dsf,m_dsf);
     }
-    dg::blas2::symv( jumpY, f, temp0);
-    dg::tensor::pointwiseDivide( temp0, volume_, temp0);
-    dg::blas1::axpby( -1., temp0, 1., dsTdsf, dsTdsf);
+    dg::join( m_dsf, dsTdsf);
+    //     add jump term 
+    if(m_apply_jumpX)
+        dg::blas2::symv( -1., jumpX, f, 1., dsTdsf);
+    if(m_apply_jumpY)
+        dg::blas2::symv( -1., jumpY, f, 1., dsTdsf);
     if( no_ == not_normed)
-    {
         dg::blas1::pointwiseDot( vol3d, dsTdsf, dsTdsf); //make it symmetric
-    }
 }
 
 //enables the use of the dg::blas2::symv function 
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index 5f9849ebf..db00e2f71 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -667,8 +667,8 @@ void FieldAligned<G, I, container>::ePlus( enum whichMatrix which, const contain
 
         cView fp( f.cbegin() + ip*perp_size_, f.cbegin() + (ip+1)*perp_size_);
         View fP( fpe.begin() + i0*perp_size_, fpe.begin() + (i0+1)*perp_size_);
-        if(which == einsPlus) cusp::multiply( plus, fp, fP);
-        else if(which == einsMinusT) cusp::multiply( minusT, fp, fP );
+        if(which == einsPlus)           cusp::multiply( plus, fp, fP);
+        else if(which == einsMinusT)    cusp::multiply( minusT, fp, fP );
         //make ghostcells i.e. modify fpe in the limiter region
         if( i0==Nz_-1 && bcz_ != dg::PER)
         {
-- 
GitLab


From 6c14846593a6a0a200d0bc61f123752a60ee82a9 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Wed, 4 Oct 2017 14:13:50 +0200
Subject: [PATCH 338/453] dg::split should be done by fieldaligned

---
 inc/dg/backend/split_and_join.h |  31 ------
 inc/dg/blas_b.cu                |   8 +-
 inc/dg/geometry/base_geometry.h |   2 +-
 inc/geometries/ds.h             | 162 +++++++++++++-------------------
 4 files changed, 69 insertions(+), 134 deletions(-)

diff --git a/inc/dg/backend/split_and_join.h b/inc/dg/backend/split_and_join.h
index 3d76c772a..cf7d02242 100644
--- a/inc/dg/backend/split_and_join.h
+++ b/inc/dg/backend/split_and_join.h
@@ -81,36 +81,5 @@ void join( const std::vector<MPI_Vector<thrust_vector> >& in, MPI_Vector<thrust_
 }
 #endif //MPI_VERSION
 
-/**
-* @brief Expand 2d to 3d vector
-* @tparam thrust_vector1 either thrust::host_vector or thrust::device_vector
-* @tparam thrust_vector2 either thrust::host_vector or thrust::device_vector
-* @param in 2d vector of 2d size 
-* @param out contiguous 3d vector (gets resized if necessary) 
-* @param grid provide dimensions in 3rd and first two dimensions
-*/
-template<class thrust_vector1, class thrust_vector2>
-void expand( const thrust_vector1& in, thrust_vector2& out, const aTopology3d& grid)
-{
-    unsigned perp_size = grid.n()*grid.n()*grid.Nx()*grid.Ny();
-    out.resize( grid.size());
-    for( unsigned i=0; i<grid.Nz(); i++)
-        thrust::copy( in.begin(), in.end(), out.begin() + i*perp_size);
-}
-#ifdef MPI_VERSION
-///@brief MPI Version of expand
-///@copydetails dg::expand()
-template<class thrust_vector1, class thrust_vector2>
-void expand( const MPI_Vector<thrust_vector1>& in, MPI_Vector<thrust_vector2>& out, const aMPITopology3d& grid)
-{
-    unsigned perp_size = grid.n()*grid.n()*grid.Nx()*grid.Ny();
-    out.data().resize( grid.size());
-    out.communicator() = grid.communicator();
-    for( unsigned i=0; i<grid.Nz(); i++)
-        thrust::copy( in.data().begin(), in.data().end(), out.data().begin() + i*perp_size);
-}
-#endif //MPI_VERSION
-
-
 ///@}
 }//namespace dg
diff --git a/inc/dg/blas_b.cu b/inc/dg/blas_b.cu
index 70780b25c..cd144d5bd 100644
--- a/inc/dg/blas_b.cu
+++ b/inc/dg/blas_b.cu
@@ -46,9 +46,11 @@ int main()
     value_type gbytes=(value_type)x.size()*sizeof(value_type)/1e9;
     std::cout << "Sizeof vectors is "<<gbytes<<" GB\n";
     std::cout << "Generate interpolation and projection\n";
-    dg::MultiMatrix<Matrix, Vector> inter, project; 
-    dg::blas2::transfer(dg::create::fast_interpolation( grid_half, 2,2), inter);
-    dg::blas2::transfer(dg::create::fast_projection( grid, 2,2), project);
+    //dg::MultiMatrix<Matrix, Vector> inter, project; 
+    //dg::blas2::transfer(dg::create::fast_interpolation( grid_half, 2,2), inter);
+    //dg::blas2::transfer(dg::create::fast_projection( grid, 2,2), project);
+    dg::IDMatrix inter = dg::create::interpolation( grid, grid_half);
+    dg::IDMatrix project = dg::create::projection( grid_half, grid);
     std::cout << "Done...\n";
     int multi=100;
     t.tic();
diff --git a/inc/dg/geometry/base_geometry.h b/inc/dg/geometry/base_geometry.h
index a539f1d96..e2bd930a5 100644
--- a/inc/dg/geometry/base_geometry.h
+++ b/inc/dg/geometry/base_geometry.h
@@ -93,7 +93,7 @@ struct aGeometry3d : public aTopology3d
     *
     *  The elements of the Tensor are (if x,y,z are the coordinates in computational space and R,Z,P are the physical space coordinates)
     \f[
-    J = \begin{pmatrix} x_R(x,y,z) & x_Z(x,y,z) & x_\varphi(x,y,z) \\ 
+    J = \begin{pmatrix} x_R(x,y,z) & x_Z(x,y,z) & x_\varphi(x,y,z) \\
     y_R(x,y,z) & y_Z(x,y,z) & y_\varphi(x,y,z) \\
     z_R(x,y,z) & z_Z(x,y,z) & z_\varphi(x,y,z)
     \end{pmatrix}
diff --git a/inc/geometries/ds.h b/inc/geometries/ds.h
index aec57660a..bd941e902 100644
--- a/inc/geometries/ds.h
+++ b/inc/geometries/ds.h
@@ -3,7 +3,6 @@
 #include "dg/blas.h"
 #include "dg/geometry/geometry.h"
 #include "dg/backend/derivatives.h"
-#include "backend/split_and_join.h"
 #include "fieldaligned.h"
 #ifdef MPI_VERSION
 #include "backend/mpi_derivatives.h"
@@ -55,9 +54,7 @@ struct DS
     * @param dsf contains result on output (write only)
     */
     void forward( double alpha, const container& f, double beta, container& dsf){
-        dg::split( f, m_f);
-        do_forward( alpha, m_f, beta, m_dsf);
-        dg::join( m_dsf, dsf);
+        do_forward( alpha, f, beta, dsf);
     }
     /**
     * @brief Apply the backward derivative on a 3d vector
@@ -67,9 +64,7 @@ struct DS
     * @param dsf contains result on output (write only)
     */
     void backward( double alpha, const container& f, double beta, container& dsf){
-        dg::split( f, m_f);
-        do_backward( alpha, m_f, beta, m_dsf);
-        dg::join( m_dsf, dsf);
+        do_backward( alpha, f, beta, dsf);
     }
     /**
     * @brief Apply the centered derivative on a 3d vector
@@ -79,9 +74,7 @@ struct DS
     * @param dsf contains result on output (write only)
     */
     void centered( double alpha, const container& f, double beta, container& dsf){
-        dg::split( f, m_f);
-        do_centered( alpha, m_f, beta, m_dsf);
-        dg::join( m_dsf, dsf);
+        do_centered( alpha, f, beta, dsf);
     }
 
     /**
@@ -91,9 +84,7 @@ struct DS
     * @param dsf contains result on output (write only)
     */
     void forwardAdj( double alpha, const container& f, double beta, container& dsf){
-        dg::split( m_temp, m_f);
-        do_forwardAdj( alpha, m_f, beta, m_dsf, dg::normed);
-        dg::join( m_dsf, dsf);
+        do_forwardAdj( alpha, f, beta, dsf, dg::normed);
     }
     /**
     * @brief Apply the negative adjoint derivative on a 3d vector
@@ -102,9 +93,7 @@ struct DS
     * @param dsf contains result on output (write only)
     */
     void backwardAdj( double alpha, const container& f, double beta, container& dsf){
-        dg::split( f, m_f);
-        do_backwardAdj( alpha, m_f, beta, m_dsf, dg::normed);
-        dg::join( m_dsf, dsf);
+        do_backwardAdj( alpha, f, beta, dsf, dg::normed);
     }
     /**
     * @brief Apply the negative adjoint derivative on a 3d vector
@@ -113,9 +102,7 @@ struct DS
     * @param dsf contains result on output (write only)
     */
     void centeredAdj(double alpha, const container& f, double beta, container& dsf){
-        dg::split( f, m_f);
-        do_centeredAdj( alpha, m_f, beta, m_dsf, dg::normed);
-        dg::join( m_dsf, dsf);
+        do_centeredAdj( alpha, f, beta, dsf, dg::normed);
     }
     /**
     * @brief Apply the negative adjoint derivative on a 3d vector with the direct method
@@ -126,9 +113,9 @@ struct DS
     */
     void forwardAdjDir( double alpha, const container& f, double beta, container& dsf)
     {
-        dg::blas1::pointwiseDivide( f, m_B, temp);
-        forward( temp, temp);
-        dg::blas1::pointwiseDot( alpha, temp, m_B, beta, dsf);
+        dg::blas1::pointwiseDivide( f, m_B, m_temp);
+        forward( m_temp, m_temp);
+        dg::blas1::pointwiseDot( alpha, m_temp, m_B, beta, dsf);
     }
     /**
     * @brief Apply the negative adjoint derivative on a 3d vector with the direct method
@@ -138,9 +125,9 @@ struct DS
     */
     void backwardAdjDir( double alpha, const container& f, double beta, container& dsf)
     {
-        dg::blas1::pointwiseDivide( f, m_B, temp);
-        backward( temp, temp);
-        dg::blas1::pointwiseDot( alpha, temp, m_B, beta, dsf);
+        dg::blas1::pointwiseDivide( f, m_B, m_temp);
+        backward( 1., m_temp, 0., m_temp);
+        dg::blas1::pointwiseDot( alpha, m_temp, m_B, beta, dsf);
     }
     /**
     * @brief Apply the negative adjoint derivative on a 3d vector with the direct method
@@ -150,9 +137,9 @@ struct DS
     */
     void centeredAdjDir( double alpha, const container& f, double beta, container& dsf)
     {
-        dg::blas1::pointwiseDivide( f, m_B, temp);
-        backward( temp, temp);
-        dg::blas1::pointwiseDot( alpha, temp, m_B, beta, dsf);
+        dg::blas1::pointwiseDivide( f, m_B, m_temp);
+        backward( 1., m_temp, m_temp);
+        dg::blas1::pointwiseDot( alpha, m_temp, m_B, beta, dsf);
     }
 
     /**
@@ -173,10 +160,7 @@ struct DS
      * @param dsTdsf contains result on output (write only)
      * @note if apply_jumpX is false then no jumpy terms will be added in the x-direction
      */
-    void symv( const container& f, container& dsTdsf){ symv( 1., f, 0., dsTdsf);}
-    void symv( double alpha, const container& f, double beta, container& dsTdsf){
-        do_symv( alpha, f, beta, dsTdsf);
-    }
+    void symv( const container& f, container& dsTdsf){ do_symv( f, dsTdsf);}
 
     /**
     * @brief Set boundary conditions in the limiter region
@@ -219,21 +203,8 @@ struct DS
         f_.set_boundaries( bcz, global, scal_left, scal_right);
     }
 
-    /**
-     * @brief Returns the weights used to make the matrix symmetric 
-     *
-     * needed by invert class
-     * @return weights
-     */
     const container& weights()const {return vol3d;}
     const container& inv_weights()const {return inv3d;}
-    /**
-     * @brief Returns the preconditioner to use in conjugate gradient
-     *
-     * needed by invert class
-     * In this case inverse weights are the best choice
-     * @return inverse weights
-     */
     const container& precond()const {return inv3d;}
 
     /**
@@ -243,20 +214,19 @@ struct DS
     */
     const FieldAligned<Geometry, IMatrix, container>& fieldaligned() const{return f_;}
     private:
-    void do_forward(double alpha, const std::vector<container>& f, double beta, std::vector<container>& dsf);
-    void do_backward(double alpha, const std::vector<container>& f, double beta, std::vector<container>& dsf);
-    void do_centered(double alpha, const std::vector<container>& f, double beta, std::vector<container>& dsf);
-    void do_forwardAdj(double alpha, const std::vector<container>& f, double beta, std::vector<container>& dsf, dg::norm no);
-    void do_backwardAdj(double alpha, const std::vector<container>& f, double beta, std::vector<container>& dsf, dg::norm no);
-    void do_centeredAdj(double alpha, const std::vector<container>& f, double beta, std::vector<container>& dsf, dg::norm no);
-    void do_symv(double alpha, const std::vector<container>& f, double beta, std::vector<container>& dsf);
+    void do_forward(double alpha, const container& f, double beta, container& dsf);
+    void do_backward(double alpha, const container& f, double beta, container& dsf);
+    void do_centered(double alpha, const container& f, double beta, container& dsf);
+    void do_forwardAdj(double alpha, const container& f, double beta, container& dsf, dg::norm no);
+    void do_backwardAdj(double alpha, const container& f, double beta, container& dsf, dg::norm no);
+    void do_centeredAdj(double alpha, const container& f, double beta, container& dsf, dg::norm no);
+    void do_symv(const container& f, container& dsf);
 
     FieldAligned<Geometry,IMatrix,container> m_fa;
     Matrix m_jumpX, m_jumpY;
     container m_temp;
-    std::vector<container> m_tempP, m_temp0, m_tempM;
-    std::vector<container> m_f, m_dsf;
-    std::vector<container> m_vol3d, m_inv3d;
+    container m_tempP, m_temp0, m_tempM;
+    container m_vol3d, m_inv3d, m_weights_wo_vol;
     container m_B;
     //container R_;
     dg::norm m_no;
@@ -270,21 +240,15 @@ struct DS
 template<class Geometry, class I, class M, class container>
 DS<Geometry, I, M,container>::DS(const dg::geo::TokamakMagneticField& mag, const Geometry& grid, dg::norm no, dg::direction dir, bool jumpX, bool jumpY, unsigned mx, unsigned my):
         m_fa( dg::geo::BinaryVectorLvl0( dg::geo::BHatR(mag), dg::geo::BHatZ(mag), dg::geo::BHatP(mag)), grid, mx, my, 1e-5, FullLimiter(), grid.bcx(), grid.bcy()),
-        jumpX( dg::create::jumpX( grid)),
-        jumpY( dg::create::jumpY( grid)),
         m_no(no), m_dir(dir), m_apply_jumpX(jumpX), m_apply_jumpY(jumpY)
 {
     dg::blas1::transfer( dg::pullback( dg::geo::Bmodule(mag), grid), m_B);
-    m_temp = m_B;
-    dg::blas1::transfer( dg::create::volume(     grid), m_temp); 
-    dg::split( m_temp, m_vol3d);
-    dg::blas1::transfer( dg::create::inv_volume( grid), m_temp); 
-    dg::split( m_temp, m_inv3d);
-    dg::split( m_temp, m_tempP);
-    dg::split( m_temp, m_temp0);
-    dg::split( m_temp, m_tempM);
-    dg::split( m_temp, m_f);
-    dg::split( m_temp, m_dsf);
+    m_temp = m_B, m_tempP = m_B, m_temp0 = m_b, m_tempM = m_b;
+    dg::blas1::transfer( dg::create::volume(     grid), m_vol3d); 
+    dg::blas1::transfer( dg::create::weights(    grid), m_weights_wo_vol); 
+    dg::blas1::transfer( dg::create::inv_volume( grid), m_inv3d); 
+    dg::blas2::transfer( dg::create::jumpX( grid), jumpX);
+    dg::blas2::transfer( dg::create::jumpY( grid), jumpY);
 }
 
 template<class G, class I, class M, class container>
@@ -298,7 +262,7 @@ inline void DS<G,I,M,container>::operator()( const container& f, container& dsf)
 }
 
 template<class G, class I, class M, class container>
-void DS<G,I,M,container>::do_forward( double alpha, const std::vector<container>& f, double beta, std::vector<container>& dsf)
+void DS<G,I,M,container>::do_forward( double alpha, const container& f, double beta, container& dsf)
 {
     //direct
     m_fa(einsPlus, f, m_tempP);
@@ -306,7 +270,7 @@ void DS<G,I,M,container>::do_forward( double alpha, const std::vector<container>
     dg::blas1::pointwiseDot( alpha, m_tempP, m_fa.hp_inv(), beta, dsf);
 }
 template<class G,class I, class M, class container>
-void DS<G,I,M,container>::do_backward( double alpha, const std::vector<container>& f, double beta, std::vector<container>& dsf)
+void DS<G,I,M,container>::do_backward( double alpha, const container& f, double beta, container& dsf)
 {
     //direct
     m_fa(einsMinus, f, m_tempM);
@@ -314,7 +278,7 @@ void DS<G,I,M,container>::do_backward( double alpha, const std::vector<container
     dg::blas1::pointwiseDot( alpha, m_tempM, m_fa.hp_inv(), beta, dsf);
 }
 template<class G, class I, class M, class container>
-void DS<G, I,M,container>::do_centered( double alpha, const std::vector<container>& f, double beta, std::vector<container>& dsf)
+void DS<G, I,M,container>::do_centered( double alpha, const container& f, double beta, container& dsf)
 {
     //direct discretisation
     m_fa(einsPlus, f, m_tempP);
@@ -323,65 +287,65 @@ void DS<G, I,M,container>::do_centered( double alpha, const std::vector<containe
     dg::blas1::pointwiseDot( alpha, m_tempM, m_fa.hz_inv(), beta, dsf);
 }
 template<class G, class I, class M, class container>
-void DS<G,I,M,container>::do_forwardAdj( double alpha, std::vector<container>& f, double beta, std::vector<container>& dsf, dg::norm no)
+void DS<G,I,M,container>::do_forwardAdj( double alpha, const container& f, double beta, container& dsf, dg::norm no)
 {    
     //adjoint discretisation
-    dg::blas1::pointwiseDot( m_vol3d, f, f);
-    dg::blas1::pointwiseDot( f, m_fa.hp_inv(), f);
-    m_fa(einsPlusT, f, m_tempP);
-    dg::blas1::axpby( -1., m_tempP, 1., f, f);
+    dg::blas1::pointwiseDot( m_vol3d, f, m_temp0);
+    dg::blas1::pointwiseDot( m_temp0, m_fa.hp_inv(), m_temp0);
+    m_fa(einsPlusT, m_temp0, m_tempP);
+    dg::blas1::axpby( -1., m_tempP, 1., m_temp0, m_temp0);
     if(no == dg::normed) 
-        dg::blas1::pointwiseDot( alpha, m_inv3d, f, beta, dsf); 
+        dg::blas1::pointwiseDot( alpha, m_inv3d, m_temp0, beta, dsf); 
 }
 template<class G,class I, class M, class container>
-void DS<G,I,M,container>::do_backwardAdj( double alpha, std::vector<container>& f, double beta, std::vector<container>& dsf, dg::norm no)
+void DS<G,I,M,container>::do_backwardAdj( double alpha, const container& f, double beta, container& dsf, dg::norm no)
 {    
     //adjoint discretisation
-    dg::blas1::pointwiseDot( m_vol3d, f, f);
-    dg::blas1::pointwiseDot( f, m_fa.hm_inv(), f);
-    m_fa(einsMinusT, f, m_tempM);
-    dg::blas1::axpby( -1., m_tempM, 1., f, f);
+    dg::blas1::pointwiseDot( m_vol3d, m_temp0, m_temp0);
+    dg::blas1::pointwiseDot( m_temp0, m_fa.hm_inv(), m_temp0);
+    m_fa(einsMinusT, m_temp0, m_tempM);
+    dg::blas1::axpby( -1., m_tempM, 1., m_temp0, m_temp0);
     if(no == dg::normed) 
-        dg::blas1::pointwiseDot( alpha, m_inv3d, f, beta, dsf); 
+        dg::blas1::pointwiseDot( alpha, m_inv3d, m_temp0, beta, dsf); 
 }
 template<class G, class I, class M, class container>
-void DS<G, I,M,container>::do_centeredAdj( double alpha, std::vector<container>& f, double beta, std::vector<container>& dsf, dg::norm no)
+void DS<G, I,M,container>::do_centeredAdj( double alpha, container& f, double beta, container& dsf, dg::norm no)
 {               
     //adjoint discretisation
-    dg::blas1::pointwiseDot( m_vol3d, f, f);
-    dg::blas1::pointwiseDot( f, m_fa.hz_inv(), f);
-    m_fa(einsPlusT,  f, m_tempP);
-    m_fa(einsMinusT, f, m_tempM);
+    dg::blas1::pointwiseDot( m_vol3d, f, m_temp0);
+    dg::blas1::pointwiseDot( m_temp0, m_fa.hz_inv(), m_temp0);
+    m_fa(einsPlusT,  m_temp0, m_tempP);
+    m_fa(einsMinusT, m_temp0, m_tempM);
     dg::blas1::axpby( 1., m_tempP, -1., m_tempM);
     if(no == dg::normed) 
         dg::blas1::pointwiseDot( alpha, m_inv3d, m_tempM, beta, dsf); 
 }
 
 template<class G,class I, class M, class container>
-void DS<G,I,M,container>::do_symv( double alpha, const container& f, double beta, container& dsTdsf)
+void DS<G,I,M,container>::do_symv( const container& f, container& dsTdsf)
 {
-    dg::split( f, m_f);
     if(m_dir == dg::centered)
     {
-        do_centered( m_f, m_tempP);
-        do_centeredAdj( m_tempP, m_dsf);
+        do_centered( f, m_tempP);
+        do_centeredAdj( m_tempP, dsTdsf, dg::not_normed);
     }
     else 
     {
-        do_forward( m_f, m_tempP);
-        do_forwardAdj( m_tempP, m_dsf);
-        do_backward( m_f, m_tempM);
-        do_backwardAdj( m_tempM, m_temp0);
-        dg::blas1::axpby(0.5,m_temp0,0.5,m_dsf,m_dsf);
+        do_forward( f, m_tempP);
+        do_forwardAdj( m_tempP, m_temp0, dg::not_normed);
+        do_backward( f, m_tempM);
+        do_backwardAdj( m_tempM, dsTdsf, dg::not_normed);
+        dg::blas1::axpby(0.5,m_temp0,0.5,dsTdsf);
     }
-    dg::join( m_dsf, dsTdsf);
+    dg::blas1::pointwiseDivide( dsTdsf, m_weights_wo_vol, dsTdsf);
     //     add jump term 
     if(m_apply_jumpX)
         dg::blas2::symv( -1., jumpX, f, 1., dsTdsf);
     if(m_apply_jumpY)
         dg::blas2::symv( -1., jumpY, f, 1., dsTdsf);
-    if( no_ == not_normed)
-        dg::blas1::pointwiseDot( vol3d, dsTdsf, dsTdsf); //make it symmetric
+    dg::blas1::pointwiseDot( m_weights_wo_vol, dsTdsf, dsTdsf); //make it symmetric
+    if( m_no == dg::normed)
+        dg::blas1::pointwiseDot( m_inv3d, dsTdsf, dsTdsf); //make it symmetric
 }
 
 //enables the use of the dg::blas2::symv function 
-- 
GitLab


From b7d958ea49c49827b51e66a24de9a399dd28d01d Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Wed, 4 Oct 2017 15:48:12 +0200
Subject: [PATCH 339/453] rewrite ePlus and eMinus in fieldaligned and
 mpi_fieldaligned

---
 inc/geometries/fieldaligned.h     | 117 +++++-----
 inc/geometries/mpi_fieldaligned.h | 347 ++++++------------------------
 2 files changed, 121 insertions(+), 343 deletions(-)

diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index db00e2f71..54dbbc465 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -479,15 +479,13 @@ struct FieldAligned
     private:
     void ePlus( enum whichMatrix which, const container& in, container& out);
     void eMinus(enum whichMatrix which, const container& in, container& out);
-    typedef cusp::array1d_view< typename container::iterator> View;
-    typedef cusp::array1d_view< typename container::const_iterator> cView;
     IMatrix m_plus, m_minus, m_plusT, m_minusT; //interpolation matrices
-    container hz_, hp_,hm_, ghostM, ghostP;
-    unsigned Nz_, perp_size_;
-    dg::bc bcz_;
-    container left_, right_;
-    container limiter_;
-    dg::Handle<Geometry> g_;
+    container m_hz, m_hp, m_hm, m_ghostM, m_ghostP;
+    unsigned m_Nz, m_perp_size;
+    dg::bc m_bcz;
+    container m_left, m_right;
+    container m_limiter;
+    dg::Handle<Geometry> m_g;
 };
 
 ///@cond 
@@ -498,8 +496,8 @@ struct FieldAligned
 template<class Geometry, class IMatrix, class container>
 template <class Limiter>
 FieldAligned<Geometry, IMatrix, container>::FieldAligned(const dg::geo::BinaryVectorLvl0& vec, const Geometry& grid, unsigned mx, unsigned my, double eps, Limiter limit, dg::bc globalbcx, dg::bc globalbcy, double deltaPhi):
-        hz_( dg::evaluate( dg::zero, grid)), hp_( hz_), hm_( hz_), 
-        Nz_(grid.Nz()), bcz_(grid.bcz()), g_(grid)
+        m_hz( dg::evaluate( dg::zero, grid)), m_hp( m_hz), m_hm( m_hz), 
+        m_Nz(grid.Nz()), m_bcz(grid.bcz()), m_g(grid)
 {
     if( deltaPhi <=0) deltaPhi = grid.hz();
     else assert( grid.Nz() == 1 || grid.hz()==deltaPhi);
@@ -633,14 +631,14 @@ container FieldAligned<G, I,container>::evaluate( BinaryOp binary, UnaryOp unary
     {
         for( unsigned i0=0; i0<Nz_; i0++)
         {
-            unsigned revi0 = (Nz_ - i0)%Nz_; //reverted index
+            unsigned revi0 = (m_Nz - i0)%m_Nz; //reverted index
             dg::blas1::axpby( 1., plus2d[i0], 0., result[i0]);
             dg::blas1::axpby( 1., minus2d[revi0], 1., result[i0]);
         }
         dg::blas1::axpby( -1., init2d, 1., result[0]);
-        for(unsigned i0=0; i0<Nz_; i0++)
+        for(unsigned i0=0; i0<m_Nz; i0++)
         {
-            int idx = ((int)i0 -(int)p0 + Nz_)%Nz_; //shift index
+            int idx = ((int)i0 -(int)p0 + m_Nz)%m_Nz; //shift index
             thrust::copy( result[idx].begin(), result[idx].end(), vec3d.begin() + i0*perp_size_);
         }
     }
@@ -658,72 +656,59 @@ void FieldAligned<G, I, container >::operator()(enum whichMatrix which, const co
 template< class G, class I, class container>
 void FieldAligned<G, I, container>::ePlus( enum whichMatrix which, const container& f, container& fpe)
 {
-    View ghostPV( ghostP.begin(), ghostP.end());
-    View ghostMV( ghostM.begin(), ghostM.end());
-    cView rightV( right_.begin(), right_.end());
-    for( unsigned i0=0; i0<Nz_; i0++)
+    dg::spit( f, m_f);
+    //1. compute 2d interpolation in every plane and store in m_temp
+    for( unsigned i0=0; i0<m_Nz; i0++)
     {
-        unsigned ip = (i0==Nz_-1) ? 0:i0+1;
-
-        cView fp( f.cbegin() + ip*perp_size_, f.cbegin() + (ip+1)*perp_size_);
-        View fP( fpe.begin() + i0*perp_size_, fpe.begin() + (i0+1)*perp_size_);
-        if(which == einsPlus)           cusp::multiply( plus, fp, fP);
-        else if(which == einsMinusT)    cusp::multiply( minusT, fp, fP );
-        //make ghostcells i.e. modify fpe in the limiter region
-        if( i0==Nz_-1 && bcz_ != dg::PER)
+        unsigned ip = (i0==m_Nz-1) ? 0:i0+1;
+        if(which == einsPlus)           dg::blas2::symv( plus,   m_f[ip], m_temp[i0]);
+        else if(which == einsMinusT)    dg::blas2::symv( minusT, m_f[ip], m_temp[i0]);
+    }
+    //2. apply right boundary conditions in last plane
+    unsigned i0=m_Nz-1;
+    if( m_bcz != dg::PER)
+    {
+        if( m_bcz == dg::DIR || m_bcz == dg::NEU_DIR)
+            dg::blas1::axpby( 2, m_right, -1., m_f[i0], m_ghostP);
+        if( m_bcz == dg::NEU || m_bcz == dg::DIR_NEU)
         {
-            cView f0( f.cbegin() + i0*perp_size_, f.cbegin() + (i0+1)*perp_size_);
-            if( bcz_ == dg::DIR || bcz_ == dg::NEU_DIR)
-            {
-                cusp::blas::axpby( rightV, f0, ghostPV, 2., -1.);
-            }
-            if( bcz_ == dg::NEU || bcz_ == dg::DIR_NEU)
-            {
-                thrust::transform( right_.begin(), right_.end(),  hp_.begin(), ghostM.begin(), thrust::multiplies<double>());
-                cusp::blas::axpby( ghostMV, f0, ghostPV, 1., 1.);
-            }
-            //interlay ghostcells with periodic cells: L*g + (1-L)*fpe
-            cusp::blas::axpby( ghostPV, fP, ghostPV, 1., -1.);
-            dg::blas1::pointwiseDot( limiter_, ghostP, ghostP);
-            cusp::blas::axpby(  ghostPV, fP, fP, 1.,1.);
+            dg::blas1::pointwiseDivide( m_right, m_hp_inv[i0], m_ghostP);
+            dg::blas1::axpby( 1., m_ghostP, 1., m_f[i0], m_ghostP);
         }
+        //interlay ghostcells with periodic cells: L*g + (1-L)*fpe
+        dg::blas1::axpby( 1., m_ghostP, -1., m_temp[i0], m_ghostP);
+        dg::blas1::pointwiseDot( 1., m_limiter, m_ghostP, 1., m_temp[i0]);
     }
+    dg::join( m_temp, fpe);
 }
 
 template< class G, class I, class container>
 void FieldAligned<G, I, container>::eMinus( enum whichMatrix which, const container& f, container& fme)
 {
-    //note that thrust functions don't work on views
-    View ghostPV( ghostP.begin(), ghostP.end());
-    View ghostMV( ghostM.begin(), ghostM.end());
-    cView leftV( left_.begin(), left_.end());
-    for( unsigned i0=0; i0<Nz_; i0++)
+    dg::split( f, m_f);
+    //1. compute 2d interpolation in every plane and store in m_temp
+    for( unsigned i0=0; i0<m_Nz; i0++)
     {
-        unsigned im = (i0==0) ? Nz_-1:i0-1;
-        cView fm( f.cbegin() + im*perp_size_, f.cbegin() + (im+1)*perp_size_);
-        View fM( fme.begin() + i0*perp_size_, fme.begin() + (i0+1)*perp_size_);
-        if(which == einsPlusT) cusp::multiply( plusT, fm, fM );
-        else if (which == einsMinus) cusp::multiply( minus, fm, fM );
-        //make ghostcells
-        if( i0==0 && bcz_ != dg::PER)
+        unsigned im = (i0==0) ? m_Nz-1:i0-1;
+        if(which == einsPlusT)          dg::blas2::symv( plusT, m_f[im], m_temp[i0]);
+        else if (which == einsMinus)    dg::blas2::symv( minus, m_f[im], m_temp[i0]);
+    }
+    //2. apply left boundary conditions in first plane
+    unsigned i0=0;
+    if( m_bcz != dg::PER)
+    {
+        if( m_bcz == dg::DIR || m_bcz == dg::DIR_NEU)
+            dg::blas1::axpby( 2., m_left,  -1., m_f[i0], m_ghostM);
+        if( m_bcz == dg::NEU || m_bcz == dg::NEU_DIR)
         {
-            cView f0( f.cbegin() + i0*perp_size_, f.cbegin() + (i0+1)*perp_size_);
-            if( bcz_ == dg::DIR || bcz_ == dg::DIR_NEU)
-            {
-                cusp::blas::axpby( leftV,  f0, ghostMV, 2., -1.);
-            }
-            if( bcz_ == dg::NEU || bcz_ == dg::NEU_DIR)
-            {
-                thrust::transform( left_.begin(), left_.end(),  hm_.begin(), ghostP.begin(), thrust::multiplies<double>());
-                cusp::blas::axpby( ghostPV, f0, ghostMV, -1., 1.);
-            }
-            //interlay ghostcells with periodic cells: L*g + (1-L)*fme
-            cusp::blas::axpby( ghostMV, fM, ghostMV, 1., -1.);
-            dg::blas1::pointwiseDot( limiter_, ghostM, ghostM);
-            cusp::blas::axpby( ghostMV, fM, fM, 1., 1.);
-
+            dg::blas1::pointwiseDivide( m_left, m_hm_inv[i0], m_ghostM);
+            dg::blas1::axpby( -1., m_ghostM, 1., m_f[i0], m_ghostM);
         }
+        //interlay ghostcells with periodic cells: L*g + (1-L)*fme
+        dg::blas1::axpby( 1., m_ghostM, -1., m_temp[i0], m_ghostM);
+        dg::blas1::pointwiseDot( 1., m_limiter, m_ghostM, 1., m_temp[i0]);
     }
+    dg::join( m_temp, fme);
 }
 
 
diff --git a/inc/geometries/mpi_fieldaligned.h b/inc/geometries/mpi_fieldaligned.h
index 88999d96e..72b6027b0 100644
--- a/inc/geometries/mpi_fieldaligned.h
+++ b/inc/geometries/mpi_fieldaligned.h
@@ -129,31 +129,27 @@ struct FieldAligned< Geometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vec
 
     void operator()(enum whichMatrix which, const MPI_Vector<LocalContainer>& in, MPI_Vector<LocalContainer>& out);
 
-    const MPI_Vector<LocalContainer>& hz()const {return hz_;}
-    const MPI_Vector<LocalContainer>& hp()const {return hp_;}
-    const MPI_Vector<LocalContainer>& hm()const {return hm_;}
+    const MPI_Vector<LocalContainer>& hz_inv()const {return m_hz_inv;}
+    const MPI_Vector<LocalContainer>& hp_inv()const {return m_hp_inv;}
+    const MPI_Vector<LocalContainer>& hm_inv()const {return m_hm_inv;}
     const Geometry& grid() const{return g_;}
   private:
-    typedef cusp::array1d_view< typename LocalContainer::iterator> View;
-    typedef cusp::array1d_view< typename LocalContainer::const_iterator> cView;
-    MPI_Vector<LocalContainer> hz_, hp_, hm_; 
-    LocalContainer ghostM, ghostP;
-    dg::Handle<Geometry> g_;
-    dg::bc bcz_;
-    LocalContainer left_, right_;
-    LocalContainer limiter_;
-    std::vector<LocalContainer> tempXYplus_, tempXYminus_, temp_; 
-    CommunicatorXY commXYplus_, commXYminus_;
-    LocalIMatrix m_plus, m_minus; //interpolation matrices
-    LocalIMatrix m_plusT, m_minusT; //interpolation matrices
+    MPI_Vector<LocalContainer> m_hz_inv, m_hp_inv, m_hm_inv; 
+    LocalContainer m_ghostM, m_ghostP;
+    dg::Handle<Geometry> m_g;
+    dg::bc m_bcz;
+    MPI_Vector<LocalContainer> m_left, m_right;
+    MPI_Vector<LocalContainer> m_limiter;
+    std::vector<LocalContainer> tempXYplus_, tempXYminus_, m_temp; 
+    MPIDistMat<LocalIMatrix, CommunicatorXY> m_plus, m_minus, m_plusT, m_minusT;
 };
 //////////////////////////////////////DEFINITIONS/////////////////////////////////////
 template<class MPIGeometry, class LocalIMatrix, class CommunicatorXY, class LocalContainer>
 template <class Limiter>
 FieldAligned<MPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vector<LocalContainer> >::FieldAligned(
     const dg::geo::BinaryVectorLvl0& vec, const MPIGeometry& grid, unsigned mx, unsigned my, double eps, Limiter limit, dg::bc globalbcx, dg::bc globalbcy, double deltaPhi):
-    hz_( dg::evaluate( dg::zero, grid)), hp_( hz_), hm_( hz_), 
-    g_(grid), bcz_(grid.bcz()), 
+    m_hz( dg::evaluate( dg::zero, grid)), m_hp( m_hz), m_hm( m_hz), 
+    m_g(grid), m_bcz(grid.bcz()), 
     tempXYplus_(g_.Nz()), tempXYminus_(g_.Nz()), temp_(g_.Nz())
 {
     if( deltaPhi <=0) deltaPhi = grid.hz();
@@ -326,294 +322,91 @@ MPI_Vector<container> FieldAligned<G,RowDistMat<M,C>, MPI_Vector<container> >::e
 }
 
 template<class G, class M, class C, class container>
-void FieldAligned<G,RowDistMat<M,C>, MPI_Vector<container> >::einsPlus( const MPI_Vector<container>& f, MPI_Vector<container>& fplus ) 
+void FieldAligned<G, RowDistMatr<M,C>, MPI_Vector<container> >::operator()(enum whichMatrix which, const MPI_Vector<container>& f, MPI_Vector<container>& fe)
 {
-    //dg::blas2::detail::doSymv( plus, f, fplus, MPIMatrixTag(), MPIVectorTag(), MPIVectorTag());
-    const container& in = f.data();
-    container& out = fplus.data();
-    int size2d = g_.n()*g_.n()*g_.Nx()*g_.Ny();
-        int dims[3], periods[3], coords[3];
-        MPI_Cart_get( g_.communicator(), 3, dims, periods, coords);
-        int sizeXY = dims[0]*dims[1];
-        int sizeZ = dims[2];
-
-    //1. compute 2d interpolation in every plane and store in temp_
-    if( sizeXY != 1) //communication needed
-    {
-        for( int i0=0; i0<(int)g_.Nz(); i0++)
-        {
-            cView inV( in.cbegin() + i0*plus.num_cols, in.cbegin() + (i0+1)*plus.num_cols);
-            View tempV( tempXYplus_[i0].begin(), tempXYplus_[i0].end() );
-            cusp::multiply( plus, inV, tempV);
-            //exchange data in XY
-            commXYplus_.global_scatter_reduce( tempXYplus_[i0], temp_[i0]);
-        }
-    }
-    else //directly compute in temp_
-    {
-        for( int i0=0; i0<(int)g_.Nz(); i0++)
-        {
-            cView inV( in.cbegin() + i0*plus.num_cols, in.cbegin() + (i0+1)*plus.num_cols);
-            View tempV( temp_[i0].begin(), temp_[i0].end() );
-            cusp::multiply( plus, inV, tempV);
-        }
-    }
-    //2. reorder results and communicate halo in z
-    for( int i0=0; i0<(int)g_.Nz(); i0++)
-    {
-        int ip = i0 + 1;
-        if( ip > (int)g_.Nz()-1) ip -= (int)g_.Nz();
-        thrust::copy( temp_[ip].begin(), temp_[ip].begin() + size2d, out.begin() + i0*size2d);
-    }
-    if( sizeZ != 1)
-    {
-        detail::sendBackward( temp_[0].begin(), temp_[0].end(), out.begin() + (g_.Nz()-1)*size2d);
-    }
-
-    //make ghostcells in last plane
-    unsigned size = g_.n()*g_.n()*g_.Nx()*g_.Ny();
-    if( bcz_ != dg::PER && g_.z1() == g_.global().z1())
-    {
-        unsigned i0 = g_.Nz()-1, im = g_.Nz()-2, ip = 0;
-        cView fp( in.cbegin() + ip*size, in.cbegin() + (ip+1)*size);
-        cView f0( in.cbegin() + i0*size, in.cbegin() + (i0+1)*size);
-        cView fm( in.cbegin() + im*size, in.cbegin() + (im+1)*size);
-        View outV( out.begin() + i0*size, out.begin() + (i0+1)*size);
-        View ghostPV( ghostP.begin(), ghostP.end());
-        View ghostMV( ghostM.begin(), ghostM.end());
-        //overwrite out
-        cusp::copy( f0, ghostPV);
-        if( bcz_ == dg::DIR || bcz_ == dg::NEU_DIR)
-        {
-            dg::blas1::axpby( 2., right_, -1, ghostP);
-        }
-        if( bcz_ == dg::NEU || bcz_ == dg::DIR_NEU)
-        {
-            //note that hp_ is 3d and the rest 2d
-            thrust::transform( right_.begin(), right_.end(),  hp_.data().begin(), ghostM.begin(), thrust::multiplies<double>());
-            dg::blas1::axpby( 1., ghostM, 1., ghostP);
-        }
-        cusp::blas::axpby(  ghostPV,  outV, ghostPV, 1.,-1.);
-        dg::blas1::pointwiseDot( limiter_, ghostP, ghostP);
-        cusp::blas::axpby(  ghostPV,  outV, outV, 1.,1.);
-    }
-
+    if(which == einsPlus || which == einsMinusT) ePlus( which, f, fe);
+    if(which == einsMinus || which == einsPlusT) eMinus( which, f, fe);
 }
 
-template<class G,class M, class C, class container>
-void FieldAligned<G,RowDistMat<M,C>,MPI_Vector<container> >::einsMinus( const MPI_Vector<container>& f, MPI_Vector<container>& fminus ) 
-{
-    const container& in = f.data();
-    container& out = fminus.data();
-    //dg::blas2::detail::doSymv( minus, f, fminus, MPIMatrixTag(), MPIVectorTag(), MPIVectorTag());
-    int size2d = g_.n()*g_.n()*g_.Nx()*g_.Ny();
-        int dims[3], periods[3], coords[3];
-        MPI_Cart_get( g_.communicator(), 3, dims, periods, coords);
-        int sizeXY = dims[0]*dims[1];
-        int sizeZ = dims[2];
-    //1. compute 2d interpolation in every plane and store in temp_
-    if( sizeXY != 1) //communication needed
-    {
-        for( int i0=0; i0<(int)g_.Nz(); i0++)
-        {
-            cView inV( in.cbegin() + i0*minus.num_cols, in.cbegin() + (i0+1)*minus.num_cols);
-            View tempV( tempXYminus_[i0].begin(), tempXYminus_[i0].end());
-            cusp::multiply( minus, inV, tempV);
-            //exchange data in XY
-            commXYminus_.global_scatter_reduce( tempXYminus_[i0], temp_[i0]);
-        }
-    }
-    else //directly compute in temp_
-    {
-        for( int i0=0; i0<(int)g_.Nz(); i0++)
-        {
-            cView inV( in.cbegin() + i0*minus.num_cols, in.cbegin() + (i0+1)*minus.num_cols);
-            View tempV( temp_[i0].begin(), temp_[i0].end() );
-            cusp::multiply( minus, inV, tempV);
-        }
-    }
-    //2. reorder results and communicate halo in z
-    for( int i0=0; i0<(int)g_.Nz(); i0++)
-    {
-        int ip = i0 -1;
-        if( ip < 0) ip += (int)g_.Nz();
-        thrust::copy( temp_[ip].begin(), temp_[ip].end(), out.begin() + i0*size2d);
-    }
-    if( sizeZ != 1)
-    {
-        detail::sendForward( temp_[g_.Nz()-1].begin(), temp_[g_.Nz()-1].end(), out.begin());
-    }
-    //make ghostcells in first plane
-    unsigned size = g_.n()*g_.n()*g_.Nx()*g_.Ny();
-    if( bcz_ != dg::PER && g_.z0() == g_.global().z0())
-    {
-        unsigned i0 = 0, im = g_.Nz()-1, ip = 1;
-        cView fp( in.cbegin() + ip*size, in.cbegin() + (ip+1)*size);
-        cView f0( in.cbegin() + i0*size, in.cbegin() + (i0+1)*size);
-        cView fm( in.cbegin() + im*size, in.cbegin() + (im+1)*size);
-        View outV( out.begin() + i0*size, out.begin() + (i0+1)*size);
-        View ghostPV( ghostP.begin(), ghostP.end());
-        View ghostMV( ghostM.begin(), ghostM.end());
-        //overwrite out
-        cusp::copy( f0, ghostMV);
-        if( bcz_ == dg::DIR || bcz_ == dg::DIR_NEU)
-        {
-            dg::blas1::axpby( 2., left_, -1, ghostM);
-        }
-        if( bcz_ == dg::NEU || bcz_ == dg::NEU_DIR)
-        {
-            thrust::transform( left_.begin(), left_.end(), hm_.data().begin(), ghostP.begin(), thrust::multiplies<double>());
-            dg::blas1::axpby( -1, ghostP, 1., ghostM);
-        }
-        cusp::blas::axpby(  ghostMV,  outV, ghostMV, 1.,-1.);
-        dg::blas1::pointwiseDot( limiter_, ghostM, ghostM);
-        cusp::blas::axpby(  ghostMV,  outV, outV, 1.,1.);
-    }
-}
-template< class G, class M, class C, class container>
-void FieldAligned<G,RowDistMat<M,C>,MPI_Vector<container> >::einsMinusT( const MPI_Vector<container>& f, MPI_Vector<container>& fpe)
+template<class G, class M, class C, class container>
+void FieldAligned<G,RowDistMat<M,C>, MPI_Vector<container> >::ePlus( enum whichMatrix which, const MPI_Vector<container>& f, MPI_Vector<container>& fpe ) 
 {
-    //dg::blas2::detail::doSymv( minusT, f, fpe, MPIMatrixTag(), MPIVectorTag(), MPIVectorTag());
-    const container& in = f.data();
-    container& out = fpe.data();
-    int size2d = g_.n()*g_.n()*g_.Nx()*g_.Ny();
-        int dims[3], periods[3], coords[3];
-        MPI_Cart_get( g_.communicator(), 3, dims, periods, coords);
-        int sizeXY = dims[0]*dims[1];
-        int sizeZ = dims[2];
+    dg::split( f, m_f);
 
-    //1. compute 2d interpolation in every plane and store in temp_
-    if( sizeXY != 1) //communication needed
+    //1. compute 2d interpolation in every plane and store in m_temp
+    for( unsigned i0=0; i0<m_Nz; i0++)
     {
-        //first exchange data in XY
-        for( int i0=0; i0<(int)g_.Nz(); i0++)
-        {
-            thrust::copy( in.cbegin() + i0*size2d, in.cbegin() + (i0+1)*size2d, temp_[i0].begin());
-            tempXYminus_[i0] = commXYminus_.global_gather( temp_[i0] );
-            cView inV( tempXYminus_[i0].cbegin(), tempXYminus_[i0].cend() );
-            View tempV( temp_[i0].begin(), temp_[i0].end() );
-            cusp::multiply( minusT, inV, tempV);
-        }
-    }
-    else //directly compute in temp_
-    {
-        for( int i0=0; i0<(int)g_.Nz(); i0++)
-        {
-            cView inV( in.cbegin() + i0*minusT.num_cols, in.cbegin() + (i0+1)*minusT.num_cols);
-            View tempV( temp_[i0].begin() , temp_[i0].end() );
-            cusp::multiply( minusT, inV, tempV);
-        }
-    }
-    //2. reorder results and communicate halo in z
-    for( int i0=0; i0<(int)g_.Nz(); i0++)
-    {
-        int ip = i0 + 1;
-        if( ip > (int)g_.Nz()-1) ip -= (int)g_.Nz();
-        thrust::copy( temp_[ip].begin(), temp_[ip].end(), out.begin() + i0*size2d);
+        unsigned ip = (i0==m_Nz-1) ? 0:i0+1;
+        if(which == einsPlus)           dg::blas2::symv( plus,   m_f[ip], m_temp[i0]);
+        else if(which == einsMinusT)    dg::blas2::symv( minusT, m_f[ip], m_temp[i0]);
     }
-    if( sizeZ != 1)
+
+    //2. communicate halo in z
+    if( m_sizeZ != 1)
     {
-        detail::sendBackward( temp_[0].begin(), temp_[0].end(), out.begin() + ( g_.Nz()-1)*size2d);
+        unsigned i0 = m_Nz-1;
+        detail::sendBackward( m_temp[i0].begin(), m_temp[i0].end(), m_buffer.begin());
+        m_temp[i0].swap( m_buffer);
     }
-    //make ghostcells in last plane
-    unsigned size = g_.n()*g_.n()*g_.Nx()*g_.Ny();
-    if( bcz_ != dg::PER && g_.z1() == g_.global().z1())
+
+    //3. apply right boundary conditions in last plane
+    unsigned i0=m_Nz-1;
+    if( m_bcz != dg::PER && m_g.z1() == m_g.global().z1())
     {
-        unsigned i0 = g_.Nz()-1, im = g_.Nz()-2, ip = 0;
-        cView fp( in.cbegin() + ip*size, in.cbegin() + (ip+1)*size);
-        cView f0( in.cbegin() + i0*size, in.cbegin() + (i0+1)*size);
-        cView fm( in.cbegin() + im*size, in.cbegin() + (im+1)*size);
-        View outV( out.begin() + i0*size, out.begin() + (i0+1)*size);
-        View ghostPV( ghostP.begin(), ghostP.end());
-        View ghostMV( ghostM.begin(), ghostM.end());
-        //overwrite out
-        cusp::copy( f0, ghostPV);
         if( bcz_ == dg::DIR || bcz_ == dg::NEU_DIR)
-        {
-            dg::blas1::axpby( 2., right_, -1, ghostP);
-        }
+            dg::blas1::axpby( 2, m_right, -1., m_f[i0], m_ghostP);
         if( bcz_ == dg::NEU || bcz_ == dg::DIR_NEU)
         {
-            //note that hp_ is 3d and the rest 2d
-            thrust::transform( right_.begin(), right_.end(),  hp_.data().begin(), ghostM.begin(), thrust::multiplies<double>());
-            dg::blas1::axpby( 1., ghostM, 1., ghostP);
+            dg::blas1::pointwiseDivide( m_right, m_hp_inv[i0], m_ghostP);
+            dg::blas1::axpby( 1., m_ghostP, 1., m_f[i0], m_ghostP);
         }
-        cusp::blas::axpby(  ghostPV,  outV, ghostPV, 1.,-1.);
-        dg::blas1::pointwiseDot( limiter_, ghostP, ghostP);
-        cusp::blas::axpby(  ghostPV,  outV, outV, 1.,1.);
+        //interlay ghostcells with periodic cells: L*g + (1-L)*fpe
+        dg::blas1::axpby( 1., m_ghostP, -1., m_temp[i0], m_ghostP);
+        dg::blas1::pointwiseDot( 1., m_limiter, m_ghostP, 1., m_temp[i0]);
     }
+    dg::join( m_temp, fpe);
 }
-template< class G,class M, class C, class container>
-void FieldAligned<G,RowDistMat<M,C>,MPI_Vector<container> >::einsPlusT( const MPI_Vector<container>& f, MPI_Vector<container>& fme)
+
+template<class G, class M, class C, class container>
+void FieldAligned<G,RowDistMat<M,C>, MPI_Vector<container> >::eMinus( enum whichMatrix which, const MPI_Vector<container>& f, MPI_Vector<container>& fme ) 
 {
-    //dg::blas2::detail::doSymv( plusT, f, fme, MPIMatrixTag(), MPIVectorTag(), MPIVectorTag());
-    const container& in = f.data();
-    container& out = fme.data();
-    int size2d = g_.n()*g_.n()*g_.Nx()*g_.Ny();
-        int dims[3], periods[3], coords[3];
-        MPI_Cart_get( g_.communicator(), 3, dims, periods, coords);
-        int sizeXY = dims[0]*dims[1];
-        int sizeZ = dims[2];
+    dg::split( f, m_f);
 
-    //1. compute 2d interpolation in every plane and store in temp_
-    if( sizeXY != 1) //communication needed
-    {
-        for( int i0=0; i0<(int)g_.Nz(); i0++)
-        {
-            //first exchange data in XY
-            thrust::copy( in.cbegin() + i0*size2d, in.cbegin() + (i0+1)*size2d, temp_[i0].begin());
-            tempXYplus_[i0] = commXYplus_.global_gather( temp_[i0]);
-            dg::blas2::symv( plusT, tempXYplus_[i0], temp_[i0]);
-        }
-    }
-    else //directly compute in temp_
-    {
-        for( int i0=0; i0<(int)g_.Nz(); i0++)
-        {
-            cView inV( in.cbegin() + i0*plus.num_cols, in.cbegin() + (i0+1)*plus.num_cols);
-            View tempV( temp_[i0].begin(), temp_[i0].end());
-            cusp::multiply( plusT, inV, tempV);
-        }
-    }
-    //2. reorder results and communicate halo in z
-    for( int i0=0; i0<(int)g_.Nz(); i0++)
+    //1. compute 2d interpolation in every plane and store in m_temp
+    for( unsigned i0=0; i0<m_Nz; i0++)
     {
-        int ip = i0 - 1;
-        if( ip < 0 ) ip += (int)g_.Nz();
-        thrust::copy( temp_[ip].begin(), temp_[ip].end(), out.begin() + i0*size2d);
+        unsigned im = (i0==0) ? m_Nz-1:i0-1;
+        if(which == einsPlusT)         dg::blas2::symv( plusT, m_f[im], m_temp[i0]);
+        else if(which == einsMinus)    dg::blas2::symv( minus, m_f[im], m_temp[i0]);
     }
-    if( sizeZ != 1)
+
+    //2. communicate halo in z
+    if( m_sizeZ != 1)
     {
-        detail::sendForward( temp_[g_.Nz()-1].begin(), temp_[g_.Nz()-1].end(), out.begin());
+        unsigned i0 = 0;
+        detail::sendForward( m_temp[i0].begin(), m_temp[i0].end(), m_buffer.begin());
+        m_temp[i0].swap( m_buffer);
     }
-    //make ghostcells in first plane
-    unsigned size = g_.n()*g_.n()*g_.Nx()*g_.Ny();
-    if( bcz_ != dg::PER && g_.z0() == g_.global().z0())
+
+    //3. apply left boundary conditions in first plane
+    unsigned i0=0;
+    if( m_bcz != dg::PER)
     {
-        unsigned i0 = 0, im = g_.Nz()-1, ip = 1;
-        cView fp( in.cbegin() + ip*size, in.cbegin() + (ip+1)*size);
-        cView f0( in.cbegin() + i0*size, in.cbegin() + (i0+1)*size);
-        cView fm( in.cbegin() + im*size, in.cbegin() + (im+1)*size);
-        View outV( out.begin() + i0*size, out.begin() + (i0+1)*size);
-        View ghostPV( ghostP.begin(), ghostP.end());
-        View ghostMV( ghostM.begin(), ghostM.end());
-        //overwrite out
-        cusp::copy( f0, ghostMV);
-        if( bcz_ == dg::DIR || bcz_ == dg::DIR_NEU)
+        if( m_bcz == dg::DIR || m_bcz == dg::DIR_NEU)
+            dg::blas1::axpby( 2., m_left,  -1., m_f[i0], m_ghostM);
+        if( m_bcz == dg::NEU || m_bcz == dg::NEU_DIR)
         {
-            dg::blas1::axpby( 2., left_, -1, ghostM);
+            dg::blas1::pointwiseDivide( m_left, m_hm_inv[i0], m_ghostM);
+            dg::blas1::axpby( -1., m_ghostM, 1., m_f[i0], m_ghostM);
         }
-        if( bcz_ == dg::NEU || bcz_ == dg::NEU_DIR)
-        {
-            thrust::transform( left_.begin(), left_.end(), hm_.data().begin(), ghostP.begin(), thrust::multiplies<double>());
-            dg::blas1::axpby( -1, ghostP, 1., ghostM);
-        }
-        cusp::blas::axpby(  ghostMV,  outV, ghostMV, 1.,-1.);
-        dg::blas1::pointwiseDot( limiter_, ghostM, ghostM);
-        cusp::blas::axpby(  ghostMV,  outV, outV, 1.,1.);
+        //interlay ghostcells with periodic cells: L*g + (1-L)*fme
+        dg::blas1::axpby( 1., m_ghostM, -1., m_temp[i0], m_ghostM);
+        dg::blas1::pointwiseDot( 1., m_limiter, m_ghostM, 1., m_temp[i0]);
     }
+    dg::join( m_temp, fme);
 }
 
+
 ///@endcond
 }//namespace dg
 
-- 
GitLab


From 67f26794f3c9ec0d3af96c2d4f0efc7ec552a117 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Wed, 4 Oct 2017 17:22:27 +0200
Subject: [PATCH 340/453] changed mpi_fieldaligned, please write
 perp_communicator in MPI Topology

---
 inc/geometries/fieldaligned.h     | 87 ++++++++++++++++---------------
 inc/geometries/mpi_fieldaligned.h | 20 +++----
 2 files changed, 54 insertions(+), 53 deletions(-)

diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index 54dbbc465..04ff12d97 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -399,18 +399,18 @@ struct FieldAligned
     */
     void set_boundaries( dg::bc bcz, double left, double right)
     {
-        bcz_ = bcz;
-        const dg::Grid1d g2d( 0, 1, 1, perp_size_);
-        left_ = dg::evaluate( dg::CONSTANT(left), g2d);
-        right_ = dg::evaluate( dg::CONSTANT(right),g2d);
+        m_bcz = bcz;
+        const dg::Grid1d g2d( 0, 1, 1, m_perp_size);
+        m_left  = dg::evaluate( dg::CONSTANT(left), g2d);
+        m_right = dg::evaluate( dg::CONSTANT(right),g2d);
     }
 
     ///@copydoc set_boundaries()
     void set_boundaries( dg::bc bcz, const container& left, const container& right)
     {
-        bcz_ = bcz;
-        left_ = left;
-        right_ = right;
+        m_bcz = bcz;
+        m_left = left;
+        m_right = right;
     }
 
     /**
@@ -437,7 +437,7 @@ struct FieldAligned
      * @return Returns an instance of container
      */
     template< class BinaryOp>
-    container evaluate( BinaryOp f, unsigned plane=0) const;
+    container evaluate( const BinaryOp& f, unsigned plane=0) const;
 
     /**
      * @brief Evaluate a 2d functor and transform to all planes along the fieldlines
@@ -457,7 +457,7 @@ struct FieldAligned
      * @return Returns an instance of container
      */
     template< class BinaryOp, class UnaryOp>
-    container evaluate( BinaryOp f, UnaryOp g, unsigned p0, unsigned rounds) const;
+    container evaluate( const BinaryOp& f, const UnaryOp& g, unsigned p0, unsigned rounds) const;
 
     /**
     * @brief Applies the interpolation 
@@ -469,22 +469,23 @@ struct FieldAligned
 
     ///@brief hz is the distance between the plus and minus planes
     ///@return three-dimensional vector
-    const container& hz()const {return hz_;}
+    const container& hz_inv()const {return hz_inv;}
     ///@brief hp is the distance between the plus and current planes
     ///@return three-dimensional vector
-    const container& hp()const {return hp_;}
+    const container& hp_inv()const {return hp_inv;}
     ///@brief hm is the distance between the current and minus planes
     ///@return three-dimensional vector
-    const container& hm()const {return hm_;}
+    const container& hm_inv()const {return hm_inv;}
     private:
     void ePlus( enum whichMatrix which, const container& in, container& out);
     void eMinus(enum whichMatrix which, const container& in, container& out);
     IMatrix m_plus, m_minus, m_plusT, m_minusT; //interpolation matrices
-    container m_hz, m_hp, m_hm, m_ghostM, m_ghostP;
+    container m_hz_inv, m_hp_inv, m_hm_inv, m_ghostM, m_ghostP;
     unsigned m_Nz, m_perp_size;
     dg::bc m_bcz;
     container m_left, m_right;
     container m_limiter;
+    std::vector<container> m_temp;
     dg::Handle<Geometry> m_g;
 };
 
@@ -496,19 +497,20 @@ struct FieldAligned
 template<class Geometry, class IMatrix, class container>
 template <class Limiter>
 FieldAligned<Geometry, IMatrix, container>::FieldAligned(const dg::geo::BinaryVectorLvl0& vec, const Geometry& grid, unsigned mx, unsigned my, double eps, Limiter limit, dg::bc globalbcx, dg::bc globalbcy, double deltaPhi):
-        m_hz( dg::evaluate( dg::zero, grid)), m_hp( m_hz), m_hm( m_hz), 
+        m_hz_inv( dg::evaluate( dg::zero, grid)), m_hp_inv( m_hz), m_hm_inv( m_hz), 
         m_Nz(grid.Nz()), m_bcz(grid.bcz()), m_g(grid)
 {
+    dg::split( m_hz_inv, m_temp);
     if( deltaPhi <=0) deltaPhi = grid.hz();
     else assert( grid.Nz() == 1 || grid.hz()==deltaPhi);
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     const aGeometry2d* grid2d_ptr = detail::clone_3d_to_perp(&grid);
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     //Resize vector to 2D grid size
-    perp_size_ = grid2d_ptr->size();
-    limiter_ = dg::pullback( limit, *grid2d_ptr);
-    right_ = left_ = dg::evaluate( zero, *grid2d_ptr);
-    ghostM.resize( perp_size_); ghostP.resize( perp_size_);
+    m_perp_size = grid2d_ptr->size();
+    m_limiter = dg::pullback( limit, *grid2d_ptr);
+    m_right = m_left = dg::evaluate( zero, *grid2d_ptr);
+    ghostM.resize( m_perp_size); ghostP.resize( m_perp_size);
     //%%%%%%%%%%%%%%%%%%%%%%%%%%Set starting points and integrate field lines%%%%%%%%%%%%%%
     std::cout << "Start fieldline integration!\n";
     dg::Timer t;
@@ -546,7 +548,7 @@ FieldAligned<Geometry, IMatrix, container>::FieldAligned(const dg::geo::BinaryVe
     dg::blas2::transfer( minus, m_minus);
     dg::blas2::transfer( minusT, m_minusT);
     //%%%%%%%%%%%%%%%%%%%%%%%project h and copy into h vectors%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-    thrust::host_vector<double> hp( perp_size_), hm(hp);
+    thrust::host_vector<double> hp( m_perp_size), hm(hp);
     dg::blas2::symv( projection, yp[2], hp);
     dg::blas2::symv( projection, ym[2], hm);
     for( unsigned i=0; i<grid.Nz(); i++)
@@ -562,15 +564,10 @@ FieldAligned<Geometry, IMatrix, container>::FieldAligned(const dg::geo::BinaryVe
 template<class G, class I, class container>
 void FieldAligned<G, I,container>::set_boundaries( dg::bc bcz, const container& global, double scal_left, double scal_right)
 {
-    cView left( global.cbegin(), global.cbegin() + perp_size_);
-    cView right( global.cbegin()+(Nz_-1)*perp_size_, global.cbegin() + Nz_*perp_size_);
-    View leftView( left_.begin(), left_.end());
-    View rightView( right_.begin(), right_.end());
-    cusp::copy( left, leftView);
-    cusp::copy( right, rightView);
-    dg::blas1::scal( left_, scal_left);
-    dg::blas1::scal( right_, scal_right);
-    bcz_ = bcz;
+    dg::split( global, m_temp);
+    dg::blas1::axpby( scal_left,  m_temp[0],      0, m_left);
+    dg::blas1::axpby( scal_right, m_temp[m_Nz-1], 0, m_right);
+    m_bcz = bcz;
 }
 
 template< class G, class I, class container>
@@ -582,22 +579,26 @@ container FieldAligned<G, I,container>::evaluate( BinaryOp binary, unsigned p0)
 
 template<class G, class I, class container>
 template< class BinaryOp, class UnaryOp>
-container FieldAligned<G, I,container>::evaluate( BinaryOp binary, UnaryOp unary, unsigned p0, unsigned rounds) const
+container FieldAligned<G, I,container>::evaluate( const BinaryOp& binary, const UnaryOp& unary, unsigned p0, unsigned rounds) const
 {
     //idea: simply apply I+/I- enough times on the init2d vector to get the result in each plane
     //unary function is always such that the p0 plane is at x=0
-    assert( g_.get().Nz() > 1);
-    assert( p0 < g_.get().Nz());
-    const typename G::perpendicular_grid g2d = g_.get().perp_grid();
-    assert( g2d.size() == perp_size_ && Nz_== g_.get().Nz());
-    container init2d = dg::pullback( binary, g2d), temp(init2d), tempP(init2d), tempM(init2d);
-    container vec3d = dg::evaluate( dg::zero, g_.get());
-    std::vector<container>  plus2d( g_.get().Nz(), (container)dg::evaluate(dg::zero, g2d) ), minus2d( plus2d), result( plus2d);
+    assert( p0 < m_g.get().Nz());
+    const aGeometry2d* g2d_ptr = m_g.get().perp_grid();
+    container init2d = dg::pullback( binary, *g2d_ptr); 
+    delete g2d_ptr;
+
+    container temp(init2d), tempP(init2d), tempM(init2d);
+    container vec3d = dg::evaluate( dg::zero, m_g.get());
+    std::vector<container>  plus2d, minus2d, result; 
+    dg::split( vec3d, plus2d);
+    dg::split( vec3d, minus2d);
+    dg::split( vec3d, result);
     unsigned turns = rounds; 
     if( turns ==0) turns++;
     //first apply Interpolation many times, scale and store results
     for( unsigned r=0; r<turns; r++)
-        for( unsigned i0=0; i0<Nz_; i0++)
+        for( unsigned i0=0; i0<m_Nz; i0++)
         {
             dg::blas1::copy( init2d, tempP);
             dg::blas1::copy( init2d, tempM);
@@ -609,15 +610,15 @@ container FieldAligned<G, I,container>::evaluate( BinaryOp binary, UnaryOp unary
                 dg::blas2::symv( minus, tempM, temp);
                 temp.swap( tempM);
             }
-            dg::blas1::scal( tempP, unary(  (double)rep*g_.get().hz() ) );
-            dg::blas1::scal( tempM, unary( -(double)rep*g_.get().hz() ) );
+            dg::blas1::scal( tempP, unary(  (double)rep*m_g.get().hz() ) );
+            dg::blas1::scal( tempM, unary( -(double)rep*m_g.get().hz() ) );
             dg::blas1::axpby( 1., tempP, 1., plus2d[i0]);
             dg::blas1::axpby( 1., tempM, 1., minus2d[i0]);
         }
     //now we have the plus and the minus filaments
     if( rounds == 0) //there is a limiter
     {
-        for( unsigned i0=0; i0<Nz_; i0++)
+        for( unsigned i0=0; i0<m_Nz; i0++)
         {
             int idx = (int)i0 - (int)p0;
             if(idx>=0)
@@ -629,7 +630,7 @@ container FieldAligned<G, I,container>::evaluate( BinaryOp binary, UnaryOp unary
     }
     else //sum up plus2d and minus2d
     {
-        for( unsigned i0=0; i0<Nz_; i0++)
+        for( unsigned i0=0; i0<m_Nz; i0++)
         {
             unsigned revi0 = (m_Nz - i0)%m_Nz; //reverted index
             dg::blas1::axpby( 1., plus2d[i0], 0., result[i0]);
@@ -649,8 +650,8 @@ container FieldAligned<G, I,container>::evaluate( BinaryOp binary, UnaryOp unary
 template<class G, class I, class container>
 void FieldAligned<G, I, container >::operator()(enum whichMatrix which, const container& f, container& fe)
 {
-    if(which == einsPlus || which == einsMinusT) ePlus( which, f, fe);
-    if(which == einsMinus || which == einsPlusT) eMinus( which, f, fe);
+    if(which == einsPlus  || which == einsMinusT) ePlus(  which, f, fe);
+    if(which == einsMinus || which == einsPlusT ) eMinus( which, f, fe);
 }
 
 template< class G, class I, class container>
diff --git a/inc/geometries/mpi_fieldaligned.h b/inc/geometries/mpi_fieldaligned.h
index 72b6027b0..0fbd0f9a5 100644
--- a/inc/geometries/mpi_fieldaligned.h
+++ b/inc/geometries/mpi_fieldaligned.h
@@ -88,24 +88,24 @@ struct FieldAligned< Geometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vec
 
     void set_boundaries( dg::bc bcz, double left, double right)
     {
-        bcz_ = bcz; 
-        const dg::Grid2d g2d( g_.x0(), g_.x1(), g_.y0(), g_.y1(), g_.n(), g_.Nx(), g_.Ny());
-        left_  = dg::evaluate( dg::CONSTANT(left), g2d);
-        right_ = dg::evaluate( dg::CONSTANT(right),g2d);
+        m_bcz = bcz; 
+        const dg::MPIGrid2d g2d( 0., 1., 0., 1., m_g.get().global().n(), m_g.get().global().Nx(), m_g.get().global().Ny(), m_g.get().perp_communicator() );
+        m_left  = dg::evaluate( dg::CONSTANT(left), g2d);
+        m_right = dg::evaluate( dg::CONSTANT(right),g2d);
     }
 
     void set_boundaries( dg::bc bcz, const MPI_Vector<LocalContainer>& left, const MPI_Vector<LocalContainer>& right)
     {
-        bcz_ = bcz; 
-        left_ = left.data();
-        right_ = right.data();
+        m_bcz = bcz; 
+        m_left = left;
+        m_right = right;
     }
 
     void set_boundaries( dg::bc bcz, const MPI_Vector<LocalContainer>& global, double scal_left, double scal_right)
     {
-        bcz_ = bcz;
-        unsigned size = g_.n()*g_.n()*g_.Nx()*g_.Ny();
-        if( g_.z0() == g_.global().z0())
+        m_bcz = bcz;
+        unsigned size = m_g.n()*m_g.n()*m_g.Nx()*m_g.Ny();
+        if( m_g.z0() == m_g.global().z0())
         {
             cView left( global.data().cbegin(), global.data().cbegin() + size);
             View leftView( left_.begin(), left_.end());
-- 
GitLab


From 623eb7317b89721ed0e3a02b10188c3ece10b580 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Thu, 5 Oct 2017 15:59:34 +0200
Subject: [PATCH 341/453] implemented get_perp_comm in mpi_topology3d

---
 inc/dg/backend/mpi_grid.h         | 20 ++++++--
 inc/dg/geometry/mpi_base.h        | 18 +------
 inc/geometries/fieldaligned.h     |  6 +--
 inc/geometries/mpi_curvilinear.h  | 20 ++------
 inc/geometries/mpi_fieldaligned.h | 80 +++++++++++--------------------
 5 files changed, 52 insertions(+), 92 deletions(-)

diff --git a/inc/dg/backend/mpi_grid.h b/inc/dg/backend/mpi_grid.h
index 1de4dda89..62ee55d84 100644
--- a/inc/dg/backend/mpi_grid.h
+++ b/inc/dg/backend/mpi_grid.h
@@ -184,9 +184,9 @@ struct aMPITopology2d
     MPI_Comm communicator() const{return comm;}
 
     /**
-     * @brief The data of dlt
+     * @brief The Discrete Legendre Transformation 
      *
-     * @return 
+     * @return DLT corresponding to n given in the constructor
      */
     const DLT<double>& dlt() const{return g.dlt();}
 
@@ -530,14 +530,24 @@ struct aMPITopology3d
     bc bcz() const {return g.bcz();}
     /**
      * @brief Return mpi cartesian communicator that is used in this grid
-     *
      * @return Communicator
      */
     MPI_Comm communicator() const{return comm;}
     /**
-     * @brief The data of dlt
+     * @brief MPI Cartesian communicator in the first two dimensions
+     * @return 2d Cartesian Communicator
+     */
+    MPI_Comm get_perp_comm() const
+    {
+        MPI_Comm planeComm;
+        int remain_dims[] = {true,true,false}; //true true false
+        MPI_Cart_sub( comm, remain_dims, &planeComm);
+        return planeComm;
+    }
+    /**
+     * @brief The Discrete Legendre Transformation 
      *
-     * @return 
+     * @return DLT corresponding to n given in the constructor
      */
     const DLT<double>& dlt() const{return g.dlt();}
     /**
diff --git a/inc/dg/geometry/mpi_base.h b/inc/dg/geometry/mpi_base.h
index d89d78a1d..850a3371c 100644
--- a/inc/dg/geometry/mpi_base.h
+++ b/inc/dg/geometry/mpi_base.h
@@ -205,15 +205,8 @@ struct CartesianMPIGrid3d : public aProductMPIGeometry3d
     }
 
     private:
-    MPI_Comm get_perp_comm( MPI_Comm src) const
-    {
-        MPI_Comm planeComm;
-        int remain_dims[] = {true,true,false}; //true true false
-        MPI_Cart_sub( src, remain_dims, &planeComm);
-        return planeComm;
-    }
     virtual CartesianMPIGrid2d* do_perp_grid()const{ 
-        return new CartesianMPIGrid2d( global().x0(), global().x1(), global().y0(), global().y1(), global().n(), global().Nx(), global().Ny(), global().bcx(), global().bcy(), get_perp_comm( communicator() ));
+        return new CartesianMPIGrid2d( global().x0(), global().x1(), global().y0(), global().y1(), global().n(), global().Nx(), global().Ny(), global().bcx(), global().bcy(), get_perp_comm( ));
     }
     virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz){
         aMPITopology3d::do_set(new_n,new_Nx,new_Ny,new_Nz);
@@ -249,14 +242,7 @@ struct CylindricalMPIGrid3d: public aProductMPIGeometry3d
     }
     private:
     virtual CartesianMPIGrid2d* do_perp_grid()const{ 
-        return new CartesianMPIGrid2d( global().x0(), global().x1(), global().y0(), global().y1(), global().n(), global().Nx(), global().Ny(), global().bcx(), global().bcy(), get_perp_comm( communicator() ));
-    }
-    MPI_Comm get_perp_comm( MPI_Comm src) const
-    {
-        MPI_Comm planeComm;
-        int remain_dims[] = {true,true,false}; 
-        MPI_Cart_sub( src, remain_dims, &planeComm);
-        return planeComm;
+        return new CartesianMPIGrid2d( global().x0(), global().x1(), global().y0(), global().y1(), global().n(), global().Nx(), global().Ny(), global().bcx(), global().bcy(), get_perp_comm( ));
     }
     virtual SparseTensor<host_vector > do_compute_metric()const{
         SparseTensor<host_vector> metric(1);
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index 04ff12d97..ccf7a3208 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -602,7 +602,7 @@ container FieldAligned<G, I,container>::evaluate( const BinaryOp& binary, const
         {
             dg::blas1::copy( init2d, tempP);
             dg::blas1::copy( init2d, tempM);
-            unsigned rep = i0 + r*Nz_;
+            unsigned rep = i0 + r*m_Nz;
             for(unsigned k=0; k<rep; k++)
             {
                 dg::blas2::symv( plus, tempP, temp);
@@ -625,7 +625,7 @@ container FieldAligned<G, I,container>::evaluate( const BinaryOp& binary, const
                 result[i0] = plus2d[idx];
             else
                 result[i0] = minus2d[abs(idx)];
-            thrust::copy( result[i0].begin(), result[i0].end(), vec3d.begin() + i0*perp_size_);
+            thrust::copy( result[i0].begin(), result[i0].end(), vec3d.begin() + i0*m_perp_size);
         }
     }
     else //sum up plus2d and minus2d
@@ -640,7 +640,7 @@ container FieldAligned<G, I,container>::evaluate( const BinaryOp& binary, const
         for(unsigned i0=0; i0<m_Nz; i0++)
         {
             int idx = ((int)i0 -(int)p0 + m_Nz)%m_Nz; //shift index
-            thrust::copy( result[idx].begin(), result[idx].end(), vec3d.begin() + i0*perp_size_);
+            thrust::copy( result[idx].begin(), result[idx].end(), vec3d.begin() + i0*m_perp_size);
         }
     }
     return vec3d;
diff --git a/inc/geometries/mpi_curvilinear.h b/inc/geometries/mpi_curvilinear.h
index 38c0d88a8..2a092f811 100644
--- a/inc/geometries/mpi_curvilinear.h
+++ b/inc/geometries/mpi_curvilinear.h
@@ -56,13 +56,6 @@ struct CurvilinearMPIGrid2d : public dg::aMPIGeometry2d
         CurvilinearGrid2d g( handle_.get(), new_n, new_Nx, new_Ny);
         divide_and_conquer(g);//distribute to processes
     }
-    MPI_Comm get_perp_comm( MPI_Comm src)
-    {
-        MPI_Comm planeComm;
-        int remain_dims[] = {true,true,false}; //true true false
-        MPI_Cart_sub( src, remain_dims, &planeComm);
-        return planeComm;
-    }
     void divide_and_conquer(const CurvilinearGrid2d& g_)
     {
         dg::SparseTensor<thrust::host_vector<double> > jacobian=g_.jacobian(); 
@@ -109,7 +102,7 @@ struct CurvilinearProductMPIGrid3d : public dg::aProductMPIGeometry3d
         handle_( generator)
     {
         map_.resize(3);
-        CurvilinearMPIGrid2d g(generator,n,Nx,Ny, bcx, bcy, get_perp_comm(comm));
+        CurvilinearMPIGrid2d g(generator,n,Nx,Ny, bcx, bcy, get_perp_comm());
         constructPerp( g);
         constructParallel(this->Nz());
     }
@@ -126,19 +119,12 @@ struct CurvilinearProductMPIGrid3d : public dg::aProductMPIGeometry3d
     }
     private:
     virtual perpendicular_grid* do_perp_grid() const { return new perpendicular_grid(*this);}
-    MPI_Comm get_perp_comm( MPI_Comm src)
-    {
-        MPI_Comm planeComm;
-        int remain_dims[] = {true,true,false}; //true true false
-        MPI_Cart_sub( src, remain_dims, &planeComm);
-        return planeComm;
-    }
     virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz)
     {
         dg::aMPITopology3d::do_set(new_n, new_Nx, new_Ny, new_Nz);
         if( !( new_n == n() && new_Nx == global().Nx() && new_Ny == global().Ny() ) )
         {
-            CurvilinearMPIGrid2d g(handle_.get(),new_n,new_Nx,new_Ny, this->bcx(), this->bcy(), get_perp_comm(communicator()));
+            CurvilinearMPIGrid2d g(handle_.get(),new_n,new_Nx,new_Ny, this->bcx(), this->bcy(), get_perp_comm());
             constructPerp( g);
         }
         constructParallel(this->Nz());
@@ -203,7 +189,7 @@ struct CurvilinearProductMPIGrid3d : public dg::aProductMPIGeometry3d
 };
 ///@cond
 CurvilinearMPIGrid2d::CurvilinearMPIGrid2d( const CurvilinearProductMPIGrid3d& g):
-    dg::aMPIGeometry2d( g.global().x0(), g.global().x1(), g.global().y0(), g.global().y1(), g.global().n(), g.global().Nx(), g.global().Ny(), g.global().bcx(), g.global().bcy(), get_perp_comm( g.communicator() )),
+    dg::aMPIGeometry2d( g.global().x0(), g.global().x1(), g.global().y0(), g.global().y1(), g.global().n(), g.global().Nx(), g.global().Ny(), g.global().bcx(), g.global().bcy(), get_perp_comm( )),
     handle_(g.generator())
 {
     map_=g.map();
diff --git a/inc/geometries/mpi_fieldaligned.h b/inc/geometries/mpi_fieldaligned.h
index 0fbd0f9a5..dbb4bebbb 100644
--- a/inc/geometries/mpi_fieldaligned.h
+++ b/inc/geometries/mpi_fieldaligned.h
@@ -103,22 +103,10 @@ struct FieldAligned< Geometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vec
 
     void set_boundaries( dg::bc bcz, const MPI_Vector<LocalContainer>& global, double scal_left, double scal_right)
     {
+        dg::split( global, m_temp);
+        dg::blas1::axpby( scal_left, m_temp[0],               0., m_left);
+        dg::blas1::axpby( scal_right, m_temp[m_g.get().Nz()], 0., m_left);
         m_bcz = bcz;
-        unsigned size = m_g.n()*m_g.n()*m_g.Nx()*m_g.Ny();
-        if( m_g.z0() == m_g.global().z0())
-        {
-            cView left( global.data().cbegin(), global.data().cbegin() + size);
-            View leftView( left_.begin(), left_.end());
-            cusp::copy( left, leftView);
-            dg::blas1::scal( left_, scal_left);
-        }
-        if( g_.z1() == g_.global().z1())
-        {
-            cView right( global.data().cbegin()+(g_.Nz()-1)*size, global.data().cbegin() + g_.Nz()*size);
-            View rightView( right_.begin(), right_.end());
-            cusp::copy( right, rightView);
-            dg::blas1::scal( right_, scal_right);
-        }
     }
 
     template< class BinaryOp>
@@ -248,76 +236,66 @@ MPI_Vector<container> FieldAligned<G,RowDistMat<M,C>, MPI_Vector<container> >::e
     //idea: simply apply I+/I- enough times on the init2d vector to get the result in each plane
     //unary function is always such that the p0 plane is at x=0
     assert( g_.Nz() > 1);
-    assert( p0 < g_.global().Nz());
-    const typename G::perpendicular_grid g2d = g_.perp_grid();
-    MPI_Vector<container> init2d = dg::pullback( binary, g2d); 
-    container temp(init2d.data()), tempP(init2d.data()), tempM(init2d.data());
-    MPI_Vector<container> vec3d = dg::evaluate( dg::zero, g_);
-    std::vector<container>  plus2d( g_.global().Nz(), (container)dg::evaluate(dg::zero, g2d.local()) ), minus2d( plus2d), result( plus2d);
-    container tXYplus( tempXYplus_[0]), tXYminus( tempXYminus_[0]);
+    assert( p0 < m_g.get().global().Nz());
+    const aGeometry2d* g2d_ptr = m_g.get().perp_grid();
+    MPI_Vector<container> init2d = dg::pullback( binary, *g2d_perp); 
+
+    MPI_Vector<container> temp(init2d), tempP(init2d), tempM(init2d);
+    MPI_Vector<container> vec3d = dg::evaluate( dg::zero, m_g.get());
+    std::vector<MPI_Vector<container> >  plus2d, minus2d, result;
+    dg.:split( vec3d, plus2d);
+    dg.:split( vec3d, minus2d);
+    dg.:split( vec3d, result);
     unsigned turns = rounds; 
     if( turns ==0) turns++;
     //first apply Interpolation many times, scale and store results
-        int dims[3], periods[3], coords[3];
-        MPI_Cart_get( g_.communicator(), 3, dims, periods, coords);
-        int sizeXY = dims[0]*dims[1];
     for( unsigned r=0; r<turns; r++)
-        for( unsigned i0=0; i0<g_.global().Nz(); i0++)
+        for( unsigned i0=0; i0<m_g.get().global().Nz(); i0++)
         {
-            dg::blas1::copy( init2d.data(), tempP);
-            dg::blas1::copy( init2d.data(), tempM);
-            unsigned rep = i0 + r*g_.global().Nz(); 
+            dg::blas1::copy( init2d, tempP);
+            dg::blas1::copy( init2d, tempM);
+            unsigned rep = i0 + r*m_g.get().global().Nz(); 
             for(unsigned k=0; k<rep; k++)
             {
-                if( sizeXY != 1){
-                    dg::blas2::symv( plus, tempP, tXYplus);
-                    commXYplus_.global_scatter_reduce( tXYplus, temp);
-                }
-                else
-                    dg::blas2::symv( plus, tempP, temp);
+                dg::blas2::symv( plus, tempP, temp);
                 temp.swap( tempP);
-                if( sizeXY != 1){
-                    dg::blas2::symv( minus, tempM, tXYminus);
-                    commXYminus_.global_scatter_reduce( tXYminus, temp);
-                }
-                else
-                    dg::blas2::symv( minus, tempM, temp);
+                dg::blas2::symv( minus, tempM, temp);
                 temp.swap( tempM);
             }
-            dg::blas1::scal( tempP, unary(  (double)rep*g_.hz() ) );
-            dg::blas1::scal( tempM, unary( -(double)rep*g_.hz() ) );
+            dg::blas1::scal( tempP, unary(  (double)rep*m_g.get().hz() ) );
+            dg::blas1::scal( tempM, unary( -(double)rep*m_g.get().hz() ) );
             dg::blas1::axpby( 1., tempP, 1., plus2d[i0]);
             dg::blas1::axpby( 1., tempM, 1., minus2d[i0]);
         }
     //now we have the plus and the minus filaments
     if( rounds == 0) //there is a limiter
     {
-        for( unsigned i0=0; i0<g_.Nz(); i0++)
+        for( unsigned i0=0; i0<m_g.get().global().Nz(); i0++)
         {
-            int idx = (int)(i0+coords[2]*g_.Nz())  - (int)p0;
+            int idx = (int)(i0+coords[2]*m_g.get().Nz())  - (int)p0;
             if(idx>=0)
                 result[i0] = plus2d[idx];
             else
                 result[i0] = minus2d[abs(idx)];
-            thrust::copy( result[i0].begin(), result[i0].end(), vec3d.data().begin() + i0*g2d.size());
+            thrust::copy( result[i0].data().begin(), result[i0].data().end(), vec3d.data().begin() + i0*g2d_ptr->size());
         }
     }
     else //sum up plus2d and minus2d
     {
         for( unsigned i0=0; i0<g_.global().Nz(); i0++)
         {
-            //int idx = (int)(i0+coords[2]*g_.Nz());
-            unsigned revi0 = (g_.global().Nz() - i0)%g_.global().Nz(); //reverted index
+            unsigned revi0 = (m_g.get().global().Nz() - i0)%m_g.get().global().Nz(); //reverted index
             dg::blas1::axpby( 1., plus2d[i0], 0., result[i0]);
             dg::blas1::axpby( 1., minus2d[revi0], 1., result[i0]);
         }
-        dg::blas1::axpby( -1., init2d.data(), 1., result[0]);
+        dg::blas1::axpby( -1., init2d, 1., result[0]);
         for(unsigned i0=0; i0<g_.Nz(); i0++)
         {
-            int idx = ((int)i0 + coords[2]*g_.Nz() -(int)p0 + g_.global().Nz())%g_.global().Nz(); //shift index
-            thrust::copy( result[idx].begin(), result[idx].end(), vec3d.data().begin() + i0*g2d.size());
+            int idx = ((int)i0 + coords[2]*g_.Nz() -(int)p0 + m_g.get().global().Nz())%m_g.get().global().Nz(); //shift index
+            thrust::copy( result[idx].data().begin(), result[idx].data().end(), vec3d.data().begin() + i0*g2d_ptr->size());
         }
     }
+    delete g2d_ptr;
     return vec3d;
 }
 
-- 
GitLab


From 598b807e78f38bb65dc2c99abfbc791a3a325b8c Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Thu, 5 Oct 2017 17:26:42 +0200
Subject: [PATCH 342/453] some documentation issues

---
 inc/geometries/mpi_curvilinear.h | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/inc/geometries/mpi_curvilinear.h b/inc/geometries/mpi_curvilinear.h
index 2a092f811..f73cebf62 100644
--- a/inc/geometries/mpi_curvilinear.h
+++ b/inc/geometries/mpi_curvilinear.h
@@ -27,11 +27,11 @@ struct CurvilinearProductMPIGrid3d;
  */
 struct CurvilinearMPIGrid2d : public dg::aMPIGeometry2d
 {
-    /// @opydoc hide_grid_parameters2d
+    /// @copydoc hide_grid_parameters2d
     /// @param comm a two-dimensional Cartesian communicator
     /// @note the paramateres given in the constructor are global parameters 
-    CurvilinearMPIGrid2d( const aGenerator2d& generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx, dg::bc bcy, MPI_Comm comm2d): 
-        dg::aMPIGeometry2d( 0, generator.width(), 0., generator.height(), n, Nx, Ny, bcx, bcy, comm2d), handle_(generator)
+    CurvilinearMPIGrid2d( const aGenerator2d& generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx, dg::bc bcy, MPI_Comm comm): 
+        dg::aMPIGeometry2d( 0, generator.width(), 0., generator.height(), n, Nx, Ny, bcx, bcy, comm), handle_(generator)
     {
         //generate global 2d grid and then reduce to local 
         CurvilinearGrid2d g(generator, n, Nx, Ny);
@@ -94,7 +94,7 @@ struct CurvilinearMPIGrid2d : public dg::aMPIGeometry2d
 struct CurvilinearProductMPIGrid3d : public dg::aProductMPIGeometry3d
 {
     typedef dg::geo::CurvilinearMPIGrid2d perpendicular_grid; //!< the two-dimensional grid
-    /// @opydoc hide_grid_parameters3d
+    /// @copydoc hide_grid_parameters3d
     /// @param comm a three-dimensional Cartesian communicator
     /// @note the paramateres given in the constructor are global parameters 
     CurvilinearProductMPIGrid3d( const aGenerator2d& generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz, MPI_Comm comm): 
-- 
GitLab


From 5d2d25708a5cf36a8cdab7a42c230d04d2502b64 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Fri, 6 Oct 2017 11:42:02 +0200
Subject: [PATCH 343/453] first writeup added to documentation

---
 inc/doc/Makefile                              |  67 ++--
 inc/geometries/Doxyfile                       |   2 +-
 inc/geometries/fieldaligned.h                 |  29 +-
 .../{solovev_doc.h => geometries_doc.h}       |   4 +
 inc/geometries/related_pages/parallel.tex     | 370 ++++++++++++++++++
 src/toefl/toefl.tex                           |   2 +-
 6 files changed, 424 insertions(+), 50 deletions(-)
 rename inc/geometries/{solovev_doc.h => geometries_doc.h} (94%)
 create mode 100644 inc/geometries/related_pages/parallel.tex

diff --git a/inc/doc/Makefile b/inc/doc/Makefile
index 590c85315..7e169dbe0 100644
--- a/inc/doc/Makefile
+++ b/inc/doc/Makefile
@@ -8,45 +8,50 @@ all: doc
 
 .PHONY: clean doc dg.tag geometries.tag dg file geometries
 
+# the semicolons and backslashes are needed by Makefile
 dg.tag:
-	(cat ../dg/Doxyfile; \
-	echo "INPUT = ../dg/"; \
-	echo "OUTPUT_DIRECTORY = ./dg"; \
-	echo "GENERATE_HTML = NO"; \
-	echo "GENERATE_TAGFILE = ./dg.tag" ) | doxygen - ;
+	cd ../dg; \
+		(cat Doxyfile; \
+		echo "OUTPUT_DIRECTORY = ../doc/dg"; \
+		echo "GENERATE_HTML = NO"; \
+		echo "GENERATE_TAGFILE = ../doc/dg.tag" ) | doxygen - ;
 
 geometries.tag:
-	(cat ../geometries/Doxyfile; \
-	echo "INPUT = ../geometries/"; \
-	echo "OUTPUT_DIRECTORY = ./geometries"; \
-	echo "GENERATE_HTML = NO"; \
-	echo "GENERATE_TAGFILE = ./geometries.tag" ) | doxygen - ;
+	cd ../geometries;  \
+		(cat Doxyfile; \
+		echo "OUTPUT_DIRECTORY = ../doc/geometries"; \
+		echo "GENERATE_HTML = NO"; \
+		echo "GENERATE_TAGFILE = ../doc/geometries.tag" ) | doxygen - ;
 
 dg: geometries.tag
-	(cat ../dg/Doxyfile; \
-	echo "INPUT = ../dg/"; \
-	echo "OUTPUT_DIRECTORY = ./dg"; \
-	echo "HTML_HEADER = header.html"; \
-    echo "EXTERNAL_GROUPS=NO" ;\
-    echo "EXTERNAL_PAGES=NO" ;\
-	echo "TAGFILES = geometries.tag=../../geometries/html") | doxygen - ; 
-	#echo ) | doxygen - ; 
+	cd ../dg; \
+		(cat Doxyfile; \
+		echo "OUTPUT_DIRECTORY = ../doc/dg"; \
+		echo "HTML_HEADER = ../doc/header.html"; \
+    	echo "EXTERNAL_GROUPS=NO" ;\
+    	echo "EXTERNAL_PAGES=NO" ;\
+		echo "TAGFILES = ../doc/geometries.tag=../../geometries/html") | doxygen - ; 
 
 geometries: dg.tag
-	(cat ../geometries/Doxyfile; \
-	echo "INPUT = ../geometries/"; \
-	echo "OUTPUT_DIRECTORY = ./geometries"; \
-	echo "HTML_HEADER = header.html"; \
-    echo "EXTERNAL_GROUPS=NO" ;\
-    echo "EXTERNAL_PAGES=NO" ;\
-	echo "TAGFILES = dg.tag=../../dg/html") | doxygen - ; 
+	cd ../geometries; \
+		(cat Doxyfile; \
+		echo "OUTPUT_DIRECTORY = ../doc/geometries"; \
+		echo "HTML_HEADER = ../doc/header.html"; \
+    	echo "EXTERNAL_GROUPS=NO" ;\
+    	echo "EXTERNAL_PAGES=NO" ;\
+		echo "TAGFILES = ../doc/dg.tag=../../dg/html") | doxygen - ; 
 
 file:
-	(cat ../file/Doxyfile; \
-	echo "INPUT = ../file/"; \
-	echo "OUTPUT_DIRECTORY = ./file"; \
-	echo "HTML_HEADER = header.html"; \
-	echo ) | doxygen - ; 
+	cd ../file; \
+		(cat Doxyfile; \
+		echo "OUTPUT_DIRECTORY = ../doc/file"; \
+		echo "HTML_HEADER = ../doc/header.html"; \
+		echo ) | doxygen - ; 
+
+pdf:
+	cd ../geometries/related_pages; \
+		pdflatex parallel.tex;
+
 
 doc: dg geometries file
 	ln -sf dg/html/modules.html index.html
@@ -54,5 +59,7 @@ doc: dg geometries file
 	#Open with: firefox index.html or on Windows: firefox dg/html/modules.html#
 	#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%#
 
+full_doc: pdf doc
+
 clean:
 	rm -rf dg file geometries dg.tag file.tag geometries.tag index.html
diff --git a/inc/geometries/Doxyfile b/inc/geometries/Doxyfile
index 803732e04..f17afa02e 100644
--- a/inc/geometries/Doxyfile
+++ b/inc/geometries/Doxyfile
@@ -1151,7 +1151,7 @@ HTML_EXTRA_STYLESHEET  =
 # files will be copied as-is; there are no commands or markers available.
 # This tag requires that the tag GENERATE_HTML is set to YES.
 
-HTML_EXTRA_FILES       =
+HTML_EXTRA_FILES       = "../geometries/related_pages/parallel.pdf"
 
 # The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen
 # will adjust the colors in the style sheet and background images according to
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index ccf7a3208..298b0763c 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -423,7 +423,13 @@ struct FieldAligned
      * @param scal_left left scaling factor
      * @param scal_right right scaling factor
      */
-    void set_boundaries( dg::bc bcz, const container& global, double scal_left, double scal_right);
+    void set_boundaries( dg::bc bcz, const container& global, double scal_left, double scal_right)
+    {
+        dg::split( global, m_temp);
+        dg::blas1::axpby( scal_left,  m_temp[0],      0, m_left);
+        dg::blas1::axpby( scal_right, m_temp[m_Nz-1], 0, m_right);
+        m_bcz = bcz;
+    }
 
     /**
      * @brief Evaluate a 2d functor and transform to all planes along the fieldlines
@@ -437,7 +443,10 @@ struct FieldAligned
      * @return Returns an instance of container
      */
     template< class BinaryOp>
-    container evaluate( const BinaryOp& f, unsigned plane=0) const;
+    container evaluate( const BinaryOp& f, unsigned plane=0) const
+    {
+        return evaluate( binary, dg::CONSTANT(1), p0, 0);
+    }
 
     /**
      * @brief Evaluate a 2d functor and transform to all planes along the fieldlines
@@ -561,22 +570,6 @@ FieldAligned<Geometry, IMatrix, container>::FieldAligned(const dg::geo::BinaryVe
     delete grid2d_ptr;
 }
 
-template<class G, class I, class container>
-void FieldAligned<G, I,container>::set_boundaries( dg::bc bcz, const container& global, double scal_left, double scal_right)
-{
-    dg::split( global, m_temp);
-    dg::blas1::axpby( scal_left,  m_temp[0],      0, m_left);
-    dg::blas1::axpby( scal_right, m_temp[m_Nz-1], 0, m_right);
-    m_bcz = bcz;
-}
-
-template< class G, class I, class container>
-template< class BinaryOp>
-container FieldAligned<G, I,container>::evaluate( BinaryOp binary, unsigned p0) const
-{
-    return evaluate( binary, dg::CONSTANT(1), p0, 0);
-}
-
 template<class G, class I, class container>
 template< class BinaryOp, class UnaryOp>
 container FieldAligned<G, I,container>::evaluate( const BinaryOp& binary, const UnaryOp& unary, unsigned p0, unsigned rounds) const
diff --git a/inc/geometries/solovev_doc.h b/inc/geometries/geometries_doc.h
similarity index 94%
rename from inc/geometries/solovev_doc.h
rename to inc/geometries/geometries_doc.h
index 45fa73b94..9c75f8107 100644
--- a/inc/geometries/solovev_doc.h
+++ b/inc/geometries/geometries_doc.h
@@ -30,6 +30,10 @@
  *   and profiles are added to the dg::geo namespace
  * - there are some miscellaneous additions like a flux surface average class
  * and one used to integrate the field lines for parallel derivatives all in the dg::geo namespace.
+ * @subsection pdf PDF writeups
+ * 
+ * We have a collection of writeups: 
+ *  - The parallel derivative <a href="./parallel.pdf" target="_blank">parallel derivative</a>
  */
  /** @class hide_container
   * @tparam container 
diff --git a/inc/geometries/related_pages/parallel.tex b/inc/geometries/related_pages/parallel.tex
new file mode 100644
index 000000000..91bc94c92
--- /dev/null
+++ b/inc/geometries/related_pages/parallel.tex
@@ -0,0 +1,370 @@
+%\documentclass[12pt]{article}
+%\documentclass[12pt]{scrartcl}
+\documentclass{hitec} % contained in texlive-latex-extra
+\settextfraction{0.9} % indent text
+\usepackage{csquotes}
+\usepackage[hidelinks]{hyperref} % doi links are short and usefull?
+\hypersetup{%
+    colorlinks=true,
+    linkcolor=blue,
+    urlcolor=magenta
+}
+\urlstyle{rm}
+\usepackage[english]{babel}
+\usepackage{mathtools} % loads and extends amsmath
+\usepackage{amssymb}
+% packages not used
+%\usepackage{graphicx}
+%\usepackage{amsthm}
+%\usepackage{subfig}
+\usepackage{bm}
+\usepackage{longtable}
+\usepackage{booktabs}
+\usepackage{ragged2e} % maybe use \RaggedRight for tables and literature?
+\usepackage[table]{xcolor} % for alternating colors
+\rowcolors{2}{gray!25}{white}
+\renewcommand\arraystretch{1.3}
+
+%%% reset bibliography distances %%%
+\let\oldthebibliography\thebibliography
+\let\endoldthebibliography\endthebibliography
+\renewenvironment{thebibliography}[1]{
+  \begin{oldthebibliography}{#1}
+    \RaggedRight % remove if justification is desired
+    \setlength{\itemsep}{0em}
+    \setlength{\parskip}{0em}
+}
+{
+  \end{oldthebibliography}
+}
+%%% --- %%%
+
+%%%%%%%%%%%%%%%%%%%%%definitions%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\newcommand{\eps}{\varepsilon}
+\renewcommand{\d}{\mathrm{d}}
+\newcommand{\T}{\mathrm{T}}
+\renewcommand{\vec}[1]{{\mathbf{#1}}}
+\newcommand{\dx}{\,\mathrm{d}x}
+%\newcommand{\dA}{\,\mathrm{d}(x,y)}
+%\newcommand{\dV}{\mathrm{d}^3{x}\,}
+\newcommand{\dA}{\,\mathrm{dA}}
+\newcommand{\dV}{\mathrm{dV}\,}
+
+\newcommand{\Eins}{\mathbf{1}}
+
+\newcommand{\ExB}{$\bm{E}\times\bm{B} \,$}
+\newcommand{\GKI}{\int d^6 \bm{Z} \BSP}	
+\newcommand{\GKIV}{\int dv_{\|} d \mu d \theta \BSP}	
+\newcommand{\BSP}{B_{\|}^*}
+\newcommand{\GA}[1]{\langle #1	 \rangle}
+
+\newcommand{\Abar}{\langle A_\parallel \rangle}
+%Vectors
+\newcommand{\bhat}{\bm{\hat{b}}}
+\newcommand{\bbar}{\overline{\bm{b}}}
+\newcommand{\chat}{\bm{\hat{c}}}
+\newcommand{\ahat}{\bm{\hat{a}}}
+\newcommand{\xhat}{\bm{\hat{x}}}
+\newcommand{\yhat}{\bm{\hat{y}}}
+\newcommand{\zhat}{\bm{\hat{z}}}
+
+\newcommand{\Xbar}{\bar{\vec{X}}}
+\newcommand{\phat}{\bm{\hat{\perp}}}
+\newcommand{\that}{\bm{\hat{\theta}}}
+
+\newcommand{\eI}{\bm{\hat{e}}_1}
+\newcommand{\eII}{\bm{\hat{e}}_2}
+\newcommand{\ud}{\mathrm{d}}
+
+%Derivatives etc.
+\newcommand{\pfrac}[2]{\frac{\partial#1}{\partial#2}}
+\newcommand{\ffrac}[2]{\frac{\delta#1}{\delta#2}}
+\newcommand{\fixd}[1]{\Big{\arrowvert}_{#1}}
+\newcommand{\curl}[1]{\nabla \times #1}
+\newcommand{\np}{\nabla_{\perp}}
+\newcommand{\npc}{\nabla_{\perp} \cdot }
+\newcommand{\nc}{\nabla\cdot }
+\newcommand{\GAI}{\Gamma_{1}^{\dagger}}
+\newcommand{\GAII}{\Gamma_{1}^{\dagger -1}}
+\newcommand{\Tp}{\mathcal T^+_{\Delta\varphi}}
+\newcommand{\Tm}{\mathcal T^-_{\Delta\varphi}}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%DOCUMENT%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\begin{document}
+%\preprint{}
+
+\title{The parallel derivative on structured grids}
+\author{M.~Wiesenberger}
+%\email{Matthias.Wiesenberger@uibk.ac.at}
+%\ead{Matthias.Wiesenberger@uibk.ac.at}
+%\affiliation{Institute for Ion Physics and Applied Physics, Association EURATOM-\"OAW,  University of
+%   Innsbruck, A-6020 Innsbruck, Austria}
+%\author{M.~Held}
+%\affiliation{Institute for Ion Physics and Applied Physics, Association EURATOM-\"OAW,  University of
+   %Innsbruck, A-6020 Innsbruck, Austria}
+%\address{Institute for Ion Physics and Applied Physics, Association EURATOM-\"OAW,  Universit\"at 
+%   Innsbruck, A-6020 Innsbruck, Austria}
+ 
+\maketitle
+\section{The parallel derivative}
+The idea of the sandwich method is to bracket the parallel derivative by interpolation and projection matrices:
+\begin{align}
+    \nabla^c_\parallel &= P\nabla_\parallel^f Q \\
+    \nabla^{c\dagger}_\parallel &= P \nabla^{f\dagger}_\parallel Q
+    \label{eq:sandwich}
+\end{align}
+In this way the projection integrals
+\begin{align}
+    \int\dV \nabla_\parallel f p_i(x)p_j(y) 
+    \label{}
+\end{align}
+are computed more precisely.
+The size of the fine grid should therefore be as large as
+possible (reasonable? what resolution is needed?).
+We first notice that the one interpolation matrix can be absorbed
+in the parallel derivative since this also consists of 
+interpolation operations. 
+\begin{align}
+    \nabla^c_\parallel &= P\nabla_\parallel^{fc} \\
+    \nabla^{c\dagger}_\parallel &= \nabla^{fc\dagger}_\parallel Q
+    \label{eq:sandwich}
+\end{align}
+
+
+In order to understand what the adjoint operators do let us denote $\mathcal T^+_{\Delta\varphi}$ as the push-forwward operator. Then we have
+\begin{align}
+    \int f(\vec x) \Tp h(\vec x) \sqrt{g(\vec x)}\d^3x \\
+    =  \int f(\vec x) h(\Tm \vec x)\sqrt{g(\vec x)}\d^3x \\
+    =  \int f(\Tp \vec x') h(\vec x')\sqrt{g(\Tp \vec x')}J^{-1}( \Tp\vec x') \d^3x' \\
+    =  \int \frac{1}{\sqrt{g(\vec x')}}\Tm\left[J^{-1}(\vec x')\sqrt{g(\vec x')}f(\vec x')\right] h(\vec x')\sqrt{g(\vec x')}   \d^3x' \\
+    \equiv  \int (\Tp)^\dagger\left[f(\vec x)\right] h(\vec x)\sqrt{g(\vec x)}   \d^3x
+    \label{}
+\end{align}
+$J$ is the determinant of the Jacobian $\partial(\vec x')/\partial(\vec x)$.
+In the last step we simply replaced the dummy variable $\vec x'$ with $\vec x$ again and identified the relevant terms
+as the adjoint operator:
+\begin{align}
+    (\Tp)^\dagger f(\vec x ) := \frac{1}{\sqrt{g(\vec x)}} \Tm\left[\sqrt{g(\vec x)} J^{-1}(\vec x) f(\vec x) \right]
+    \label{}
+\end{align}
+This means that numerically the adjoint of the push-forward 
+operator should be a valid discretization of its inverse. (how can this be exploited as a test?)
+Note that $\sqrt{g}J^{-1}(\vec x) = \sqrt{g'(\Tm \vec x)}$.
+With this we can write
+\begin{align}
+    (\Tp)^\dagger f(\vec x ) := \sqrt{\frac{g'(\vec x)}{g(\vec x)}} \Tm\left[f(\vec x) \right]
+    \label{}
+\end{align}
+Also note that while $\Tp [fh] = \Tp f \Tp h$ this might not 
+hold on the discrete level. Also the question is how $J$ enters 
+on the discrete level. We have to multiply $\sqrt{g}$ artificially when we form the adjoint. 
+Theoretically $J$ could be hidden somehow when we integrate the fieldlines, so the information could be contained in the discrete version? (Maybe in the back-projection?)
+
+If the streamlines are divergence free, we have $J=1$.
+A numerical test could be ( if we neglect the volume form in the adjoint)
+\begin{align}
+    (\Tp)^\dagger \left[J(\vec x)\Tp f(\vec x)\right] - f(\vec x) = 0
+    \label{}
+\end{align}
+The numerical computation of $J$ might a bit tricky at the boundaries. 
+In a flux-aligned $\zeta, \eta$ it should be feasible but in cylindrical coordinates I don't know how. Maybe we can simply cut the last few cells before the boundary.
+Even easier might be
+\begin{align}
+    \left(\Tp\right)^\dagger J(\vec x ) = 1
+    \label{}
+\end{align}
+
+If we integrate streamlines of the vector field $\vec B/B^\varphi$, then we have
+\begin{align}
+    \frac{\d J}{\d \varphi} = J(\vec x ) \nabla\cdot\left( \vec B/B^\varphi\right)
+    \label{}
+\end{align}
+along these streamlines.
+Also we have that $\d s = B/B^\varphi \d \varphi $ and the interpolation/projection of $\triangle s$ can probably be neglected. (We want to neglect it because it's memory intensive to store all combinations)
+Also this means that when we transpose $\nabla_\parallel$ we get a 
+multiplication by $B^\varphi/B$ in the beginning.
+In any case this means that we want to have 
+\begin{align}
+\left(\Tp\right)^\dagger B^\varphi  =  B^\varphi
+\end{align}
+\section{Boundary conditions}
+The question is what to do when a fieldline intersects with the boundary
+of the simulation domain before reaching the next plane.
+The problem with simply putting a value exactly where the fieldline 
+reaches the boundary is first that a true interpolation at that 
+position requires a 3d interpolation (we 
+are between planes) instead of a 2d interpolation.
+Secondly, for Dirichlet boundaries the small 
+distance seriously deteriorates the CFL condition.
+A simple solution is to set the perpendicular field zero on the 
+boundary such that the field becomes purely 
+toroidal on the boundary. The fieldlines then have a kink on the 
+boundary. On the other hand we can implement boundary conditions consistent with 
+the perpendicular ones since the fieldlines never leave the domain. 
+We simply interpolate the quantity to derive on the inner side of the
+domain boundary (Neumann conditions = "No boundary condition") or 
+set the value to zero (Dirichlet condition).
+
+
+\section{Geometry}
+When we are on a different geometry than $(R,Z)$ the question is how to integrate the field lines. There are two possibilities. 
+First, interpolate $R(\zeta_i, \eta_i), Z(\zeta_i, \eta_i)$ for 
+all $i$, then integrate in $(R,Z)$ space and finally use
+Newton iteration to find $\zeta(R^\pm_i, Z^\pm_i), \eta(R^\pm_i, Z^\pm_i)$. 
+The downside here is that it is difficult to tell when and where the fieldline leaves the simulation domain and even worse in MPI the next points might belong to another process. 
+
+The second possibiliy is to integrate entirely in the 
+transformed coordinate system $\zeta, \eta$. 
+The magnetic field can be easily transformed since we have the
+Jacobian of the coordinate transformation
+\begin{align}
+    B^\zeta(\zeta, \eta) &= \left(\frac{\partial \zeta}{\partial R} B^{R} + \frac{\partial \zeta}{\partial Z}B^Z\right)_{R(\zeta, \eta), Z(\zeta, \eta)} \\
+    B^\eta(\zeta, \eta) &= \left(\frac{\partial \eta}{\partial R} B^{R} + \frac{\partial \eta}{\partial Z}B^Z\right)_{R(\zeta, \eta), Z(\zeta, \eta)} \\
+    B^\varphi(\zeta, \eta) &= B^\varphi({R(\zeta, \eta), Z(\zeta, \eta)})
+    \label{eq:field_trafo}
+\end{align}
+The advantage is that we can do this for any coordinate
+system. Only for cylindrical coordinates, we integrate directly in physical space. The equations to integrate
+are
+\begin{subequations}
+\begin{align}
+\frac{\d \zeta}{\d\varphi} &= \frac{B^\zeta}{B^\varphi}\\
+\frac{\d \eta}{\d\varphi} &= \frac{B^\eta}{B^\varphi}\\
+\frac{\d s}{\d\varphi} &= \frac{|B|}{|B^\varphi|}
+\end{align}
+\label{eq:fieldlines}
+\end{subequations}
+The downside here is that when integrating fieldlines we
+have to interpolate the magnetic field at arbitrary points. 
+However, the error should vanish with $3rd$ order in the 
+perpendicular plane. (check this). In order to mitigate this error
+we could maybe transform the B-Field on a finer grid/higher order polynomials for more accurate
+integration. Also, we need interpolation in the 
+first algorithm as well. 
+
+Note that the matrix-matrix multiplications in Eq.~\eqref{eq:sandwich} can
+be precomputed and stored. The memory requirements 
+in the final computations are 
+therefore the same  as in the old version. (Not entirely, since
+the diagonal $1/\Delta s$ matrix does not commute with $Q$ or $P$).
+
+Finally remember that the adjoint of a matrix in the modified geometry 
+involves the volume element. This means that after you've adjoined the 
+parallel derivative the normal way simply bracket the result 
+by $1/\sqrt{g}$ and $\sqrt{g}$. 
+\section{Algorithm}
+Given are the components $v^i(R,Z)$ for $i\in\{R,Z,\varphi\}$ and a compuational grid (in the following the ``coarse grid``)
+\begin{itemize}
+  \item generate a fine grid by multiplying the cell numbers of the given coarse grid (only topologcially, metric and Jacobian are not needed)
+  \item integrate the fieldlines for the fine grid:
+    \begin{itemize}
+      \item evaluate the starting points on the coarse grid in computational space 
+      \item For a curvilinear grid set up a (higher order) grid for the 
+        interpolation of the vector components $v^i$ and push forward the vector components
+        to the curvilinear coordinate system
+      \item Integrate the fieldline equations 
+\begin{subequations}
+\begin{align}
+\frac{\d \zeta}{\d\varphi} &= \frac{v^\zeta}{v^\varphi}\\
+\frac{\d \eta}{\d\varphi} &= \frac{v^\eta}{v^\varphi}\\
+\frac{\d s}{\d\varphi} &= \frac{1}{|v^\varphi|}
+\end{align}
+\label{eq:fieldlines_converted}
+\end{subequations}
+    with the given starting points and $s(0)=0$ from $\varphi=0$ until $\varphi = \pm\Delta \varphi$.
+      \item create an interpolation matrix that interpolates from the coarse grid 
+        to the fine grid
+      \item use the interpolation matrix to generate the plus/minus points for the fine grid
+    \end{itemize}
+  \item create the interpolation matrices that interpolate from the given coarse grid 
+    to the plus/minus points 
+  \item create a projection matrix that projects from the fine grid to the coarse grid
+  \item compute the matrix-matrix multiplications $P\cdot I^\pm$ as well as the transpose
+  \item project the $s$ vectors to the coarse grid
+\end{itemize}
+\section{MPI implementation}
+Let us also note the mpi-implementation, which is not entirely
+trivial due to the matrix-matrix multiplications involed in Eq.~\eqref{eq:sandwich}.
+\subsection{Row and column distributed sparse matrices}
+In Feltor each mpi process gets an equally sized chunk of a 
+vector.
+Contrary to a vector
+a matrix can be distributed in two ways, row-wise and column wise. 
+In a row-distributed matrix each process gets the complete 
+rows of the matrix that correspond to the indices in the 
+vector it holds. 
+In a column-distributed matrix each process gets the complete 
+columns of the matrix corresponding to the indices in the 
+vector it holds. 
+When we implement a matrix-vector multiplication the order 
+of communication and computation depends on the distribution 
+of the matrix.
+For the row-distributed matrix each process first has to gather all elements of the input vector it needs to be able to compute the elements of the output. This requires MPI communication.
+Formally, the gather operation can be written as a matrix $G$
+of $1'$s and $0'$s where the output vector is of equal or larger size than the input vector.
+After the elements have been gathered the local matrix-vector
+multiplications can be executed.
+\begin{align}
+M = R\cdot G
+\end{align}
+where $R$ is the row-distributed matrix with modified indices 
+and $G$ is the gather matrix, in which the MPI-communication takes place.
+
+In a column distributed matrix the local matrix-vector multiplication can be executed first because each processor already
+has all vector elements it needs. 
+However the resuling elements have to be communicated back to 
+the process they belong to. Furthermore, a process has to sum
+all elements it receives from other processes on the same
+index. This is a scatter and reduce operation and
+it can be written as a scatter matrix $S$. The transpose
+of the scatter matrix is a gather matrix and vice-versa.
+\begin{align}
+M = S\cdot C
+\end{align}
+where $S$ is the scatter matrix and $C$ is the column distributed
+matrix with modified indices. 
+
+It turns out that a row-distributed matrix can be transposed
+by transposition of the local matrices and the gather matrix.
+The result is then a column distributed matrix.
+The transpose of a column distributed matrix is a row-distributed matrix and vice-versa.
+
+\subsection{Matrix-Matrix multiplication}
+We note that we normally construct $\nabla_\parallel^{fc}$ as a column 
+distributed
+matrix. The advantage is then that the gather operation is bijective, i.e. the transpose of the gather matrix is its inverse. 
+This advantage is lost in the present problem. 
+It turns out that it is advantageous to construct $\nabla_\parallel^{fc}$
+as s row-distributed matrix with global indices. 
+This is because a column distributed matrix can be easily (without mpi-communication) multiplied
+with a row distributed matrix especially if the indices are global indices. 
+Each process just multiplies its local matrices.
+\begin{align}
+M = C\cdot R
+\end{align}
+This is not true the other way round. 
+The result is then a row distributed matrix with global indices. 
+From the global indices the gather map/matrix and the local
+indices can be constructed.
+We note here that we even don't need to construct the gather matrix
+for $\nabla_\parallel^{fc}$, only the one for $\nabla_\parallel^c$ is
+needed.
+
+
+\section*{Acknowledgements} 	
+This work was supported by the Austrian Science Fund (FWF) W1227-N16 and Y398, and by 
+the European Commission under the Contract of Association between EURATOM and \"OAW, carried out within the framework of the European Fusion Development Agreement (EFDA).
+
+%..................................................................
+\bibliography{refs}
+%\bibliographystyle{aipnum4-1.bst}
+\bibliographystyle{model1-num-names}
+
+
+%..................................................................
+
+
+\end{document}
+
diff --git a/src/toefl/toefl.tex b/src/toefl/toefl.tex
index 20f2abd43..7b7313996 100644
--- a/src/toefl/toefl.tex
+++ b/src/toefl/toefl.tex
@@ -1,6 +1,6 @@
 %\documentclass[12pt]{article}
 %\documentclass[12pt]{scrartcl}
-\documentclass{hitec}
+\documentclass{hitec} % contained in texlive-latex-extra 
 \settextfraction{0.9} % indent text
 \usepackage{csquotes}
 \usepackage[hidelinks]{hyperref} % doi links are short and usefull?
-- 
GitLab


From 616e5883ed8d5d7ca2df1fad6ff267bb9789d5df Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 9 Oct 2017 17:18:53 +0200
Subject: [PATCH 344/453] work on conversion functions for global to MPI
 matrices

---
 inc/dg/backend/mpi_collective.h           | 33 ++---------
 inc/dg/backend/mpi_matrix.h               | 10 ++++
 inc/dg/backend/mpi_projection.h           | 67 +++++++++++++++++++++++
 inc/dg/backend/transpose.h                | 18 ++----
 inc/geometries/related_pages/parallel.tex |  3 -
 5 files changed, 89 insertions(+), 42 deletions(-)

diff --git a/inc/dg/backend/mpi_collective.h b/inc/dg/backend/mpi_collective.h
index 8873d79d6..7a3d36029 100644
--- a/inc/dg/backend/mpi_collective.h
+++ b/inc/dg/backend/mpi_collective.h
@@ -361,11 +361,11 @@ struct GeneralComm : public aCommunicator<Vector>
     /**
     * @brief Construct from local indices and PIDs gather map
     *
-    * The indices in the gather map is written with respect to the buffer vector (unlike in BijectiveComm, where it is given wrt the source vector) 
-    * @param localGatherMap The gather map containing local vector indices ( local buffer size)
-    * @param pidGatherMap The gather map containing the pids from where to gather the local index.
-    Same size as localGatherMap.
-     *   The rank needs to be element of the given communicator.
+    * The indices in the gather map are written with respect to the buffer vector (unlike in BijectiveComm, where it is given wrt the source vector).
+    * Each location in the source vector is uniquely specified by a local vector index and the process rank. 
+    * @param localGatherMap Each element localGatherMap[i] represents a local vector index from where to gather the value. There are "local buffer size" elements.
+    * @param pidGatherMap Each element pidGatherMap[i] represents the pid/rank from where to gather the corresponding local index localGatherMap[i].  Same size as localGatherMap.
+     *   The pid/rank needs to be element of the given communicator.
     * @param comm The MPI communicator participating in the scatter/gather operations
     */
     GeneralComm( const thrust::host_vector<int>& localGatherMap, const thrust::host_vector<int>& pidGatherMap, MPI_Comm comm) {
@@ -383,7 +383,7 @@ struct GeneralComm : public aCommunicator<Vector>
      *
      * Uses the global2localIdx() member of MPITopology to generate localGatherMap and pidGatherMap 
      * @tparam MPITopology any implementation of an MPI Topology (aMPITopology2d, aMPITopology3d, ...)
-     * @param globalGatherMap The gather map containing global vector indices (local buffer size)
+     * @param globalGatherMap Each element globalGatherMap[i] represents a global vector index from where to take the value. There are "local buffer size" elements.
      * @param g a grid object
      */
     template<class MPITopology>
@@ -439,25 +439,4 @@ struct GeneralComm : public aCommunicator<Vector>
     Index scatterMap_;
 };
 
-
-//given global indices -> make a sorted unique indices vector -> make a gather map into the unique vector
-void global2bufferIdx( const thrust::host_vector<int>& global_idx, thrust::host_vector<int>& idx_in_buffer, thrust::host_vector<int>& unique_global_idx)
-{
-    thrust::host_vector<int> copy(global_idx);
-    thrust::host_vector<int> index(copy);
-    thrust::sequence( index.begin(), index.end());
-    thrust::stable_sort_by_key( copy.begin(), copy.end(), index.begin());//note: this also sorts the pids
-    thrust::host_vector<int> ones( index.size(), 1);
-    thrust::host_vector<int> unique_global( index.size()), howmany( index.size());
-    thrust::pair<int*, int*> new_end;
-    new_end = thrust::reduce_by_key( copy.begin(), copy.end(), ones.begin(), unique_global.begin(), howmany.begin());
-    unique_global_idx.assign( unique_global.begin(), new_end.first);
-    thrust::host_vector<int> gather_map;
-    for( int i=0; i<unique_global_idx.size(); i++)
-        for( int j=0; j<howmany[i]; j++)
-            gather_map.append(i );
-    assert( gather_map.size() == global_idx.size());
-    idx_in_buffer.resize( global_idx.size());
-    thrust::scatter( gather_map.begin(), gather_map.end(), index.begin(), idx_in_buffer.begin());
-}
 }//namespace dg
diff --git a/inc/dg/backend/mpi_matrix.h b/inc/dg/backend/mpi_matrix.h
index bc2f91e9c..54a5aae59 100644
--- a/inc/dg/backend/mpi_matrix.h
+++ b/inc/dg/backend/mpi_matrix.h
@@ -336,6 +336,16 @@ struct MatrixTraits<const MPIDistMat<L, C> >
     typedef typename MatrixTraits<L>::value_type value_type;//!< value type
     typedef MPIMatrixTag matrix_category; //!< 
 };
+namespace detail
+{
+template <class Matrix, class Collective>
+MPIDistMat<Matrix, Collective> doTranspose( const MPIDistMat<Matrix, Collective>& src, MPIMatrixTag)
+{
+    Matrix tr = doTranspose( src.matrix(), typename MatrixTraits<Matrix>::matrix_category());
+    MPIDistMat<Matrix, Collective> out( tr, src.collective());
+    return out;
+}
+} //namespace detail
 ///@endcond
 
 
diff --git a/inc/dg/backend/mpi_projection.h b/inc/dg/backend/mpi_projection.h
index 2a42d8766..52213449e 100644
--- a/inc/dg/backend/mpi_projection.h
+++ b/inc/dg/backend/mpi_projection.h
@@ -1,6 +1,8 @@
 #pragma once
 
 #include "mpi_matrix.h"
+#include "mpi_vector.h"
+#include "mpi_collective.h"
 #include "projection.cuh"
 #include "typedefs.cuh"
 
@@ -15,6 +17,31 @@ namespace dg
 //interpolation matrices
 typedef MPIDistMat< dg::IHMatrix, GeneralComm< dg::iHVec, dg::HVec > > MIHMatrix; //!< MPI distributed CSR host Matrix
 typedef MPIDistMat< dg::IDMatrix, GeneralComm< dg::iDVec, dg::DVec > > MIDMatrix; //!< MPI distributed CSR device Matrix
+
+
+namespace detail{
+//given global indices -> make a sorted unique indices vector -> make a gather map into the unique vector
+//buffer_idx -> (gather map/ new column indices) same size as global_idx ( can alias global_idx
+//unique_global_idx -> (idx to be used in a Collective Communication object)
+void global2bufferIdx( cusp::array1d<int, cusp::host_memory>& global_idx, cusp::array1d<int, cusp::host_memory>& buffer_idx, thrust::host_vector<int>& unique_global_idx)
+{
+    thrust::host_vector<int> index(global_idx_begin, global_idx_end);
+    thrust::sequence( index.begin(), index.end());
+    thrust::stable_sort_by_key( global_idx.begin(), global_idx.end(), index.begin());
+    thrust::host_vector<int> ones( index.size(), 1);
+    thrust::host_vector<int> unique_global( index.size()), howmany( index.size());
+    thrust::pair<int*, int*> new_end;
+    new_end = thrust::reduce_by_key( global_idx.begin(), global_idx.end(), ones.begin(), unique_global.begin(), howmany.begin());
+    unique_global_idx.assign( unique_global.begin(), new_end.first);
+    thrust::host_vector<int> gather_map;
+    for( int i=0; i<unique_global_idx.size(); i++)
+        for( int j=0; j<howmany[i]; j++)
+            gather_map.append(i );
+    assert( gather_map.size() == global_idx.size());
+    buffer_idx.resize( global_idx.size());
+    thrust::scatter( gather_map.begin(), gather_map.end(), index.begin(), buffer_idx.begin());
+}
+}//namespace detail
 ///@}
 
 namespace create
@@ -55,6 +82,46 @@ dg::MIHMatrix projection( const aMPITopology3d& g_new, const aMPITopology3d& g_o
 {
     return MIHMatrix( projection( g_new.local(), g_old.local()), GeneralComm<iHVec, HVec>());
 }
+
+/**
+ * @brief Convert a matrix with global indices to a row distributed MPI matrix
+ *
+ * @param global the column indices need to be global, the row indices local
+ * @param topology the topology defines how the indices are converted from global to local
+ *
+ * @return 
+ */
+dg::MIHMatrix convert( const dg::IHMatrix& global, const aMPITopology2d& topology) 
+{
+    dg::iHVec unique_global_idx;
+    cusp::array1d<int, cusp::host_memory> buffer_idx;
+    dg::detail::global2bufferIdx( global.column_indices, buffer_idx, unique_global_idx);
+
+    dg::IHMatrix local( global.num_rows, topology.size(), global.num_values);
+    local.row_offsets=global.row_offsets;
+    local.column_indices=buffer_idx;
+    local.values=global.values;
+
+    dg::GeneralComm<dg::iHVec, dg::HVec> comm( unique_global_idx, topology);
+    dg::MIHMatrix matrix(   local, comm, dg::row_dist);
+    return matrix;
+}
+
+
+/**
+ * @brief Create an MPI row distributed interpolation matrix 
+ *
+ * @copydetails interpolation(const thrust::host_vector<double>&,const thrust::host_vector<double>&,const aTopology2d&,dg::bc,dg::bc)
+ */
+dg::MIHMatrix interpolation( const MPI_Vector<dg::HVec>& x, const MPI_Vector<dg::HVec>& y, const aMPITopology2d& grid, dg::bc bcx = dg::NEU, dg::bc bcy = dg::NEU)
+{
+    return convert(  
+            dg::create::interpolation( x.data(), y.data(), grid.global(), bcx, bcy), 
+            grid);
+}
+
+
+
 ///@}
 
 }//namespace create
diff --git a/inc/dg/backend/transpose.h b/inc/dg/backend/transpose.h
index 9874a7bf9..947512bc7 100644
--- a/inc/dg/backend/transpose.h
+++ b/inc/dg/backend/transpose.h
@@ -2,10 +2,12 @@
 
 //some thoughts on how to transpose a MPI matrix 
 
-///@cond
 namespace dg
 {
 
+///@cond
+namespace detail
+{
 template <class Matrix>
 Matrix doTranspose( const Matrix& src, CuspMatrixTag)
 {
@@ -13,21 +15,13 @@ Matrix doTranspose( const Matrix& src, CuspMatrixTag)
     cusp::transpose( src, out);
     return out;
 }
-template <class Matrix, class Collective>
-ColDistMat doTranspose( const RowDistMat<Matrix, Collective>& src, MPIMatrixTag)
-{
-    Matrix tr( src.matrix());
-    cusp::transpose( src.matrix(), tr);
-    ColDistMat out( tr, src.collective());
-    return out;
-}
-
+}//namespace detail
+///@endcond
 
 template<class Matrix>
 Matrix transpose( const Matrix& src)
 {
-    out = doTranspose( src, typename MatrixTraits<Matrix>::matrix_category());
+    return detail::doTranspose( src, typename MatrixTraits<Matrix>::matrix_category());
 }
 
 } //namespace dg
-///@endcond
diff --git a/inc/geometries/related_pages/parallel.tex b/inc/geometries/related_pages/parallel.tex
index 91bc94c92..ed7166655 100644
--- a/inc/geometries/related_pages/parallel.tex
+++ b/inc/geometries/related_pages/parallel.tex
@@ -353,9 +353,6 @@ for $\nabla_\parallel^{fc}$, only the one for $\nabla_\parallel^c$ is
 needed.
 
 
-\section*{Acknowledgements} 	
-This work was supported by the Austrian Science Fund (FWF) W1227-N16 and Y398, and by 
-the European Commission under the Contract of Association between EURATOM and \"OAW, carried out within the framework of the European Fusion Development Agreement (EFDA).
 
 %..................................................................
 \bibliography{refs}
-- 
GitLab


From f41bd284ae034c9399a8872cc4125004de532e56 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Tue, 10 Oct 2017 16:12:14 +0200
Subject: [PATCH 345/453] no-communication test passed but communication test
 failed

---
 inc/dg/backend/cusp_matrix_blas.cuh |  4 ++
 inc/dg/backend/grid.h               | 32 +++++------
 inc/dg/backend/mpi_collective.h     | 27 +++++++---
 inc/dg/backend/mpi_communicator.h   | 13 +++++
 inc/dg/backend/mpi_grid.h           | 69 ++++++++++++------------
 inc/dg/backend/mpi_init.h           | 29 +++++-----
 inc/dg/backend/mpi_matrix.h         |  9 ++--
 inc/dg/backend/mpi_projection.h     | 44 +++++++++------
 inc/dg/backend/projection_mpit.cu   | 83 +++++++++++++++++++++++++++++
 inc/dg/backend/typedefs.cuh         |  6 +++
 inc/dg/functors.h                   |  8 +--
 inc/geometries/fieldaligned.h       |  5 +-
 inc/geometries/mpi_fieldaligned.h   |  6 ++-
 13 files changed, 237 insertions(+), 98 deletions(-)
 create mode 100644 inc/dg/backend/projection_mpit.cu

diff --git a/inc/dg/backend/cusp_matrix_blas.cuh b/inc/dg/backend/cusp_matrix_blas.cuh
index 48849e22e..75f7984a5 100644
--- a/inc/dg/backend/cusp_matrix_blas.cuh
+++ b/inc/dg/backend/cusp_matrix_blas.cuh
@@ -84,6 +84,10 @@ inline void doSymv( Matrix& m,
                     ThrustVectorTag  )
 {
 #ifdef DG_DEBUG
+    std::cout << "num rows "<<m.num_rows<<std::endl;
+    std::cout << "num rows "<<y.size()<<std::endl;
+    std::cout << "num cols "<<m.num_cols<<std::endl;
+    std::cout << "num cols "<<x.size()<<std::endl;
     assert( m.num_rows == y.size() );
     assert( m.num_cols == x.size() );
 #endif //DG_DEBUG
diff --git a/inc/dg/backend/grid.h b/inc/dg/backend/grid.h
index 871f5c56f..fc356f300 100644
--- a/inc/dg/backend/grid.h
+++ b/inc/dg/backend/grid.h
@@ -452,22 +452,6 @@ struct aTopology3d
 {
     typedef SharedTag memory_category;
     typedef ThreeDimensionalTag dimensionality;
-    ///@copydoc aTopology2d::multiplyCellNumbers()
-    void multiplyCellNumbers( double fx, double fy){
-        set(n(), round(fx*(double)Nx()), round(fy*(double)Ny()), Nz());
-    }
-    /**
-    * @brief Set the number of polynomials and cells
-    *
-    * @param new_n new number of %Gaussian nodes
-    * @param new_Nx new number of cells in x 
-    * @param new_Ny new number of cells in y
-    * @param new_Nz new number of cells in z
-    */
-    void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz) {
-        if(!( new_n==n() && new_Nx ==Nx() && new_Ny == Ny() && new_Nz==Nz())) 
-            do_set(new_n,new_Nx,new_Ny,new_Nz);
-    }
 
     /**
      * @brief left boundary in x
@@ -666,6 +650,22 @@ struct aTopology3d
             return true; 
         return false;
     }
+    ///@copydoc aTopology2d::multiplyCellNumbers()
+    void multiplyCellNumbers( double fx, double fy){
+        set(n(), round(fx*(double)Nx()), round(fy*(double)Ny()), Nz());
+    }
+    /**
+    * @brief Set the number of polynomials and cells
+    *
+    * @param new_n new number of %Gaussian nodes
+    * @param new_Nx new number of cells in x 
+    * @param new_Ny new number of cells in y
+    * @param new_Nz new number of cells in z
+    */
+    void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz) {
+        if(!( new_n==n() && new_Nx ==Nx() && new_Ny == Ny() && new_Nz==Nz())) 
+            do_set(new_n,new_Nx,new_Ny,new_Nz);
+    }
     protected:
     ///disallow deletion through base class pointer
     ~aTopology3d(){}
diff --git a/inc/dg/backend/mpi_collective.h b/inc/dg/backend/mpi_collective.h
index 7a3d36029..552777e4a 100644
--- a/inc/dg/backend/mpi_collective.h
+++ b/inc/dg/backend/mpi_collective.h
@@ -152,11 +152,11 @@ struct BijectiveComm : public aCommunicator<Vector>
     ///@copydoc GeneralComm::GeneralComm()
     BijectiveComm( ){ }
     /**
-     * @brief Construct from a given map with respect to the source/data vector
+     * @brief Construct from a given scatter map with respect to the source/data vector
      *
-     * @param pids Gives to every point of the values/data vector (not the buffer vector!) 
-     *   the rank to which to send this data element. 
-     *   The rank needs to be element of the given communicator.
+     * @param pids Gives to every index i of the values/data vector (not the buffer vector!) 
+     *   the rank pids[i] to which to send the data element data[i]. 
+     *   The rank pids[i] needs to be element of the given communicator.
      * @param comm An MPI Communicator that contains the participants of the scatter/gather
      * @note The actual scatter/gather map is constructed from the given map so the result behaves as if pids was the actual scatter/gather map on the buffer
      */
@@ -177,6 +177,15 @@ struct BijectiveComm : public aCommunicator<Vector>
     const thrust::host_vector<int>& get_pids()const{return pids_;}
     virtual BijectiveComm* clone() const {return new BijectiveComm(*this);}
     private:
+    bool do_isCommunicating() const{
+        int rank;
+        MPI_Comm_rank( do_communicator(), &rank);
+        bool communicating = false;
+        for( unsigned i=0; i<pids_.size(); i++)
+            if( pids_[i] != rank) 
+                communicating = true;
+        return communicating;
+    }
     MPI_Comm do_communicator() const {return p_.communicator();}
     unsigned do_size() const { return p_.store_size();}
     Vector do_make_buffer()const{ 
@@ -254,7 +263,9 @@ template< class Index, class Vector>
 struct SurjectiveComm : public aCommunicator<Vector>
 {
     ///@copydoc GeneralComm::GeneralComm()
-    SurjectiveComm(){}
+    SurjectiveComm(){
+        buffer_size_ = store_size_ = 0;
+    }
     ///@copydoc GeneralComm::GeneralComm(const thrust::host_vector<int>&,const thrust::host_vector<int>&,MPI_Comm)
     ///@note we assume that the gather map is surjective
     SurjectiveComm( const thrust::host_vector<int>& localGatherMap, const thrust::host_vector<int>& pidGatherMap, MPI_Comm comm)
@@ -270,7 +281,7 @@ struct SurjectiveComm : public aCommunicator<Vector>
         thrust::host_vector<int> local(globalGatherMap.size()), pids(globalGatherMap.size());
         bool success = true;
         for(unsigned i=0; i<local.size(); i++)
-            if( !g.global2localIdx(globalGatherMap[i], pids[i], local[i]) ) success = false;
+            if( !g.global2localIdx(globalGatherMap[i], local[i], pids[i]) ) success = false;
 
         assert( success);
         construct( local, pids, g.communicator());
@@ -290,6 +301,7 @@ struct SurjectiveComm : public aCommunicator<Vector>
     const Index& getSortedGatherMap() const {return sortedGatherMap_;}
     virtual SurjectiveComm* clone() const {return new SurjectiveComm(*this);}
     private:
+    bool do_isCommunicating() const{ return bijectiveComm_.isCommunicating();}
     Vector do_make_buffer()const{
         Vector tmp(do_size());
         return tmp;
@@ -392,7 +404,7 @@ struct GeneralComm : public aCommunicator<Vector>
         thrust::host_vector<int> local(globalGatherMap.size()), pids(globalGatherMap.size());
         bool success = true;
         for(unsigned i=0; i<local.size(); i++)
-            if( !g.global2localIdx(globalGatherMap[i], pids[i], local[i]) ) success = false;
+            if( !g.global2localIdx(globalGatherMap[i], local[i], pids[i]) ) success = false;
         assert( success);
         construct( local, pids, g.communicator());
     }
@@ -403,6 +415,7 @@ struct GeneralComm : public aCommunicator<Vector>
     const thrust::host_vector<int>& getPidGatherMap() const {return surjectiveComm_.getPidGatherMap();}
     virtual GeneralComm* clone() const {return new GeneralComm(*this);}
     private:
+    bool do_isCommunicating() const{ return surjectiveComm_.isCommunicating();}
     Vector do_make_buffer() const{ 
         Vector tmp(do_size());
         return tmp;
diff --git a/inc/dg/backend/mpi_communicator.h b/inc/dg/backend/mpi_communicator.h
index d9ae42abc..939ddac86 100644
--- a/inc/dg/backend/mpi_communicator.h
+++ b/inc/dg/backend/mpi_communicator.h
@@ -135,6 +135,16 @@ struct aCommunicator
     * @note we assume that, contrary to size(), the vector size is always the local size of a dg::MPI_Vector
     */
     unsigned size() const{return do_size();}
+    /**
+     * @brief True if the gather/scatter operation involves actual MPI communication
+     *
+     * This test can be used to avoid the gather operation alltogether in e.g. the construction of a MPI distributed matrix.
+     * @return False, if the global gather can be done without MPI communication (i.e. the indices are all local to each calling process). True else. 
+     */
+    bool isCommunicating() const{ 
+        if( do_size() == 0) return false;
+        return do_isCommunicating();
+    }
     /**
     * @brief The internal MPI communicator used 
     *
@@ -163,6 +173,9 @@ struct aCommunicator
     virtual LocalContainer do_make_buffer( )const=0;
     virtual void do_global_gather( const LocalContainer& values, LocalContainer& gathered)const=0;
     virtual void do_global_scatter_reduce( const LocalContainer& toScatter, LocalContainer& values) const=0;
+    virtual bool do_isCommunicating( ) const {
+        return true;
+    }
 };
 
 
diff --git a/inc/dg/backend/mpi_grid.h b/inc/dg/backend/mpi_grid.h
index 62ee55d84..0346d2f57 100644
--- a/inc/dg/backend/mpi_grid.h
+++ b/inc/dg/backend/mpi_grid.h
@@ -39,26 +39,6 @@ struct aMPITopology2d
     typedef MPITag memory_category;
     typedef TwoDimensionalTag dimensionality;
 
-    /**
-    * @brief Multiply the number of cells with a given factor
-    *
-    * With this function you can resize the grid ignorantly of its current size
-    * @param fx new global number of cells is fx*global().Nx()
-    * @param fy new global number of cells is fy*global().Ny()
-    */
-    void multiplyCellNumbers( double fx, double fy){
-        set(g.n(), floor(fx*(double)g.Nx()+0.5), floor(fy*(double)g.Ny()+0.5));
-    }
-    /**
-    * @copydoc Grid2d::set(unsigned,unsigned,unsigned)
-    * @attention these are global parameters, i.e. set( g.n(), 2*g.Nx(), 2*g.Ny()) is NOT(!) what you want
-    *           use the multiplyCellNumbers function instead, or set( g.n(), 2*g.global().Nx(), 2*g.global().Ny())
-    */
-    void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny) {
-        check_division( new_Nx, new_Ny, g.bcx(), g.bcy());
-        if( new_n == g.n() && new_Nx == g.Nx() && new_Ny == g.Ny()) return;
-        do_set( new_n,new_Nx,new_Ny);
-    }
 
     /**
      * @brief Return local x0
@@ -224,6 +204,27 @@ struct aMPITopology2d
      */
     int pidOf( double x, double y) const;
 
+    /**
+    * @brief Multiply the number of cells with a given factor
+    *
+    * With this function you can resize the grid ignorantly of its current size
+    * @param fx new global number of cells is fx*global().Nx()
+    * @param fy new global number of cells is fy*global().Ny()
+    */
+    void multiplyCellNumbers( double fx, double fy){
+        set(g.n(), floor(fx*(double)g.Nx()+0.5), floor(fy*(double)g.Ny()+0.5));
+    }
+    /**
+    * @copydoc Grid2d::set(unsigned,unsigned,unsigned)
+    * @attention these are global parameters, i.e. set( g.n(), 2*g.Nx(), 2*g.Ny()) is NOT(!) what you want
+    *           use the multiplyCellNumbers function instead, or set( g.n(), 2*g.global().Nx(), 2*g.global().Ny())
+    */
+    void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny) {
+        check_division( new_Nx, new_Ny, g.bcx(), g.bcy());
+        if( new_n == g.n() && new_Nx == g.Nx() && new_Ny == g.Ny()) return;
+        do_set( new_n,new_Nx,new_Ny);
+    }
+
     /**
     * @brief Map a local index plus the PID to a global vector index
     *
@@ -356,20 +357,6 @@ struct aMPITopology3d
 {
     typedef MPITag memory_category;
     typedef ThreeDimensionalTag dimensionality;
-    ///@copydoc aMPITopology2d::multiplyCellNumbers()
-    void multiplyCellNumbers( double fx, double fy){
-        set(g.n(), round(fx*(double)g.Nx()), round(fy*(double)g.Ny()), g.Nz());
-    }
-    /**
-     * @copydoc Grid3d::set(unsigned,unsigned,unsigned,unsigned)
-     * @attention these are global parameters, i.e. set( g.n(), 2*g.Nx(), 2*g.Ny(), 2*g.Nz()) is NOT(!) what you want
-     *           use the multiplyCellNumbers function instead, or set( g.n(), 2*g.global().Nx(), 2*g.global().Ny(), 2*g.global().Nz())
-     */
-    void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz) {
-        check_division( new_Nx,new_Ny,new_Nz,g.bcx(),g.bcy(),g.bcz());
-        if( new_n == g.n() && new_Nx == g.Nx() && new_Ny == g.Ny() && new_Nz == g.Nz()) return;
-        do_set(new_n,new_Nx,new_Ny,new_Nz);
-    }
 
 
     /**
@@ -581,6 +568,20 @@ struct aMPITopology3d
      * @return pid of a process, or -1 if non of the grids matches
      */
     int pidOf( double x, double y, double z) const;
+    ///@copydoc aMPITopology2d::multiplyCellNumbers()
+    void multiplyCellNumbers( double fx, double fy){
+        set(g.n(), round(fx*(double)g.Nx()), round(fy*(double)g.Ny()), g.Nz());
+    }
+    /**
+     * @copydoc Grid3d::set(unsigned,unsigned,unsigned,unsigned)
+     * @attention these are global parameters, i.e. set( g.n(), 2*g.Nx(), 2*g.Ny(), 2*g.Nz()) is NOT(!) what you want
+     *           use the multiplyCellNumbers function instead, or set( g.n(), 2*g.global().Nx(), 2*g.global().Ny(), 2*g.global().Nz())
+     */
+    void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz) {
+        check_division( new_Nx,new_Ny,new_Nz,g.bcx(),g.bcy(),g.bcz());
+        if( new_n == g.n() && new_Nx == g.Nx() && new_Ny == g.Ny() && new_Nz == g.Nz()) return;
+        do_set(new_n,new_Nx,new_Ny,new_Nz);
+    }
     ///@copydoc aMPITopology2d::local2globalIdx(int,int,int&)const
     bool local2globalIdx( int localIdx, int PID, int& globalIdx)const
     {
diff --git a/inc/dg/backend/mpi_init.h b/inc/dg/backend/mpi_init.h
index 29ba2b58b..655669de0 100644
--- a/inc/dg/backend/mpi_init.h
+++ b/inc/dg/backend/mpi_init.h
@@ -1,5 +1,6 @@
 #pragma once
 
+#include <iostream>
 
 /*!@file
 @brief convenience mpi init functions
@@ -16,13 +17,14 @@ namespace dg
 * Also sets the GPU a process should use in case THRUST_DEVICE_SYSTEM_CUDA
 * @param bcx if bcx==dg::PER then the communicator is periodic in x 
 * @param bcy if bcy==dg::PER then the communicator is periodic in y 
-* @param n read in from std::cin and broadcasted to all processes in MPI_COMM_WORLD
-* @param Nx read in from std::cin and broadcasted to all processes in MPI_COMM_WORLD
-* @param Ny read in from std::cin and broadcasted to all processes in MPI_COMM_WORLD
+* @param n read in from is and broadcasted to all processes in MPI_COMM_WORLD
+* @param Nx read in from is and broadcasted to all processes in MPI_COMM_WORLD
+* @param Ny read in from is and broadcasted to all processes in MPI_COMM_WORLD
 * @param comm (write only) a 2d Cartesian MPI communicator
+* @param is Input stream to read parameters from (npx, npy, n, Nx, Ny)
 * @ingroup misc
 */
-void mpi_init2d( dg::bc bcx, dg::bc bcy, unsigned& n, unsigned& Nx, unsigned& Ny, MPI_Comm& comm  )
+void mpi_init2d( dg::bc bcx, dg::bc bcy, unsigned& n, unsigned& Nx, unsigned& Ny, MPI_Comm& comm, std::istream& is = std::cin  )
 {
     int periods[2] = {false,false};
     if( bcx == dg::PER) periods[0] = true;
@@ -35,7 +37,7 @@ void mpi_init2d( dg::bc bcx, dg::bc bcy, unsigned& n, unsigned& Nx, unsigned& Ny
     if( rank == 0)
     {
         std::cout << "Type npx and npy\n";
-        std::cin >> np[0] >> np[1];
+        is >> np[0] >> np[1];
         std::cout<< "Computing with "<<np[0] <<" x "<<np[1]<<" = "<<size<<" processes! "<<std::endl;
         assert( size == np[0]*np[1]);
     }
@@ -44,7 +46,7 @@ void mpi_init2d( dg::bc bcx, dg::bc bcy, unsigned& n, unsigned& Nx, unsigned& Ny
     if( rank == 0)
     {
         std::cout << "Type n, Nx and Ny\n";
-        std::cin >> n >> Nx >> Ny;
+        is >> n >> Nx >> Ny;
         std::cout<< "On the grid "<<n <<" x "<<Nx<<" x "<<Ny<<std::endl;
     }
     MPI_Bcast(  &n,1 , MPI_UNSIGNED, 0, comm);
@@ -72,14 +74,15 @@ void mpi_init2d( dg::bc bcx, dg::bc bcy, unsigned& n, unsigned& Nx, unsigned& Ny
 * @param bcx if bcx==dg::PER then the communicator is periodic in x 
 * @param bcy if bcy==dg::PER then the communicator is periodic in y 
 * @param bcz if bcz==dg::PER then the communicator is periodic in z 
-* @param n read in from std::cin and broadcasted to all processes in MPI_COMM_WORLD
-* @param Nx read in from std::cin and broadcasted to all processes in MPI_COMM_WORLD
-* @param Ny read in from std::cin and broadcasted to all processes in MPI_COMM_WORLD
-* @param Nz read in from std::cin and broadcasted to all processes in MPI_COMM_WORLD
+* @param n read in from is and broadcasted to all processes in MPI_COMM_WORLD
+* @param Nx read in from is and broadcasted to all processes in MPI_COMM_WORLD
+* @param Ny read in from is and broadcasted to all processes in MPI_COMM_WORLD
+* @param Nz read in from is and broadcasted to all processes in MPI_COMM_WORLD
 * @param comm (write only) a 3d Cartesian MPI communicator
+* @param is Input stream to read parameters from (npx, npy, npz, n, Nx, Ny, Nz)
 * @ingroup misc
 */
-void mpi_init3d( dg::bc bcx, dg::bc bcy, dg::bc bcz, unsigned& n, unsigned& Nx, unsigned& Ny, unsigned& Nz, MPI_Comm& comm  )
+void mpi_init3d( dg::bc bcx, dg::bc bcy, dg::bc bcz, unsigned& n, unsigned& Nx, unsigned& Ny, unsigned& Nz, MPI_Comm& comm, std::istream& is = std::cin  )
 {
     int periods[3] = {false,false, false};
     if( bcx == dg::PER) periods[0] = true;
@@ -92,7 +95,7 @@ void mpi_init3d( dg::bc bcx, dg::bc bcy, dg::bc bcz, unsigned& n, unsigned& Nx,
     if( rank == 0)
     {
         std::cout << "Type npx and npy and npz\n";
-        std::cin >> np[0] >> np[1]>>np[2];
+        is >> np[0] >> np[1]>>np[2];
         std::cout<< "Computing with "<<np[0] <<" x "<<np[1]<<" x "<<np[2]<<" = "<<size<<" processses! "<<std::endl;
         assert( size == np[0]*np[1]*np[2]);
     }
@@ -101,7 +104,7 @@ void mpi_init3d( dg::bc bcx, dg::bc bcy, dg::bc bcz, unsigned& n, unsigned& Nx,
     if( rank == 0)
     {
         std::cout << "Type n, Nx and Ny and Nz\n";
-        std::cin >> n >> Nx >> Ny >> Nz;
+        is >> n >> Nx >> Ny >> Nz;
         std::cout<< "On the grid "<<n <<" x "<<Nx<<" x "<<Ny<<" x "<<Nz<<std::endl;
     }
     MPI_Bcast(  &n,1 , MPI_UNSIGNED, 0, MPI_COMM_WORLD);
diff --git a/inc/dg/backend/mpi_matrix.h b/inc/dg/backend/mpi_matrix.h
index 54a5aae59..d2f0f2558 100644
--- a/inc/dg/backend/mpi_matrix.h
+++ b/inc/dg/backend/mpi_matrix.h
@@ -2,6 +2,7 @@
 
 #include "mpi_vector.h"
 #include "memory.h"
+#include "matrix_traits.h"
 
 /*!@file
 
@@ -253,9 +254,9 @@ struct MPIDistMat
         }
         int result;
         MPI_Comm_compare( x.communicator(), y.communicator(), &result);
-        assert( result == MPI_CONGRUENT);
+        assert( result == MPI_CONGRUENT || result == MPI_IDENT);
         MPI_Comm_compare( x.communicator(), m_c.get().communicator(), &result);
-        assert( result == MPI_CONGRUENT);
+        assert( result == MPI_CONGRUENT || result == MPI_IDENT);
         if( m_dist == row_dist){
             m_c.global_gather( x.data(), m_buffer.data());
             dg::blas2::detail::doSymv( alpha, m_m, m_buffer.data(), beta, y.data(), 
@@ -283,9 +284,9 @@ struct MPIDistMat
         }
         int result;
         MPI_Comm_compare( x.communicator(), y.communicator(), &result);
-        assert( result == MPI_CONGRUENT);
+        assert( result == MPI_CONGRUENT || result == MPI_IDENT);
         MPI_Comm_compare( x.communicator(), m_c.get().communicator(), &result);
-        assert( result == MPI_CONGRUENT);
+        assert( result == MPI_CONGRUENT || result == MPI_IDENT);
         if( m_dist == row_dist){
             m_c.get().global_gather( x.data(), m_buffer.data());
             dg::blas2::detail::doSymv( m_m, m_buffer.data(), y.data(), 
diff --git a/inc/dg/backend/mpi_projection.h b/inc/dg/backend/mpi_projection.h
index 52213449e..539b2728d 100644
--- a/inc/dg/backend/mpi_projection.h
+++ b/inc/dg/backend/mpi_projection.h
@@ -1,5 +1,7 @@
 #pragma once
 
+#include "cusp_matrix_blas.cuh"
+#include "mpi_grid.h"
 #include "mpi_matrix.h"
 #include "mpi_vector.h"
 #include "mpi_collective.h"
@@ -23,20 +25,21 @@ namespace detail{
 //given global indices -> make a sorted unique indices vector -> make a gather map into the unique vector
 //buffer_idx -> (gather map/ new column indices) same size as global_idx ( can alias global_idx
 //unique_global_idx -> (idx to be used in a Collective Communication object)
-void global2bufferIdx( cusp::array1d<int, cusp::host_memory>& global_idx, cusp::array1d<int, cusp::host_memory>& buffer_idx, thrust::host_vector<int>& unique_global_idx)
+void global2bufferIdx( const cusp::array1d<int, cusp::host_memory>& locally_global_idx, cusp::array1d<int, cusp::host_memory>& buffer_idx, thrust::host_vector<int>& unique_global_idx)
 {
-    thrust::host_vector<int> index(global_idx_begin, global_idx_end);
+    thrust::host_vector<int> index(locally_global_idx.begin(), locally_global_idx.end()), global_idx(index);
     thrust::sequence( index.begin(), index.end());
-    thrust::stable_sort_by_key( global_idx.begin(), global_idx.end(), index.begin());
+    thrust::stable_sort_by_key( global_idx.begin(), global_idx.end(), index.begin());//this changes both global_idx and index
     thrust::host_vector<int> ones( index.size(), 1);
     thrust::host_vector<int> unique_global( index.size()), howmany( index.size());
-    thrust::pair<int*, int*> new_end;
+    typedef typename thrust::host_vector<int>::iterator iterator;
+    thrust::pair<iterator, iterator> new_end;
     new_end = thrust::reduce_by_key( global_idx.begin(), global_idx.end(), ones.begin(), unique_global.begin(), howmany.begin());
     unique_global_idx.assign( unique_global.begin(), new_end.first);
     thrust::host_vector<int> gather_map;
-    for( int i=0; i<unique_global_idx.size(); i++)
+    for( int i=0; i<(int)unique_global_idx.size(); i++)
         for( int j=0; j<howmany[i]; j++)
-            gather_map.append(i );
+            gather_map.push_back(i);
     assert( gather_map.size() == global_idx.size());
     buffer_idx.resize( global_idx.size());
     thrust::scatter( gather_map.begin(), gather_map.end(), index.begin(), buffer_idx.begin());
@@ -84,25 +87,34 @@ dg::MIHMatrix projection( const aMPITopology3d& g_new, const aMPITopology3d& g_o
 }
 
 /**
- * @brief Convert a matrix with global indices to a row distributed MPI matrix
+ * @brief Convert a matrix with local row and global column indices to a row distributed MPI matrix
  *
+ * Checks if communication is actually needed
  * @param global the column indices need to be global, the row indices local
  * @param topology the topology defines how the indices are converted from global to local
  *
- * @return 
+ * @return a row distributed MPI matrix
  */
-dg::MIHMatrix convert( const dg::IHMatrix& global, const aMPITopology2d& topology) 
+dg::MIHMatrix convert_row_dist( const dg::IHMatrix& global, const aMPITopology2d& topology) 
 {
     dg::iHVec unique_global_idx;
     cusp::array1d<int, cusp::host_memory> buffer_idx;
     dg::detail::global2bufferIdx( global.column_indices, buffer_idx, unique_global_idx);
-
-    dg::IHMatrix local( global.num_rows, topology.size(), global.num_values);
+    dg::GeneralComm<dg::iHVec, dg::HVec> comm( unique_global_idx, topology);
+    dg::IHMatrix local( global.num_rows, topology.size(), global.values.size());
     local.row_offsets=global.row_offsets;
     local.column_indices=buffer_idx;
     local.values=global.values;
-
-    dg::GeneralComm<dg::iHVec, dg::HVec> comm( unique_global_idx, topology);
+    if( !comm.isCommunicating() ) 
+    {
+        cusp::array1d<int, cusp::host_memory> local_idx(global.column_indices), pids(local_idx);
+        bool success = true;
+        for(unsigned i=0; i<local_idx.size(); i++)
+            if( !topology.global2localIdx(global.column_indices[i], local_idx[i], pids[i]) ) success = false;
+        assert( success);
+        local.column_indices=local_idx;
+        comm = dg::GeneralComm< dg::iHVec, dg::HVec>();
+    }
     dg::MIHMatrix matrix(   local, comm, dg::row_dist);
     return matrix;
 }
@@ -113,10 +125,10 @@ dg::MIHMatrix convert( const dg::IHMatrix& global, const aMPITopology2d& topolog
  *
  * @copydetails interpolation(const thrust::host_vector<double>&,const thrust::host_vector<double>&,const aTopology2d&,dg::bc,dg::bc)
  */
-dg::MIHMatrix interpolation( const MPI_Vector<dg::HVec>& x, const MPI_Vector<dg::HVec>& y, const aMPITopology2d& grid, dg::bc bcx = dg::NEU, dg::bc bcy = dg::NEU)
+dg::MIHMatrix interpolation( const dg::HVec& x, const dg::HVec& y, const aMPITopology2d& grid, dg::bc bcx = dg::NEU, dg::bc bcy = dg::NEU)
 {
-    return convert(  
-            dg::create::interpolation( x.data(), y.data(), grid.global(), bcx, bcy), 
+    return convert_row_dist(  
+            dg::create::interpolation( x, y, grid.global(), bcx, bcy), 
             grid);
 }
 
diff --git a/inc/dg/backend/projection_mpit.cu b/inc/dg/backend/projection_mpit.cu
new file mode 100644
index 000000000..80bec4299
--- /dev/null
+++ b/inc/dg/backend/projection_mpit.cu
@@ -0,0 +1,83 @@
+#include <iostream>
+#include <sstream>
+#include <mpi.h>
+#include "mpi_projection.h"
+#include "mpi_init.h"
+#include "mpi_evaluation.h"
+
+
+double shift = 0.2;
+double function( double x, double y){ return sin(2*M_PI*x)*sin(2*M_PI*y);}
+double shifted_function(double x, double y){return function( x-shift, y-shift);}
+
+int main(int argc, char* argv[])
+{
+
+    MPI_Init( &argc, &argv);
+    int rank, size;
+    MPI_Comm_size( MPI_COMM_WORLD, &size);
+    if(size!=4) 
+    {
+        std::cerr << "Please run with 4 processes!\n";
+        MPI_Finalize();
+        return 0;
+    }
+    unsigned n, Nx, Ny;
+    MPI_Comm comm;
+    std::stringstream ss;
+    ss<< "2 2 3 8 8";
+    mpi_init2d( dg::PER, dg::PER, n, Nx, Ny, comm, ss);
+    MPI_Comm_rank( comm, &rank);
+    if(rank==0) std::cout << "Test non-communicating MPI matrix-creation!\n";
+    dg::MPIGrid2d g2d( 0,1,0,1, n,Nx,Ny, comm);
+    dg::MPIGrid2d g2d_half = g2d;
+    g2d_half.multiplyCellNumbers(0.5, 0.5);
+    dg::MIHMatrix direct_p= dg::create::interpolation( g2d, g2d_half);
+    dg::HVec x = dg::evaluate( dg::cooX2d, g2d.local());
+    dg::HVec y = dg::evaluate( dg::cooY2d, g2d.local());
+    dg::IHMatrix global_projection = dg::create::interpolation( x,y, g2d_half.global());
+    dg::MIHMatrix converted_p = dg::create::convert_row_dist(global_projection, g2d_half);
+
+    //now compare
+    bool equal_cols=true, equal_rows=true, equal_values=true;
+    for( unsigned i=0; i<direct_p.matrix().values.size(); i++)
+    {
+        if( direct_p.matrix().column_indices[i] - converted_p.matrix().column_indices[i] > 1e-15) equal_cols = false;
+        if( direct_p.matrix().values[i] - converted_p.matrix().values[i] > 1e-15) equal_values = false;
+    }
+    for( unsigned i=0; i<direct_p.matrix().num_rows+1; i++)
+        if( direct_p.matrix().row_offsets[i] - converted_p.matrix().row_offsets[i] > 1e-15) equal_rows = false;
+
+    if( !equal_cols || !equal_rows || !equal_values || direct_p.collective().size() != 0 || converted_p.collective().size() != 0 )
+        std::cout << "FAILED from rank "<<rank<<"!\n";
+    else
+        std::cout << "SUCCESS from rank "<<rank<<"!\n";
+
+    MPI_Barrier(comm);
+    if(rank==0) std::cout << "Now test communicating MPI matrix-creation!\n";
+    x = dg::evaluate( dg::cooX2d, g2d.local());
+    y = dg::evaluate( dg::cooY2d, g2d.local());
+    for( unsigned i=0; i<x.size(); i++)
+    {
+        x[i] -=shift;
+        y[i] -=shift;
+        g2d.global().shift_topologic( x[i], y[i], x[i], y[i]);
+    }
+    dg::MIHMatrix converted_i = dg::create::interpolation( x,y,g2d);
+    dg::MHVec sine = dg::evaluate( function, g2d); 
+    dg::MHVec temp(sine);
+    dg::MHVec shifted_sine = dg::evaluate( shifted_function, g2d);
+    converted_i.symv( sine, temp);
+    //now compare
+    bool success = true;
+    for( unsigned i=0; i<temp.size(); i++)
+        if( temp.data()[i] - shifted_sine.data()[i] > 1e-14) 
+            success = false; 
+    if( !success) 
+        std::cout << "FAILED from rank "<<rank<<"!\n";
+    else
+        std::cout << "SUCCESS from rank "<<rank<<"!\n";
+
+    MPI_Finalize();
+    return 0;
+}
diff --git a/inc/dg/backend/typedefs.cuh b/inc/dg/backend/typedefs.cuh
index 37f01a1df..810fcd970 100644
--- a/inc/dg/backend/typedefs.cuh
+++ b/inc/dg/backend/typedefs.cuh
@@ -1,5 +1,9 @@
 #ifndef _DG_TYPEDEFS_CUH_
 #define _DG_TYPEDEFS_CUH_
+#include <thrust/host_vector.h>
+#include <thrust/device_vector.h>
+#include "sparseblockmat.h"
+#include "sparseblockmat.cuh"
 
 /*! @file
   @brief Useful typedefs of commonly used types.
@@ -21,6 +25,8 @@ typedef EllSparseBlockMatDevice<double> DMatrix; //!< Device Matrix for derivati
 typedef EllSparseBlockMat<double> HMatrix; //!< Host Matrix for derivatives
 
 #ifdef MPI_VERSION
+#include "mpi_vector.h"
+#include "mpi_matrix.h"
 //typedef MPI_Vector<thrust::device_vector<double> >  MDVec; //!< MPI Device Vector s.a. dg::DVec
 typedef MPI_Vector<dg::DVec >  MDVec; //!< MPI Device Vector s.a. dg::DVec
 typedef MPI_Vector<dg::HVec >  MHVec; //!< MPI Host Vector s.a. dg::HVec
diff --git a/inc/dg/functors.h b/inc/dg/functors.h
index 992359078..ae4942269 100644
--- a/inc/dg/functors.h
+++ b/inc/dg/functors.h
@@ -396,7 +396,7 @@ struct GaussianZ
 };
 /**
  * @brief Functor for a sin prof in x and y-direction
- * \f[ f(x,y) =B+ A sin(k_x x) sin(k_y y) \f]
+ * \f[ f(x,y) =B+ A \sin(k_x x) \sin(k_y y) \f]
  */
 struct SinXSinY
 {
@@ -423,7 +423,7 @@ struct SinXSinY
 };
 /**
  * @brief Functor for a cos prof in x and y-direction
- * \f[ f(x,y) =B+ A cos(k_x x) cos(k_y y) \f]
+ * \f[ f(x,y) =B+ A \cos(k_x x) \cos(k_y y) \f]
  */
 struct CosXCosY
 {
@@ -450,7 +450,7 @@ struct CosXCosY
 };
 /**
  * @brief Functor for a sin prof in x- and and cos prof in  y-direction
- * \f[ f(x,y) =B+ A sin(k_x x) cos(k_y y) \f]
+ * \f[ f(x,y) =B+ A \sin(k_x x) \cos(k_y y) \f]
  */
 struct SinXCosY
 {
@@ -477,7 +477,7 @@ struct SinXCosY
 };
 /**
  * @brief Functor for a sin prof in x-direction
- * \f[ f(x,y) =B+ A sin(k_x x) \f]
+ * \f[ f(x,y) =B+ A \sin(k_x x) \f]
  */
 struct SinX
 {
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index 298b0763c..9e2d0c9b5 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -10,6 +10,7 @@
 #include "dg/backend/functions.h"
 #include "dg/backend/typedefs.cuh"
 #include "dg/backend/split_and_join.h"
+#include "dg/backend/transpose.h"
 
 #include "dg/geometry/geometry.h"
 #include "dg/functors.h"
@@ -550,8 +551,8 @@ FieldAligned<Geometry, IMatrix, container>::FieldAligned(const dg::geo::BinaryVe
     t.toc();
     std::cout<< "Multiplication of P*I took: "<<t.diff()<<"s\n";
     //%Transposed matrices work only for csr_matrix due to bad matrix form for ell_matrix!!!
-    cusp::transpose( plus, plusT);
-    cusp::transpose( minus, minusT);     
+    dg::transpose( plus, plusT);
+    dg::transpose( minus, minusT);     
     dg::blas2::transfer( plus, m_plus);
     dg::blas2::transfer( plusT, m_plusT);
     dg::blas2::transfer( minus, m_minus);
diff --git a/inc/geometries/mpi_fieldaligned.h b/inc/geometries/mpi_fieldaligned.h
index dbb4bebbb..0d8a64388 100644
--- a/inc/geometries/mpi_fieldaligned.h
+++ b/inc/geometries/mpi_fieldaligned.h
@@ -178,8 +178,10 @@ FieldAligned<MPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vector<L
     t.toc();
     std::cout<< "Multiplication of P*I took: "<<t.diff()<<"s\n";
     //%Transposed matrices work only for csr_matrix due to bad matrix form for ell_matrix!!!
-    cusp::transpose( plus, plusT);
-    cusp::transpose( minus, minusT);     
+    dg::convert( plus, m_plus, *grid2d_ptr);
+    dg::convert( minus, m_minus, *grid2d_ptr);
+    dg::transpose( m_plus, m_plusT);
+    dg::transpose( m_minus, m_minusT);     
     dg::blas2::transfer( plus, m_plus);
     dg::blas2::transfer( plusT, m_plusT);
     dg::blas2::transfer( minus, m_minus);
-- 
GitLab


From 8ff4e735b32f17d435d76b7e44ce66e6cdca1c04 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Tue, 10 Oct 2017 23:15:13 +0200
Subject: [PATCH 346/453] success for convert function

---
 inc/dg/backend/cusp_matrix_blas.cuh |  4 --
 inc/dg/backend/mpi_projection.h     | 92 +++++++++++++++--------------
 inc/dg/backend/projection_mpit.cu   | 19 +++---
 inc/dg/dg_doc.h                     |  4 +-
 4 files changed, 62 insertions(+), 57 deletions(-)

diff --git a/inc/dg/backend/cusp_matrix_blas.cuh b/inc/dg/backend/cusp_matrix_blas.cuh
index 75f7984a5..48849e22e 100644
--- a/inc/dg/backend/cusp_matrix_blas.cuh
+++ b/inc/dg/backend/cusp_matrix_blas.cuh
@@ -84,10 +84,6 @@ inline void doSymv( Matrix& m,
                     ThrustVectorTag  )
 {
 #ifdef DG_DEBUG
-    std::cout << "num rows "<<m.num_rows<<std::endl;
-    std::cout << "num rows "<<y.size()<<std::endl;
-    std::cout << "num cols "<<m.num_cols<<std::endl;
-    std::cout << "num cols "<<x.size()<<std::endl;
     assert( m.num_rows == y.size() );
     assert( m.num_cols == x.size() );
 #endif //DG_DEBUG
diff --git a/inc/dg/backend/mpi_projection.h b/inc/dg/backend/mpi_projection.h
index 539b2728d..3ccb075b2 100644
--- a/inc/dg/backend/mpi_projection.h
+++ b/inc/dg/backend/mpi_projection.h
@@ -20,33 +20,73 @@ namespace dg
 typedef MPIDistMat< dg::IHMatrix, GeneralComm< dg::iHVec, dg::HVec > > MIHMatrix; //!< MPI distributed CSR host Matrix
 typedef MPIDistMat< dg::IDMatrix, GeneralComm< dg::iDVec, dg::DVec > > MIDMatrix; //!< MPI distributed CSR device Matrix
 
-
 namespace detail{
 //given global indices -> make a sorted unique indices vector -> make a gather map into the unique vector
-//buffer_idx -> (gather map/ new column indices) same size as global_idx ( can alias global_idx
-//unique_global_idx -> (idx to be used in a Collective Communication object)
-void global2bufferIdx( const cusp::array1d<int, cusp::host_memory>& locally_global_idx, cusp::array1d<int, cusp::host_memory>& buffer_idx, thrust::host_vector<int>& unique_global_idx)
+//buffer_idx -> (gather map/ new column indices) same size as global_idx ( can alias global_idx, index into unique_global_idx
+//unique_global_idx -> (list of unique global indices to be used in a Collective Communication object)
+void global2bufferIdx( const cusp::array1d<int, cusp::host_memory>& global_idx, cusp::array1d<int, cusp::host_memory>& buffer_idx, thrust::host_vector<int>& locally_unique_global_idx)
 {
-    thrust::host_vector<int> index(locally_global_idx.begin(), locally_global_idx.end()), global_idx(index);
+    thrust::host_vector<int> index(global_idx.begin(), global_idx.end()), m_global_idx(index);
     thrust::sequence( index.begin(), index.end());
-    thrust::stable_sort_by_key( global_idx.begin(), global_idx.end(), index.begin());//this changes both global_idx and index
+    //1. sort input global indices
+    thrust::stable_sort_by_key( m_global_idx.begin(), m_global_idx.end(), index.begin());//this changes both global_idx and index
+    //2. now reduce on multiple indices
     thrust::host_vector<int> ones( index.size(), 1);
     thrust::host_vector<int> unique_global( index.size()), howmany( index.size());
     typedef typename thrust::host_vector<int>::iterator iterator;
     thrust::pair<iterator, iterator> new_end;
-    new_end = thrust::reduce_by_key( global_idx.begin(), global_idx.end(), ones.begin(), unique_global.begin(), howmany.begin());
-    unique_global_idx.assign( unique_global.begin(), new_end.first);
+    new_end = thrust::reduce_by_key( m_global_idx.begin(), m_global_idx.end(), ones.begin(), unique_global.begin(), howmany.begin());
+    //3. copy unique indices
+    locally_unique_global_idx.assign( unique_global.begin(), new_end.first);
+    //4. manually make gather map into locally_unique_global_idx
     thrust::host_vector<int> gather_map;
-    for( int i=0; i<(int)unique_global_idx.size(); i++)
+    for( int i=0; i<(int)locally_unique_global_idx.size(); i++)
         for( int j=0; j<howmany[i]; j++)
             gather_map.push_back(i);
     assert( gather_map.size() == global_idx.size());
+    //5. buffer idx is the new index 
     buffer_idx.resize( global_idx.size());
     thrust::scatter( gather_map.begin(), gather_map.end(), index.begin(), buffer_idx.begin());
 }
 }//namespace detail
 ///@}
 
+/**
+ * @brief Convert a matrix with local row and global column indices to a row distributed MPI matrix
+ *
+ * Checks if communication is actually needed
+ * @param global the column indices need to be global, the row indices local
+ * @param topology the topology defines how the indices are converted from global to local
+ *
+ * @return a row distributed MPI matrix
+ * @ingroup misc
+ */
+dg::MIHMatrix convert_row_dist( const dg::IHMatrix& global, const aMPITopology2d& topology) 
+{
+    int rank;
+    MPI_Comm_rank( MPI_COMM_WORLD, &rank);
+    dg::iHVec unique_global_idx;
+    cusp::array1d<int, cusp::host_memory> buffer_idx;
+    dg::detail::global2bufferIdx( global.column_indices, buffer_idx, unique_global_idx);
+    dg::GeneralComm<dg::iHVec, dg::HVec> comm( unique_global_idx, topology);
+    dg::IHMatrix local( global.num_rows, comm.size(), global.values.size());
+    local.row_offsets=global.row_offsets;
+    local.column_indices=buffer_idx;
+    local.values=global.values;
+    if( !comm.isCommunicating() ) 
+    {
+        cusp::array1d<int, cusp::host_memory> local_idx(global.column_indices), pids(local_idx);
+        bool success = true;
+        for(unsigned i=0; i<local_idx.size(); i++)
+            if( !topology.global2localIdx(global.column_indices[i], local_idx[i], pids[i]) ) success = false;
+        assert( success);
+        local.column_indices=local_idx;
+        comm = dg::GeneralComm< dg::iHVec, dg::HVec>();
+    }
+    dg::MIHMatrix matrix(   local, comm, dg::row_dist);
+    return matrix;
+}
+
 namespace create
 {
 
@@ -86,40 +126,6 @@ dg::MIHMatrix projection( const aMPITopology3d& g_new, const aMPITopology3d& g_o
     return MIHMatrix( projection( g_new.local(), g_old.local()), GeneralComm<iHVec, HVec>());
 }
 
-/**
- * @brief Convert a matrix with local row and global column indices to a row distributed MPI matrix
- *
- * Checks if communication is actually needed
- * @param global the column indices need to be global, the row indices local
- * @param topology the topology defines how the indices are converted from global to local
- *
- * @return a row distributed MPI matrix
- */
-dg::MIHMatrix convert_row_dist( const dg::IHMatrix& global, const aMPITopology2d& topology) 
-{
-    dg::iHVec unique_global_idx;
-    cusp::array1d<int, cusp::host_memory> buffer_idx;
-    dg::detail::global2bufferIdx( global.column_indices, buffer_idx, unique_global_idx);
-    dg::GeneralComm<dg::iHVec, dg::HVec> comm( unique_global_idx, topology);
-    dg::IHMatrix local( global.num_rows, topology.size(), global.values.size());
-    local.row_offsets=global.row_offsets;
-    local.column_indices=buffer_idx;
-    local.values=global.values;
-    if( !comm.isCommunicating() ) 
-    {
-        cusp::array1d<int, cusp::host_memory> local_idx(global.column_indices), pids(local_idx);
-        bool success = true;
-        for(unsigned i=0; i<local_idx.size(); i++)
-            if( !topology.global2localIdx(global.column_indices[i], local_idx[i], pids[i]) ) success = false;
-        assert( success);
-        local.column_indices=local_idx;
-        comm = dg::GeneralComm< dg::iHVec, dg::HVec>();
-    }
-    dg::MIHMatrix matrix(   local, comm, dg::row_dist);
-    return matrix;
-}
-
-
 /**
  * @brief Create an MPI row distributed interpolation matrix 
  *
diff --git a/inc/dg/backend/projection_mpit.cu b/inc/dg/backend/projection_mpit.cu
index 80bec4299..334967267 100644
--- a/inc/dg/backend/projection_mpit.cu
+++ b/inc/dg/backend/projection_mpit.cu
@@ -8,7 +8,6 @@
 
 double shift = 0.2;
 double function( double x, double y){ return sin(2*M_PI*x)*sin(2*M_PI*y);}
-double shifted_function(double x, double y){return function( x-shift, y-shift);}
 
 int main(int argc, char* argv[])
 {
@@ -28,7 +27,7 @@ int main(int argc, char* argv[])
     ss<< "2 2 3 8 8";
     mpi_init2d( dg::PER, dg::PER, n, Nx, Ny, comm, ss);
     MPI_Comm_rank( comm, &rank);
-    if(rank==0) std::cout << "Test non-communicating MPI matrix-creation!\n";
+    if(rank==0) std::cout << "Test NON-COMMUNICATING MPI matrix-creation!\n";
     dg::MPIGrid2d g2d( 0,1,0,1, n,Nx,Ny, comm);
     dg::MPIGrid2d g2d_half = g2d;
     g2d_half.multiplyCellNumbers(0.5, 0.5);
@@ -36,7 +35,7 @@ int main(int argc, char* argv[])
     dg::HVec x = dg::evaluate( dg::cooX2d, g2d.local());
     dg::HVec y = dg::evaluate( dg::cooY2d, g2d.local());
     dg::IHMatrix global_projection = dg::create::interpolation( x,y, g2d_half.global());
-    dg::MIHMatrix converted_p = dg::create::convert_row_dist(global_projection, g2d_half);
+    dg::MIHMatrix converted_p = dg::convert_row_dist(global_projection, g2d_half);
 
     //now compare
     bool equal_cols=true, equal_rows=true, equal_values=true;
@@ -54,29 +53,33 @@ int main(int argc, char* argv[])
         std::cout << "SUCCESS from rank "<<rank<<"!\n";
 
     MPI_Barrier(comm);
-    if(rank==0) std::cout << "Now test communicating MPI matrix-creation!\n";
+    if(rank==0) std::cout << "Now test COMMUNICATING MPI matrix-creation!\n";
     x = dg::evaluate( dg::cooX2d, g2d.local());
     y = dg::evaluate( dg::cooY2d, g2d.local());
     for( unsigned i=0; i<x.size(); i++)
     {
-        x[i] -=shift;
-        y[i] -=shift;
+        x[i] +=shift;
+        y[i] +=shift;
         g2d.global().shift_topologic( x[i], y[i], x[i], y[i]);
     }
+    dg::IHMatrix  direct_i = dg::create::interpolation( x,y,g2d.global());
     dg::MIHMatrix converted_i = dg::create::interpolation( x,y,g2d);
     dg::MHVec sine = dg::evaluate( function, g2d); 
     dg::MHVec temp(sine);
-    dg::MHVec shifted_sine = dg::evaluate( shifted_function, g2d);
+    dg::HVec global_sine = dg::evaluate( function, g2d.global()); 
+    dg::HVec g_temp( x.size());
     converted_i.symv( sine, temp);
+    dg::blas2::detail::doSymv( direct_i, global_sine, g_temp, dg::CuspMatrixTag(), dg::ThrustVectorTag(), dg::ThrustVectorTag());
     //now compare
     bool success = true;
     for( unsigned i=0; i<temp.size(); i++)
-        if( temp.data()[i] - shifted_sine.data()[i] > 1e-14) 
+        if( temp.data()[i] - g_temp[i] > 1e-14) 
             success = false; 
     if( !success) 
         std::cout << "FAILED from rank "<<rank<<"!\n";
     else
         std::cout << "SUCCESS from rank "<<rank<<"!\n";
+    //Finally test transpose
 
     MPI_Finalize();
     return 0;
diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index 0d83a433c..0d6c38d47 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -50,13 +50,13 @@
  *             The function discretisation routines compute the DG discretisation
  *             of analytic functions on a given grid. In 1D the discretisation
  *             simply consists of n function values per grid cell ( where n is the number
- *             of Legendre coefficients used; currently 1, 2, 3, 4 or 5) evaluated at
+ *             of Legendre coefficients used; currently 1 <= n <= 20 ) evaluated at
  *             the Gaussian abscissas in the respective cell. In 2D and 3D we simply 
  *             use the product space. We choose x to be the contiguous direction.
  *             The first elements of the resulting vector lie in the cell at (x0,y0) and the last
  *             in (x1, y1).
  *         @defgroup highlevel create weights 
- *              overloads for the create::weights and create::inv_weights functions for all
+ *              overloads for the dg::create::weights and dg::create::inv_weights functions for all
  *              available topologies
  *         @defgroup creation create derivatives 
  *
-- 
GitLab


From 0d1d6151c363b4a56e51c9399e14a7df59a2902a Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Wed, 11 Oct 2017 11:32:39 +0200
Subject: [PATCH 347/453] documentation of convert_row_dist and trranspose

---
 inc/dg/Doxyfile                 |  1 -
 inc/dg/backend/mpi_collective.h | 29 +++++++++++++++++------------
 inc/dg/backend/mpi_projection.h | 20 +++++++++++++-------
 inc/dg/backend/transpose.h      | 11 +++++++++++
 4 files changed, 41 insertions(+), 20 deletions(-)

diff --git a/inc/dg/Doxyfile b/inc/dg/Doxyfile
index 3c809011f..bbf0519b5 100644
--- a/inc/dg/Doxyfile
+++ b/inc/dg/Doxyfile
@@ -829,7 +829,6 @@ EXCLUDE                = ../dg/backend/creation.cuh \
                          ../dg/backend/cusp_thrust_backend.h \
                          ../dg/backend/sparseblockmat_gpu_kernels.cuh \
                          ../dg/backend/sparseblockmat_omp_kernels.h \
-                         ../dg/backend/transpose.h \
                          ../dg/backend/average.h \
                          ../dg/backend/ell_interpolation.cuh \
 
diff --git a/inc/dg/backend/mpi_collective.h b/inc/dg/backend/mpi_collective.h
index 552777e4a..eaf755f2a 100644
--- a/inc/dg/backend/mpi_collective.h
+++ b/inc/dg/backend/mpi_collective.h
@@ -273,18 +273,18 @@ struct SurjectiveComm : public aCommunicator<Vector>
         construct( localGatherMap, pidGatherMap, comm);
     }
 
-    ///@copydoc GeneralComm::GeneralComm(const thrust::host_vector<int>&,const MPITopology&)
+    ///@copydoc GeneralComm::GeneralComm(const thrust::host_vector<int>&,const ConversionPolicy&)
     ///@note we assume that the gather map is surjective
-    template<class MPITopology>
-    SurjectiveComm( const thrust::host_vector<int>& globalGatherMap, const MPITopology& g)
+    template<class ConversionPolicy>
+    SurjectiveComm( const thrust::host_vector<int>& globalGatherMap, const ConversionPolicy& p)
     {
         thrust::host_vector<int> local(globalGatherMap.size()), pids(globalGatherMap.size());
         bool success = true;
         for(unsigned i=0; i<local.size(); i++)
-            if( !g.global2localIdx(globalGatherMap[i], local[i], pids[i]) ) success = false;
+            if( !p.global2localIdx(globalGatherMap[i], local[i], pids[i]) ) success = false;
 
         assert( success);
-        construct( local, pids, g.communicator());
+        construct( local, pids, p.communicator());
     }
 
     ///@copydoc GeneralComm::GeneralComm(const GeneralComm<OtherIndex,OtherVector>&)
@@ -394,19 +394,24 @@ struct GeneralComm : public aCommunicator<Vector>
      * @brief Construct from global indices gather map
      *
      * Uses the global2localIdx() member of MPITopology to generate localGatherMap and pidGatherMap 
-     * @tparam MPITopology any implementation of an MPI Topology (aMPITopology2d, aMPITopology3d, ...)
-     * @param globalGatherMap Each element globalGatherMap[i] represents a global vector index from where to take the value. There are "local buffer size" elements.
-     * @param g a grid object
+     * @tparam ConversionPolicy has to have the members: 
+     *  - global2localIdx(unsigned,unsigned,unsigned) const;
+     * where the first parameter is the global index and the 
+     * other two are the pair (local idx, rank). 
+     *  - MPI_Comm %communicator() const;  returns the communicator to use in the gather
+     * @param globalGatherMap Each element globalGatherMap[i] represents a global vector index from where to take the value. There are "local buffer size == size()" elements.
+     * @param p the conversion object
+     * @sa basictopology the MPI %grids defined in Level 3 can all be used as a ConversionPolicy
      */
-    template<class MPITopology>
-    GeneralComm( const thrust::host_vector<int>& globalGatherMap, const MPITopology& g)
+    template<class ConversionPolicy>
+    GeneralComm( const thrust::host_vector<int>& globalGatherMap, const ConversionPolicy& p)
     {
         thrust::host_vector<int> local(globalGatherMap.size()), pids(globalGatherMap.size());
         bool success = true;
         for(unsigned i=0; i<local.size(); i++)
-            if( !g.global2localIdx(globalGatherMap[i], local[i], pids[i]) ) success = false;
+            if( !p.global2localIdx(globalGatherMap[i], local[i], pids[i]) ) success = false;
         assert( success);
-        construct( local, pids, g.communicator());
+        construct( local, pids, p.communicator());
     }
 
     ///@brief read access to the local index gather map
diff --git a/inc/dg/backend/mpi_projection.h b/inc/dg/backend/mpi_projection.h
index 3ccb075b2..30214a559 100644
--- a/inc/dg/backend/mpi_projection.h
+++ b/inc/dg/backend/mpi_projection.h
@@ -54,21 +54,27 @@ void global2bufferIdx( const cusp::array1d<int, cusp::host_memory>& global_idx,
 /**
  * @brief Convert a matrix with local row and global column indices to a row distributed MPI matrix
  *
- * Checks if communication is actually needed
+ * @tparam ConversionPolicy has to have the members: 
+ *  - global2localIdx(unsigned,unsigned,unsigned) const;
+ * where the first parameter is the global index and the 
+ * other two are the pair (local idx, rank). 
+ *  - MPI_Comm %communicator() const;  returns the communicator to use in the gather
  * @param global the column indices need to be global, the row indices local
- * @param topology the topology defines how the indices are converted from global to local
+ * @param policy the conversion object
  *
- * @return a row distributed MPI matrix
- * @ingroup misc
+ * @return a row distributed MPI matrix. If no MPI communication is needed the collective communicator will have zero size. 
+ * @sa basictopology the MPI %grids defined in Level 3 can all be used as a ConversionPolicy
+ * @ingroup mpi_structures
  */
-dg::MIHMatrix convert_row_dist( const dg::IHMatrix& global, const aMPITopology2d& topology) 
+template<class ConversionPolicy>
+dg::MIHMatrix convert_row_dist( const dg::IHMatrix& global, const ConversionPolicy& policy) 
 {
     int rank;
     MPI_Comm_rank( MPI_COMM_WORLD, &rank);
     dg::iHVec unique_global_idx;
     cusp::array1d<int, cusp::host_memory> buffer_idx;
     dg::detail::global2bufferIdx( global.column_indices, buffer_idx, unique_global_idx);
-    dg::GeneralComm<dg::iHVec, dg::HVec> comm( unique_global_idx, topology);
+    dg::GeneralComm<dg::iHVec, dg::HVec> comm( unique_global_idx, policy);
     dg::IHMatrix local( global.num_rows, comm.size(), global.values.size());
     local.row_offsets=global.row_offsets;
     local.column_indices=buffer_idx;
@@ -78,7 +84,7 @@ dg::MIHMatrix convert_row_dist( const dg::IHMatrix& global, const aMPITopology2d
         cusp::array1d<int, cusp::host_memory> local_idx(global.column_indices), pids(local_idx);
         bool success = true;
         for(unsigned i=0; i<local_idx.size(); i++)
-            if( !topology.global2localIdx(global.column_indices[i], local_idx[i], pids[i]) ) success = false;
+            if( !policy.global2localIdx(global.column_indices[i], local_idx[i], pids[i]) ) success = false;
         assert( success);
         local.column_indices=local_idx;
         comm = dg::GeneralComm< dg::iHVec, dg::HVec>();
diff --git a/inc/dg/backend/transpose.h b/inc/dg/backend/transpose.h
index 947512bc7..5fcf5d07f 100644
--- a/inc/dg/backend/transpose.h
+++ b/inc/dg/backend/transpose.h
@@ -18,6 +18,17 @@ Matrix doTranspose( const Matrix& src, CuspMatrixTag)
 }//namespace detail
 ///@endcond
 
+/**
+ * @brief Generic matrix transpose method
+ *
+ * @tparam Matrix one of 
+ *  - any cusp matrix
+ *  - any MPIDistMatrix with a cusp matrix as template parameter
+ * @param src the marix to transpose
+ *
+ * @return the matrix that acts as the transpose of src
+ * @ingroup lowlevel 
+ */
 template<class Matrix>
 Matrix transpose( const Matrix& src)
 {
-- 
GitLab


From fc724468b0655fa0fa25a08ed06176437630b0f9 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Wed, 11 Oct 2017 14:41:53 +0200
Subject: [PATCH 348/453] rename convert_row_dist to convert

---
 inc/dg/backend/mpi_collective.h   | 2 +-
 inc/dg/backend/mpi_projection.h   | 6 +++---
 inc/dg/backend/projection_mpit.cu | 2 +-
 inc/geometries/fieldaligned.h     | 1 -
 inc/geometries/mpi_fieldaligned.h | 1 +
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/inc/dg/backend/mpi_collective.h b/inc/dg/backend/mpi_collective.h
index eaf755f2a..6e4978607 100644
--- a/inc/dg/backend/mpi_collective.h
+++ b/inc/dg/backend/mpi_collective.h
@@ -398,7 +398,7 @@ struct GeneralComm : public aCommunicator<Vector>
      *  - global2localIdx(unsigned,unsigned,unsigned) const;
      * where the first parameter is the global index and the 
      * other two are the pair (local idx, rank). 
-     *  - MPI_Comm %communicator() const;  returns the communicator to use in the gather
+     *  - MPI_Comm %communicator() const;  returns the communicator to use in the gather/scatter
      * @param globalGatherMap Each element globalGatherMap[i] represents a global vector index from where to take the value. There are "local buffer size == size()" elements.
      * @param p the conversion object
      * @sa basictopology the MPI %grids defined in Level 3 can all be used as a ConversionPolicy
diff --git a/inc/dg/backend/mpi_projection.h b/inc/dg/backend/mpi_projection.h
index 30214a559..996aff44d 100644
--- a/inc/dg/backend/mpi_projection.h
+++ b/inc/dg/backend/mpi_projection.h
@@ -58,7 +58,7 @@ void global2bufferIdx( const cusp::array1d<int, cusp::host_memory>& global_idx,
  *  - global2localIdx(unsigned,unsigned,unsigned) const;
  * where the first parameter is the global index and the 
  * other two are the pair (local idx, rank). 
- *  - MPI_Comm %communicator() const;  returns the communicator to use in the gather
+ *  - MPI_Comm %communicator() const;  returns the communicator to use in the gather/scatter
  * @param global the column indices need to be global, the row indices local
  * @param policy the conversion object
  *
@@ -67,7 +67,7 @@ void global2bufferIdx( const cusp::array1d<int, cusp::host_memory>& global_idx,
  * @ingroup mpi_structures
  */
 template<class ConversionPolicy>
-dg::MIHMatrix convert_row_dist( const dg::IHMatrix& global, const ConversionPolicy& policy) 
+dg::MIHMatrix convert( const dg::IHMatrix& global, const ConversionPolicy& policy) 
 {
     int rank;
     MPI_Comm_rank( MPI_COMM_WORLD, &rank);
@@ -139,7 +139,7 @@ dg::MIHMatrix projection( const aMPITopology3d& g_new, const aMPITopology3d& g_o
  */
 dg::MIHMatrix interpolation( const dg::HVec& x, const dg::HVec& y, const aMPITopology2d& grid, dg::bc bcx = dg::NEU, dg::bc bcy = dg::NEU)
 {
-    return convert_row_dist(  
+    return convert(  
             dg::create::interpolation( x, y, grid.global(), bcx, bcy), 
             grid);
 }
diff --git a/inc/dg/backend/projection_mpit.cu b/inc/dg/backend/projection_mpit.cu
index 334967267..000b77280 100644
--- a/inc/dg/backend/projection_mpit.cu
+++ b/inc/dg/backend/projection_mpit.cu
@@ -35,7 +35,7 @@ int main(int argc, char* argv[])
     dg::HVec x = dg::evaluate( dg::cooX2d, g2d.local());
     dg::HVec y = dg::evaluate( dg::cooY2d, g2d.local());
     dg::IHMatrix global_projection = dg::create::interpolation( x,y, g2d_half.global());
-    dg::MIHMatrix converted_p = dg::convert_row_dist(global_projection, g2d_half);
+    dg::MIHMatrix converted_p = dg::convert(global_projection, g2d_half);
 
     //now compare
     bool equal_cols=true, equal_rows=true, equal_values=true;
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index 9e2d0c9b5..6b79c0346 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -1,6 +1,5 @@
 #pragma once
 #include <cmath>
-#include <cusp/transpose.h>
 #include <cusp/csr_matrix.h>
 
 #include "dg/backend/grid.h"
diff --git a/inc/geometries/mpi_fieldaligned.h b/inc/geometries/mpi_fieldaligned.h
index 0d8a64388..9caa62d3d 100644
--- a/inc/geometries/mpi_fieldaligned.h
+++ b/inc/geometries/mpi_fieldaligned.h
@@ -7,6 +7,7 @@
 #include "dg/backend/mpi_matrix_blas.h"
 #include "dg/backend/mpi_collective.h"
 #include "dg/backend/mpi_grid.h"
+#include "dg/backend/mpi_projection.h"
 #include "dg/backend/interpolation.cuh"
 #include "dg/backend/functions.h"
 #include "dg/runge_kutta.h"
-- 
GitLab


From c9e6d1baa39e9a27d42bd786a76337feb5186bb9 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Wed, 11 Oct 2017 17:21:02 +0200
Subject: [PATCH 349/453] moved documentation to feltor/doc and added hector
 writeup

---
 doc/Makefile                                 |  70 ++
 {inc/doc => doc}/header.html                 |   0
 doc/related_pages/hector/drawing_magenta.pdf | Bin 0 -> 16041 bytes
 doc/related_pages/hector/hector.tex          | 929 +++++++++++++++++++
 inc/doc/Makefile                             |  65 --
 inc/geometries/Doxyfile                      |   2 +-
 inc/geometries/geometries_doc.h              |   3 +-
 inc/geometries/related_pages/parallel.tex    | 367 --------
 8 files changed, 1002 insertions(+), 434 deletions(-)
 create mode 100644 doc/Makefile
 rename {inc/doc => doc}/header.html (100%)
 create mode 100644 doc/related_pages/hector/drawing_magenta.pdf
 create mode 100644 doc/related_pages/hector/hector.tex
 delete mode 100644 inc/doc/Makefile
 delete mode 100644 inc/geometries/related_pages/parallel.tex

diff --git a/doc/Makefile b/doc/Makefile
new file mode 100644
index 000000000..2584bd9fb
--- /dev/null
+++ b/doc/Makefile
@@ -0,0 +1,70 @@
+###Packages needed on linux:
+###doxygen
+###libjs-mathjax
+###graphviz
+#(shouldn't require latex according to doxygen documentation)
+
+all: doc
+
+.PHONY: clean doc dg.tag geometries.tag dg file geometries
+
+# the semicolons and backslashes are needed by Makefile
+dg.tag:
+	cd ../inc/dg; \
+		(cat Doxyfile; \
+		echo "OUTPUT_DIRECTORY = ../../doc/dg"; \
+		echo "GENERATE_HTML = NO"; \
+		echo "GENERATE_TAGFILE = ../../doc/dg.tag" ) | doxygen - ;
+
+geometries.tag:
+	cd ../inc/geometries;  \
+		(cat Doxyfile; \
+		echo "OUTPUT_DIRECTORY = ../../doc/geometries"; \
+		echo "GENERATE_HTML = NO"; \
+		echo "GENERATE_TAGFILE = ../../doc/geometries.tag" ) | doxygen - ;
+
+dg: geometries.tag
+	cd ../inc/dg; \
+		(cat Doxyfile; \
+		echo "OUTPUT_DIRECTORY = ../../doc/dg"; \
+		echo "HTML_HEADER = ../../doc/header.html"; \
+    	echo "EXTERNAL_GROUPS=NO" ;\
+    	echo "EXTERNAL_PAGES=NO" ;\
+		echo "TAGFILES = ../../doc/geometries.tag=../../geometries/html") | doxygen - ; 
+
+geometries: dg.tag parallel.pdf hector.pdf
+	cd ../inc/geometries; \
+		(cat Doxyfile; \
+		echo "OUTPUT_DIRECTORY = ../../doc/geometries"; \
+		echo "HTML_EXTRA_FILES  = ../../doc/related_pages/parallel/parallel.pdf"; \
+		echo "HTML_EXTRA_FILES += ../../doc/related_pages/hector/hector.pdf"; \
+		echo "HTML_HEADER = ../../doc/header.html"; \
+    	echo "EXTERNAL_GROUPS=NO" ;\
+    	echo "EXTERNAL_PAGES=NO" ;\
+		echo "TAGFILES = ../../doc/dg.tag=../../dg/html") | doxygen - ; 
+
+file:
+	cd ../inc/file; \
+		(cat Doxyfile; \
+		echo "OUTPUT_DIRECTORY = ../../doc/file"; \
+		echo "HTML_HEADER = ../../doc/header.html"; \
+		echo ) | doxygen - ; 
+
+%.pdf: 
+	cd related_pages/$*; \
+		pdflatex $*.tex; \
+		bibtex $*.aux; \
+		pdflatex $*.tex; \
+		pdflatex $*.tex;
+
+
+doc: dg geometries file
+	ln -sf dg/html/modules.html index.html
+	#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%#
+	#Open with: firefox index.html or on Windows: firefox dg/html/modules.html#
+	#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%#
+
+full_doc: pdf doc
+
+clean:
+	rm -rf dg file geometries dg.tag file.tag geometries.tag index.html
diff --git a/inc/doc/header.html b/doc/header.html
similarity index 100%
rename from inc/doc/header.html
rename to doc/header.html
diff --git a/doc/related_pages/hector/drawing_magenta.pdf b/doc/related_pages/hector/drawing_magenta.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..0c28fb77a9633fc32276ba5276f6ca3197ceb506
GIT binary patch
literal 16041
zcmdsebzGEN7cL+z-6<m=ib~ATCEeYP3`5t@-3TI$Aky8^EsFFI64EUtNS6X42zLgL
zM?IeJe)rx#?(hEYz%ab)-Fv@#uQ%57JZsG@6$vR8AS)O4t-77^Gi(k3I{<28jV&k$
zU{f%&w{WopaKcS$*Z=^4P0GsF#mou*X$y8SlQ4roP0g?$JivB#aWVtjVSA+2>Afwd
zX&{_G(RN@)+vI-Og+X8n24ETkuHT;NHKJIV3e0*{89a5c@G*^knq!{By`cLES$mjk
zuSu0ojC$9Fl@pB0;^LV6Q<wY1`Im}b)~<7Y-A|+!zG)A9Y_?4XPSiR5xr8Z|>dhy(
zy4ZjuTP1wW3WJZk)+|f(4Z_#DaDqOWVW~ycxW(y3##e`ZVmiw=ZNO7z>RZl9)e1k6
zm-~3+J9U}~(`i?8UXQ>%z8`!vCSB;d#eE~_=$Sy9@>=Gk#p$b4uO@dl{lpI;pK?(%
znu*ig<(=BbDm~hUA$qoQOx<0Rukn<nOtFR)f(bU&)D5*yz#sjnRz8qDw0=lEqPA=-
zST}67JsT=s^v3>nZHBi$UdHTZG5Cf%$zt{*lcSDUpU#G}QOSpAhZ}Kty<#TZZ7YY_
zIj=o1(D-p`PEYBi6K~U0C*GR8DbnXhfVWGhphS+t*eL4JkAzNnum2wGmUzqJJ7!+B
zC#y&pI;OF>kCpiPv+07k<5=;Bv7*{G_fHg>Ci3%qZobW;Nkr4v*Jo88Wv0dp<3=;l
zew3C;U=@B+oHywqwUwCm+6%pGtE;pB%=MmGnCl~T%xBdS{CNGiWXjI+H`8`@FjHmj
zy|u1WyisLEkU`X~&0kqFa{7n_V~L%vF||VVS+okqgLIK}VTKsvg>AMkF6}iwUqhZe
ztd5U3+kB)vOE9W7M4MZJ89P<RXc-asGKRL^aX%jAT5z2JIpBU!SgT0~zhE-Sa}6^B
zrY1?T35vI8mCnLZjxgFuPBQtfsE=*0qL>tRy-$SQ+rEyJP2tp?1q=4Ev-eYlGo!^N
zy9r|sw$;Q*y|~Y{Y<iOoZ!=_iOl{(3G!gTBtr|*x+1BBM?QLa9q(=6V1Ubi^DINcw
z6^=4w@4=0yM|A?dTm#$qpVs%0-xw+!$@B?&Q6JuxPAVcD&zA4m&9X3T!s9Xw?*WLQ
zv4g4FbQVIR&a802naD5B^q}N)S`&7h%$1hjCoS&1UuUAGymoi^ZlqIXPz#Ag80IL_
zs>Ch&Y-2RYOKm=0rI(hHdu~RHtPIwZ8<i8~u3mlp8eiaSltMno#jXtb^I^7g25h}{
zcMeOuvfkzPNvnElt{{ICl2YrFw1MmA_|gkR2C$3rHr(v9Yn6;(3XyQ^xlc5jeiy?A
zkI3l+QNNHf@8@E(Gb}e!4Yu$+LcSBMJH%_FVL&YuB+bduFT}nSiG|nLM}7;llSt)$
zmhJ`w73;pE%M)B_&3E=OY?ltn1sH%!51x54gJktau?F$_d7_{w?9Hc|j0A%B8_=-x
z2Ft7J893dlj*K9Ns_vgjEzuhpJL{j4;xbt2U<fedaF=AiQ^;?<g^Vjhpo3ZXZr_@&
zW35q2^=q`vr^0VaB@N=6PE{!S;`sJe;r85+q1xL!=r^BQM_dH7NJ%sa%=9hC+m4Zn
zV^&|hkK73h?ui4)sp!UL5M?=3m0Y)!wpgt0Q?p?}niKn$1m3{smyozC!O*tNTc7l5
zHm}$^)q`&R{^JZGUC%%?yT_22gpc8>olnPSSd{uuvbvW_$&Xt%;v{vBZt~aXRniJ{
z=U{l^k*f#6CW?@etF%4<-KR8BqUDT;zeP+(+l=Z*rsz;esyMWxsx!yvcH}xaGChk<
zuw)3o*27?fDuQ~<-C9aQ(msY~B~eL%jHXfzZdZAZr-jm3S@fbB<zZ3)SxvO;q6@{x
z(Y(G>xoom~Qw%y?_>I8^#XC9Lo3_^O;SN5Zm6GFZAiHTnvPn_54lV2Kvn?d%<MhGy
zf;ldEc~tl66iALuh$h+&OjMsCr|IfQBJ~rqG!R#EJ#{HL3YZD(H8y#t5UH8Xw(ON9
zq+cRrjoE5P+%j??k)3@V<8z>A#_IJ=+mx>B>gJeln6Eukwbg!`i1CTqZ_3}H(L-w+
zf7*NU3;lbwyINVQQK;5n=SQ}2$WL!X`a|5?bQy;~kRn$?>CJO$JoE5kMDmP6L&mS+
z1+=^q98+CJ4m7!rVi8EMo+eFMY08f}IK-r+*0GOzmv%rKw<aA_#+cYyPg<bTeC)EQ
zSj+I3AS$69m*QniM9&0Tt=dErU98R$$rlD)&m;=(yI4Kf+H|g4=c!Mn9(6aKwPQwM
zci0ead!ep8iPdc;iITt+)j_>*u#w8k;_<Y+7S$A{Wdd2qIx-1h4iIy>rqxpk)DoS`
z*2HVPmWgj02+;C*!AyCnub=bmn3vSPYn0k&f>*R+Y%NBMp;M+CFOGV>-wTa<OQ~u}
za!;QQPnxmm(-$-G*!;6$fue!kc-VzwM_2`p&`K!edc#VnmCUJ#x#+F#W0;w2N?kL$
z0=B7UHIp>#0p=^MysBh;9AQiL4<07j`P!qC$4f77*g;61-S7UYXT)YQR3z;>wxw!6
zcR828Dc=A(@p5qve|#%>(tiFWlMUno<W$KnC(_->{UO(bwQoix0@N93l@CBxJ>B}s
z{rXNHFG-`qx;*k%!UE@yf>BEwuU#2%dS_<J=5QtKm7Qu=v1T=1B9oOAV1?>M&7<bJ
zj!sXF14I&*aEifcgwZIn`a&bpXWz_evQTm5XB1MW(+EONsw#1GZ--4dG8+48ye&df
zvs-6UH09k-ys25Wu*{pnL?HbRZNMr3^W)f4NK;{i9&!YmOQ4Do4Q@6w1gD}F*_)+Q
zeR@7ec}OCtvt1$k30J7nZ~@vUu5Qe`Mdc2TDBsp&Lpz-C)~Z*EzD_|81^Pf*r8o>C
z#QDwPwF(c@3JR4gw23tH5;QqXqxNa(NA|yFZta+t)wTLdnMdi8c0?;Pp)HEw<`1;R
z5nm_ck%=s;sg#gxw<A&J==FN;Jy??UVm{+et4`mF$<$5f5=xSy4<UftYJ1j9V;&pA
z0TZpRQxL6GtB!lEJv;<s4>2P$o|-Hu^j^nHE4r30BH(}>_{_L@J(B7(SVX{*V^%Bd
zm42v3w_n79s>$?(>j<{kb>;_YRrxs9P`Qe`r-woTRM*=}y3t%|J>_$iwvV$cueFo(
z99@`wU4A@0XmD}ycw@l-{hLdfaDC>5*Bg1C^8EL1`+9uZ-D%isnd$$!zxG`8Rnp18
ztos^lY{6c^R{+EQO{Pz?0QZ8dhYi%WS;;vn;mOje2q!UGRwvTY*+$HS-bvv?Zm5kY
zp!-=#{oVDP{pG;^TK^HV{Y=Hs@^Il6;l~RCZYzu1Cba$1H-j)=OcZVxWFFL;EX=34
zt!$6Y?3xPXw}n4=MR4X>nh*~O3$q=<{OoEa#(ux^)(4J`uw^u^)7712^k@%OdgNB?
zQa$S|oql92PycJyST)AfvPklGB#}^^)%)3o3>w^2>96aKp-)jEudLbC;D=g0<D-4o
zlDIgxOV#w0KvNvqxZ5>bO!|=|np1JU7jrLv08J&|5LrL{P2xB62`hEM5-4bSXd3j5
zc~s}Tk6%@X%%Y6lB9%a!$O@Yep#-zFLQhadZj@HyA>n%wOvrnv1+|_yWm2uPJt6Ih
zH6B_VFQQ9IqBp3wR?kNg7QFrFF*6^S0ZkTfvJ8Bv7jA)*QkJhme6OOf!H8sfSGKrT
z&2=25lf>jQA)o^KwbMpS!y0|uXZU>yFV?E=&Aa+^U7=ZZuO;w9wZ&}H_u8&|TMe1n
z(s|j%H?Wa@AtNZA!x*T`KqdfJlj=L&0IUZ&Y*~ev4fhW|b0ban;};LmK2i@<Hct<3
z(jOK`T?wMMIR0|YY)$7&mS}Q(l8`n_X}KFe9-a{OVwnNX*U~|B##BK@jgv?A^?kgT
zC+IFx!%tPo0d8zY0`F@4EQw{qBkl!?c&0xJ*r@F@S&@`W)i_m{xWm}GX~69b3s0-y
zeuWf)PZLfm+0U#ljKpzs0?_c9lLnb|viF=yW3H!8yVnuVCkR~3{Rzx0r_&bIhwR#B
zHlqu>Bo?rJdX4@Xb()E408Sx|z_hph4ka$ghT|~`YG7d3noY_(!`1r%m6;@HNkMPH
z4zUkH?;+34u2c<j#N70++G!P}j8z$}xk*u1`{Bc>5&B@Zh9@p^sT!>xgG7pnW$(U+
za7yv^+x;E-_hnM@i(A7X`U5_*5-D%VFJEcK(A!tHL(PgHY2qZJqGfS`-`cafJX57(
z-ofNvsDsH8oXzP7r+ldxClfdKGW=i2SGfDqf9cFo+z9h_l@t*0LGI~9=RO@$O+Jir
zH1~cGH8U18v>j|#ZCzKi5>Y)_a2t9z!jF#g#lSdsDe+5<Z_DOGb>u~_(pmYbViZzW
zp}4kYQs{0^maIn2Up4YS;NJgAf38?ge5#2j<YuMiCEs+|`$X}5lVuDl)8P9t3t38|
z1-mip_y+?cO>ta8&>_h;DMhZC9}kYimb0whhR|Dd+s??mVR2O(I6}96&r|Q4bI@Lh
z#(j5Mx(r)T2HbCQox`8|19I+)be)5epH)t#>wa6WvmIK{q|1ysGu4u|B(}5;=Ef)=
zoqyqj3C%DDAM+_^`-L<EJ9pl$!*8OXe%UXYIMJ~-z7rFtOLy7nSt?m=za)O;%yPej
zt{Tjty`1x8eQR{;W$Q<@qm+-I!o|L(hIEmQ&>iu2PUY)W7(LhQ7JX=(tM(<@l}+aY
zUnXiOW&Wbh6SLoJ`oOvz;v8(fRXWp>ilj(Z$Lh;;U%k-7FzLB*xrcjg{lfl}_l?j@
zg^X{C%wGpK^IK?#hn+8A7=T7a5fP$qoPiew0SOC=(<8nwlu;<GGdVm_2-49i?U-m+
z>@C9QA0*8J?RVYkW|!uOKJScRJxto*ijMJB)*6a?n1inI&S&j$OQO$wU!y)fdbfKZ
zJ@e?Ktw`HxEdgEu38Br{q;2^7XWFB)@k5wFsrP(bm9XdPAsDSHr^rU{p8HA(`n96?
z3|!1#9?Y_il_<#T#onHL^U#5NeYU#rz(LIEYew>sJewbSu1aEsW*zQn#*o7E&o;I4
zp|yE0grXrNI@Y1U14%uOYc1K0R;ZaRj$Z`1PjHJj2dw38PmJD%DOrJ+w>DZ(6qH=B
z&FoG8BCzoP5Y!jJcM+d(#>>IO&5dY)*aZE}jIX$`q=$>Nx(l2K0|W&T2Kb8&%nsiG
zu!)NTfB-fK82%T|j^R7L|7DYc+PnPH+MI=*g&n~4gIyyAFlPb6P29hm*ja!8p5IMC
zxQX|N2|haf_pe#<!L2{|yN0KO8Gua%Y+-gaV->KI8GHi3AN|-A%}lMpVo(o&KHS6(
z;NatAW#{AM<OKnMydYLi4qkR3AHeXcuQ+__@XpQvgg)?H43Kt$x;p%_obN}wvi|Ug
zI)F{X32g7|fH*S5^M_d$;Ns+ZH7CTH#o?_a%-pOXW@^%6h$eq|OU=v~>gohBa|R$h
zulCCtfr#1uSma+K1@Pyx{!OJo97x^O#O2B(h?xS#e^V~<6YOl&%}BbTljVZZe2?_r
z+?juX?U+YT#jeXq`RGefk$kNCHq5enqWZ|y*Z*Y$+ewBu)O&Wf+)#e@iA{rR_C3gb
zbt+7;u{#G061iARj9=QqfK}S^ISrNgK8vZh`98`H#@`;|yL>npXAGibijzq=<1(KY
zNI8ffs5#3OEi;pVbTWc)Iv8_Wm|h29vB^DRVF{AE>(Zyhg>$KKF;QzzxK|FfgxwqB
zT4|t8t<uLSqvi&e5j|08{umrwVXu9XL0d=txR&2}QvRC$E4QNYSFhZQ9;&+UP;%m0
z;Trm06AumzY<vFdJvWZ6zaF#l+IwM}F&n8)f72DR7qlIC5R9srDh3^!2pyZ}rqXYz
z)n0D97GZhYJWa72nOXbF4KA-zH1f7o51)SQJ?W}(PMV#2_obJt2u~i*)~CK&K@5_C
zL-k3mHy3zoOKG^FZPH#0N|#~v!#3mmIAn#C1jt6A#^RBg*Qf%rt%muSU~5m4Y5KFC
zG6=c#@p2ZS@N7kUj}}mNgA|aj$(p;F6vU<uY>^Eb1;15ExvQ9aYEs4TnH)WQzmfod
z*xmsN4IFd!ZukVR6?5uPMJhJW(>!QkY_mI8Q227A-{mQUV(}&#)?LD7`YKEn%lpPu
z7_=a>bPjT1eI31F*5hI+kv&k<vIiptJq<Z$ey#|TDB&2^GI8_8qwck_ID1slL{o;M
z+~pV8Ls~qfiHZK0bE<*_=#Px@F$6{ZTlWef-=Ylb=c9c|WJ3*Bc<t+^=J<J$-+)T#
zWkk}H9LP_tPg&LGhVFA<^|X`jggkf`JPx7dNm2Ng5zi<Zaua3A^3hFI^<7Mf_;V+>
zY?Y)GIY&wn;3=c{lt7zYNU3k=jRh5z$0YbN_>bDq-7rTa44!k5rlT#;s6Nt6O3mOK
zax}m}QcO?g^2f7MT}M?{Bz=Be7>LDxw<HB8_<EX%8e^ajReO+t4^zyfWXeq!yg_js
zUjEQUw{KES93RshiumzP$wF|SDf*vNG#V~=synva_{<Cy-8np*Z_n4b<;&p8WfVxS
z=VCavaTH2O`Xx7z(Fk;CPMDPX8Lcc=-*xXehQIlJr6NgdQd`8Gv;vPFpgi$-*8=gW
z$Bi5r$Hz;SH=gcy8J`kF2GP2Exa7j(cOi@yB-x;c1LX;xN<PG-ZO+J5w}m%wy^9I#
zZ<fj|tO(H3kc}5Q$mNB)`AuKel&pA&tCzi)Y@E5|XL?$8GN(F6{8h3Muky@Mno%Ya
zmxp`CZTbY+YJyscPPgN0(%t0}`iX~~Db52am!>(fS@8~r`n%Vz=?K;zR9#*>c^~hf
zj-fT=!^ZvGoN1cxO)RN&U^{Lc5~$hrOGn2p5+yo5z+g87X(eL0lt&bZ#5qtJ1yDQx
z>VchYsvPqK<t1j!<PIKR8;;uq^@+ui{IRA;1bgc1*lz^}ulRiLWn3yw;dr_X4x2)g
z$&NW(FFifQ#VcOajZX;cS%mYSy{^pCsFOYAnavD5=GII-UpSUNE0ery^Q;0o<KKGU
z&G*kJboD^{D+~dFT%121@W7vO=x>D<`17g!Zx28C1NCo-)==fGeFuQx^(Rtq1$;uK
zguJLn_ceRF9zW1%pbG#y##xKl`0Ot{mrjrqLaON>wB$S=F%+<1*mplH>BfZdEApO&
zkhBNny4p5i*lE62T-A!liM;LDGDuiBNICQf=_0O&Zv;ElVh6*=7^!tZhC!F7eQj%G
zvceB*3FEd4A=N~E)-dBMa!RAsN39Vavm2eWjrQ@|&=qG~%rnrQHRCGRpy*x34Jp$c
zD?I{OEl&DHH*JXt`=&k-ag4H6jnzo}e$_GrRv4~KnR)L4c}prt;3M{K+_+f6NMeRu
zKg~P-*V>dmJta<l$TEGdm;d6|9||`-B!D=9f4lWZ)JFvH@1YD4t|iQzAx>5fE>Ned
zFo-ZIf$iW?RZ>$+Lr&wKBG|=J4(#y|@yViQX5nfJ{@GFrY-eTb3ApoXBj)ep*n%yb
z0i1u2DfT@IvvBYMSU^Bt0Fa#v9`k`be??ze7qG1rMAY8G)(pUY)k)MDf{5ktCRdw?
zX(F}}ov#eyU<Vm9D+|k?P1Ie??6d&9-@79=WPWTQ76rd-%sAkO{YRulfRi7A_NP;>
zj_`Xly)ub`oz1Q=&VTogm6NlJxFy&LzyZ4IssR3L8wdhk8Gd&@0QfTrA!!fKJ6PFU
z0NAvx>_zRJt$rA;`e|F4x>!2@g=i4V`G5b1iw6YY<VCpp>OVgA?`Iu;8A3Jftq@5T
zz|~Fv8wC3G%=~~rf8)k~P1O7bfy8Ac5FSRHQ}*AHke#}|Ewv=cd=wp}+S?*WTq2cj
zNdOl!b9O%6YH_$0eTYU5ZV{%aqisA!Ic;G%arePJzf$+T-I*<SBH^c;n{NvoY4U6H
zr7(-WUPBGX76q*0e-QyN8uQrR()sw<&q12QV$H~EH<^Q<m+3i#H`>M<Cpxerw2Bya
zThMw}*bW$J+(^^GZjdsBiI0sRD0kbo8<m!|mJ7=(xl-JosLY?uo`M88-Z2YQDAyFC
zM?Nmw97dmOYN#rQcBMEP6u<wveGf{7pJeO1W*<MjN#W9Xyu0hw#=T8NxlQM=?oKs(
z<lWSC?7q9<l_T%}&d0iPp+wl6hEqRkt^4!5*2?ZBgNOTWnB9lGwji(~(Gt@%lZ`@j
zFSWE#dAI*FR}n*AjqWYAp3Rd3+IF-maorY<l4sTKcw8@LyBf%O-hVwW9G*C7S;lAc
ze)A6V{3z<IZZWUf5kDpJjvFm0(DAY47^*@il8*60{`=I4&2_e8TLP<#J7|aY6zHj=
z8}tjv%FCv4#xndUH!%q%7^w<^Y*D+!KF;Sm022!2*stL*COdh7SsyBqzGT)3;C#ju
z-}8_%ow_f3*Rx9n_Vi{F&l;SbQzpFBbK7X&TR3-p6|I&=FPu?eT`5Ms^3pa4LhOS5
z?94KrHIJr{o`A8#G%4o}(H(3gE7Zjv)I8(x-kMR)>rrud&JRQi0oM8NC9UJ%8FfJ`
zs%%~?7=AN?LHz7)U~iaatP)ua9)L8K=wwVu_vUq)JXq3Ad3(f2tCiS;pQiw>6-k?t
zHs?H}2xb?`_(YbC8KU46x(a$Oy!fzee|GQ?lmZ#~%=zJ6*(4k7STm~BtcdX887av-
z7wXDvl(-~b8)gb)R3FiX+110TW?1Dt)5mutAWzdQ<#ZE{t2CD0Zoh0NA^(_!nh@X|
zd<{K=%*Obd_JdfAytUMCnxclQFNY_lSMLdXOu%f;-$rclaQc5e^%aYmz$?$1<@r#V
z?$9C?dPl73S(xPBW~^?nwz)-Nq4OM0RwL-`BLlm|Jo-UM2ZqmD59Uai-`%k<hU+?9
zm+SH(VnkF$BAF+RCmyXxjIpT+&V5$KR!n#<Yfii3oR`j2Bs}1{^WN@z_g1$;VwPX_
z*M0Hv2}}<WEK$6FH2Zj4oL{yJlN#K?$+vnhDxY@b?Q2zhAlUz~xcg-0+c40Hh(I(b
zkJ@n6-Dv6Ic}hjTbt^J?fzaYL>>Nhf(4C<tnK|E<`O&aZ461sQ?(NEZajcvgKM_<m
zRM0GGxhWf;Ov=Pu*TS!}GhTf8sksEaG#J|8UYC(U>_zdwL5i7bea%bW_pP~NN6qY`
zaIH!Dw}KT-7#b{1WUIm@S_*+HilCQbFY^g6Q67sQk^M{LL_A-9AQE<tpNIrOPJcv+
zAJ3w{+3CMNi+)EWudF-S3AT?&^^L;C{Y0L&Q?%zh#$h-fB3IWsp;j?YI4!lryyj<(
zMZ+?;u%M7K3WX;IQ4iCDZU6&a-xHLB8KT&iwqF-fvnv(eTYkuP7dyFY<qeYoIy88l
zKb9`pfOyBj3j}T#Kt?)@!g>0MBkc_<L^n!5Qn)coS>?$aj9~8B=AthpWT?H%pNv6!
zBYuO`EfGUP^qi4fM6wvcHX2m2r<9V~s~g!ho?e0P&sS!1Qc5qhIwxJ<l+SFuu^1!y
ztfKN(M!0%Z)7j$&+8oMO)E$e}MBWdCL!PfmDa8df&i#;UPu#Ts#is~A{pC{-2hZPV
z<WCy^dqDg<Mt(;la-xzFlJft*p^@*>$WLtpgox!oF0}syM}CcLS6=#wO8%Rd;9b={
zo#D_(*4`Y7=!?LDa8PFjN6&B=CkmH=%&zboB9tmSnVLBvV8|U=Q#kapa`F6om%6Ki
zgRPkz0+3z7t-l0~-_IZbuC4r#QnUg3>~P2e;$jEDRTls!9}j>V4gd`ee-S-?y~+O5
z+ONX-|K_59peF#^k9DxfTn+IZQ<{R|;1G@!5RmH1|7za_p<l4ccWvsI(}a*>ew?Pi
zo(G`s8V<s7|He=d_U|o$01hrb?%%YT9W7rMjd8-vGrzrErs-{S_ZlpT!>XIC;RHcg
zHyI^fb*kLFgYqP0Y6M-^MB+|cR*)5H6f!cfs1g&!o=XyU%$`MzvtII)0M|+rv+yoU
z4xp|eh%DQ1J+B5>_OVv_(#~6HZ{G7{5V|%vncDm@jy{K1PYkfv|B_?#y-;0obPq49
z%V@s<xyQyttr-2zZ8ZaR_qTJWxw~nKrlcA5??D!-ijD&*tkY29Z@{`5b5s+1?+*z*
z-l)`u_X56QZK32OMTE38y6pyqkQ+WWY*!|D;5zs^nZs0A`VQc=`)*{2>l(^ipQXK+
zL)V6(5^9@o3iZG;B5@W<jSkTosRp@Pt)?D2XvXcgo}{_t)J&_}d}GdheE3H+o0Q4x
zeJ-aU!(=Wd!=mo2>$iC=UC&oP&9aEgswN5ZP~Eq$Ru2?zJESOco+?9a2q9Xb8DxlW
ze?t0<P0e7h9H<%Jj^*op7*oujF}rg2J_!c7m(<;exDSr_Jya$iVw}h+aOL-Gtntt$
z%XKXr{X4ZY1bZRMJ~_wD{u(<Qg#M}C^VU2-E4hPi2nUz0aR~{PC87FbJRhqn02`L>
zlS2IZ_X1H32dt~loy&+mzJ9~YagH@%nWiPl-H=QnPGoVAT(K<MFYk8S6US!4(*5F2
zuBMnSH4YaKm+qX9fzM}O)~_EP&3GT?d1=a7y{mqYKS1W2V#^s&ef}gdHdX8Z{}mk#
zMQTcH^jMzgwXU^fOO}|-4M#tIHC%23C3mCp*;P)~-doS(tI?%1VSQ1l+`Z7zf=_o7
zhth?IK5CR^v>W<}6v?ml(&QhC84SsLQck|mw2o`iFMFz$1Cdx$Y!cAZ^H;+XD<B*>
z(PJi|jt9%4ncYeKq~d?1S^HJUn^i`HPw1sZPKCRgr`fo1oe(ls0k9b`n*Yhyv%Eh~
zLCfs5(SVouGeb)|j;-6Z%3&@IsxEgnk}M0m<E<1Dxzf{ucrAfPB~D!8EE*?8(viL}
z!aagL8EBll{@UXE_m=o-GK%Ws^w90+x5HoWXC7A2$@$QzQjba1>6#b$@5aA;xI_N(
z6#8{tKg!Q2ewBLX(*Il==b7@(81pAm>)a^o2nQ~dA#z4jLFECL$ADLwg?jd;*Xu)+
zMn#7Wq|z-pWS^;KWJ&B<1`ift&#-c1Ka(caV_`~;AJ|#rteF?Ia#Ib|g}Nn=n>Y7$
z#D7I^mMkLh#^q^Nq8uxaRugJ|6chLO`GDG}sPu3MH~&qeob(A<mN7Y8q6q1M+%BnL
zpknN#3%Ok^pYjL8k$j9>P0uxmT}L78QCPLQuJT6dE{Q2B;QLHs8LA8iEll-IAI{NB
zcx2sQ@R~r9jES1RG`-wQ4z-=1=TUy!QW|)-Xne%}K&PftzScy4%zhl=nIPsuu+Zi6
zx-ezn{wGRjlsX?>WDWI*srLjzkYd#c#*=m1Fg>1K&<jm!Ut2AGDA}xcF6EP<9T;bC
zQK%7F)7#~QG+2rG{pT{dBb=MniOGCUM)!57Z`tH{_7qN6+hl!Q_w193)$3Gy_$Cp>
zS4+vBGe&lh#rm0JVv~xTcrol;!&>o14q#a6g=8u_3GNAhM<~<;u$`&5G5_#cve5ZM
zor=*-+|h(e-?)b626wr9J{j#0S8IwlNtGLK`Hx-OA=4eV6^lQIUUI*wUD*7xsn*ta
z&y0E%mCVP-D)as(XQ}UDst)b(VChl0LwV0#y`?nG1Vvw-EY-S;4IT^JOFyv#@&v-s
z$qwkezx|@{g!5xIgO#FiA$nVlqBpiq#ghg~3{LwmJK~C?OZ?7XfS=)&iK2%@J*(B7
z(9)H~&oUG1{^}~13zd<1gR5aByc6Tc^I5khCn!|4Z0=p?+`v}u`vFD<U9FmVo#j2D
zp)ikqZMuu65zeA%u8Y@Ul(0<SFLQxILA*O1#iT2rnPG*Z=geUlm=B&T#ze+M>^~+-
zGxbS`Anty9^aKzcv3%g<^F=iJf;|Y|$szLK!HOlT-(K5Z5O2H3P?S6Mff9z_Ti*Md
zkEw@!*RAL0!^E{$7H5M$i^67u_sLciVNi+|a~CwnEDl3DW0!)JD1+C7Q<K{xXKGbE
zl-?AtC@Vll;BAxh@B?`m8wDE&KLy*%{bjXGzqsKyHR25%Zo5h4?rz2(iJZHyKWK(D
zwU7&(uU&s&YjG|Z_K{K;@`zH{Opv@)nP#efys!bSvTmTP;p@9KSPjwV8bZMikCdvA
z0|<{ErLxe*JLT17SqwiC5g83JQNgyU*UUyxbKCH4ShxCT7}AY}#j%(|<6z>gm3<0`
z;k)ulV31qsGjW?>oliyL6)#C*`1fT{@hf)U;d5v;C``T(sPuX=eHg3UpdcNw;M4P1
zGZI5P`W-$<E4me1*13!KzJF`MbMO5C+_$JYWUXq&V`_9MhK-rSE}8N?z+LAhBN!3P
zB(&uu119B*pIb#JjFXj3FU#}Afd@he$-d^^wKB*%*QP@t>WkVS%`P0o4-<spQ#gYh
zsUh_*B!*bK%OYo_=I9l0A!H1#NxhB9z+B96LLaspqm&Eaxz3`J$Wcu~4cHRw1Bow$
z)!x{EEu(@>RFbIgnfi15PEP!S#9N!ik<8o%-AO)W>dNK@Tb*2yvh>NFcT(5K%$_7A
znNJd@CmDB-M?x`x55m-7no{*|?gl-@i%dloUeS-dSK+XAPpC16SM=eM)Pc?`*5eZ<
z<5F75NHJIvO3V#n(KgFM)_r4_H;OTC^b4V?8$C4)Cr+t@W(M{hJv6B|tfZrH8LA;i
zn76ATKtb{j)=wK?WCQN8GPY0PBI@Tx%cIYuJ8nI|Ihg(&i~7a3F)6N#A}v6nmtzzc
zibP46Cy%{>oHreJmsG(FCwwu7C=WBfor0njsdx-Ys+vYlNtCFgL&mgGvTeXYw!EB(
zmpYChAQ&8@JmDtw+Oqq02Q18)K9P{Y8i?}(BQd~-MWBwEHWpR1&1sguE)I7U6{ztF
zo3lc#<ANOhTj8}Yu()IQWPQk0+V1!4;}uf+!3aP+>^~?0=!b6kOJy$Trz-YusOmSC
z6)SJk2_Wd+^9$#*$B$sTo`3yjr7auZXVFnVq+G~d&b!3UJq;BxX4i#^S{&De+5BiT
zFs^i8>rRC3+xRt#r2SEiYpIo}J}Fe|<v9Jd6<NVun@%$aSWWr0i?}qKJX$>Ro!fuw
zz8y<j{1X+9*OxHJT?kRPyx&=>H6e80pw@^lyz3tt9EF*tA9jAcK`>*&Q+I2Xs~n;G
znwzE*2)V#M!*$<}-lgHaMNrvNn@qJlf(_ZyyD$1b_I>Qnsh066@wq#vVK0}i4>JH(
ztUU;y2^%3X^Uwb_*XZi;&~g9fi@sB|P-U;0KMB{B%YL|*llO1${;H?_gxCL3{#8~L
zlhc;DCt+p{)^b&cr#P=5GXg085PiSz0wDJNs`>t<cOqK;5PgB{aA3{C$qi4t^6<e$
zSP%#3zv;ewpzkv8Pu-V;>uLr+bzj6VSEm0i{DJ_W@7dFz82$&<|6L9LKHlGHRCp2>
z;i@a^PnPu`T=O3lUrxTO0sg^`K;Ow9f*AZ{DF0Kzmjf=B|1S1&z@=2g8}ajVFE<A_
z0P()!F~5qeS9jR2A}i=uFZ>`f-`URp3XuVE@$&vgWX3gYG&CkDzwOSS^aDC@I(P#l
zhDBe$K~g2aEW#ox&4QH}qm*flff=QO!-iDoqn3j^jdPOk5JATEf``>2#5(iptVa^w
zaUTp9RMkn@S=;2w5-Uod#X;Ydmyg%Yv=|Cm45rtdoXlCId&@H9+|UpQe4I0r7yPVa
z%M@2HMs`LvpsxdTn)ac`Eco_`tn~8$cG?`%nUwCww{jNV;y3)4_~U>g4qnLw$J(W<
zK^B-dRO%gi!(*vk<Q$?FW6@TfE#!@27LS5D;ECC)o;wX1nD6PQYAhW$a#Saj&1`B3
zia{g2-F_W~$1?6(O>6@Fjf#o0dSQC`)CoR^{dI&N1f6&K>z0HVmg)N&!+0}A-R};c
z4U!(~(?0a8p8`oZ^=~#kink{6v}`op5w<j*>52S2maty5{<$_T7#()fP>^5fm6J<9
zuQhKyJU6Ub%uKlf3TBFM^H_-tx#f({k{)M2BeW6|<!>UJOjgg!veq`w<s%5>V-w(F
zV*!>lf)ytMrZ)1+p!U-#nXx>su%=!NqLZUnFF%NMS=-zAsNLHyuwW8W(Nj6!3cfCp
zohW5DcJmW2muhakcC7L`HrQ?RaZzF-wzJsQJMMMa_MLoc7d^pARg4sz5hj(AN(s6<
z=?^BcizS^*t?gMxt${sBwzU+ak=!eg+;~u=T<+){;N3cnZMB%9=4z0mocplASj<fA
zbq!f*N77<m^AA*bO#w=r!=%L~G8cFyg+dy)nj7n!^zyYgMjv_VyQ&5EJ5{+`3)GH|
z%&XdQrRSHie-@m{PA4^_b&1!?Re5;p<hJIC>?KsbT<HlkOD=d!mnA_*nV>xX?7_f>
zAB<i$eJ6v*@|gx}o=_rp#4Ht?X3*_=ZGmjV8ry<&7b*^wK>RvAhWAqzA5_216N_Mt
z%kPF7M|KsKk?geTlJ=Lg@yOLOHCZR<e3aQ$7nM}2k<4TY_i<5}aEegpj2f}766MF$
zgA|YQx5*@W^ct6ujZ3V`<epZ)$O(1{d%9TY)jL);`83;J&NSCim2=XD`Y3hmi2;?^
zgB4;v8z<t^CwgUMW+Ib!hF5GS27I8-$7A$EL*Xk=i7s)y(zKL06s_tjKG(XsDl&t-
zUM!oB7gCzb)PP|R&y`^jvQLN$Njy_7fvhJ+6OOlDwv&?da)&)4^E0q-V9}}7O+I|f
zW_8@p@@b53Yq@4KEnipGNmsJU=ylma)Tghd`@;`aS_rVCPbTIzkA^PwKkh0xk(}_-
z`3!B?OPq{0#X_FY_-q*%m*jrAaaoc57WatqfOG4{Ag-VG?u7WwTPl<xz0})zc5S6n
zc{grdN~Idb&%34)I;`w5y-{dm5w_>wM_X}*EhzY`wHEr#u{$=~mo^Q4>5OUl>c|q%
zg1T=38^3?fj6uh7krZ_A`SmeUVb+M1RTP7-cXSQ07jsPwOwR)yZ=$uhuekfAS@^8C
zLth+-3ix&K-roIK;&|Ne;aX2+0bBWe);w+nJ8M3;v7#OM5aidk{&{6S`pu@Hl=Q=M
zL9{*g_a#v((4)5EJ87nA#^(u+X$Qox7)mJUy)g}>mLpnf@u;rAD}8ukrhJB?OxnC;
z!mW0KLK-i1v!{QXm8O>)FVzx!-V-ApNe4=y6gH;$yc}??Vuxqk#%LSrRQ!!8#lS-|
z#g8k%H*$~T81&|2`wiHxAzqyT+AEqb<>UfgW`%SW>~61V3f$O@@WfY+DaO~kj?oKe
zZUm#>t`9M}w{ok+u(@-8_8<qfdA&Pv0I$xBom*gE3~2~ztv$rSJwBw7I#X2)EAk)I
zIO0&vv=g*2VOW{A2ragt9KsP#?<u8@z}>R54yN9+)4AQr&o<E~TgIhI9mo|qt}I<N
z9$Cn+Pb-BBA!T^ARlI3^M3UP?wQ!tM!mg&7?D4efX;pjRDp-dsJU{XlvIcr3Z$LN_
z8L*F1nVqb2$q{xQM#I&D6a(xl0V)T?sP^rH)<A3Q<`&tE0niXI(@JV2ls!B`Ow$ob
zGTVdmu9Q$pkTk_;{^4gD5~&;!=aL|!BzQ?Mme9BcjD-5lVA6S|x}v$@b|)iDqDsM3
zp~t-hODK-cM*CFh!!$7CfJ+#58u!*iMp+FXEp{}zia2h-Ble(`;<;CA6fn1AoPt~`
zruxP3wBZtknKT(gyrX}p{yd$CRFk?{V_b&0`i4^8G<!9h+Kvn*Se-eNkQ+z2zIe%v
zahCml@nfnr2{nAi0sy$EnCLYHe`kJZ3dM5!qG$n?AX%^i4yT>9F|PtOI{Px^w+_PF
z{3{0g9>Z(_Z|Jit$gA8w#}RLLtkbso@`Vu+bgZvR)~0E2GROtxm=N?jaCm_XmKRMV
z!Y0CKcwZ3ih6IE0Bh=iak~|(uw!tu6b1)br=ENwpc<vifm~ehTmk`sb^pQ-LQw~}~
z1=ey8=)Y2x|Kd;axy635C3>m?>CCpR>R-Uz4^b2zj)A;ff6M2fAFBGVSox>i`rol~
zTR6N%B6ROT7tO3V@PGzI{^Y6nEBP8<I8xTKluAw#fA1HC%d#EN-hb(TC~!FE>-K@T
z2Z~qFjXD@_me$dZ4sIQeFCx`}j=X<1tH6()yyTZotVH_~kb?e!CBk``6pxM@$Abh3
z&@x5uO||48d@`ik()pzoi&^y64B7pr^Et`x7a0Vmfy(IvP0Cx$5NP)h6`{t)mIWj6
z{Xtb~Q$>dGg1&LF6{k9s-iuLpYlk&$XUECuGL8Ij?@Z|2aj*8B8upKZX2iF8$l8eC
zB$A5H6**lXaV5av+8+gvpdZ-=xEn#Bzw<gA|1P)xM@%dssw1f+`hS3l|Ez=fe*qGI
z&!+!`#2mlM+CM46@0j?P?8A5T{Sz1eyL<lc<{Yk0soK?)|H?UVv%|{{5QQec<{Eyz
zeg4x*5%%xJB>!!RKk@s2%`jZ~_>T<3)$RN%F#q#RAc|bSb1-=;Q$()f>TdswMu|gR
z5p}P>BvarF>t~8W9RBT9c!3Y#4g_rF1O)(Dft;+|fIF5hE)M)`@G@Sw(ZUJrU}*($
zW`#OgFn_0WDo#*SR|x#uu>b4bez}%@CG+1sEDnYj`9LkM&dt?o{wmb{*~!Dn%p4oe
zWH_<e|9-*qL|hzP0CT{18<2|~;Vi@pVE>nmor9Mb?y%o&?Cd}|!TR0C!OQhWUv3}|
ze5${-1HxJAAMJR#f$;m~_ja7T+<&y=WaorGvwv&H&I#iAV=i1=f3AZIZv1CoE?#)?
z%s<BC;^q5eJZ>&{Rr5dE@$ewDmfz>X!^8Q{b+GfY!>Rc{+VQd@3U+?)%ge$2PhYX~
z^6>t1JRrQ9@1NtrZ9IRr<6{5kc!;ilv;(no!LRn;<^tqoN0bcx(*|d~|Fm(yOV0kV
z{jpvkCpX6*bKzo#s}=v~%LPJ635XZs`g3st!|TeNuD(pJZslc$xONfDSq%zx0U!qa
ltCSp(Z-rNaU2Vg!cV`!{lgriB4qq-0C-$vdlFCxp{{tHIO?ChP

literal 0
HcmV?d00001

diff --git a/doc/related_pages/hector/hector.tex b/doc/related_pages/hector/hector.tex
new file mode 100644
index 000000000..af3f98224
--- /dev/null
+++ b/doc/related_pages/hector/hector.tex
@@ -0,0 +1,929 @@
+%\documentclass[12pt]{article}
+%\documentclass[12pt]{scrartcl}
+\documentclass{hitec} % contained in texlive-latex-extra
+\settextfraction{0.9} % indent text
+\usepackage{csquotes}
+\usepackage[hidelinks]{hyperref} % doi links are short and usefull?
+\hypersetup{%
+    colorlinks=true,
+    linkcolor=blue,
+    urlcolor=magenta
+}
+\urlstyle{rm}
+\usepackage[english]{babel}
+\usepackage{mathtools} % loads and extends amsmath
+\usepackage{amssymb}
+% packages not used
+%\usepackage{graphicx}
+%\usepackage{amsthm}
+%\usepackage{subfig}
+\usepackage{bm}
+\usepackage{longtable}
+\usepackage{booktabs}
+%\usepackage{ragged2e} % maybe use \RaggedRight for tables and literature?
+%\usepackage[table]{xcolor} % for alternating colors
+%\rowcolors{2}{gray!25}{white}
+%\renewcommand\arraystretch{1.3}
+\usepackage{doi}
+\usepackage[sort,square,numbers]{natbib}
+\bibliographystyle{abbrvnat}
+
+%%%%%%%%%%%%%%%%%%%%%definitions%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\newcommand{\eps}{\varepsilon}
+\renewcommand{\d}{\mathrm{d}}
+\newcommand{\T}{\mathrm{T}}
+\renewcommand{\vec}[1]{{\mathbf{#1}}}
+\newcommand{\dx}{\,\mathrm{d}x}
+%\newcommand{\dA}{\,\mathrm{d}(x,y)}
+%\newcommand{\dV}{\mathrm{d}^3{x}\,}
+\newcommand{\dA}{\,\mathrm{dA}}
+\newcommand{\dV}{\mathrm{dV}\,}
+
+\newcommand{\Eins}{\mathbf{1}}
+
+\newcommand{\ExB}{$\bm{E}\times\bm{B} \,$}
+\newcommand{\GKI}{\int d^6 \bm{Z} \BSP}	
+\newcommand{\GKIV}{\int dv_{\|} d \mu d \theta \BSP}	
+\newcommand{\BSP}{B_{\|}^*}
+\newcommand{\GA}[1]{\langle #1	 \rangle}
+
+\newcommand{\Abar}{\langle A_\parallel \rangle}
+%Vectors
+\newcommand{\bhat}{\bm{\hat{b}}}
+\newcommand{\bbar}{\overline{\bm{b}}}
+\newcommand{\chat}{\bm{\hat{c}}}
+\newcommand{\ahat}{\bm{\hat{a}}}
+\newcommand{\xhat}{\bm{\hat{x}}}
+\newcommand{\yhat}{\bm{\hat{y}}}
+\newcommand{\zhat}{\bm{\hat{z}}}
+
+\newcommand{\Xbar}{\bar{\vec{X}}}
+\newcommand{\phat}{\bm{\hat{\perp}}}
+\newcommand{\that}{\bm{\hat{\theta}}}
+
+\newcommand{\eI}{\bm{\hat{e}}_1}
+\newcommand{\eII}{\bm{\hat{e}}_2}
+\newcommand{\ud}{\mathrm{d}}
+
+%Derivatives etc.
+\newcommand{\pfrac}[2]{\frac{\partial#1}{\partial#2}}
+\newcommand{\ffrac}[2]{\frac{\delta#1}{\delta#2}}
+\newcommand{\fixd}[1]{\Big{\arrowvert}_{#1}}
+\newcommand{\curl}[1]{\nabla \times #1}
+\newcommand{\np}{\nabla_{\perp}}
+\newcommand{\npc}{\nabla_{\perp} \cdot }
+\newcommand{\nc}{\nabla\cdot }
+\newcommand{\GAI}{\Gamma_{1}^{\dagger}}
+\newcommand{\GAII}{\Gamma_{1}^{\dagger -1}}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%DOCUMENT%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\begin{document}
+%\preprint{}
+
+\title{Streamline integration as a method for two-dimensional elliptic grid generation}
+\author{M.~Wiesenberger, M.~Held, and L.~Einkemmer}
+%\email{Matthias.Wiesenberger@uibk.ac.at}
+%\affiliation{Institute for Ion Physics and Applied Physics, Association EURATOM-\"OAW,  University of
+%   Innsbruck, A-6020 Innsbruck, Austria}
+%\affiliation{Institute for Ion Physics and Applied Physics, Association EURATOM-\"OAW,  University of
+   %Innsbruck, A-6020 Innsbruck, Austria}
+%\address{Institute for Ion Physics and Applied Physics,  Universit\"at 
+   %Innsbruck,  A-6020 Innsbruck, Austria}
+%\address{Numerical Analysis group,  Universit\"at Innsbruck, A-6020 Innsbruck, Austria}
+ 
+\maketitle
+\begin{abstract}
+  [This writeup is based/copied on the recent article \cite{Wiesenberger2017}] \\
+We propose a new numerical algorithm to construct a structured
+numerical elliptic grid of a doubly connected domain. Our method is 
+applicable to domains with boundaries defined by two contour lines of a two-dimensional function. 
+Furthermore, we can adapt any analytically given boundary
+aligned structured
+grid, which specifically includes polar and Cartesian grids.
+The resulting coordinate lines are orthogonal to the boundary. 
+Grid points as well as
+the elements of the Jacobian matrix can be computed efficiently and up to machine precision. 
+In the simplest case we construct conformal grids, yet with the help of weight functions and monitor metrics we can control the
+distribution of cells across the domain. 
+Our algorithm is parallelizable and easy to implement with
+elementary numerical methods. 
+We assess the quality of grids by considering both the distribution of cell sizes and the accuracy of the solution to 
+elliptic problems. 
+Among the tested grids these key properties are best fulfilled by the grid constructed with the monitor metric approach.
+%All in all, the grid constructed with a monitor metric
+%yields the best accuracy and cell sizes compared
+
+%Simple flux aligned orthogonal grids are suitable for the solution
+%of flux aligned problems, but they exhibit 
+%very large ratios of maximal to minimal cell size. 
+%If we construct a conformal grid, the aspect ratio of the cells is constant by construction. However, the variation in cell size is large 
+%and the errors of the elliptic equation are high. 
+%The adapted grid and the grid with monitor metric yield smaller variation in size and smaller errors, where
+%the monitor grid overall has the smallest ratios of maximal to minimal cell size.
+%The errors and cell sizes are competitive with a previously suggested near conformal grid.
+%
+%It is based on the integration of the streamlines of the two vector fields that
+%form the basis of the coordinate system. 
+%These vector fields are either built directly from the given function or from the solution of a suitably chosen elliptic equation (which 
+%can be solved once an initial grid has been constructed). 
+%We are able to construct conformal, orthogonal and curvilinear coordinates.
+%The method is parallelizable and the metric elements can be computed with high accuracy. 
+%Furthermore, it is easy to implement as only the integration 
+%of well-behaved ordinary differential equations and the inversion
+%of a linear elliptic equation are required.
+%All our grids are orthogonal to the boundary of the domain, which is the major
+%advantage over previously suggested grids. 
+\end{abstract}
+
+\section{High precision elliptic grid generation} \label{sec:geometry}
+
+Given is a function $\psi(x,y)$ in Cartesian coordinates. 
+We want to construct a grid on the region 
+bounded by the two lines
+$\psi(x,y) = \psi_0$ and $\psi(x,y)=\psi_1$ with $\psi_0\neq\psi_1$.
+The derivative of the function $\psi$ may not vanish within this region and on the boundary and 
+we further assume that the region is topologically a ring. 
+Note here that this excludes the description of domains with an X-point (saddle point) or O-point (local extremum). 
+The numerical grid is described by a mapping of the discretization of the rectangular
+computational domain $(u,v) \in [0,u_1]\times[0,2\pi]$ to the physical domain $(x,y)$. $u_1$ is an unknown, which the grid generation process has to provide. 
+
+%Together with 
+%this mapping described by $x_i = x(u_i, v_i),\ y_i = y(u_i, v_i)$ we construct the derivatives
+%$u_x(u_i,v_i),\ v_x(u_i, v_i),\ u_y(u_i, v_i),\ v_y(u_i,v_i)$ in order to 
+%transform tensors to the curvilinear coordinate system. 
+%Here and in the following we use the notation $u_x\equiv \frac{\partial u}{\partial x}$.
+%The index $i$ is given by $i=0,1,\dots N_uN_v$, where $N_u$ and $N_v$ are the number of points
+%in $u$ and $v$ respectively. The discretization of the computational domain need 
+%not be equidistant. We will for example use the Gauss-Legendre nodes 
+%needed for a discontinuous Galerkin discretization. 
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{figure}[htbp]
+\centering
+\includegraphics[trim = 0px 0px 0px 0px, clip, scale=1.0]{./drawing_magenta}
+\caption{
+  Sketch of the coordinate systems, coordinate lines and basis vector fields 
+  involved in our method.
+}
+\label{fig:sketch}
+\end{figure}
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+As mentioned in the introduction and illustrated in Fig.~\ref{fig:sketch} the idea of our algorithm are two consecutive coordinate transformations constructed by streamline integration. 
+We denote the first transformation with $x(\zeta, \eta)$, $y(\zeta, \eta)$, where
+the $\zeta$ coordinate is aligned with $\psi$ and $\eta$ is an angle-like coordinate. The solution of the 
+elliptic equation on this flux-aligned coordinate system is denoted with $\bar u(\zeta, \eta)$. 
+The second coordinate transformation is then denoted $\zeta(u,v)$, $\eta(u,v)$ with $u$ aligned to $\bar u$.
+
+When deriving the algorithm we use basic methods and notational 
+conventions of differential geometry.  
+Here, we recommend the excellent Reference~\cite{Frankel} as an introduction to the topic.  
+We do so since in this approach the separate roles of the metric tensor, 
+the coordinate system and its base vector fields are very clear. This is 
+paramount for a concise description of our method. 
+
+
+In this Section we first show how the
+basis vector fields can be integrated to construct coordinate lines
+ with the example of orthogonal coordinates in Section~\ref{sec:orthogonal}. This follows
+an introduction of some helpful quantities, our notation and streamlines
+in Section~\ref{sec:preliminaries}. 
+%In Section~\ref{sec:orthogonal} we review the
+%construction of orthogonal coordinates via the integration of 
+%streamlines, that is ordinary differential 
+%equations. 
+In the next step we transform, discretize and solve an elliptic equation on the flux aligned
+coordinate system.
+The solution $\bar u(\zeta, \eta)$ then takes the role of a new flux function in the flux aligned coordinates. 
+We can therefore repeat the streamline integration method in the flux aligned
+coordinate system in order to construct coordinate lines of the final $u,v$ coordinates.
+We discuss three examples of this method.
+%In Section~\ref{sec:laplace} we discretize and 
+%solve the Laplace equation.
+%with local discontinuous Galerkin (dG) methods. 
+In 
+Section~\ref{sec:conformal} we consider the simple Laplace and the Cauchy-Riemann equations~\eqref{eq:CR}
+in order to construct conformal coordinates.
+In Section~\ref{sec:adaption} we modify the elliptic equation to allow grid adaption, before
+we introduce the monitor metric in Section~\ref{sec:metric}. 
+Finally, we present our algorithm in Section~\ref{sec:elliptic}. 
+
+\subsection{Preliminaries} \label{sec:preliminaries}
+In a curvilinear
+coordinate system $(\zeta,\eta)$ the components of the metric tensor and its inverse
+take the form 
+\begin{align}
+  \vec g(\zeta,\eta) = \begin{pmatrix}
+    g_{\zeta\zeta} & g_{\zeta\eta}  \\
+    g_{\zeta\eta} & g_{\eta\eta}  
+  \end{pmatrix}
+  \quad
+  \vec g^{-1}(\zeta,\eta) = \begin{pmatrix}
+    g^{\zeta\zeta} & g^{\zeta\eta}  \\
+    g^{\zeta\eta} & g^{\eta\eta}  
+  \end{pmatrix}
+  \label{}
+\end{align}
+We denote $g = \det{\vec g} = g_{\zeta\zeta}g_{\eta\eta}-g_{\zeta\eta}^2$.
+For Cartesian coordinates $(x,y)$ the elements of the inverse metric tensor are 
+transformed by
+\begin{subequations}
+\begin{align}
+  g^{\zeta\zeta} &=  \zeta_x^2+\zeta_y^2\\
+  g^{\zeta\eta} &=  \zeta_x \eta_x+\zeta_y \eta_y\\
+  g^{\eta\eta} &=  \eta_x^2+\eta_y^2
+  \label{}
+\end{align}
+\end{subequations}
+since $g^{xx}=g^{yy}=1$ and $g^{xy}=0$. 
+Given $\zeta(x,y)$, $\eta(x,y)$ and its inverse $x(\zeta,\eta)$, $y(\zeta,\eta)$ recall that their Jacobian matrices are related by
+\begin{align}
+  \begin{pmatrix}
+    x_\zeta & x_\eta \\
+    y_\zeta & y_\eta
+  \end{pmatrix} \bigg\rvert_{\zeta(x,y),\eta(x,y)}
+  =
+  \frac{1}{\zeta_x\eta_y - \zeta_y\eta_x }\begin{pmatrix}
+    \eta_y & -\zeta_y \\
+    -\eta_x & \zeta_x
+  \end{pmatrix}\bigg\rvert_{x,y} 
+  \label{eq:inverse}
+\end{align}
+With the rules of tensor transformation it is easy to prove that the element of the volume form $\sqrt{g}$ is related to the Jacobian via
+\begin{align}
+  %\mathcal V^2:=\d x\wedge \d y = (x_uy_v - y_ux_v)\d u\wedge\d v = \frac{\d u\wedge\d v}{u_xv_y - u_yv_x} \equiv \sqrt{g} \d u\wedge \d v
+%\iint \d V:= \iint \d x \d y = \iint (x_uy_v - y_ux_v)\d u\d v &=  \nonumber\\
+%\iint\frac{\d u\d v}{u_xv_y - u_yv_x} &\equiv \iint \sqrt{g} \d u\d v
+\sqrt{g} = (x_\zeta y_\eta-y_\zeta x_\eta) = (\eta_y\zeta_x -\eta_x\zeta_y)^{-1}
+  \label{eq:vol}
+\end{align}
+The components of the gradient operator and the divergence in an arbitrary coordinate
+system read
+\begin{subequations}
+\begin{align}
+  \left( \nabla f \right)^i &= g^{ij}\partial_j f \\
+  \nabla \cdot \vec A &= \frac{1}{\sqrt g}\partial_i \left( \sqrt{g} A^i  \right),
+\end{align}
+  \label{eq:arbitrary}
+\end{subequations}
+where we sum over repeated indices $i,j\in\{\zeta,\eta\}$ and define $\partial_\zeta \equiv \partial/\partial \zeta$, $\partial_\eta \equiv \partial/\partial \eta$.
+%Note that 
+%\begin{align}
+%  \d x = x_R \d R + x_Z \d Z\\
+%  \d y = y_R \d R + y_Z \d Z
+%  \label{}
+%\end{align}
+%and
+%\begin{align}
+%  \partial_x = R_x \partial_R + Z_x \partial_Z\\
+%  \partial_y = R_y \partial_R + Z_y \partial_Z
+%  \label{}
+%\end{align}
+%Figuratively $\d x$ and $\d y$ are the one-forms that form the lines of constant $x$ and $y$, while the coordinate lines are given by the streamlines of $\partial_x$ and $\partial_y$.
+%Recall that the coordinate line $x$ is implicitly defined by the points with $y=const$ and vice versa\cite{Frankel}. 
+% Through the inverse derivatives
+%Eq.~\eqref{eq:inverse} $\partial_y$ is related with $\d x$ and $\partial_x$ is 
+%related with $\d y$. 
+%
+%The contravariant vectors $\nabla x$ and $\nabla y$ are the vectors that are everywhere perpendicular to $\d x$ and $\d y$. 
+%$\nabla x$ and $\nabla y$ are in general not parallel to $\partial_x$ and $\partial_y$, yet\footnote{ Some textbooks use the notation $e_x = \partial_x$ and $e^x=\nabla x$}
+%\begin{align}
+% \partial_y \cdot \nabla y = \partial_x \cdot \nabla x = 1\\
+% \partial_x \cdot \nabla y = \partial_y \cdot \nabla x =0
+%  \label{}
+%\end{align}
+%Now note that in order to reconstruct $R(x,y)$, $Z(x,y)$ we have to integrate the 
+%coordinate lines i.e. $\partial_x$ and $\partial_y$.
+Finally, we introduce the 
+geometrical poloidal angle 
+\begin{align}
+  \theta (x,y) = \begin{cases}
+    +\arccos\left( \frac{x-x_0}{\sqrt{(x-x_0)^2 + (y-y_0)^2}} \right) \text{ for } y\geq y_0 \\
+    -\arccos\left( \frac{x-x_0}{\sqrt{(x-x_0)^2 + (y-y_0)^2}} \right) \text{ for } y< y_0 
+  \end{cases}
+  \label{eq:deftheta}
+\end{align}
+such that the differential 1-form
+\begin{align}
+  \d \theta = 
+    -\frac{y-y_0}{(x-x_0)^2+(y-y_0)^2} \d x
+    +\frac{x-x_0}{(x-x_0)^2+(y-y_0)^2} \d y,
+  \label{eq:theta}
+\end{align}
+where $(x_0,y_0)$ is any point inside the region bounded by $\psi_0$.
+
+Streamline integration is the central part of our algorithm. Recall that
+given a vector field $v(x,y)=v^x(x,y)\partial_x + v^y(x,y)\partial_y$ its streamlines are given by the equation
+\begin{subequations}
+\begin{align}
+  \frac{\d x}{\d t } = v^x(x,y)|_{x(t), y(t)} \\
+  \frac{\d y}{\d t } = v^y(x,y)|_{x(t), y(t)}
+\end{align}
+\label{eq:streamlines}
+\end{subequations}
+where $t$ is a parameter. Recall here that in differential geometry 
+the directional derivatives $\partial_x$ and
+$\partial_y$ are the base vector fields of the coordinate system\footnote{ 
+  Some textbooks (e.g.~\cite{haeseleer}) introduce the notation $\vec e_i := \partial \vec x /\partial x^i$ 
+  and $\vec e^i := \nabla x^i$. 
+  While this formulation is suitable in many situations we refrain from 
+  using it since it mixes the metric into the basis vectors through the use of 
+  the gradient (cf. Eq.~\eqref{eq:arbitrary}). 
+  This is unpractical for our purposes.}.
+Now recall that if we have a function $f(x,y)$ 
+such that $f(x(t), y(t))$ is a one-to-one map from $t$ to $f$ we can re-parameterize
+Eq.~\eqref{eq:streamlines} by
+\begin{subequations}
+\begin{align}
+  \frac{\d x}{\d f } = \frac{\d x/\d t|_{t(f)}}{\d f/\d t|_{t(f)}} = 
+  \frac{v^x(x,y) }{ (v^x\partial_xf + v^y\partial_yf)(x,y) }\bigg |_{x(f), y(f)}\\
+  \frac{\d y}{\d f } = \frac{\d y/\d t|_{t(f)}}{\d f/\d t|_{t(f)}} = 
+  \frac{v^y(x,y) }{ (v^x\partial_xf + v^y\partial_yf)(x,y) }\bigg |_{x(f), y(f)}
+\end{align}
+\label{eq:reparameter}
+\end{subequations}
+Furthermore, the derivative of any function $g(x,y)$ along the streamlines of $v$ 
+parameterized by $f$ reads
+\begin{align}
+  \frac{\d g}{\d f }\bigg |_{x(f),y(f)} = 
+  \frac{(v^x\partial_x g + v^y\partial_y g)(x,y) }{ (v^x\partial_xf + v^y\partial_yf)(x,y) }\bigg |_{x(f),y(f)}
+  \label{}
+\end{align}
+
+
+Figuratively, in any coordinate system $(\zeta,\eta)$ the 1-forms $\d \zeta$ and $\d \eta$ (the contravariant basis) are visualized by the lines (surfaces in higher dimensions) 
+of constant $\zeta$ and $\eta$. 
+At the same time the streamlines of the vector fields $\partial_\zeta$ and $\partial_\eta$ (the covariant basis) 
+give the coordinate lines of $\zeta$ and $\eta$ (cf. Fig.~\ref{fig:sketch}). 
+For example, if we hold $\eta$ constant and vary $\zeta$, we go along a streamline of 
+$\partial_\zeta$. This implies that in two dimensions $\d \eta$ and $\partial_\zeta$ 
+trace the same line. The vector fields $\nabla \zeta$ and $\nabla \eta$ are associated to 
+$\d \zeta$ and $\d \eta$ through the metric tensor by Eq.~\eqref{eq:arbitrary} and are the vector fields that
+are everywhere perpendicular to the lines of constant $\zeta$ and $\eta$, respectively. 
+It is important to realize that in general curvilinear coordinates the vector
+fields $\nabla \zeta$ and $\nabla \eta$ point in 
+different directions than $\partial_\zeta$ and $\partial_\eta$.
+The central point in our algorithm is the realization that once we can 
+express $\partial_\zeta$ and $\partial_\eta$ in terms of $\partial_x$ and 
+$\partial_y$ we can immediately construct the coordinate transformation 
+by integrating 
+streamlines of $\partial_\zeta$ and $\partial_\eta$ using Eq.~\eqref{eq:streamlines}. This 
+holds true even if $(x,y)$ were curvilinear coordinates. The components of the 
+one-forms $\d \zeta$ and $\d\eta$ in terms of $\d x$ and $\d y$ form 
+the elements of the Jacobian matrix of the transformation. These 
+are also necessary in order to transform any tensor (including the metric) 
+from the old to the new coordinate system. 
+
+
+\subsection{Orthogonal coordinates} \label{sec:orthogonal}
+In general, orthogonal coordinates $\zeta, \eta$ with $\zeta$ aligned to $\psi$ are described by
+\begin{subequations}
+\begin{align}
+  \d \zeta & = \zeta_x\d x +\zeta_y\d y =  f(\psi)(\psi_x \d x + \psi_y \d y) \\
+  \d \eta  & = \eta_x \d x + \eta_y \d y = h(x,y) ( -\psi_y \d x + \psi_x \d y)
+\end{align}
+  \label{eq:orthogonal}
+\end{subequations}
+With Eq.~\eqref{eq:orthogonal} we have $g^{\zeta\eta} = \zeta_x\eta_x + \zeta_y\eta_y = 0$,
+$g^{\zeta\zeta} = (\nabla\psi)^2 f^2$, $g^{\eta\eta} = (\nabla\psi)^2h^2$ and $\sqrt{g}^{\,-1} = (\nabla\psi)^2 h f$. 
+From Eq.~\eqref{eq:inverse} we directly see that the basis vector fields are
+\begin{subequations}
+\begin{align}
+  \partial_\zeta&= x_\zeta\partial_x + y_\zeta\partial_y = \frac{1}{(\nabla\psi)^2f} (\psi_x \partial_x + \psi_y\partial_y) \\
+  \partial_\eta &=  x_\eta\partial_x + y_\eta\partial_y =  \frac{1}{(\nabla\psi)^2h} (-\psi_y \partial_x + \psi_x\partial_y) 
+\label{eq:orthogonal_linesb}
+\end{align}
+\label{eq:orthogonal_lines}
+\end{subequations}
+i.e. $\partial_\zeta$ points into the direction of the gradient of $\psi$ and $\partial_\eta$ into the direction of constant $\psi=\text{const}$ surfaces.
+Now, the coordinate system is defined up to the functions $f(\psi)$ and $h(x,y)$.
+Note that $f$ must be a function of $\psi$ only since the restriction $\d(\d \zeta)=0$ must hold.
+Furthermore, $f(\psi) = \d\zeta/ \d\psi \neq 0 $ is in principle an arbitrary function, yet we choose $f(\psi) = f_0 = \text{const}$.
+With this choice we directly get
+\begin{align}
+  \zeta(x,y) = f_0(\psi(x,y)-\psi_0)
+  \label{eq:zeta}
+\end{align}
+Note that $\zeta_0=0$ and $\zeta_1=f_0(\psi_1-\psi_0)$.
+The up to now undefined function $h(x,y)$ is not arbitrary since $\d(\d \eta) = 0$ must hold. This is the requirement that $\d \eta$ must be
+a closed form in order for the potential $\eta$ to exist. 
+We can express this as
+\begin{align}
+  (\psi_x\partial_x + \psi_y\partial_y) h = f(\nabla\psi)^2 \partial_\zeta h= -h\Delta \psi 
+  \label{eq:hequation}
+\end{align}
+where $\Delta \psi = \psi_{xx} + \psi_{yy}$ is the two-dimensional Laplacian.
+Let us remark here that Eq.~\eqref{eq:hequation} can be written as $\nabla\cdot\left( h\nabla\psi \right)=0$, which makes the orthogonal grid an elliptic grid
+with adaption function $h$ as becomes evident later. 
+In order to integrate this equation we need an initial condition for $h$. 
+We choose to first discretize the line given by $\psi(x,y) = \psi_0$. 
+
+As already mentioned $\partial_\eta$ is the vector field the streamlines of which give the 
+coordinate lines for $\eta$. We choose $h(x,y) = \text{const}$ on $\psi_0$. To this end
+we parameterize the coordinate line by $\theta$ (cf. Eq.~\eqref{eq:reparameter})
+\begin{subequations}
+\begin{align}
+  \left . \frac{\d x}{\d \theta}\right|_{\zeta=0}=\frac{x_\eta}{\theta_\eta} = \frac{-\psi_y}{\psi_x\theta_y - \psi_y \theta_x}\\
+  \left . \frac{\d y}{\d \theta}\right|_{\zeta=0}=\frac{y_\eta}{\theta_\eta} = \frac{\psi_x}{\psi_x\theta_y - \psi_y \theta_x}\\
+  \left . \frac{\d \eta}{\d \theta}\right|_{\zeta=0}=\frac{1}{\theta_\eta} = \frac{(\nabla\psi)^2h(\psi_0)}{\psi_x\theta_y - \psi_y \theta_x}
+\end{align}
+  \label{eq:etaline}
+\end{subequations}
+Let us define $h(\psi_0)$ such that $\eta \in [0,2\pi]$, that is,
+\[ 2\pi = \oint_{\psi=\psi_0}\d \eta = \oint_0^{2\pi} \frac{\d \eta}{\d\theta}\bigg|_{\zeta=0}\d \theta \nonumber \]
+  or 
+\begin{align}
+  f_0 := h(\psi_0) = \frac{2\pi}{\int_0^{2\pi}\d\theta \frac{(\nabla\psi)^2}{\psi_x\theta_y - \psi_y \theta_x}}
+  \label{eq:definef}
+\end{align}
+Here, we also fixed the constant $f_0$ such that our coordinate system $\zeta, \eta$ fulfills the Cauchy--Riemann condition~\eqref{eq:CR} on the boundary line $\psi_0$. 
+As initial point for the integration of Eq.~\eqref{eq:etaline} we can use any point with $\psi(x,y) = \psi_0$.
+We then use $h(\psi_0)$ on the flux-surface $\psi_0$ as initial condition for the integration 
+of Eq.~\eqref{eq:hequation}.
+
+We obtain coordinate lines by integrating the vector fields $\partial_\zeta$ and $\partial_\eta$ given in \eqref{eq:orthogonal_lines}. We start the construction by integrating $\partial_\eta$ for $\psi = \psi_0$, i.e. $\zeta=0$. This can be done since $h|_{\psi_0}$ is known. 
+The obtained points serve as starting points for the integration of $\partial_\zeta = f_0^{-1} \partial_\psi$.
+In order to get $h$ we simply integrate Eq.~\eqref{eq:hequation}
+%We use Nemov's algorithm \cite{Nemov1988} to compute $\eta_x$ and $\eta_y$ along the coordinate lines. Starting from $\nabla \psi\cdot \nabla \eta = 0$ we differentiate with respect to $x$ and $y$ and get
+\begin{subequations}
+\begin{align}
+    \left .\frac{\d x}{\d \zeta}\right |_{\eta=\text{const}} &= \frac{\psi_x}{f_0(\nabla\psi)^2}\\
+    \left .\frac{\d y}{\d \zeta}\right|_{\eta=\text{const}} &= \frac{\psi_y}{f_0(\nabla\psi)^2}\\
+  %\frac{\d \eta_x}{\d \psi} &= -\frac{\psi_{xx}}{(\nabla\psi)^2}\eta_x - \frac{\psi_{xy}}{(\nabla\psi)^2}\eta_y, \\ 
+  %\frac{\d \eta_y}{\d \psi} &= -\frac{\psi_{yx}}{(\nabla\psi)^2}\eta_x - \frac{\psi_{yy}}{(\nabla\psi)^2}\eta_y, \\
+    \left .\frac{\d h}{\d \zeta}\right|_{\eta=\text{const}} &= - \frac{\Delta \psi}{f_0(\nabla\psi)^2} h%\\
+  %\frac{\d h_x}{\d \zeta}|_{\eta=const} &= -\frac{ 2\psi_{xx}+ \psi_{yy} }{f_0(\nabla\psi)^2}h_x - \frac{\psi_{xy}}{f_0(\nabla\psi)^2}h_y - \frac{(\Delta\psi)_x}{f_0(\nabla\psi)^2} h, \\ 
+  %\frac{\d h_y}{\d \zeta}|_{\eta=const} &= -\frac{\psi_{yx}}{f_0(\nabla\psi)^2}h_x - \frac{ 2\psi_{yy} + \psi_{xx}}{f_0(\nabla\psi)^2}h_y - \frac{(\Delta\psi)_y}{f_0(\nabla\psi)^2} h
+\end{align}
+  \label{eq:etacoordinates}
+\end{subequations}
+%The initial condition for $h_x$ and $h_y$ on $\psi_0$ are obtained as $\nabla h = - h \frac{\Delta\psi }{(\nabla\psi)^2} \nabla\psi$ from $\nabla\psi\cdot \nabla h$ and $\{\psi, h\} = 0$.
+Note that if $\Delta\psi=0$, we directly get a conformal grid with this algorithm. This can be seen as then $h(\zeta,\eta) = f_0$. In~\ref{app:conf-field}
+we briefly study the class of functions $\psi$ that are solutions of the Grad--Shafranov equation and satisfy $\Delta \psi=0$. This, however, is not true in general.
+
+Let us further remark on the sign of $f_0$. It is our goal to construct a right handed
+coordinate system and to have $\zeta_1>\zeta_0=0$.
+The curves of constant $\theta$ surround $x_0, y_0$ in a mathematically positive direction. 
+That means that Eq.~\eqref{eq:definef} implies that $f_0>0$ if $\nabla \psi$ points away from $x_0, y_0$. If this is not the case, we obtain $f_0 < 0$. 
+On the other hand if $\zeta$ should increase from $\psi_0$ to $\psi_1$, we
+need $f_0 <0$ for $\psi_1<\psi_0$ and $f_0>0$ for $\psi_1>\psi_0$. 
+We thus simply take the absolute value of Eq.~\eqref{eq:definef} and multiply 
+by $-1$ if $\psi_1<\psi_0$. 
+For ease of notation we do so also in the following without further notice.
+
+Let us finally summarize the grid generation in the following algorithm; we 
+assume that the $\zeta$ coordinate is discretized by a list of 
+not necessarily equidistant values $\zeta_i$ with $i = 0,1,\dots N_\zeta-1$ 
+and $\eta$ is discretized by a list of $\eta_j$ with $j = 0,1,\dots N_\eta-1$:
+\begin{enumerate}
+  \item Find an arbitrary point $(x,y)$ with $\psi(x,y) = \psi_0$
+    and a point $x_0, y_0$ within the region bound by $\psi(x,y) = \psi_0$ for the definition of $\theta$ in Eq.~\eqref{eq:deftheta}
+  \item Integrate Eq.~\eqref{eq:etaline} with $h=1$ over $\Theta=[0,2\pi]$ and use Eq.~\eqref{eq:definef} to compute $f \equiv f_0$ and $h(\psi_0)$.
+    Use any convenient method for the integration of ordinary differential equations.
+  \item Integrate one streamline of Eq.~\eqref{eq:orthogonal_linesb} with $h=f_0$
+    from $\eta = 0\dots\eta_j$ for all $j$.
+    The result is a list of $N_\eta$ coordinates $x(0,\eta_j), y(0,\eta_j)$ on the $\psi_0$ surface.
+  \item Using this list and $h=f_0$ as starting values integrate Eq.~\eqref{eq:etacoordinates}
+    from $\zeta=0\dots\zeta_i$ for all $i$ and all $\eta_j$. This gives the map $x(\zeta_i, \eta_j), y(\zeta_i, \eta_j)$ as well as $h(\zeta_i,\eta_j)$ for all $i$ and $j$.
+  \item Last, using these results and 
+    Eq.~\eqref{eq:orthogonal} evaluate the derivatives 
+    $\zeta_x(\zeta_i,\eta_j)$, $\zeta_y(\zeta_i, \eta_j)$, $\eta_x(\zeta_i,\eta_j)$, and $\eta_y(\zeta_i, \eta_j)$ for all $i$ and $j$.
+\end{enumerate}
+
+%\subsection{The Laplace equation} \label{sec:laplace}
+\subsection{Conformal coordinates} \label{sec:conformal}
+A conformal mapping $u(x,y), v(x,y)$ has to satisfy the Cauchy--Riemann equations given in Eq.~\eqref{eq:CR}.
+A direct consequence is that $u$ and $v$ are harmonic functions 
+\begin{align}
+  \Delta u = \Delta v = 0
+  \label{eq:harmonic}
+\end{align}
+Here, $\Delta=\nabla^2$ is the two-dimensional Laplacian with the divergence
+and gradient operators defined in~\eqref{eq:arbitrary}.
+First, we note that Eq.~\eqref{eq:harmonic} holds in every coordinate system.
+Let us assume that we have constructed flux aligned  coordinates $(\zeta, \eta)$. These can be, but not necessarily have to be,
+the orthogonal coordinates introduced in the last section. 
+Now, in order to construct conformal coordinates $u$, $v$ we first define 
+\begin{align}
+u(\zeta,\eta):=c_0(\bar u(\zeta,\eta)-\psi_0)
+  \label{eq:ubar}
+\end{align}
+and thus
+\begin{align}
+  %\bar u(\zeta,\eta) = \tilde u (\zeta, \eta) + \zeta 
+  \Delta \bar u(\zeta,\eta) = 0 %\tilde u (\zeta, \eta) + \psi(\zeta)
+  %v = \tilde v + \eta
+  \label{eq:ubar_harmonic}
+\end{align}
+where $\bar u( 0, \eta) = \psi_0$ and $\bar u( \zeta_1, \eta) =\psi_1$ fulfills Dirichlet boundary conditions in $\zeta$. 
+In $\eta$ we have periodic boundary conditions. 
+%With these definitions we get
+%\begin{align}
+%  %\Delta \tilde u &= -\Delta \zeta = -f_0\Delta \psi. %\\
+%  \Delta \tilde u &=  -\Delta \psi. %\\
+%  %\Delta \tilde v &= -\Delta \eta = \psi_x h_y - \psi_y h_x \equiv \{ \psi, h\} 
+%  \label{eq:harmonic_u}
+%\end{align}
+%We can discretize and solve Eq.~\eqref{eq:ubar_harmonic} in the $\zeta, \eta$ coordinate system by any high order method that
+%quickly converges to a solution. 
+%We choose a high order local discontinuous Galerkin method~\cite{Cockburn2001, Held2016} in Section~\ref{sec:numerics}. 
+
+Note the analogy between Eq.~\eqref{eq:ubar} and Eq.~\eqref{eq:zeta}. Now, $\bar u$ is
+equal to $\psi$ at the boundaries and its Laplacian vanishes in between. 
+In fact, $\bar u$ takes the role of $\psi$ in the following coordinate
+transformation. 
+We introduce $c_0$ as a normalization constant with the same role as $f_0$ in the orthogonal coordinate transformation. 
+%Once we have determined $\tilde u$ and $\tilde v$ we can construct 
+%the inverse $\zeta( u,v)$ and $\eta(u,v)$ by Newton iterations of the form
+%\begin{align}
+%  \zeta^{n+1} = \zeta^n - \frac{1}{\det J|_{\zeta^n, \eta^n}} (v_y u - u_y v)|_{\zeta^n, \eta^n}\\
+%  \eta^{n+1} = \eta^n - \frac{1}{\det J|_{\zeta^n, \eta^n}} (- v_x u + u_x v)|_{\zeta^n, \eta^n}\\
+%  \label{}
+%\end{align}
+%which is in principle a repeated interpolation of the results of the elliptic equation. The interpolation is given by and has the same order as the dG method. 
+%With $\zeta, \eta$ at hand we determine $x(\zeta, \eta)$, $y(\zeta, \eta)$ by 
+%the previous transformation.
+Having $\bar u(\zeta, \eta)$, 
+our idea is to construct the basis one-forms $\d u$ and $\d v$  in terms 
+of $\d \zeta$ and $\d \eta$ by transforming the Cauchy-Riemann equations to the
+$\zeta, \eta$ coordinate system. Analogues to the algorithm in Section~\ref{sec:orthogonal} we can then construct the basis vector fields $\partial_u$ and $\partial_v$, appropriately choose a normalization and then use streamline
+integration in the $\zeta, \eta$ coordinate system to construct the coordinates. 
+From the basis one-forms $\d u$ and $\d v$ we get the elements of the Jacobian 
+matrix of the transformation. 
+Before we do this in detail however, let us first discuss some alternative elliptic
+equations to the simple Eq.~\eqref{eq:harmonic}.
+
+
+%\subsection{Conformal coordinates} \label{sec:conformal}
+%For simplicity of the argument let us assume that $(\zeta, \eta)$ are 
+%the orthogonal coordinates from Section~\ref{sec:orthogonal}. We will generalize
+%the approach in Section~\ref{sec:elliptic}.
+%
+%We can use $\bar u(\zeta, \eta)$ as well as its derivatives $\bar u_\zeta(\zeta,\eta)$ and $\bar u_\eta(\zeta,\eta)$  and Eq.~\eqref{eq:CR}
+%as a starting point to construct conformal coordinates $(u,v)$ 
+%\begin{subequations}
+%\begin{align}
+% \d u &= c_0 \left( \bar u_\zeta \d \zeta + \bar u_\eta \d \eta\right)= u_x\d x + u_y\d y  \\
+% \d v &= c_0 \sqrt{g} \left(-g^{\eta\eta} \bar u_\eta \d \zeta + g^{\zeta\zeta} \bar u_\zeta \d \eta\right)= v_x\d x + v_y\d y = -u_y\dx + u_x\d y  
+%\end{align}
+%\label{eq:conformal_coords}
+%\end{subequations}
+%%The prefactor $a$ is given by $a^2 \equiv g^{\eta\eta}/g^{\zeta\zeta} = h^2/f_0^2$.
+%The rightmost equalities are proven using the orthogonality of the $(\zeta, \eta)$
+%coordinates and the relation for the inverse derivatives Eq.~\eqref{eq:inverse}.
+%With this it is easy to prove that $g^{uu} = g^{vv} = (\nabla u)^2 = g^{\zeta\zeta}u_\zeta^2 + g^{\eta\eta}u_\eta^2$ and $g^{uv} = 0$.
+%Furthermore, we get 
+%\begin{subequations}
+%\begin{align}
+%  \partial_u &= \frac{1}{c_0(\nabla u)^2} \left(g^{\zeta\zeta}u_\zeta \partial_\zeta + g^{\eta\eta}\bar u_\eta\partial_\eta\right) \label{eq:basis_conformala} \\
+%  %= \frac{1}{c_0}\frac{ \bar u_\zeta\partial_\zeta + a^2 \bar u_\eta\partial_\eta }{ \left( \bar u_\zeta^2 + a^2 \bar u_\eta^2 \right)} \\
+%  \partial_v &= \frac{1}{c_0\sqrt{g}(\nabla u)^2} \left( -\bar u_\eta \partial_\zeta + \bar u_\zeta\partial_\eta\right)
+%  %= \frac{1}{c_0}\frac{   -a\bar u_\eta \partial_\zeta + a\bar u_\zeta \partial_\eta }{\left( \bar u_\zeta^2 + a^2 \bar u_\eta^2   \right)}
+%  \label{eq:basis_conformal}
+%\end{align}
+%\end{subequations} 
+%%with 
+%%$J = c_0^2/a \left( \bar u_\zeta^2 + a^2 \bar u_\eta^2\right)$.
+%As for the orthogonal coordinates we have to integrate these two vector fields to construct our coordinates. 
+%We begin with the integration of $\partial_v$ along the $u=0$ line. 
+%There we have
+%\begin{align}
+% \partial_v|_{u=0} = \eta_v \partial_\eta = \frac{1}{c_0 \bar u_\zeta} \partial_\eta, 
+%  \label{eq:normalize}
+%\end{align}
+%where we used $\bar u_\eta|_{\zeta=0} = 0$ and $\sqrt{g}g^{\zeta\zeta}|_{\zeta=0} = h(\zeta_0, \eta)/f_0 = 1$, since $h(\psi_0) = f_0$. 
+%We can use Eq.~\eqref{eq:normalize} to define $c_0$ such that $v\in[0,2\pi]$. 
+%In order to do so we simply integrate 
+%\begin{align}
+%  v_1 = \int_0^{2\pi} \left .\frac{\d v}{\d \eta}\right|_{\zeta=0} \d \eta = \int_0^{2\pi} \frac{1}{\eta_v}\d \eta = c_0 \int_0^{2\pi} \bar u_\zeta(0,\eta) \d \eta := 2\pi
+%  \label{eq:computec0}
+%\end{align}
+% with $v_0 = 0$ and choose $c_0$ such that $v_1=2\pi$. 
+%The integration can be done by evaluating $\bar u_\zeta$ at the Gaussian nodes and using Gauss--Legendre quadrature. 
+%Having done this we integrate, analog to Section~\ref{sec:orthogonal}, 
+%$\partial_v $ to get starting points for the integration of $\partial_u$ 
+%from $\bar u_0 = 0$ to $\bar u_1= c_0 \zeta_1$.
+%In order to avoid out-of-bound errors we can artificially make the $\zeta,\eta$ box periodic.
+%Note, that we can compute $u_x(\zeta, \eta)$ and $u_y(\zeta, \eta)$ from $\bar u(\zeta,\eta)$ by using the equations $u_x = u_\zeta \zeta_x + u_\eta\eta_x$, $u_y = u_\zeta \zeta_y + u_\eta\eta_y$ as soon as the constant $c_0$ becomes available. 
+%
+%For the integration of the vector fields Eq.~\eqref{eq:basis_conformal} and the evaluation of derivatives Eq.~\eqref{eq:conformal_coords} we need to evaluate its components at points 
+%unequal to the Gaussian nodes. An interpolation method 
+%is thus needed. We use interpolation with 
+%the same order as the dG method we used for the solution of $\bar u(\zeta,\eta)$.
+%
+%The result of the algorithm is the list of points 
+%$\zeta(u_i,v_i)$, $\eta(u_i,v_i)$, which we can insert into
+%$x(\zeta,\eta)$ and $y(\zeta, \eta)$ as well as $u_x(\zeta,\eta)$ and $u_y(\zeta,\eta)$ which have been computed previously. The derivatives $v_x$ and $v_y$ are given by the Cauchy--Riemann equations~\eqref{eq:CR}.
+
+\subsection{Grid adaption} \label{sec:adaption}
+Although the conformal grid is advantageous for elliptic equations (due to the vanishing metric
+coefficients) the cell distribution is not very flexible; once the boundary is set
+the conformal map is unique. We therefore have little control over the 
+distribution of cells. 
+We can use grid adaption techniques to overcome this restriction. 
+The idea is to modify the elliptic equations that $u$ and $v$ have 
+to fulfill. That is, we choose 
+\begin{align}
+\nabla\cdot\left( \frac{\nabla u}{w}\right) = \nabla\cdot\left( w \nabla v\right) = 0
+\label{eq:adaption}
+\end{align}
+where $w$ is an appropriately chosen weight function. The 
+cell size will be small in regions where $w$ is large and spread out in regions
+where $w$ is small.
+The Cauchy--Riemann equations~\eqref{eq:CR} are changed accordingly to 
+\begin{align}
+v_x = -\frac{u_y}{w}\quad v_y = \frac{u_x}{w}
+\label{eq:CR_adaption}
+\end{align}
+Let us remark here that it is straightforward to implement the weight function in the orthogonal grid generation. 
+In Section~\ref{sec:orthogonal} we simply replace the function $h$ by $h/w$.
+Then we have
+\begin{subequations}
+\begin{align}
+\partial_\zeta &= \frac{1}{f_0(\nabla\psi)^2} (\psi_x\partial_x + \psi_y \partial_y)\\
+\partial_\eta &= \frac{w}{h(\nabla\psi)^2} (-\psi_y\partial_x + \psi_x \partial_y)
+\end{align}
+\end{subequations}
+and $\nabla \psi\cdot \nabla (h/w) = -h/w \Delta\psi$.
+A suitable choice for $w$ is 
+\begin{align}
+w = |\nabla\psi|
+\label{eq:weight_adaption}
+\end{align}
+as then the angle-like coordinate $\eta$ becomes the arc length on the $\psi_0$ line. 
+
+%For completeness, we explicitly state the formulas for the adaption of the conformal case here.
+%These become apparent in 
+%the next section, where we generalize this approach.
+%In a boundary adapted grid (i.e.~a grid constructed from a coordinate transform that exactly resolves the boundary) we solve the equation
+%\begin{align}
+%\nabla\cdot\left( \frac{1}{w} \nabla \bar u \right) = 0
+%\label{eq:ubar_adapted}
+%\end{align}
+%with the familiar Dirichlet boundary conditions $\bar u|_{\partial\Omega} = \psi$. 
+%Then the coordinate lines are given by the vector fields
+%\begin{subequations}
+%\begin{align}
+%  \partial_u &= \frac{1}{c_0(\nabla \bar u)^2} 
+%        (g^{\zeta\zeta}\bar u_\zeta \partial_\zeta + g^{\eta\eta}\bar u_\eta\partial_\eta)\label{eq:basis_adapteda}\\
+%  \partial_v &= \frac{w}{c_0\sqrt{g}(\nabla \bar u)^2} 
+%          (-\bar u_\eta \partial_\zeta + \bar u_\zeta\partial_\eta)
+%  \label{eq:basis_adapted}
+%\end{align}
+%\end{subequations} 
+%where $(\nabla \bar u)^2 = g^{\zeta\zeta}\bar u_\zeta^2 + g^{\eta\eta}\bar u_\eta^2$.
+%The relation between $u$ and $v$ is given by
+%\begin{align}
+% v_x = -\frac{u_y}{w},\quad v_y = \frac{u_x}{w}
+%\end{align}
+%These formulas are very similar to the ones given in Section~\ref{sec:conformal}.
+
+
+
+\subsection{Monitor metric and the heat conduction tensor} \label{sec:metric}
+We follow Reference~\cite{Liseikin, Glasser2006, Vaseva2009} and replace the canonical metric tensor $\vec g$ by
+a specifically tailored tensor $\vec G$ that takes the form
+\begin{align}
+\vec G(x,y) = \vec T \vec T + k^2\vec N \vec N + \eps(x,y)\vec I
+\label{eq:metric}
+\end{align}
+with $\vec T = (-\psi_y, \psi_x)$ and $\vec N = -(\psi_x, \psi_y)$. 
+The vector $\vec T$ is tangential to the contour lines of $\psi$ while $\vec N$ is 
+normal to it. 
+We have 
+\begin{align}
+  \sqrt{G} = \left[(\eps+k^2(\psi_x^2+\psi_y^2))(\eps+(\psi_x^2+\psi_y^2))\right]^{-1/2}
+  \label{}
+\end{align}
+where $k<1$ is a constant and $\eps$ is a function that is nonzero in the neighborhood
+of singularities, i.e.~where $\nabla \psi(x,y)=0$.
+In our work we choose $k=0.1$ and $\eps(x,y)\equiv\eps =0.001$.
+Note that the scalar product induced by $\vec G$ conserves perpendicularity 
+with respect to $\vec T$, i.e.~if
+and only if
+ any vector $\vec v \perp \vec T$ in the canonical metric, then it is also perpendicular to $\vec T$ in $\vec G$.
+In our application this is important at the boundary.\\
+%
+Now, we consider the elliptic equation
+\begin{align}
+  \nabla\cdot(\sqrt{G}\vec G \nabla u) = 0\nonumber\\
+  \partial_x(\sqrt{G} (G^{xx}\partial_x u + G^{xy}\partial_y u )) + \partial_y(\sqrt{G}(G^{yx}\partial_x u + G^{yy}\partial_y u )) = 0
+  \label{eq:elliptic}
+\end{align}
+with Dirichlet boundary conditions. % $u|_{\partial\Omega} = \psi$.
+The resulting grid coordinate $u$ is almost perfectly aligned in regions
+far away from singularities and breaks the alignment in regions where $|\nabla\psi|$ is small or vanishes.
+
+We now take a slightly more general approach and rewrite Eq.~\eqref{eq:elliptic}
+\begin{align}
+  \nabla\cdot(\vec \chi \nabla u) = 0
+  \label{eq:elliptic_mod}
+\end{align}
+where $\chi(x,y)$ is a symmetric positive-definite contravariant tensor. 
+Then the conformal grid from Section~\ref{sec:conformal}, the grid adaption from Section~\ref{sec:adaption} as 
+well as the monitor metric can be considered special cases. 
+The grid adaption is recovered by setting $\vec \chi = 1/w\vec I$,
+while the monitor metric is simply $\vec \chi = \sqrt{G}\vec G$. 
+The true conformal case is, of course, recovered by setting $\vec \chi = \vec I$. 
+
+This allows us to provide a commonly known physical interpretation of Eq.~\eqref{eq:elliptic_mod}.
+If $u$ is a temperature, then the Dirichlet boundary condition 
+fixes a temperature at the boundary of our domain. The tensor $\vec \chi(x,y)$ is 
+then the anisotropic heat conduction tensor and the coordinate lines for $u$ are 
+the isothermal lines of the steady state solution to the heat diffusion problem. 
+This interpretation allows us to intuitively estimate how the
+coordinate lines look like when a specific $\vec\chi$ is chosen. 
+In the case of grid adaption, if the weight function is large, the heat conduction 
+is low resulting in small temperature gradients and thus closely spaced grid cells. 
+On the other hand, let us reconsider Eq.~\eqref{eq:metric} for the case $\eps=0$. 
+Then we can write 
+\begin{align}
+\vec \chi = \frac{1}{k}\hat t\hat t + k \hat n\hat n 
+= \chi_\parallel \hat t \hat t + \chi_\perp \hat n \hat n
+  \label{eq:heat_conduction}
+\end{align}
+which for $k<1$ simply means that the heat conduction parallel to the magnetic field
+is far stronger than perpendicular to it, which is in fact the case in an actual 
+fusion reactor. 
+The coordinate lines will thus tend to align with the magnetic flux
+surfaces with the degree of alignment given by $k$ resulting 
+in an almost aligned grid. 
+In the limit of vanishing $k$ the alignment should be perfect. 
+If $|\nabla\psi|$ vanishes, the tensor~\eqref{eq:metric} reduces to $\vec \chi = \vec I$.
+
+
+\subsection{The elliptic grids} \label{sec:elliptic}
+Suppose that we have constructed a boundary aligned grid $(\zeta,\eta)$, which
+may but not necessarily has to be the orthogonal grid from Section~\ref{sec:orthogonal}. 
+Now, we solve the general elliptic equation
+\begin{align}
+ \nabla\cdot(\vec \chi \nabla \bar u) = \partial_i(\sqrt{g} \chi^{ij}\partial_j \bar u) = 0
+  \label{eq:general_elliptic}
+\end{align}
+in the transformed coordinate system with boundary conditions $\bar u|_{\partial\Omega}=\psi$. We set $u=c_0(\bar u -\psi_0)$. This means that we have to 
+transform the conduction tensor $\chi$ from Cartesian to flux coordinates,
+which is done by the well known rules of tensor transformation
+\begin{subequations}
+\begin{align}
+  \chi^{\zeta\zeta}(\zeta, \eta) &= (\zeta_x\zeta_x \chi^{xx} + 2\zeta_x\zeta_y\chi^{xy} + \zeta_y\zeta_y \chi^{yy})|_{x(\zeta,\eta), y(\zeta,\eta)}\\
+  \chi^{\zeta\eta}(\zeta, \eta) &= (\zeta_x\eta_x \chi^{xx} + (\zeta_x\eta_y + \eta_x\zeta_y)\chi^{xy} + \zeta_y\eta_y \chi^{yy})|_{x(\zeta,\eta), y(\zeta,\eta)}\\
+  \chi^{\eta\eta} (\zeta, \eta) &= (\eta_x\eta_x \chi^{xx} + 2\eta_x\eta_y\chi^{xy} + \eta_y\eta_y \chi^{yy}))|_{x(\zeta,\eta), y(\zeta,\eta)}
+\end{align}
+\label{eq:transformationChi}
+\end{subequations}
+The equivalent of the Cauchy--Riemann equations in this formulation reads
+\begin{subequations}
+\begin{align}
+  v_\zeta = -\sqrt{g}(\chi^{\eta\zeta}u_\zeta + \chi^{\eta\eta}u_\eta)\\
+  v_\eta = +\sqrt{g}(\chi^{\zeta\zeta}u_\zeta + \chi^{\zeta\eta}u_\eta)
+\end{align}
+  \label{eq:hodge_dual}
+\end{subequations}
+These are constructed such that
+ $\nabla\cdot((\vec \chi/ \det \chi) \nabla v) = 0$.
+The interested reader might notice that Eq.~\eqref{eq:hodge_dual} just 
+defines the components of the Hodge dual $\d v = \star\d u$  
+if $\chi$  is interpreted as a metric.
+%In order to repeat the formalism in Section~\ref{sec:conformal}
+%we simply replace $\psi$ by $u$ and $h$ by $\sqrt{g} h$ in this 
+%Section and note that $h(\zeta,\eta)=cte$ because of Eq.~\ref{eq:general_elliptic}.
+%It becomes clear now that our algorithm for orthogonal and
+%the conformal grid generation is actually the same. 
+We note that these equations are now valid for any grid that  
+we use to solve Eq.~\eqref{eq:general_elliptic}. If we find a boundary
+aligned grid analytically, we can start the grid construction 
+directly with the solution of Eq.~\eqref{eq:general_elliptic} 
+and then proceed with the conformal grid generation. 
+
+The relevant equations for the streamline integration now read
+\begin{subequations}
+\begin{align}
+    \d u &= c_0(\bar u_\zeta\d \zeta + \bar u_\eta \d \eta) \\
+    \d v &= c_0\sqrt{g}(-(\chi^{\eta\zeta}\bar u_\zeta + \chi^{\eta\eta}\bar u_\eta)\d \zeta + (\chi^{\zeta\zeta}\bar u_\zeta + \chi^{\zeta\eta}\bar u_\eta)\d \eta)
+\end{align}
+\label{eq:contravariant_monitor}
+\end{subequations} 
+which just means that $v$
+is orthogonal to $u$ in the scalar product 
+generated by the symmetric tensor $\vec \chi$, which we denote by $\langle . , .\rangle_\chi$.
+We have 
+$J=c_0^2 \sqrt{g} \langle \nabla \bar u,\nabla \bar u \rangle_\chi
+  := c_0^2\sqrt{g} (\bar u_\zeta^2\chi^{\zeta\zeta} + 2 \bar u_\zeta \bar u_\eta \chi^{\zeta\eta} + \bar u_\eta^2\chi^{\eta\eta})$ 
+and 
+\begin{subequations}
+\begin{align}
+  \partial_u &= \frac{1}{c_0\langle\nabla \bar u,\nabla \bar u\rangle_\chi} 
+        (\chi^{\zeta\zeta}\bar u_\zeta + \chi^{\zeta\eta}\bar u_\eta)\partial_\zeta + 
+        (\chi^{\eta\zeta}\bar u_\zeta + \chi^{\eta\eta}\bar u_\eta)\partial_\eta\label{eq:basis_monitora}\\
+  \partial_v &= \frac{1}{c_0\sqrt{g}\langle\nabla \bar u,\nabla \bar u\rangle_\chi} 
+          (-\bar u_\eta \partial_\zeta + \bar u_\zeta\partial_\eta)
+  \label{eq:basis_monitorb}
+\end{align}
+\label{eq:basis_monitor}
+\end{subequations} 
+As for the orthogonal coordinates we have to integrate these two vector fields to construct our coordinates. 
+We begin with the integration of $\partial_v$ along the $\zeta=0$ line. 
+It is important to note that $\bar u_\eta|_{\zeta=0} = 0$ and thus
+\begin{align}
+  \eta_v(0, \eta) = \left( c_0\sqrt{g}\bar u_\zeta \chi^{\zeta\zeta} \right)|_{\zeta= 0, \eta}
+  \label{eq:normalize}
+\end{align}
+We can use Eq.~\eqref{eq:normalize} to define $c_0$ such that $v\in[0,2\pi]$. 
+In order to do so we simply integrate 
+\begin{align}
+  v_1 = \int_0^{2\pi} \left .\frac{\d v}{\d \eta}\right|_{\zeta=0} \d \eta = \int_0^{2\pi} \frac{1}{\eta_v}\d \eta = c_0 \int_0^{2\pi} \sqrt{g}\chi^{\zeta\zeta}\bar u_\zeta(0,\eta) \d \eta := 2\pi
+  \label{eq:computec0}
+\end{align}
+ with $v_0 = 0$ and choose $c_0$ such that $v_1=2\pi$. This is the analogous
+ equation to Eq.~\eqref{eq:definef}, with the difference that we can integrate
+ Eq.~\eqref{eq:computec0} directly using numerical quadrature. 
+%The integration can be done by evaluating $\bar u_\zeta$ at the Gaussian nodes and using Gauss--Legendre quadrature. 
+Having done this we integrate, analogous to Section~\ref{sec:orthogonal}, 
+$\partial_v $ on $\zeta=0$ to get starting points for the integration of $\partial_u$ 
+from $\bar u_0 = 0$ to $\bar u_1= c_0 \zeta_1$.
+Note, that we can compute the components of $\d u$ and $\d v$ in terms of $\d x$ and $\d y$ by using the transformation
+\begin{subequations}
+\begin{align}
+  u_x(\zeta, \eta)  &= u_\zeta \zeta_x + u_\eta\eta_x,\ u_y(\zeta, \eta) = u_\zeta \zeta_y + u_\eta\eta_y\\
+  v_x(\zeta, \eta)  &= v_\zeta \zeta_x + v_\eta\eta_x,\ v_y(\zeta, \eta) = v_\zeta \zeta_y + v_\eta\eta_y
+\end{align} 
+  \label{eq:chain_rule}
+\end{subequations}
+as soon as the constant $c_0$ becomes available. 
+
+Note that the resulting grid will, in general, not be orthogonal 
+in the Euclidean metric. However, for all the cases we discuss
+in this paper, the grid is orthogonal at the boundary. 
+
+
+The final algorithm now reads, assuming that $u$ is discretized
+by not necessarily equidistant points $u_i$ with $i=0,1,\dots N_u-1$ and 
+$v$ by $v_j$ with $j=0,1,\dots N_v-1$ and that $x(\zeta,\eta)$, $y(\zeta, \eta)$
+as well as the components of the Jacobian $\zeta_x(\zeta,\eta)$, $\zeta_y(\zeta,\eta)$, $\eta_x(\zeta,\eta)$ and $\eta_y(\zeta,\eta)$ are available from the first coordinate transformation (e.g. Section~\ref{sec:orthogonal}):
+\begin{enumerate}
+  \item Choose either $\vec \chi = \vec I$, $\vec \chi=1/w\vec I$ or $\vec \chi = \sqrt{G}\vec G$ depending on whether a conformal, adapted or monitor grid is desired.
+  \item Discretize and solve the elliptic equation~\eqref{eq:general_elliptic} on the $\zeta,\eta$ grid for $\bar u(\zeta, \eta)$ with any
+    method that converges.  
+    Use Eq.~\eqref{eq:transformationChi} to transform $\chi$ from Cartesian 
+    to the $\zeta, \eta$ coordinate system.
+    The chosen resolution
+    determines the accuracy of the subsequent streamline integration.
+  \item Numerically compute the derivatives $\bar u_\zeta$ and $\bar u_\eta$ 
+    and construct $\eta_v^{-1}(0, \eta)$ using Eq.~\eqref{eq:normalize} for $c_0=1$
+  \item Integrate Eq.~\eqref{eq:computec0} to determine $c_0$.
+  \item On the $\zeta, \eta$ grid compute $\eta_v$, $\zeta_u$ and $\zeta_v$ according to Eq.~\eqref{eq:basis_monitor} as well as $u_\zeta$, $u_\eta$, $v_\zeta$ and
+    $v_\eta$ according to Eq.~\eqref{eq:contravariant_monitor}. Use Eq.~\eqref{eq:chain_rule} and the Jacobian of the $\zeta, \eta$ coordinates to compute $u_x$, $u_y$, $v_x$ and $v_y$.
+  \item Integrate the streamline of $\partial_v$ for $\zeta=0$ 
+    from $v=0\dots v_j$ for all $j$ on the $\zeta, \eta$ grid using the normalized component $\eta_v$. 
+    Interpolate $\eta_v(\zeta, \eta)$ when necessary.
+  \item Using the resulting points as start values integrate $\partial_u$ from 
+    $u=0\dots u_i$ for all $i$ and all points. The result is the list
+    of coordinates $\zeta(u_i, v_j)$, $\eta(u_i, v_j)$ for all $i$ and $j$.
+  \item Interpolate $x\left( \zeta,\eta \right)$, $y\left( \zeta, \eta \right)$, 
+    $u_x(\zeta, \eta)$, $u_y(\zeta, \eta)$, $v_x(\zeta, \eta)$ and $v_y(\zeta, \eta)$ on this list. 
+\end{enumerate}
+There are two differences in this algorithm from the one presented in 
+Section~\ref{sec:orthogonal}.
+For the integration of the vector fields Eq.~\eqref{eq:basis_monitor} and the evaluation of derivatives Eq.~\eqref{eq:contravariant_monitor} we need to evaluate its components at arbitrary points.
+An interpolation method is thus needed. 
+In order to avoid out-of-bound errors we can artificially make the $\zeta,\eta$ box periodic.
+Second, the existence of the coordinate $v$ is guaranteed by the Cauchy--Riemann equations
+and thus the $h$ function does not appear. 
+
+A suitable test for the implementation is e.g. the volume/area of the domain. 
+Being an invariant
+the volume must be the same regardless of the coordinate system in use. 
+
+%Finally, we note that with the metric proposed in Eq.~\eqref{eq:metric}
+%the orthogonal grid generated is the same as the orthogonal 
+%grid generated by the identity tensor in the case $\eps=0$. 
+Finally, as already mentioned several times the $\zeta,\eta$ coordinates in the algorithm can 
+be replaced by any structured, boundary aligned grid. In especially this means
+that for example a Cartesian grid ($\psi(x,y) = y$, $\zeta=x$, $\eta=y$) 
+can be used if the boundary 
+is a rectangle. The periodic boundary condition in $\eta$ in Eq.~\eqref{eq:general_elliptic} can easily be replaced 
+by Neumann boundaries if necessary. 
+The proposed adaption function $w$ or the monitor metric $\vec G$
+are in the same way only suggestions, are in principle independent of $\psi(x,y)$ and can be replaced by any suitable quantity. 
+
+
+
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+
+
+
+\section*{Acknowledgements} 	
+This work was supported by the Austrian Science Fund (FWF) W1227-N16 and Y398. 
+This work has been carried out within the framework of the EUROfusion Consortium and has received funding from the Euratom research and training programme 2014‐2018 under grant agreement No 633053. The views and opinions expressed herein do not necessarily reflect those of the European Commission.
+
+%..................................................................
+\bibliography{../references}
+
+
+%..................................................................
+
+
+\end{document}
+
diff --git a/inc/doc/Makefile b/inc/doc/Makefile
deleted file mode 100644
index 7e169dbe0..000000000
--- a/inc/doc/Makefile
+++ /dev/null
@@ -1,65 +0,0 @@
-###Packages needed on linux:
-###doxygen
-###libjs-mathjax
-###graphviz
-#(shouldn't require latex according to doxygen documentation)
-
-all: doc
-
-.PHONY: clean doc dg.tag geometries.tag dg file geometries
-
-# the semicolons and backslashes are needed by Makefile
-dg.tag:
-	cd ../dg; \
-		(cat Doxyfile; \
-		echo "OUTPUT_DIRECTORY = ../doc/dg"; \
-		echo "GENERATE_HTML = NO"; \
-		echo "GENERATE_TAGFILE = ../doc/dg.tag" ) | doxygen - ;
-
-geometries.tag:
-	cd ../geometries;  \
-		(cat Doxyfile; \
-		echo "OUTPUT_DIRECTORY = ../doc/geometries"; \
-		echo "GENERATE_HTML = NO"; \
-		echo "GENERATE_TAGFILE = ../doc/geometries.tag" ) | doxygen - ;
-
-dg: geometries.tag
-	cd ../dg; \
-		(cat Doxyfile; \
-		echo "OUTPUT_DIRECTORY = ../doc/dg"; \
-		echo "HTML_HEADER = ../doc/header.html"; \
-    	echo "EXTERNAL_GROUPS=NO" ;\
-    	echo "EXTERNAL_PAGES=NO" ;\
-		echo "TAGFILES = ../doc/geometries.tag=../../geometries/html") | doxygen - ; 
-
-geometries: dg.tag
-	cd ../geometries; \
-		(cat Doxyfile; \
-		echo "OUTPUT_DIRECTORY = ../doc/geometries"; \
-		echo "HTML_HEADER = ../doc/header.html"; \
-    	echo "EXTERNAL_GROUPS=NO" ;\
-    	echo "EXTERNAL_PAGES=NO" ;\
-		echo "TAGFILES = ../doc/dg.tag=../../dg/html") | doxygen - ; 
-
-file:
-	cd ../file; \
-		(cat Doxyfile; \
-		echo "OUTPUT_DIRECTORY = ../doc/file"; \
-		echo "HTML_HEADER = ../doc/header.html"; \
-		echo ) | doxygen - ; 
-
-pdf:
-	cd ../geometries/related_pages; \
-		pdflatex parallel.tex;
-
-
-doc: dg geometries file
-	ln -sf dg/html/modules.html index.html
-	#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%#
-	#Open with: firefox index.html or on Windows: firefox dg/html/modules.html#
-	#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%#
-
-full_doc: pdf doc
-
-clean:
-	rm -rf dg file geometries dg.tag file.tag geometries.tag index.html
diff --git a/inc/geometries/Doxyfile b/inc/geometries/Doxyfile
index f17afa02e..803732e04 100644
--- a/inc/geometries/Doxyfile
+++ b/inc/geometries/Doxyfile
@@ -1151,7 +1151,7 @@ HTML_EXTRA_STYLESHEET  =
 # files will be copied as-is; there are no commands or markers available.
 # This tag requires that the tag GENERATE_HTML is set to YES.
 
-HTML_EXTRA_FILES       = "../geometries/related_pages/parallel.pdf"
+HTML_EXTRA_FILES       =
 
 # The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen
 # will adjust the colors in the style sheet and background images according to
diff --git a/inc/geometries/geometries_doc.h b/inc/geometries/geometries_doc.h
index 9c75f8107..d903fab8c 100644
--- a/inc/geometries/geometries_doc.h
+++ b/inc/geometries/geometries_doc.h
@@ -33,7 +33,8 @@
  * @subsection pdf PDF writeups
  * 
  * We have a collection of writeups: 
- *  - The parallel derivative <a href="./parallel.pdf" target="_blank">parallel derivative</a>
+ *  - The <a href="./parallel.pdf" target="_blank">parallel derivative</a>
+ *  - The <a href="./hector.pdf" target="_blank">HECTOR algorithm</a>
  */
  /** @class hide_container
   * @tparam container 
diff --git a/inc/geometries/related_pages/parallel.tex b/inc/geometries/related_pages/parallel.tex
deleted file mode 100644
index ed7166655..000000000
--- a/inc/geometries/related_pages/parallel.tex
+++ /dev/null
@@ -1,367 +0,0 @@
-%\documentclass[12pt]{article}
-%\documentclass[12pt]{scrartcl}
-\documentclass{hitec} % contained in texlive-latex-extra
-\settextfraction{0.9} % indent text
-\usepackage{csquotes}
-\usepackage[hidelinks]{hyperref} % doi links are short and usefull?
-\hypersetup{%
-    colorlinks=true,
-    linkcolor=blue,
-    urlcolor=magenta
-}
-\urlstyle{rm}
-\usepackage[english]{babel}
-\usepackage{mathtools} % loads and extends amsmath
-\usepackage{amssymb}
-% packages not used
-%\usepackage{graphicx}
-%\usepackage{amsthm}
-%\usepackage{subfig}
-\usepackage{bm}
-\usepackage{longtable}
-\usepackage{booktabs}
-\usepackage{ragged2e} % maybe use \RaggedRight for tables and literature?
-\usepackage[table]{xcolor} % for alternating colors
-\rowcolors{2}{gray!25}{white}
-\renewcommand\arraystretch{1.3}
-
-%%% reset bibliography distances %%%
-\let\oldthebibliography\thebibliography
-\let\endoldthebibliography\endthebibliography
-\renewenvironment{thebibliography}[1]{
-  \begin{oldthebibliography}{#1}
-    \RaggedRight % remove if justification is desired
-    \setlength{\itemsep}{0em}
-    \setlength{\parskip}{0em}
-}
-{
-  \end{oldthebibliography}
-}
-%%% --- %%%
-
-%%%%%%%%%%%%%%%%%%%%%definitions%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\newcommand{\eps}{\varepsilon}
-\renewcommand{\d}{\mathrm{d}}
-\newcommand{\T}{\mathrm{T}}
-\renewcommand{\vec}[1]{{\mathbf{#1}}}
-\newcommand{\dx}{\,\mathrm{d}x}
-%\newcommand{\dA}{\,\mathrm{d}(x,y)}
-%\newcommand{\dV}{\mathrm{d}^3{x}\,}
-\newcommand{\dA}{\,\mathrm{dA}}
-\newcommand{\dV}{\mathrm{dV}\,}
-
-\newcommand{\Eins}{\mathbf{1}}
-
-\newcommand{\ExB}{$\bm{E}\times\bm{B} \,$}
-\newcommand{\GKI}{\int d^6 \bm{Z} \BSP}	
-\newcommand{\GKIV}{\int dv_{\|} d \mu d \theta \BSP}	
-\newcommand{\BSP}{B_{\|}^*}
-\newcommand{\GA}[1]{\langle #1	 \rangle}
-
-\newcommand{\Abar}{\langle A_\parallel \rangle}
-%Vectors
-\newcommand{\bhat}{\bm{\hat{b}}}
-\newcommand{\bbar}{\overline{\bm{b}}}
-\newcommand{\chat}{\bm{\hat{c}}}
-\newcommand{\ahat}{\bm{\hat{a}}}
-\newcommand{\xhat}{\bm{\hat{x}}}
-\newcommand{\yhat}{\bm{\hat{y}}}
-\newcommand{\zhat}{\bm{\hat{z}}}
-
-\newcommand{\Xbar}{\bar{\vec{X}}}
-\newcommand{\phat}{\bm{\hat{\perp}}}
-\newcommand{\that}{\bm{\hat{\theta}}}
-
-\newcommand{\eI}{\bm{\hat{e}}_1}
-\newcommand{\eII}{\bm{\hat{e}}_2}
-\newcommand{\ud}{\mathrm{d}}
-
-%Derivatives etc.
-\newcommand{\pfrac}[2]{\frac{\partial#1}{\partial#2}}
-\newcommand{\ffrac}[2]{\frac{\delta#1}{\delta#2}}
-\newcommand{\fixd}[1]{\Big{\arrowvert}_{#1}}
-\newcommand{\curl}[1]{\nabla \times #1}
-\newcommand{\np}{\nabla_{\perp}}
-\newcommand{\npc}{\nabla_{\perp} \cdot }
-\newcommand{\nc}{\nabla\cdot }
-\newcommand{\GAI}{\Gamma_{1}^{\dagger}}
-\newcommand{\GAII}{\Gamma_{1}^{\dagger -1}}
-\newcommand{\Tp}{\mathcal T^+_{\Delta\varphi}}
-\newcommand{\Tm}{\mathcal T^-_{\Delta\varphi}}
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%DOCUMENT%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\begin{document}
-%\preprint{}
-
-\title{The parallel derivative on structured grids}
-\author{M.~Wiesenberger}
-%\email{Matthias.Wiesenberger@uibk.ac.at}
-%\ead{Matthias.Wiesenberger@uibk.ac.at}
-%\affiliation{Institute for Ion Physics and Applied Physics, Association EURATOM-\"OAW,  University of
-%   Innsbruck, A-6020 Innsbruck, Austria}
-%\author{M.~Held}
-%\affiliation{Institute for Ion Physics and Applied Physics, Association EURATOM-\"OAW,  University of
-   %Innsbruck, A-6020 Innsbruck, Austria}
-%\address{Institute for Ion Physics and Applied Physics, Association EURATOM-\"OAW,  Universit\"at 
-%   Innsbruck, A-6020 Innsbruck, Austria}
- 
-\maketitle
-\section{The parallel derivative}
-The idea of the sandwich method is to bracket the parallel derivative by interpolation and projection matrices:
-\begin{align}
-    \nabla^c_\parallel &= P\nabla_\parallel^f Q \\
-    \nabla^{c\dagger}_\parallel &= P \nabla^{f\dagger}_\parallel Q
-    \label{eq:sandwich}
-\end{align}
-In this way the projection integrals
-\begin{align}
-    \int\dV \nabla_\parallel f p_i(x)p_j(y) 
-    \label{}
-\end{align}
-are computed more precisely.
-The size of the fine grid should therefore be as large as
-possible (reasonable? what resolution is needed?).
-We first notice that the one interpolation matrix can be absorbed
-in the parallel derivative since this also consists of 
-interpolation operations. 
-\begin{align}
-    \nabla^c_\parallel &= P\nabla_\parallel^{fc} \\
-    \nabla^{c\dagger}_\parallel &= \nabla^{fc\dagger}_\parallel Q
-    \label{eq:sandwich}
-\end{align}
-
-
-In order to understand what the adjoint operators do let us denote $\mathcal T^+_{\Delta\varphi}$ as the push-forwward operator. Then we have
-\begin{align}
-    \int f(\vec x) \Tp h(\vec x) \sqrt{g(\vec x)}\d^3x \\
-    =  \int f(\vec x) h(\Tm \vec x)\sqrt{g(\vec x)}\d^3x \\
-    =  \int f(\Tp \vec x') h(\vec x')\sqrt{g(\Tp \vec x')}J^{-1}( \Tp\vec x') \d^3x' \\
-    =  \int \frac{1}{\sqrt{g(\vec x')}}\Tm\left[J^{-1}(\vec x')\sqrt{g(\vec x')}f(\vec x')\right] h(\vec x')\sqrt{g(\vec x')}   \d^3x' \\
-    \equiv  \int (\Tp)^\dagger\left[f(\vec x)\right] h(\vec x)\sqrt{g(\vec x)}   \d^3x
-    \label{}
-\end{align}
-$J$ is the determinant of the Jacobian $\partial(\vec x')/\partial(\vec x)$.
-In the last step we simply replaced the dummy variable $\vec x'$ with $\vec x$ again and identified the relevant terms
-as the adjoint operator:
-\begin{align}
-    (\Tp)^\dagger f(\vec x ) := \frac{1}{\sqrt{g(\vec x)}} \Tm\left[\sqrt{g(\vec x)} J^{-1}(\vec x) f(\vec x) \right]
-    \label{}
-\end{align}
-This means that numerically the adjoint of the push-forward 
-operator should be a valid discretization of its inverse. (how can this be exploited as a test?)
-Note that $\sqrt{g}J^{-1}(\vec x) = \sqrt{g'(\Tm \vec x)}$.
-With this we can write
-\begin{align}
-    (\Tp)^\dagger f(\vec x ) := \sqrt{\frac{g'(\vec x)}{g(\vec x)}} \Tm\left[f(\vec x) \right]
-    \label{}
-\end{align}
-Also note that while $\Tp [fh] = \Tp f \Tp h$ this might not 
-hold on the discrete level. Also the question is how $J$ enters 
-on the discrete level. We have to multiply $\sqrt{g}$ artificially when we form the adjoint. 
-Theoretically $J$ could be hidden somehow when we integrate the fieldlines, so the information could be contained in the discrete version? (Maybe in the back-projection?)
-
-If the streamlines are divergence free, we have $J=1$.
-A numerical test could be ( if we neglect the volume form in the adjoint)
-\begin{align}
-    (\Tp)^\dagger \left[J(\vec x)\Tp f(\vec x)\right] - f(\vec x) = 0
-    \label{}
-\end{align}
-The numerical computation of $J$ might a bit tricky at the boundaries. 
-In a flux-aligned $\zeta, \eta$ it should be feasible but in cylindrical coordinates I don't know how. Maybe we can simply cut the last few cells before the boundary.
-Even easier might be
-\begin{align}
-    \left(\Tp\right)^\dagger J(\vec x ) = 1
-    \label{}
-\end{align}
-
-If we integrate streamlines of the vector field $\vec B/B^\varphi$, then we have
-\begin{align}
-    \frac{\d J}{\d \varphi} = J(\vec x ) \nabla\cdot\left( \vec B/B^\varphi\right)
-    \label{}
-\end{align}
-along these streamlines.
-Also we have that $\d s = B/B^\varphi \d \varphi $ and the interpolation/projection of $\triangle s$ can probably be neglected. (We want to neglect it because it's memory intensive to store all combinations)
-Also this means that when we transpose $\nabla_\parallel$ we get a 
-multiplication by $B^\varphi/B$ in the beginning.
-In any case this means that we want to have 
-\begin{align}
-\left(\Tp\right)^\dagger B^\varphi  =  B^\varphi
-\end{align}
-\section{Boundary conditions}
-The question is what to do when a fieldline intersects with the boundary
-of the simulation domain before reaching the next plane.
-The problem with simply putting a value exactly where the fieldline 
-reaches the boundary is first that a true interpolation at that 
-position requires a 3d interpolation (we 
-are between planes) instead of a 2d interpolation.
-Secondly, for Dirichlet boundaries the small 
-distance seriously deteriorates the CFL condition.
-A simple solution is to set the perpendicular field zero on the 
-boundary such that the field becomes purely 
-toroidal on the boundary. The fieldlines then have a kink on the 
-boundary. On the other hand we can implement boundary conditions consistent with 
-the perpendicular ones since the fieldlines never leave the domain. 
-We simply interpolate the quantity to derive on the inner side of the
-domain boundary (Neumann conditions = "No boundary condition") or 
-set the value to zero (Dirichlet condition).
-
-
-\section{Geometry}
-When we are on a different geometry than $(R,Z)$ the question is how to integrate the field lines. There are two possibilities. 
-First, interpolate $R(\zeta_i, \eta_i), Z(\zeta_i, \eta_i)$ for 
-all $i$, then integrate in $(R,Z)$ space and finally use
-Newton iteration to find $\zeta(R^\pm_i, Z^\pm_i), \eta(R^\pm_i, Z^\pm_i)$. 
-The downside here is that it is difficult to tell when and where the fieldline leaves the simulation domain and even worse in MPI the next points might belong to another process. 
-
-The second possibiliy is to integrate entirely in the 
-transformed coordinate system $\zeta, \eta$. 
-The magnetic field can be easily transformed since we have the
-Jacobian of the coordinate transformation
-\begin{align}
-    B^\zeta(\zeta, \eta) &= \left(\frac{\partial \zeta}{\partial R} B^{R} + \frac{\partial \zeta}{\partial Z}B^Z\right)_{R(\zeta, \eta), Z(\zeta, \eta)} \\
-    B^\eta(\zeta, \eta) &= \left(\frac{\partial \eta}{\partial R} B^{R} + \frac{\partial \eta}{\partial Z}B^Z\right)_{R(\zeta, \eta), Z(\zeta, \eta)} \\
-    B^\varphi(\zeta, \eta) &= B^\varphi({R(\zeta, \eta), Z(\zeta, \eta)})
-    \label{eq:field_trafo}
-\end{align}
-The advantage is that we can do this for any coordinate
-system. Only for cylindrical coordinates, we integrate directly in physical space. The equations to integrate
-are
-\begin{subequations}
-\begin{align}
-\frac{\d \zeta}{\d\varphi} &= \frac{B^\zeta}{B^\varphi}\\
-\frac{\d \eta}{\d\varphi} &= \frac{B^\eta}{B^\varphi}\\
-\frac{\d s}{\d\varphi} &= \frac{|B|}{|B^\varphi|}
-\end{align}
-\label{eq:fieldlines}
-\end{subequations}
-The downside here is that when integrating fieldlines we
-have to interpolate the magnetic field at arbitrary points. 
-However, the error should vanish with $3rd$ order in the 
-perpendicular plane. (check this). In order to mitigate this error
-we could maybe transform the B-Field on a finer grid/higher order polynomials for more accurate
-integration. Also, we need interpolation in the 
-first algorithm as well. 
-
-Note that the matrix-matrix multiplications in Eq.~\eqref{eq:sandwich} can
-be precomputed and stored. The memory requirements 
-in the final computations are 
-therefore the same  as in the old version. (Not entirely, since
-the diagonal $1/\Delta s$ matrix does not commute with $Q$ or $P$).
-
-Finally remember that the adjoint of a matrix in the modified geometry 
-involves the volume element. This means that after you've adjoined the 
-parallel derivative the normal way simply bracket the result 
-by $1/\sqrt{g}$ and $\sqrt{g}$. 
-\section{Algorithm}
-Given are the components $v^i(R,Z)$ for $i\in\{R,Z,\varphi\}$ and a compuational grid (in the following the ``coarse grid``)
-\begin{itemize}
-  \item generate a fine grid by multiplying the cell numbers of the given coarse grid (only topologcially, metric and Jacobian are not needed)
-  \item integrate the fieldlines for the fine grid:
-    \begin{itemize}
-      \item evaluate the starting points on the coarse grid in computational space 
-      \item For a curvilinear grid set up a (higher order) grid for the 
-        interpolation of the vector components $v^i$ and push forward the vector components
-        to the curvilinear coordinate system
-      \item Integrate the fieldline equations 
-\begin{subequations}
-\begin{align}
-\frac{\d \zeta}{\d\varphi} &= \frac{v^\zeta}{v^\varphi}\\
-\frac{\d \eta}{\d\varphi} &= \frac{v^\eta}{v^\varphi}\\
-\frac{\d s}{\d\varphi} &= \frac{1}{|v^\varphi|}
-\end{align}
-\label{eq:fieldlines_converted}
-\end{subequations}
-    with the given starting points and $s(0)=0$ from $\varphi=0$ until $\varphi = \pm\Delta \varphi$.
-      \item create an interpolation matrix that interpolates from the coarse grid 
-        to the fine grid
-      \item use the interpolation matrix to generate the plus/minus points for the fine grid
-    \end{itemize}
-  \item create the interpolation matrices that interpolate from the given coarse grid 
-    to the plus/minus points 
-  \item create a projection matrix that projects from the fine grid to the coarse grid
-  \item compute the matrix-matrix multiplications $P\cdot I^\pm$ as well as the transpose
-  \item project the $s$ vectors to the coarse grid
-\end{itemize}
-\section{MPI implementation}
-Let us also note the mpi-implementation, which is not entirely
-trivial due to the matrix-matrix multiplications involed in Eq.~\eqref{eq:sandwich}.
-\subsection{Row and column distributed sparse matrices}
-In Feltor each mpi process gets an equally sized chunk of a 
-vector.
-Contrary to a vector
-a matrix can be distributed in two ways, row-wise and column wise. 
-In a row-distributed matrix each process gets the complete 
-rows of the matrix that correspond to the indices in the 
-vector it holds. 
-In a column-distributed matrix each process gets the complete 
-columns of the matrix corresponding to the indices in the 
-vector it holds. 
-When we implement a matrix-vector multiplication the order 
-of communication and computation depends on the distribution 
-of the matrix.
-For the row-distributed matrix each process first has to gather all elements of the input vector it needs to be able to compute the elements of the output. This requires MPI communication.
-Formally, the gather operation can be written as a matrix $G$
-of $1'$s and $0'$s where the output vector is of equal or larger size than the input vector.
-After the elements have been gathered the local matrix-vector
-multiplications can be executed.
-\begin{align}
-M = R\cdot G
-\end{align}
-where $R$ is the row-distributed matrix with modified indices 
-and $G$ is the gather matrix, in which the MPI-communication takes place.
-
-In a column distributed matrix the local matrix-vector multiplication can be executed first because each processor already
-has all vector elements it needs. 
-However the resuling elements have to be communicated back to 
-the process they belong to. Furthermore, a process has to sum
-all elements it receives from other processes on the same
-index. This is a scatter and reduce operation and
-it can be written as a scatter matrix $S$. The transpose
-of the scatter matrix is a gather matrix and vice-versa.
-\begin{align}
-M = S\cdot C
-\end{align}
-where $S$ is the scatter matrix and $C$ is the column distributed
-matrix with modified indices. 
-
-It turns out that a row-distributed matrix can be transposed
-by transposition of the local matrices and the gather matrix.
-The result is then a column distributed matrix.
-The transpose of a column distributed matrix is a row-distributed matrix and vice-versa.
-
-\subsection{Matrix-Matrix multiplication}
-We note that we normally construct $\nabla_\parallel^{fc}$ as a column 
-distributed
-matrix. The advantage is then that the gather operation is bijective, i.e. the transpose of the gather matrix is its inverse. 
-This advantage is lost in the present problem. 
-It turns out that it is advantageous to construct $\nabla_\parallel^{fc}$
-as s row-distributed matrix with global indices. 
-This is because a column distributed matrix can be easily (without mpi-communication) multiplied
-with a row distributed matrix especially if the indices are global indices. 
-Each process just multiplies its local matrices.
-\begin{align}
-M = C\cdot R
-\end{align}
-This is not true the other way round. 
-The result is then a row distributed matrix with global indices. 
-From the global indices the gather map/matrix and the local
-indices can be constructed.
-We note here that we even don't need to construct the gather matrix
-for $\nabla_\parallel^{fc}$, only the one for $\nabla_\parallel^c$ is
-needed.
-
-
-
-%..................................................................
-\bibliography{refs}
-%\bibliographystyle{aipnum4-1.bst}
-\bibliographystyle{model1-num-names}
-
-
-%..................................................................
-
-
-\end{document}
-
-- 
GitLab


From 9aeab3d49970f94d0572ad947b0760fb6ca9c21c Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Thu, 12 Oct 2017 17:40:05 +0200
Subject: [PATCH 350/453] added SIRK scheme documentation and init
 dg_introduction

---
 .../dg_introduction/boundary_terms.tex        |  11 +
 .../dg_introduction/dg_introduction.tex       | 572 ++++++++++++++++++
 .../dg_introduction/discretization.pdf        | Bin 0 -> 16584 bytes
 .../dg_introduction/jump_terms.tex            |  10 +
 .../dg_introduction/table_polarisation.tex    |  42 ++
 inc/dg/multistep.h                            |  21 +
 6 files changed, 656 insertions(+)
 create mode 100644 doc/related_pages/dg_introduction/boundary_terms.tex
 create mode 100644 doc/related_pages/dg_introduction/dg_introduction.tex
 create mode 100644 doc/related_pages/dg_introduction/discretization.pdf
 create mode 100644 doc/related_pages/dg_introduction/jump_terms.tex
 create mode 100644 doc/related_pages/dg_introduction/table_polarisation.tex

diff --git a/doc/related_pages/dg_introduction/boundary_terms.tex b/doc/related_pages/dg_introduction/boundary_terms.tex
new file mode 100644
index 000000000..ec18d2102
--- /dev/null
+++ b/doc/related_pages/dg_introduction/boundary_terms.tex
@@ -0,0 +1,11 @@
+\begin{longtable}{ccccccc}
+  \toprule
+ & \multicolumn{ 2}{c}{$D_x^+$ (forward)} & \multicolumn{ 2}{c}{$D_x^-$ (backward)} & \multicolumn{ 2}{c}{ $D_x^0$ (centered)} \\ 
+ & left & right & left  & right & left  & right \\ 
+periodic & $-(M+L)^\mathrm{T}$ & $-(M+L)^\mathrm{T}$ & $(M+L)$ & $(M+L)$ & $\frac{1}{2}(M-M^\mathrm{T})$ & $\frac{1}{2}(M-M^\mathrm{T})$ \\ 
+Dirichlet & $-M^\mathrm{T}$ & $-(M+L)^\mathrm{T}$ & $(M+L)$ & $-M^\mathrm{T}$ & $\frac{1}{2}(M-M^\mathrm{T}+L)$ & $\frac{1}{2}(M-M^\mathrm{T}-R)$ \\ 
+Neumann & $-(M+L)^\mathrm{T}$ & $M$ & $M$ & $(M+L)$ & $\frac{1}{2}(M-M^\mathrm{T}-L)$ & $\frac{1}{2}(M-M^\mathrm{T}+R)$ \\ 
+\bottomrule
+\caption{Upper left and lower right matrix entries for various boundary conditions. For Dirichlet and von Neumann BC the upper right and lower left entries are zero.}
+\label{tab:boundary_terms}
+\end{longtable}
diff --git a/doc/related_pages/dg_introduction/dg_introduction.tex b/doc/related_pages/dg_introduction/dg_introduction.tex
new file mode 100644
index 000000000..d33ad3db6
--- /dev/null
+++ b/doc/related_pages/dg_introduction/dg_introduction.tex
@@ -0,0 +1,572 @@
+%\documentclass[12pt]{article}
+%\documentclass[12pt]{scrartcl}
+\documentclass{hitec} % contained in texlive-latex-extra
+\settextfraction{0.9} % indent text
+\usepackage{csquotes}
+\usepackage[hidelinks]{hyperref} % doi links are short and usefull?
+\hypersetup{%
+    colorlinks=true,
+    linkcolor=blue,
+    urlcolor=magenta
+}
+\urlstyle{rm}
+\usepackage[english]{babel}
+\usepackage{mathtools} % loads and extends amsmath
+\usepackage{amssymb}
+% packages not used
+%\usepackage{graphicx}
+%\usepackage{amsthm}
+%\usepackage{subfig}
+\usepackage{bm}
+\usepackage{longtable}
+\usepackage{booktabs}
+\usepackage{ragged2e} % maybe use \RaggedRight for tables and literature?
+\usepackage[table]{xcolor} % for alternating colors
+%\rowcolors{2}{gray!25}{white}
+\renewcommand\arraystretch{1.3}
+\usepackage{doi}
+\usepackage[sort,square,numbers]{natbib}
+\bibliographystyle{abbrvnat}
+
+%%%%%%%%%%%%%%%%%%%%%definitions%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\newcommand{\eps}{\varepsilon}
+\renewcommand{\d}{\mathrm{d}}
+\newcommand{\T}{\mathrm{T}}
+\renewcommand{\vec}[1]{{\mathbf{#1}}}
+\newcommand{\dx}{\,\mathrm{d}x}
+%\newcommand{\dA}{\,\mathrm{d}(x,y)}
+%\newcommand{\dV}{\mathrm{d}^3{x}\,}
+\newcommand{\dA}{\,\mathrm{dA}}
+\newcommand{\dV}{\mathrm{dV}\,}
+
+\newcommand{\Eins}{\mathbf{1}}
+
+\newcommand{\ExB}{$\bm{E}\times\bm{B} \,$}
+\newcommand{\GKI}{\int d^6 \bm{Z} \BSP}	
+\newcommand{\GKIV}{\int dv_{\|} d \mu d \theta \BSP}	
+\newcommand{\BSP}{B_{\|}^*}
+\newcommand{\GA}[1]{\langle #1	 \rangle}
+
+\newcommand{\Abar}{\langle A_\parallel \rangle}
+%Vectors
+\newcommand{\bhat}{\bm{\hat{b}}}
+\newcommand{\bbar}{\overline{\bm{b}}}
+\newcommand{\chat}{\bm{\hat{c}}}
+\newcommand{\ahat}{\bm{\hat{a}}}
+\newcommand{\xhat}{\bm{\hat{x}}}
+\newcommand{\yhat}{\bm{\hat{y}}}
+\newcommand{\zhat}{\bm{\hat{z}}}
+
+\newcommand{\Xbar}{\bar{\vec{X}}}
+\newcommand{\phat}{\bm{\hat{\perp}}}
+\newcommand{\that}{\bm{\hat{\theta}}}
+
+\newcommand{\eI}{\bm{\hat{e}}_1}
+\newcommand{\eII}{\bm{\hat{e}}_2}
+\newcommand{\ud}{\mathrm{d}}
+
+%Derivatives etc.
+\newcommand{\pfrac}[2]{\frac{\partial#1}{\partial#2}}
+\newcommand{\ffrac}[2]{\frac{\delta#1}{\delta#2}}
+\newcommand{\fixd}[1]{\Big{\arrowvert}_{#1}}
+\newcommand{\curl}[1]{\nabla \times #1}
+\newcommand{\np}{\nabla_{\perp}}
+\newcommand{\npc}{\nabla_{\perp} \cdot }
+\newcommand{\nc}{\nabla\cdot }
+\newcommand{\GAI}{\Gamma_{1}^{\dagger}}
+\newcommand{\GAII}{\Gamma_{1}^{\dagger -1}}
+\newcommand{\Tp}{\mathcal T^+_{\Delta\varphi}}
+\newcommand{\Tm}{\mathcal T^-_{\Delta\varphi}}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%DOCUMENT%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\begin{document}
+
+\title{DG methods on structured grids}
+\author{M.~Wiesenberger}
+
+\section{ Introduction to discontinuous Galerkin methods} \label{sec:discretization}
+This writeup is based on Reference~\cite{WiesenbergerPhD}. 
+
+In recent years, discontinuous Galerkin (dG) methods have been investigated 
+as an excellent alternative to finite difference and finite volume schemes 
+in numerical simulations involving both parabolic as well as hyperbolic problems 
+(for the advection dominated case see, for example, the review article~\cite{Cockburn2001runge}). 
+Such methods combine many advantages of finite element methods (such as the ease of handling 
+complicated geometry) with properties more commonly associated with 
+finite difference approximations. Examples of the latter includes the absence 
+of a global mass matrix. 
+The main idea of a dG method is to approximate the solution 
+of a differential equation by a polynomial in each grid-cell. 
+Higher/lower
+order methods can be constructed by simply increasing/decreasing the degree of 
+the polynomials used in each cell. 
+In classical finite element methods continuity is required across cell boundaries. 
+In contrast, dG methods allow discontinuities across cell boundaries, which adds to the flexibility of the method.
+
+For the discretization of second derivatives we discuss the so-called local discontinuous Galerkin (LDG) method~\cite{Cockburn1998}.
+The LDG method and its advantages can also be used for the discretization of 
+elliptic equations (including Poisson's equation). 
+Reference~\cite{Arnold2001} highlights the relation 
+of the method to interior penalty and other alternative methods. 
+A superconvergence result for the 
+LDG approach was proven on Cartesian grids~\cite{Cockburn2001},
+where the order of convergence is $1/2$ better than on arbitrary meshes.
+Reference~\cite{Yadav2013} later showed a similar result for general, nonlinear elliptic equations.
+
+The downside of the dG methods is their rather complex and unintuitive notation 
+in the existing mathematical literature. Often, algorithms
+are described in terms of arbitrary sets of polynomials and spatial grids. 
+We propose an adapted, simplified, and in our view more practical notation for orthogonal grids and Legendre polynomials, which can be implemented straightforwardly.
+We reformulate the LDG method in terms of adjoint matrices and
+ naturally develop the symmetry of the resulting discretization.
+Doing so, we also propose a new discretization for the general elliptic equation. 
+Unfortunately, our discretization has a wider stencil than the existing ones, but 
+our numerical experiments indicate superconvergent properties and the resulting
+matrix equation is better conditioned than the old one. To the knowledge
+of the author these findings are unpublished to date.
+
+\subsection{ The Legendre polynomials} \label{sec:legendre}
+First, let us consider the one-dimensional case.
+For simplicity and ease of implementation we choose an equidistant grid with $N$ cells $C_n = [x_{n-1/2},x_{n+1/2}]$ of size $h$;
+with this choice we are able to construct basis functions of $P(C_n)$, the space of 
+polynomials of degree at most $P-1$ on $C_n$, by using orthogonal Legendre polynomials\footnote{ Often, in the literature the order of the polynomials is denoted by $k$.
+Pay attention, that we use the number of polynomial coefficients $P$ instead. We have $P=k+1$}. 
+The Legendre polynomials can be recursively defined on $[-1,1]$ by setting
+$p_0(x) = 1$, $p_1(x) = x$ and (see e.g.~\cite{AS})
+\begin{align}
+    (k+1)p_{k+1}(x) = (2k+1)xp_k(x) - kp_{k-1}(x).
+    \label{eq:recursion}
+\end{align}
+The so constructed Legendre polynomials are orthogonal on $[-1,1]$.
+We write 
+$x^a_j$ and $w_j$, $j=0,\dots,P-1$ denoting the abscissas and weights of 
+the Gauss--Legendre quadrature on the interval $[-1,1]$. Then we note that for $k,l=0, \dots, P-1$
+\begin{align}
+    \int_{-1}^1 p_k(x)p_l(x) \dx = \sum_{j=0}^{P-1} w_jp_k (x^a_j)p_l(x^a_j) = \frac{2}{2k+1}\delta_{kl}, 
+    \label{}
+\end{align}
+ since Gauss--Legendre quadrature is exact for polynomials of degree at most $2P-1$.
+
+The discrete completeness relation can then be written as 
+\begin{align}
+    \sum_{k=0}^{P-1} \frac{2k+1}{2}w_j p_k(x^a_i)p_k(x^a_j) = \delta_{ij}.
+    \label{eq:completeness}
+\end{align}
+Given a real function $f:[-1,1]\rightarrow \mathbb{R}$ we define $f_j:=f(x^a_j)$ and
+\begin{align} \label{eq:ex2}
+    \bar f^k := \frac{2k+1}{2}\sum_{j=0}^{P-1}w_j p_k(x^a_j) f_j 
+\end{align}
+Now let us define the forward transformation matrix by $F^{kj}:=\frac{2k+1}{2}w_jp_k(x^a_j)$ and 
+the backward transformation matrix by $B_{kj}:= p_j(x^a_k)$. Then, 
+using Eq.~\eqref{eq:ex2}, we get
+\begin{subequations}
+\begin{align}
+    \bar f^k = \sum_{j=0}^{P-1}F^{kj}f_j \\
+    f_j = \sum_{k=0}^{P-1} B_{jk}\bar f^k,
+\end{align}
+\end{subequations}
+We call $\bar f^k$ the values of $f$ in $L$-space and $f_j$ the values of $f$ in $X$-space.
+
+Let us now consider an interval $[a,b]$ and an equidistant discretization
+by $N$ cells with cell center $x_n$ and grid size $h=\frac{b-a}{N}$; in addition, we set $x_{nj}^a := x_n + \frac{h}{2}x^a_j$.
+Given a function $f:[a,b]\rightarrow \mathbb{R}$ we then define
+$f_{nj} := f(x^a_{nj})$ and note that
+\begin{subequations}
+\begin{align}
+    \bar{ \vec f} &= (\Eins\otimes F) \vec f \\
+    \vec f &= (\Eins\otimes B) \bar{\vec f},
+    \label{}
+\end{align}
+\end{subequations}
+where $f_{nj}$ are the elements of $\vec f$,
+ $\Eins\in\mathbb{R}^{N\times N}$ is the identity matrix and $F,B\in\mathbb{R}^{P\times P}$. Furthermore, we use
+$\otimes$ to denote the Kronecker product which is bilinear and associative. 
+The discontinuous Galerkin expansion $f_h$ of a function $f$ in the interval $[a,b]$ can 
+then readily be given as
+\begin{align}
+    f_h(x) = \sum_{n=1}^N \sum_{k=0}^{P-1} \bar f^{nk} p_{nk}(x),
+    \label{eq:dgexpansion}
+\end{align}
+where
+\begin{align}
+    p_{nk}(x) := \begin{cases}
+        p_k\left(  \frac{2}{h}(x-x_n)\right),& \ \text{for } x-x_n\in\left[ -\frac{h}{2}, \frac{h}{2} \right]\\
+        0,& \ \text{else}.
+    \end{cases}
+    \label{}
+\end{align}
+As an example, we plot Eq.~\eqref{eq:dgexpansion} for $f(x)=\sin(2x)$ in Fig.~\ref{fig:discretization}. 
+\begin{figure}[htpb]
+    \includegraphics[width= 0.9\textwidth]{discretization.pdf}
+    \caption{ 
+    Discretization of a sine function with second order polynomials, $P=3$, 
+    on $N=5$ grid cells. Dotted lines depict the cell boundaries. 
+    }
+    \label{fig:discretization}
+\end{figure}
+Already with a very low resolution of $P=3$ and $N=5$ 
+we get an acceptable function approximation. We clearly see the discontinuities
+at the cell boundaries. 
+
+The use of Legendre polynomials yields a natural approximation of the integrals of $f$
+via Gauss--Legendre quadrature
+\begin{subequations}
+\begin{align}
+    \langle f_h,g_h\rangle:=\int_a^b f_hg_h \dx &= \sum_{n=1}^N\sum_{j=0}^{P-1} \frac{hw_j}{2} f_{nj} g_{nj} 
+    = \sum_{n=1}^N\sum_{k=0}^{P-1}\frac{h}{2k+1}\bar f^{nk}\bar g^{nk}  \\
+    \|f_h\|^2_{L_2} := \int_a^b |f_h|^2 \dx &= \sum_{n=1}^N\sum_{j=0}^{P-1} \frac{h w_j}{2}f_{nj}^2 
+    = \sum_{n=1}^N\sum_{k=0}^{P-1} \frac{h}{2k+1}\left(\bar f^{nk}\right)^2. 
+    \label{eq:def_norm}
+\end{align}
+\label{eq:gausslegendre}
+\end{subequations}
+With these formulas we have a simple, accurate, and fast 
+method to evaluate integrals. This is applied, for example, to compute
+errors in the $L_2$-norm.
+
+We now define some useful quantities that simplify our notation (note that $i,j=0,\dots,P-1$) 
+\begin{subequations}
+    \begin{align}
+        S_{ij} &:= \int_{-h/2}^{h/2} p_i\left(\frac{2}{h} x\right)p_j\left(\frac{2}{h} x\right) \dx = \frac{h}{2i+1}\delta_{ij} \\%=: s_i \delta_{ij}\\ 
+        T^{ij} &:= S^{-1}_{ij} = \frac{2i+1}{h}\delta_{ij} \\%=: t_i \delta_{ij}\\
+        W^{ij} &:= \frac{h w_j}{2}\delta_{ij}\\
+        V_{ij} &:= W_{ij}^{-1} = \frac{2}{hw_j}\delta_{ij}. 
+    \end{align}
+    \label{eq:diagonal}
+\end{subequations}
+Employing these relations we can write
+    \begin{align}
+		\langle f_h,g_h\rangle =& \vec f^{\mathrm{T}}(\Eins\otimes W)\vec g 
+		= \bar{\vec f}^{\mathrm{T}}(\Eins\otimes S)\bar{\vec g} 
+    \end{align}
+    and \begin{align}
+        F = TB^{\mathrm{T}}W.
+        \label{eq:transformation}
+    \end{align}
+Furthermore, we note that
+\begin{subequations}
+    \begin{align}
+        M_{ij} &:= \int_{-h/2}^{h/2} p_i\left(\frac{2}{h} x\right)\partial_xp_j\left(\frac{2}{h} x\right) \dx\\
+		R_{ij} &:= p_i(1)p_j(1) = 1 = R^{\mathrm{T}}_{ij}\\
+		L_{ij} &:= p_i(-1)p_j(-1) = (-1)^{i+j} = L^{\mathrm{T}}_{ij}\\
+        RL_{ij}&:= p_i(1)p_j(-1) = (-1)^j\\
+		LR_{ij}&:= p_i(-1)p_j(1) = (-1)^i = RL^{\mathrm{T}}_{ij}.
+    \end{align}
+    \label{eq:legendre_operators}
+\end{subequations}
+In order to compute the elements of $M_{ij}$ we first note that $M_{ij} = 0$ for
+$i > j-1$ as $\partial_x p_j(x)$ is a polynomial of degree $j-1$. Then
+we use integration by parts to show that 
+\begin{align}
+	(M+L) = (R-M)^{\mathrm{T}}.
+    \label{eq:legendre_derivative}
+\end{align}
+Therefore, we conclude that $M_{ij} = 1 - (-1)^{i+j}$ for $i\le (j-1)$. 
+
+We introduce the notation~\eqref{eq:diagonal} and~\eqref{eq:legendre_operators} mainly for ease of implementation. If a block-matrix class is written and the
+operations $+$, $-$ and $*$ are defined on it, the assembly of the derivative
+matrices is simplified to a large extent. 
+
+%Note that $\forall n\in\mathbb{N}$ and $x\in[-1,1]$
+%    $p_n(1) = 1$ and 
+%    $p_n(-x) = (-1)^np_n(x)$.
+%For $P=4$ we have e.g.
+%\begin{align}
+%    D = \begin{pmatrix}
+%        0 & 2 & 0 & 2 \\
+%        0 & 0 & 2 & 0 \\
+%        0 & 0 & 0 & 2 \\
+%        0 & 0 & 0 & 0 
+%    \end{pmatrix}
+%    \label{eq:example_derivative}
+%\end{align}
+%
+
+
+
+\subsection{ Discretization of first derivatives} \label{sec:firstderivatives}
+From here on we write
+$ f_h(x) = \bar f^{ni}p_{ni}(x)$ and imply the summation over cell index $n$ and polynomial
+index $i$.
+The first naive idea to get an approximation to the first derivative of $f_h(x)$
+is to simply set $f_x(x) = \partial_x f_h(x) = \bar f^{ni}\partial_x p_{ni}(x)$ in 
+the interior of each cell $n$. 
+Unfortunately, in this approach we loose one polynomial order and the discretization for
+$P=1$ is plain wrong. We would 
+like our discretization to become finite differences in the limit $P=1$.
+
+On cell boundaries the derivative
+of $p_{ni}(x)$ is actually not well defined, which is why we now retain to 
+a weak formulation of derivatives. 
+Consider 
+\begin{align}
+    \int_{C_n} \partial_x f_h(x) p_{ni}(x) \dx = f_h p_{ni}|_{x_{n-1/2}}^{x_{n+1/2}}  -
+    \int_{C_n} f_h(x) \partial_x p_{ni}(x) \dx.
+    \label{}
+\end{align}
+The approximation $f_h(x)$ is double valued on the cell boundaries, which is why we replace
+the boundary terms by
+\begin{align}
+    \int_{C_n} f_x p_{ni}(x) \dx
+    = \hat f p_{ni}|_{x_{n-1/2}}^{x_{n+1/2}}  -
+    \int_{C_n} \bar f^{nk} p_{nk} \partial_x p_{ni} \dx,
+    \label{}
+\end{align}
+where $\hat f(x)$ is the numerical flux across cell boundaries and we call $f_x(x)$ the numerical approximation to the first derivative. We will use three different fluxes in this work. 
+\begin{subequations}
+\begin{align}
+	\hat{f_C}(x) &= \tfrac{1}{2}(\lim_{\eps\to 0,\eps>0}f_h(x+\eps)+\lim_{\eps\to 0,\eps>0}f_h(x-\eps)),\\
+	\hat{f_F}(x) &= \lim_{\eps\to 0,\eps>0}f_h(x+\eps), \\
+	\hat{f_B}(x) &= \lim_{\eps\to 0,\eps>0}f_h(x-\eps),
+\end{align}
+\end{subequations}
+which we call the centered, the forward and the backward flux respectively.
+For $f\colon [a,b]\to\mathbb{R}$ and periodic boundary conditions, we assume that
+\begin{equation}
+	\lim_{\eps\to 0,\eps>0} f(b+\eps) = \lim_{\eps\to 0,\eps>0} f(a+\eps), \qquad
+	\lim_{\eps\to 0,\eps>0} f(a-\eps) = \lim_{\eps\to 0,\eps>0} f(b-\eps),
+\end{equation}
+for homogeneous Dirichlet boundary conditions we assume that
+\begin{equation}
+    \hat f(a) = \hat f(b) = 0,
+\end{equation}
+and for homogeneous Neumann boundaries we assume that
+\begin{align}
+    \hat f(a) = \lim_{\eps\to 0,\eps>0} f_h(a+\eps), \qquad 
+    \hat f(b) = \lim_{\eps\to 0,\eps>0} f_h(b-\eps).
+\end{align}
+As we see the choice
+of $\hat f$ is not unique. It actually is the crucial ingredient in every dG method. 
+Depending on what flux we choose, we arrive at various 
+approximations to the derivative,
+e.g. for $P=1$ (i.e. a piecewise constant approximation in each cell)
+our scheme reduces to the classic centered, forward and backward finite difference
+schemes respectively.
+In a way all the ingenuity of a dG method lies in the choice of the numerical flux.
+
+For the following discussion we choose the centered flux $\hat f_C$ and note
+that the derivation is analogous for $\hat f_F$ and $\hat f_B$.
+We arrive at
+\begin{align}
+    \bar f_x^{ni}= T^{ij}&\left[ \quad \frac{1}{2} \left(\bar f^{(n+1)k}p_k(-1)+ \bar f^{nk}p_k(1)\right)p_j(1)\right. \nonumber\\
+        &\ \ - \left.\frac{1}{2} \left(\bar f^{nk}p_k(-1) + \bar f^{(n-1)k}p_k(1)\right)p_j(-1) - \bar f^{nk} M_{kj} \vphantom{\frac{1}{2}}\right]
+    \label{}
+\end{align}
+where we used that $p_{nk}(x_{n+1/2})\equiv p_k(1)$ and $p_{nk}(x_{n-1/2}) \equiv p_k(-1)$ holds true for all $n$.
+Together with the previously defined quantities in~\eqref{eq:legendre_operators} we can write
+\begin{align}
+	\bar{\mathbf f}_x  &= (\Eins\otimes T)\circ \left[ \frac{1}{2}(\Eins^+\otimes RL + \Eins \otimes (M-M^{\mathrm{T}}) - \Eins^-\otimes LR)\right] \bar{\mathbf f}\nonumber \\
+    &=: (\Eins\otimes T) \circ \bar D^0_{x,per} \bar {\mathbf f},
+    \label{eq:discrete_der}
+\end{align}
+using $M+M^{\mathrm{T}} = R-L$ from Eq.~\eqref{eq:legendre_derivative}
+and 
+\begin{align}
+    \Eins^{-}f^{n} &:= f^{n-1}\\
+    \Eins^{+}f^{n} &:= f^{n+1}.
+    \label{eq:operator_one}
+\end{align}
+We define
+\begin{align*}
+    D^0_{x,per} := (\Eins\otimes F^T)\circ \bar D^0_{x,per}\circ (\Eins\otimes F).
+    \label{}
+\end{align*}
+If our coefficients are given in $X$-space, we note with the help of Eq.~\eqref{eq:transformation}
+\begin{align}
+	\mathbf f_x %&= (\Eins\otimes V) (\Eins\otimes F^{\mathrm{T}}) 
+    %\bar M_x^{per}
+%\circ (\Eins\otimes F) \mathbf f%\nonumber\\
+ = (\Eins\otimes V)\circ D^0_{x,per} \mathbf f,
+    \label{eq:matrix_xspace}
+\end{align}
+where $\bar D_x^{per}$ and $D_x^{per}$ are skew-symmetric matrices.
+
+From Eq.~\eqref{eq:discrete_der} we are now able to show the matrix representation of the one-dimensional discrete derivative for 
+periodic boundary conditions that can be used
+in the implementation
+\begin{align}
+    \bar D^0_{x,per} = \frac{1}{2}\begin{pmatrix}
+		(M-M^{\mathrm{T}}) & RL      &    &   & -LR \\
+		-LR  & (M-M^{\mathrm{T}}) & RL &   &     \\
+             &  -LR    & \dots   &   &     \\
+             &         &    & \dots  & RL    \\
+			 RL &         &    & -LR&(M-M^{\mathrm{T}}) 
+    \end{pmatrix}
+    \label{eq:dxcentered}
+\end{align}
+We also write down the expressions resulting from the forward and backward fluxes $\hat f_F$ and $\hat f_B$ respectively:
+\begin{align}
+    \bar D^+_{x,per} = \begin{pmatrix}
+        -(M+L)^{\mathrm{T}} & RL      &    &   & 0 \\
+		 0   & -(M+L)^{\mathrm{T}} & RL &   &     \\
+             &    0   & \dots   &   &     \\
+             &         &    & \dots  & RL    \\
+			 RL  &         &    & 0 & -(M+L)^{\mathrm{T}}
+    \end{pmatrix}
+    \label{eq:dxplus}
+\end{align}
+and 
+\begin{align}
+    \bar D^-_{x,per} = \begin{pmatrix}
+		(M+L) & 0      &    &   & -LR \\
+		-LR  & (M+L) & 0 &   &     \\
+             &   -LR   & \dots   &   &     \\
+             &         &    & \dots  & 0    \\
+			 0  &         &    & -LR &(M+L)
+    \end{pmatrix}.
+    \label{eq:dxminus}
+\end{align}
+Note that for $P=1$ we recover the familiar finite difference approximations of the first derivative. 
+\input{boundary_terms.tex}
+Finally we note the boundary terms for homogeneous Dirichlet and Neumann boundaries
+in Table~\eqref{tab:boundary_terms} noticing that only the 
+corner entries of the matrices change.
+
+In our notation the local character of the dG method is apparent.
+To compute the derivative in one cell we only use values of neighboring
+cells. Therefore, the method is well suited for parallelization which we
+will exploit in our implementation.
+
+The generalization to higher dimensions is immediate.  
+All the 
+matrices derived above can readily be extended via the appropriate Kronecker products.
+The space complexity of 
+the matrices derived is $\mathcal{O}(P^2 N)$ in one and $\mathcal{O}(P^3N^2)$ in two dimensions.
+
+Finally, let us remark that we found it practical to always operate on 
+coefficients in $X$-space, i.e.~we use Eq.~\eqref{eq:matrix_xspace}
+for our implementations and thus use $\vec f$ rather than $\bar{\vec f}$ to represent the approximation. Function products are easily computed coefficient-wise in $X$-space, i.e.~we use
+\begin{align}
+    (fg)_{ni}=f_{ni}g_{ni}
+    \label{}
+\end{align}
+to represent the corresponding products.
+
+\subsection{Discretization of elliptic equations} \label{sec:elliptic}
+We are now ready to discretize the one-dimensional general elliptic equation 
+\begin{align}
+    -\frac{\partial}{\partial x}\left( \chi(x)\frac{\partial \phi}{\partial x}(x)\right) = \rho(x)
+    \label{eq:elliptic}
+\end{align}
+on the interval $[a,b]$. Here, $\chi(x)$ and $\rho(x)$ are given functions.
+%We assume that the function $\chi(x)$ is given in dG coefficients $\chi^{in}$ in X-space.
+
+We either choose periodic, Dirichlet, or Von-Neumann boundary conditions on the 
+left and right border for $\phi$. 
+As a first step we rewrite Eq.~\eqref{eq:elliptic} into two first order differential equations and a function product:
+\begin{subequations}
+\begin{align}
+    j'&= \partial_x \phi, \label{eq:polarisationa}\\
+    j &= \chi j', \label{eq:polarisationb}\\
+    \rho &= -\partial_x j. \label{eq:polarisationc}
+\end{align}
+\end{subequations}
+%Our plan is to now simply use a forward discretization\footnote{
+%The choice of forward discretizing the first equation is arbitrary. A backward discretization 
+%works equally well} for the first equation 
+Our plan is to simply use one of the discretizations developed in the last 
+section for the first equation~\eqref{eq:polarisationa} 
+and its negative adjoint for the third equation~\eqref{eq:polarisationc}. 
+Recall that the adjoint of a square matrix $A$ is defined by the scalar product, i.e. 
+\begin{align}
+    \vec f^\mathrm{T} (\Eins\otimes W)\circ A\vec g = 
+    \vec g^\mathrm{T} (A^\mathrm{T}\circ(1\otimes W)) \vec f =:     \vec g^\mathrm{T} (\Eins\otimes W) A^\dagger \vec f. \nonumber
+    \label{}
+\end{align}
+From here we immediately get the relation
+\begin{align}
+    A^\dagger \equiv (\Eins\otimes V)\circ A^\mathrm{T}\circ (\Eins\otimes W).
+    \label{eq:adjoint}
+\end{align}
+There is a close connection between symmetric, $A=A^\mathrm{T}$, and self-adjoint, $A=A^\dagger$, matrices.
+If and only if the matrix $A$ is symmetric, then $(\Eins\otimes V)\circ A$ is self-adjoint. Of course, we have $(AB)^\dagger = B^\dagger A^\dagger$ and $(A^\dagger)^\dagger = A$. 
+
+The function product in Eq.~\eqref{eq:polarisationb} is computed pointwisely on the Gaussian abscissas. 
+\begin{align}
+    \vec j' &= (\Eins\otimes V)\circ D_x \vec \phi, \nonumber \\
+    \vec j &= \vec \chi \vec j', \nonumber\\
+    \vec \rho &= (\Eins\otimes V)\circ D_x^\mathrm{T}\vec j = (\Eins\otimes V) D_x^\mathrm{T}\circ\chi\circ (\Eins\otimes V)\circ D_x \vec \phi,
+    \label{eq:naive}
+\end{align}
+where $D_x$ is either $D_x^+$, $D_x^-$, or $D_x^0$ with the correct boundary terms.
+Note, that~\cite{Cockburn2001} originally only proposed to use the forward or backward discretization for $D_x$. 
+Equation~\eqref{eq:naive} is indeed a self-adjoint
+discretization for the second derivative. However, it turns out that 
+Eq.~\eqref{eq:naive} is inconsistent for given $\rho(x)$. The solution does not converge. 
+This problem is solved according to~\cite{Cockburn2001} by adding a jump term
+to the flux of $j$: 
+\begin{align}
+    \hat j(x_{n+1/2}) \rightarrow \hat j(x_{n+1/2}) + [\phi(x_{n+1/2})],
+\end{align}
+where $[\phi(x_{n+1/2}]:=\phi^n(x_{n+1/2}) - \phi^{n+1}(x_{n+1/2})$ is the jump term of
+$\phi$ at $x_{n+1/2}$. That means we have to alter our discretization~\eqref{eq:naive} according to
+\begin{align}
+    \rho =  (\Eins\otimes V)[D_x^\mathrm{T}\circ \chi\circ(\Eins\otimes V)\circ D_x  + J]\phi
+    \label{eq:discreteelliptic}
+\end{align}
+where 
+\begin{align}
+    \bar J = \begin{pmatrix}
+		(L+R) & -RL      &    &   & -LR \\
+		-LR  & (L+R) & -RL &   &     \\
+             &   -LR   & \dots   &   &     \\
+             &         &    & \dots  & -RL    \\
+			 -RL  &         &    & -LR &(L+R)
+    \end{pmatrix}.
+\end{align}
+for periodic boundaries. Again we give the correct boundary terms for Dirichlet
+and Neumann boundary conditions in Table~\ref{tab:jump_terms}.
+\input{jump_terms.tex}
+Note that $J$ is symmetric, thus the overall discretization remains self-adjoint. Indeed, with $D_x = D_x^+$ Eq.~\eqref{eq:discreteelliptic} recovers
+the discretization proposed by~\cite{Cockburn2001}. 
+In addition, we remark that the centered discretization in Eq.~\eqref{eq:discreteelliptic} is symmetric with respect to an inversion of the coordinate system $x\rightarrow -x$ even for double Dirichlet or Neumann boundaries, while the forward and backward discretization is not. 
+
+We note again that the generalization to two or more dimensions is straightforward
+in a rectangular grid using Kronecker products.
+\subsection{Numerical experiments} \label{sec:ellipticexperiments}
+As an example we solve Eq.~\eqref{eq:elliptic} in two dimensions for 
+\begin{align*}
+\chi(x,y) &= 1 + \sin(x)\sin(y)\\
+\rho(x,y) &= 2\sin(x)\sin(y)\left[\sin(x)\sin(y)+1\right]-\sin^2(x)\cos^2(y)-\cos^2(x)\sin^2(y)
+    \label{}
+\end{align*}
+on the domain $D=[ 0, \pi]\times[0,\pi]$ for double Dirichlet boundary conditions. 
+The analytical solution is given by $\phi(x,y) = \sin(x)\sin(y)$.  
+
+Do not try to insert $\phi$ into Eq.~\eqref{eq:discreteelliptic} directly. The result sub-optimally converges to the analytical $\rho$ or may not converge at all, a phenomenon that is called supraconvergence. Only as a discretization for 
+elliptic or parabolic equations the stencil~\eqref{eq:discreteelliptic} works fine. 
+
+We note that Eq.~\eqref{eq:discreteelliptic}, when multiplied by $(\Eins\otimes W)$, has the form of a symmetric matrix equation
+$A\vec x=\vec b$. 
+This equation is solved by a conjugate gradient method with
+$\Eins\otimes V$ as a diagonal preconditioner. 
+We use a truncation criterion based on the norm of the residuum
+\begin{align}
+    ||\vec r_k|| < \eps_{res}||\vec b||_{L_2} + \eps_{res}
+    \label{}
+\end{align}
+where $\vec r_k=A\vec x_k - \vec b$ is the residuum of
+the $k-th$ iteration. In Table~\ref{tab:polarisation} we summarize our results.
+\input{table_polarisation.tex}
+We observe that the forward and backward discretizations give the same results, 
+which is due to the symmetry of the sine functions. The order of the 
+relative error in the $L_2$-norm coincides with the predicted convergence of order $P$
+for all $P$. 
+
+On the other side, the centered discretization needs significantly less iterations
+to achieve the same error in the residuum. This indicates that the centered
+discretization is better conditioned than the forward and backward discretizations. 
+Also, we observe a superconvergent error of order $P+1$ for $P=3$ and $P=5$. In order to exclude symmetry reasons for these
+phenomena, we repeated the computations on the domain $D=[0, \pi/2]\times[0,\pi/2]$ using Dirichlet boundaries on the left
+and Neumann boundaries on the right side. We find 
+equal results; only
+the equality in the results for forward and 
+backward discretization is broken. 
+To the knowledge of the author this superconvergence has not yet been observed before. 
+
+
+\bibliography{../references}
+%..................................................................
+\end{document}
diff --git a/doc/related_pages/dg_introduction/discretization.pdf b/doc/related_pages/dg_introduction/discretization.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..fe20df31ceba8ca8351b77bcbb8c49206a9468f0
GIT binary patch
literal 16584
zcmb`vby!@@wmt~Ko#0L*!QI`1ySuwK&;)mf;O_3$1b6p92<{RH4#7f#Pv<+|Ip^Lp
zcjh<qJa_-mUA^jEQfsZM+E4Ah`kktjG&4I37xKHYvxH(~ZU7s=#oQiQNQhO@%Gm~F
z3*doJ)LEtN96?rY09I*7Gmw>(m4%C?m52zkJIKw-%n8{mx6Xg6-iqCMv%B}3-@wF@
z!NVJvtW`owOiN8yK}2Ubpv}%U#<D(6nR|_ZGmJPI^8Tjp>3cOp>XDm<hQmB#jF<aY
z(ZJ{5Dz99|uh)SArT8z;KTQIJ8h&1Q*WcRL2{#zd-H`qKxg7X%l|lacKvj3$edzPO
zBEhq|A?>iM>_y(c*QT|?`1R+{zQ9uJVXx;L|DS=+URxeQd-RTXb{@j?Juk=PXBy0X
zZYQe`*QaC7V}vH6FH;GD&3A4S$9Z432!8Y$GbebSE*a;zI1H@sCC=U(N(UsEn|NP$
zF$Z|MpGm4TVH|Q4c*+ItmxkNqHNo@U3#afM=i&8sZ!Jx8-1u@QK5wENz08!JcFioS
zbc46*su<y1YOgPT@2AXqlpJ<d6rRV~tK50#SE?+Z-icGuNDsq1LSK4)BD)F=?P*wA
zj^TBrap*dZq2DK-nRtQioyfBNfnefA)?(Jwz48v^V}9qj>bu5yvy(MPhiH}RIV!O<
zh=ka=+I@pP0KPIOlh&Z>=_a<Bq1~pg{{30c6M#&0HT}u4Xcp;F-9DRtJZwI<<BsKC
zC^*=S<{Slv(+-BE@X<#!%f(T-iI$J8n{%mvp9a>#JtpE)ZW3SR0x^Eun*Kf!nx4+G
zU|HAhr!~P}kI|^~`<Gt{h9evn_^Fj;0>4z{vD5pA1UtEok#zo8b?91_OI}?)j%?CC
z@WWtP@zeP+H$<y(p1Shs#CI2^#k?8!TNIf#vBNTGT|HRC5u|aW${jPe-C_=1`*yB%
zJkfaFp(UKe6`BDD(F%=IR|%fc>I?M+rIUil8tFi=DWM%Po7}^@bIFH2`XB18&Ah(X
zjojm@7k9>lTv{1u8k`r+t!F=NQWQXE>~;pvKaE<Ro~}gekeHIE^g>&aq)&8zHRRj?
zX@m2(&Qa&I5o*lxa#CN`xB-r!qd5DJ<XCw4IK8?$?)N%M-;e5vCLh1LWwtKt1Cv?M
zr7c3?$M7&nAM)ILtQ7Ls#xiP<e7KZHQ-^<CEWYo7@-geC`7j+h&p)-uqpM8hcFa1r
zAz3cZ&Nk~VJI{kQE5#qESk0*)uKe=gx^?^^=JWK|8K$}Tu0><f$MC$Gfb;zIWUz^^
zrL_BZzoS->gyxxiQoDLz_`pL|iQeAW&-i8IGMtR@?Iw|C$$W=n3VrUkbIZRPE{CSW
z6st{5L)SBMW<oyR5HYl_I(KO~FmCwEG_^jMzUCoRpTeJt#0OiaEOgS?squ52$}q?n
zwP;hvE+H}X7m;tT4n+TSY6WwLBl1Pwg`glEX~kn^HC|tkn=fUjyP}vk-Tmqv$jFXH
z#^_s-nLgl3GgzBm@H+JV5jD-<Ri1?u_95WV(B=F0o&?qmA}2d4ZjHPilV`^H^}Cmh
zpX@@|wXWxsOq!jv<F-|CJ+b{oZv^MN`8~uqw@A*JT0enyTE6+)uURWx2OHCFTN}Mu
z$Y>=@b@^z>wKXn)^$qD%`K~VZ+%qiwZs3c;N0qTqs<O3wg)$;&gIo2g>4QQOhL+VT
zBi6idqa+bc`=%R*fLficdEX_?T!p}FrQZvpt}Jl^3bJL8gx$KqgKbKDh62VRqwFqK
zN++nqb;h{hFIR%s*2>h*#R5gRt6fhbr9DQ!RzBvHsQuDulisXzj_Z8%;5xauU#j%A
zG$?tPO_1Jfa=$&yAK2zMD2ZTu8ckd)VK}oM7E=*k3OVmG!M9-|=<`Zg3O-h;W_~31
z_h*xjTFDHk*Yd;RFlT5ylMCC4k9?CC6n*f1oxL%A3y7YI&oedK#@;^6o9;;IP~*^4
z$eW3o(-Fq&D2=kE`>Ej$z+sB8&6kV7Llmji!gA+)$eaJGDFJ=96<G)gK_|M7XRTc5
z-ES#xjN3s?`>t^|#88V-l7zHpXvvT|Bc*<!1C%WUN8Is+i#-?(uD)JY{J1*j8zz{z
z_0#2D)eHI-VZoT@zJA@a4iv=9E5zv2@(+zrRxY>kBSo%{?KrmwP-CgZ!fWTIcJEJY
zLqK(oulW8&v<0=b_3N)J;1%B-60f|3*a(k_3Ua+Y>oSY#t_erN-kpVj&9Lz8Hu^yn
z>oV1Lor-`T9{?4AtL|gl>~D|3=A9KM{!wMYzUIZ2l?*ist@c6;=`%ixL}=s&@9)^B
zEW?qiXm~<xC=O7yeV(aov~(OCdF4RRw!VZy{D8_h7zZ5a3jh5cBNb4a-!yUv+8lJ2
zA-KmvGLu7TO-|agsuF1tIIjC~`f|P|r>@7Z(KY6$bWMb;Ez`sl4gt#1{5kCY13@#r
z!p)SiHG}dD%i;P=^|u~J=O2kT9(;%UziH0q%vwbl10A6bb8pJltLs4ptee_}?dQ<k
zi%Kk&>WWr7l|SGUqH6Lf`bEwbcAD1?xCa_lP>1vv2*Pa!GxB#SSavNr6(WirT|Vvt
zzQ%_>nDZJoNhTE)>>d3qAX0b+=Y=-mKC;ahy#&Ye7MDa?xY3=4EYb1{KcRtTB1vDk
z@H=OJ6y^M=9I6UC7%H^BO8NRCQ_?YNxC6KD7Pe8ODLu^*ORc+)>anOQ#I*zVR~mWK
zHr{C`$oTuKsh6k{R>4~f?ltC}%&%shb^f1if#&e%rxq{Ku3L@hi!T%2)a<?KCH%Kb
z@)+}(68ed-cNU%GRwz?+N;T%bGQH_v!)W8*hcxi;o<i+M_}6z7h3U1~@wvrogY(a~
zP;%TqECwy+y$pA3t@(HH-5C8~4!0v2VEZIH7dQHeJX`Q1CzwA@z-3`fY2iKg8>j21
zh4dlf4)q+8v31$^(kH9w)<+`o$%TE9{yph6X|VyFt#=1F^(){G|BEE*Uom-lE!qWN
z?i)8}U;5iY%N6;>7esjhxl=F5$}1)?)++ooD_~CcX^RlFqmOPmrb8gbEp>}m6+!wV
zE)Q&`BesE%JsN^OAQKyx_*Zlk#}x{`yWo%bXRH~}tun5t;GtFc6#^xj2x3!<&eH)j
zFgy3-7?fm*gce6X8#c&g914X46};09jYkfIVai^CU%43K>Hk3927Tw$6Y%mo_UQ`y
z&;;XE6usW@$7|oyYsR{$IrQr-#vMILtN&@ViKzS*-k=O(p{1a`dje*V6PY0Qt2JUF
zPEKlay#@#0C5g7<jycnIAR$q4>c%r^e06H+GeP};dSUqW&oIKuIbxxo3K1>eFMTD@
zKc-k#)2`vYQ)wgN5dT^<jpvWIy+Sg-PxJ9f;+N!y<PUaZPl`Vox~5`myu>lTUxyN(
zvqznOX-w*zSy6FxTJ}cG>o@@&zII!?sILbak)#~Pbg(%|*Vu8N8O&8pDKnl_FjKS;
z`c)Cup+i$}5*@@&v2Wn)BiZLhZpJ7L$2Z*Bptq6nDG{G|Dw8ncN7loX<b1aB#*hv4
z!uU8(b*;|UjW;Gfgkqg`u!l8*@&P@59iDS0F~^B<dM7OYiKT5e+orruE~V2X*>G&f
zR&XjjF2&Q7Eh-&O4!YyDvup(ZwmEi*-fv1{O-1n+B5xti<4Z|s0`*9h{P<olp>ZX2
zFq3oQYGk?(ta}zBv-p52S9_9r5#6vcX)j~%H1uIv>o*%=Dk~n=Maz0SUzSJv=_&t5
zQsQF3OJQgqW3@TQn)kvvE#~dYE<gGv*O423%H2TXqK$$5?hh&j-+`qXd?WAMOSr?8
zTlZ2)nYd@(CZa-svVi1eh(~bB;Ul7vU;MZ`CLcoHD4_N#)}kqql!AR})VmlKa3m#+
zM*@m42y;Ha%9{tlxBeX1ao$WdV-U<i&+>0V2z^ncip_OOfhy{!QYMAdiAX5!A>GL)
za67jJ>Q`L(&9K8G!|vrDeo;v;ARmQZZq$(4l_SbtltK7is4{r4nWqfrMo^lyr&Y7_
zk+LOjhbr$0d@YADP@(C;rjJo=xT7);rEw*z8S)t)wtl}$sTYAwLSpn;Drpz-CmWX}
za+M}U;?M*-ISYD;hV-dn1YbIZzM1~R(tTW!ciU=xVp_4)r)S+2gn}^%uK{Ca`now>
z5L>(qW#A#^9=_B&g6Rk8{6kU+Ba$)EEf)UJ(=I`4<gCZQU&M{UUy`6TH!Zf;%)Thi
z=Jpiy9Vf81IV(zW*I=w=TQ4rm%+HDBp6zcQpefQCJO(}{nY^wtW6*Gw(&S7ihehW`
zAuz<b6_^osh==xjaeiEEG;=&`(a+-S4?}eM*4I-ACOaiAXsvkZ;=UHGFW+odpI-#Y
zHTHC3!bt+$I2GRl(M=WKG0D^%7EQO4o2O2p$9mavUPRK(@%*})wcfzcs&42~nQ14t
z|C~ec7iO?jtLdS<N61X`yN*%#z-AaM&go_#^%eEOLDd545*?R3{MZsXM--hjCkiEn
zY&d7d>aGRUZ5juft%YJ(gVS&aXWibZjEs5I`HAKnj#%iSQ^?m0Wfs0)iehIyWnm%C
z@|<SJbrmj-2v8gx`ud;V>?mX&IdNhO4j91t#RWx9ET;zgis9yPG^Nm3Bm_QZ$|`u9
zX0V=!@8Lvsa*LSi7u7{OSsZ>Gh^Hpslwy#Ex8e4;`3eC4P)w2G>g5=H17BR?2kW#3
zoo@OCdz?8JxOC6ugvPkV!N@;E&4sB2cHGH4R;NH^Yr*H|S*C$$Ed|kp!nBbP_;bRm
zNY_8q?vGnr8_x}uS8As4>E#Y@=FfVP*}}t~<a7=7rc4Bhe2XM*W#RLX)8+h0$roC$
zc*-t{fc{Ak5C&8YN#gvPjMn{N<7Sl{p`;vK8*Gv8c?e5GFO(wmfORfb7xN*TQZhf9
zv|lnLMcztwXa-!Zr(#8&+h%9!LX2QSgEhq$|1p#V|18P436^6~-o(m&iWeUag8}MF
zGkcN8Swbeo?;wr<A&CXKDw_5CFqB-dUnW3QjBW=Mk&gA*h8dWm!;@AqVVBvt3gU@y
zJXKt_3%S|sMNw(dZp!(X6FvQ1HDQ-lO5qE-11+D)*-!4|zz>LB4W&J>Gg9d54%vq5
zVZoeAla6IH55*WNum_JQ0TqKzRtm$I?vc+*G8xPz%%5whFvTtt@!VB9nLX!lSYf56
zYACYAQd$Hm`t%e!t{7)6X&!}+I7oTRqkzq3{xrUh2+9>&6KJyVchqsrzB&p0fEJk`
z1G*8eqPGLm3$g*0xk+FGS6e!9q>J~a?2ZK9dOM4ekqN781Yvp!ihkhTY_aPpLOK{9
zfc}>|QdziKnsAt=3Br8>Q}a(>2)!i}-(g>tatYWfj2nCyBZs2TmtEjA&D1acpb&CE
zNY$D(%xHwZZf<Tvgh!`aQ?)?C&@T*oU}QFyDW)Z&p)gl&FHuMM0Y>;_X{SV{T<ETK
zm1t3Ow3^-29?OIB7gRCKSRCcRxHcOnhMgFhDe=PRAnQ#FQjB?~ASf<|Fil6!!W<)P
zWG%FKIc1Y4MRL?<K<x=R*-=dv6!<&WNN5s1iO8mKRh%9wsW<U-j}lvuV6AQoT1Lj|
z;-z^!jdD2+G0;w{m#e7!!9hY4Re6R)ac~HlEjeGjH&qY*@*9@Xq1AS<s!)~OVm565
z2hNer#<YI1sCjC@z>vhMerq2QUz04_ilRtr4r)ry(z(TRzpI&(1X3srLW}San)n`8
zKUyRzs$1tfOkgm0z%;9w`fV$_11ClLG*v_Y^jAQ0IaZZD%8)tc!~{+xYkLtHy*dH3
zAC1o7-9nN3@bM)y^=}35-x=NQMiIF!a6C}dIL@u=Cg?w0ZNEew=66A%Hm(TWo~P?8
ze&x*asB~cUxSNTe_gr}nm45ebG4AXmHk<G#4Egs5^yuqP?PKsr=tA!(BA%3|3@P$T
zNuW!awDOV_T-11;sc{jU5dtGlX70*!9IhFIE$Z;|O7%bI&&Df7MZ|d#qFY4dhz;p0
z6TIcLA7IDEJj8pSNb;GpqR6$epI(S)nk7p_T9+LhtT;_ytsN1F(5u*Eg-6{gF=$-a
zVj6*RlQrihG+ktR%|4XJg2z^le#Y|KiR9K_oN567vyFCAX4~~|0Nz|&tziBf)J!R(
zFtnZP<mpYnD#zP=xq+$&oQP2DCmJP^Fo2oOnx$qBHz7}H;{dY>;cF{t_~!vR_~xb9
z2#iOabH570>od(IWDrKvi#OS;bGm{5u3uS)R@252_E+Bk8s5=FTFi%8h4N-aZ1I_t
z{33G_R@>?dAmXKZ))RowAtsna<JVNY{)+a-HoP(+*hNzWH+sBvs>qLA2?i=c&ali?
zW4sD9+<#Rfu0xGSyfJh?O@ikeV!@<4A*Vcby(>o=sz(1R)cG6N+5@1yVKk*hq#%o*
z#!!94-K48%q_NLvO36#Y#@iRtczxSQDrHt{G-JRw*;MYFDRkAoK`zY{(<++#C6^W+
z30q~MSRCEb{210qbF&p)j8{OT<MlN2GarHstr02E`Iux0$pjh`d<s>LG<*u{I3Oo<
z^cA6E&P}pkEkY|VN=B@X0#t329`7L7oXKy}_%$ut%A`cBcCz&N&3q<|N&d-3V#XAb
zp%{JC@`{hvU=8hrFKmlAoIRD?ZSzno#R)r1+GPtSeGE*(y^q$J#0eOlH?AjvdQEz$
zfr8F~q&)9f5V3JBnG4FTUMiH{5(P<?cflv(>#M{a0+4|KZj=c}<2##D5#?JLvkB#O
zuo)LmcQj%%yq2T-8A-w9&b!YfKo;>l5zX{E(8EGTRlWZ0M;LRN8mQqw#!@V2C~DZi
z9euuzp7#i?tIWT|T7iXCBSEcB$GW1kc+j4=oQ3DNlC*`)^>C)z7|lirfT@Aj*5u|F
z^wt&|I%0(NZ}~_CNRF|-u(k}~bJ>k3-qt!v+R=g55+p(R){#@q9Q;<Jxo3n89?=Jk
zc2c>@6POL;Q4-)3>L>>AkuM$zm=h5?hUT17(QdqR=L*EzS>g&8J05LaT|OCXtsli`
z9SI27l)c+oinApDa)&}4E2(QsS6Ca@79g?8)4Yt4%DjmXQ0VA`((X*XzNOX9o#^Nq
z<(KGqjU=qvYKw?4p-s+fSgu*2I_g6MK6i70<|x*1pJ$03I%A%9l@#FBY9gv+MEF6N
zR5BsX1BOROK?x1{*|c;Z=HZ!-^$qoNU<h}KT)kT)^*$z2x^wL&vC-mAKXt!B)Th{{
zufHnSWP*EhJ{3COqG42mKNSA>S?+^;o40!(XtYDzG0}kHE7@@hJ+>8K^lfyj#j!H)
z2wL}YP$|H=#C0^!g%8~&u~50+)nL1-jU;}EjZr~$eV1<``M`R}HKp-gM&yv|3XeX)
zfGbl2%q!a1+qqeJI1=)BOPKg|U|A(h{164p<sPsbncamc9<%D@Om&0nQ{Z4xyBVhF
z+?yDfNL=AP)$ZlE-#j>D6XiL!=p)c_1sHVbdx~RPVy+)00UnaB8h{D4W*C>06)AC1
zvGpZ&G1oo&Gx~SVj)q2Hlc+<UdYW!cV_>mY|FZFOL}(CC?~jBXO@;mOg^4I=@yI<j
zG@RU`wtRX&<BqL;TU@C9&(72pkDUHpF@5`>nP)fLDKsUOhdo;uXk^`|eXsG5Br;c=
zA6OBXGTUEgC}k2S&F~4{psTW=6$F1+R3MJhqG8z-tHq0?h8_l(Hn>#A%2EQfv(Ybk
z3Vk|fM4_uYcyvFA!M#^P5{;UKEeoJzAeP1tSc_s%l5rzeLwmfWEhh382t$(6+6eZ?
z#HBX*J@HnnfY=|txO)DppIEijz;lphUI#gTvANKTv}Vn{%^S4=U#U=UDuytkN-p$b
zj`sjx5EX;2h2|oH8vp)!5W_n4M9@}H^;PwEw92!}6CX<$22M}!WY|Ip60P=8XxydN
zn&Y_2?AlxuHTLKeY&GF^pY14QC?lReDh9wDyC^1Z1ZCzM$&OhZJUUS7dm;I7P1i-2
zZm^^_ZpbH!6+<0Q4xNpn(BQac%GU_}#?T_+5VLkic9U+4q~U>g`W;dp_L>Yy(mFc0
zj);C48**Aj@5_JU(NmBIVTdV?GhuE6(LP|Bq~kw8i4?NB@_aj8V?ns8L$fEjTCLAp
z(1yO}01NefOJwD|dBVP~+uTLF-&w91G(*4N5k498i@)@_E1||T-u7X`G#-n<vZ3n6
zrJH?A#=<QZ&og)`_ecw6%sMHK<j31e%ze4}QLgWDGj-1tMriBZt^xYx_2u3(RQFg<
zBlIN{N!29G&aR<_2IMHaF;mPEM(BVB2~VJ3A>pj7-}AEDRmRD0?2+*tW}D(Txu)%k
z)HE{pmUx?8DML*0H@e=m#8B`LV3TeGU+>iW-moA~_un1(ztLa1%leBijeH#E?N%UQ
z1?(?R&uZTSX^H&c_+FtX(`F*6C|x%YUA>eyw>ypckn5lyCKY*KLl82tCzwvh{0Gq<
zkuM6$6m7hs85fpIn`o(eZD7nuu+q^e9m|WvS<73j`iXPup;OE+k;pz5Lf>*q%|0Of
zt|SgaQBNGF<ES$wT9XI>BHs<s?PizvcycCII8M_|c<#zqQTdT0^5DZW8rg8Micp}C
z`kc=uq8vOsD+eDd=V?gcm$MO_!<+|vH~Sja^~hBT+$EAQeY1b^6m;M=PgEWVO_Iwc
z`)m*jOuXRR&t!^_HC~X~k{KKdCSI?pOrVLBeF`U@J}6@IgIW;I%25<V$~{3&*<%x4
zP@|#{tT3xD#K9GuQbi`bdm6flI?pD6`k-m~STCLjFThxZ>x_EJ!RO5V#xLjhw{K(W
zECTtB!9#k2xbFIw9;q1U9Pwv$;WBv^H^1KWUJ*o3(LvE}ND)?Uq*7l*icOF{6iKP?
z4_{rVOAAJXeoSJi8U^+cd+qJ!t#?^qXbcEo4JY<16?jqhBu<iZUFgcs=7n|?L%l<`
z8~fhexPa0`X7+;+xI-)pXggj(pZSsFb=dnP{BUspi+g%ElU}XNj`9n2zCiAcW@T1b
zs+hrm9%$=11Rr{totKIN?JK&1es)%3@NR1~nMF1BHQp6m*vXDQ9YZ#|U^RBBq{=qz
zfSB9z4$KqO6E73*H)d0xtisMN@DUP%*d!BDBQP&$)M~e9S!oY#(^EucC$~>zHY?e$
z#grHqSGyP`Q~=$`@V?lU)J>^}0v#Y+A!ky^$yh+crUblfQaOS-aOG%~P7;}a>OgM~
zlOdb+uL4zBI3+5CkI)GP*QLqCHhTR=hzp_DqrygzC(ceYq07vQl<+;caHVKnwMriM
zresebFbQHbowt?jq~b$E=Nr_r=2y4e*?r_ek3*Yb{U|zb_j(F+WSZy^pdon1Kn*-R
zZEFCSEDraW<~wZwqs3mZ8^BI8%Ok<z)H<p*llKhkM~aIUpZZ)6pvKK^JUCbnF>-5|
z15Z8CsrxtWax{(;DV(q)YjG7_t2<8+r1&#SGgZys>_r&huqokN)6St6VVbV}k{}ur
zF;<->@xrhQq%lOSw?u@6rLx8c6xNHn%_zjB310m8#-f(cI-xfkS>c&i^x-xiOG{9N
zs<W&)urVSP&amhkvEjV_cd4yXn#;pq+!+LgVRA7o_qZ5FvG3tg0iHVkz6dAT7ZQTm
zjg5@X$|UeV8O?qUuhF{%HRthxf1$8u5=7FQV4DGjub{rESY8aB#x8s+8<{`qx)4<~
z)RZ5lR7@DFH?gV^FxMh-D4{moAx(sb!#+-9j_ZWI!Yb){VR@hxO4kzP$TyWs=y=vn
zAtT`8D{XSE@&e1Cu%wlTIn@$;0GTI;J@-&z6&}`5XX>?3S5@FtP{-W~!ws5vUBacD
zex^lMWK?uekL|`x;QrF30QaI(E2;Mq;e=QCxE3)lZC#$4;#gl;^V%V9rxk7x;(WV$
z&fA=k1YgBMxJpODZq3&;KZk<mWkE?V)eH2gt)efWB<iw3e?#64&F95uvfoXUqH%<G
zI068S%*X|Kv}QUi)*KMx!|_MW=$$GykJaLq8-r-z#cs<ns&4|;EQ>`g%WgRMw3BH`
zj|4h+1szC|Qd+$Er`f8Scb{Rx^-Hdr4{r!(s1a3y6B>U~>nUu#c{;$-c7Z~a{ie60
zBEkL^k;;t<&&2flrTiYgLC#o(nv@SCw4>^4S`?`|SzPC}JnRD(F0MsK*5;`shL7xX
z98=ycsxVu#4dIDBpDTZCeK64#ucz3-+K>!a>0AeglD^~wk6)3!-A@0ln#5Z5U`j4r
z<yRWMA>PcZH__IK@i#Qn&G{@u_Cv;>tIo?Vb3>So?_?g=5hC$vd1Xj(m5O94*GHjk
zEaxAzJBMkXk3~HFihNqy*V_Ddcw{xmzLKPr^OP{$W(NRcYKIT-XU-UPzx^D)4+$OX
zY(!3Cv4J5b#?v>@n76v4OP*vDs!`$%=TYMRtdExiIgnqrtWB5zD$m=Jmrw3ScoK+G
zH%gAV&5!b_Yqpz=UoI!>OE}{Rwh}Cj5YGE4uta2<#sRd&vw2`u!}@~#<a3ibL?(i%
znTm<$Fi3Jj<y<43LDcue`|Tpes45bvw!okdUu?5I`oNZPabsgj;l11_=0<uCYrljS
zJ+Kk`lS0vSHetgue$cn^f0T?lEL9pCf_;g!5eRYT(|&tcCw`M!()8jJ;)z|4!It;i
zzL>EPE3aQ1C1(?R3)5~acLlS20oLM2bZD(&zb12z|50K-2Z-~BUs4m7YbrwSIla3%
z%9p$0VcRycfB;_F-T<4Y?;CFr5k2`32CWK<iQ5~i_uLZ=MqA|U?J3$mURn|(bbSA8
zItK47{b3?2$Y7SZ9d0N%r>{?Z{w1KWdNUZUYehVzGj5VE#Cw2ebb9hRh*JCzMz^CY
zc(jDZr$6in7tDMV<TT*A(%J_$%V`5z+IkL{5&LFd@W1H7rYqCC6_PHb#}392w^)_Z
zwvtscAI)awG^rVAcRHAU%IrW}BM5C)0+NZfo59(x-$Fwq&02HK_w(95ilCx7%{KXo
z835muG?ssRqM|h&{`I4h&A|Gn0tpwm4{%#`DHM7VQV@m9WfD6PdY%|-$6K2!3-7lH
zT;4s=hacDRmbdi08`$x-Gfv0SDRAxhZU(0smbd>NN`9hyF)hRzt&`=fyTH^h!z5@o
zIqB-MQn`jlcd>_;zq^}^f)$Rrvl1gO%lzp^2B<m)0wdAzoVh2&nBE@?X^uum|6(oO
z5KC(cqxeExW(Bud+I}D5PL?L+`0jK1M@&_+K}rQ-3FJ=zo*G{tggf}kPII3RV$wJS
zmo)jZ8gm7Nq8e4QtNI?^(F)c`1M<wd=@n7=pKCEXJsx=PBlr-PgbWRrxJTHTR)6Er
zEGAg)!+N7#a(ucbm%pLx$S@d1m{fc>nJ`b^stAO4ve1T>ijdZRAdMx;#tPOkCJ4uZ
zS;mu0Vsal}w{KKp_x0Z-?l{5v;Z6C)=n>ypF8zAgiOuq<S{~iGS{>}+62GWT0pdB2
zemBh%N6;|3OJZo}lZ_majR}y9ZW&2ju+;o^c5_c*@cE&TyJDhGQ{%$jn0v9k>bJzs
zy;|JJXATu`bY(^(z6_cn_U*fpoSWYZ5>Z6wDAYXmKa|(@+U<AWDVV8K4iH=EBxbA_
z95o0p$`my2@tyA>t#LfPxt6CR*Kq2;b%zc*HZXH7(UIpZE|d?GYBLICC73*((!$?y
z6_1u-YAOejbo7yPSTZF2B;v5Dp8K#MROApAsnDV<LX+JJbKV@Vfg4$r(;@Xjzk!=|
zaP+;-bg@Zf$RKH#C@6HS2e+1{l{-wnN$HpHgu%drMU{(7_+(1ll#y93(_!!iuC%C|
z_5@*45?gD$dq~U1K8BzQP94rBF5$gp<|0tX63Qw-;w1((^Rx1lqRr2V#rVFEk7})b
zA=FCm?ipq0A2wf*-FBMX0^#h|vafZ{9@585AjcO&CP_KpwAM|;iNMRGc*X8~Oj413
zl;h|1E2zJ;D`7Y{aM|~PI^b@a4T+2T9qKfa3*@=}?%jIRj;|Z6Ua8O+?2Q+V<5FA8
zGc?Yp8DkRf6>Zv?K9YTEZ283*M6O%`)JqIy-ep0V1Eoz_CFR6qk!Ha0dqjK*;i`4}
zr3VJv7i7#+!UV)$>RB4<74?^G!YZVaSHQ)3ED<Wlo^`xme>b3?r@IN%2O}TT{leCR
z&&b7-?$QDIGIOx-?F02IyKQ2O$$bQw(Pe&GW?$GjSHKw*COG*ruOKLX7I{hflxyI{
zymGjfuuMrM8C65o02j%x?Zy0&Y?x%XWoK9{wsScIJJJmD$+aNk#oUh9<S(mdMzo4>
z7**uc0eS9$4a*>emx}-z5XQ@;p$qt>3ywHD2eU<NSnuYmo^QF37=KttkNs$WuWQWD
zv_cccL4ucS06_Ys3sj}rc#G4a%@X;3D<BxC3wxLydDTIRkV4Yd7Q#idJ{>T3i5l4z
zLUKg!sbjf6SWg6Tt%czQvYU?=qwJfAt~Lo!PdDzqDbJ<>MQ3E&x7l?ASHAi<nKTFO
zE#x`O#7k{k9L{ds>_|6yT$*EY8@Ml-LW3p)4&nUaH9?EwE}n20RYWuJRC;Ftf>gSx
zO$Sp(1NOzt+bUe%n0qLa2*ZoRw}8k_s95`po-y~f<|zYl<$~QhK#v;<zfebkhd1N+
z1amiTX$Qx(cB1o`-qm#G?}xmr42?KIE^efIE-sVvykHTQ;%}1yel@m*FT3&8>Vq@8
z48lT1s&3Kt0gcyc@^6wfI;#K_UUZk5-SwIVZ?nw(X-4^^nhMc1DwMfSfjn&7@J*60
z!_jRV#l+7s!Hu1of@<uG71>`~YEOD9PTR5I%vLP+aI4(l(U)U<hhU8-s&4W{4w;q=
zlyg$TU_W8m@gnFDIYzH2D-WY@u~50@+)e7M^yX(OqQY(J#yZU5P8eVtx!4Bml&Q_E
z?e1t^mAzmhESjh{R_+-hDz{@Hmz<re<Ti<H_m;S*@ScBZ<3%Vm`CgN5&i=_%cRfPf
z31$i=Tq!7CfB2&wG3`9jc3Ut?;9>EsSwu=CF|_=fhpb;yoIj*5w<5LQ-f!Hh``(i+
zhC|<GdmD2pQU{8Yc%tHN^`6|)K_#*6c3SW!YD^~!iMazi8~afi57?UWxk=zB@#w2u
z=4%^=+6TSA(RY3jYm*-M;wU6hI8l?%{7sT4+wX)fS|JZs306LD0zda^)@yWM1#jaC
z2EoQXgGaYkT-p;Q;sMpn&f5a{+H$$`iJRoXa2b0N#_(MuA5_il4q9nlO&C7scZ4#+
zd)=~0g7=%KmTB<*%uzWtvZC*I=?#+<_1!nz4?`^1(2U+Cv%Sih-&D2|-Bci`%rZ?F
zToX<q?zI;PriI0o@148|<T?=h#Rz$DCye;?@0zK4rrcgq^qcLfiL!!pgSxdbK(-+5
zOtB*{mk}ujNEdTgk!scqnxn?5WN0wRs%zHFti2IIfC+KGHYfn@>(B-(P#5!_@OwZp
z!;36XH%L#uqo1NdpA%PN2aK7t02K)rMSTc^KCm(lhVuJNG@f`nj?Y0|aVQp1Y5*mf
z{Btneav!%{6@+Xko1OK3Fub$?^6M7S)axh?HFR%Q)yNha&7vO~Z9<Q+u&CS?j*BVC
zLuVh=1t)xY2#*;&G~jBh8zizL1o^V1SVkejTlwC6YItdu(O@xxo4HcT%SByL5fo91
z!9g9JhdrLCbLD-FUf`G}4>72}+T>g!0#5GyRBY%CHJh85WJy!0o#ar6f@KK;7ZV*T
zY%pj?=h7G99)eo7*fkf%SRlnj;Na4yrwK+y!G4i}aFDMupWGGkatV=0NCxZxBwJP2
z!cpTiaPsjuEDQ9}Ep3|(R^n0lqqJZM-BMw5BqDMl=c2qE`$A=85NFf|qsAS$IFnkA
zFUozF98Z*coFGt4b3=bz9fOY~!3&)yCUHB8--ANuSDW=lM@I?xQXG#$iidZ4jE~0^
ze>pk5>%!>-ifVAQ9mgy16PNsO%|GB6Z%^tdn-%K938vuYq%tBVhCj@Xxnswj5f?<q
zs1mS2^O7a1X569R=bojIlPxcioXf{r_LF%t0(k;hI)>;)Ct!35KD$Maj4W2@ACh$O
z^5UyczRD1bJkwyn7gzx37fLD)m1(YFCa<qR(gb;RB~b=l$qBk?E;TzKZ;Y?ROBs3n
z%6Qd1WL0SXp>x)(sLyp%1=R{_6u+GEs}3=;lr*dA2$LbFkQX~GG{%G;^A1JANhrEp
zdO(q+8)8$6!qE7JKs>r!2?8-$0Ovn$H5ZBm<)#MsFTs?_sG4stf$r$NReHE57(&z)
zi(+!zr#g5%G1X$6&mnraz(acCh{f8IC*m@#B|?$Pb6V%ntHIp?pU9?Z2F~j~v(cgG
zwoijT2o%~ILD~1sam6w{1@&k@>_DU}mkzd}NkIf10S3D}l5WzEnWC=fA{#EGegq_A
zk@g<&4>jFk`K*590U4*z#%IW0x~@qTpHavzCipKefjmM_Ea*W%T%9!S4uB!>4XU>E
z&FeUS?9I90!`+S(@30y$4t+bnd!FwK1%m*8YJ}wtw;^wJzf=>#vj+YT@bHzC`6`()
zX<lKI-|XzDoD0qi6VeXl=PZ3KIFG@5t5^aql6(H+xKF}X?PDXXZ11Gu8W_~b%OpM+
ze#iLvZVl<l+y`e$%Qig&>##GV4Y#tU;3*=;o<E`3*1C^SxOAy<#QwM&wOi#<AEZS-
zNmlMKi+Bm{2LjUpt|J~z)g)M@sYjG$>Ox_k>JZ*zHPeZ=(0SN}3RI?`+-UW$CWzYL
zWtr5%q>6D?4h$;P&XbcoRezgxk|iYD=KJ_GiZ(nsb#qpUe_8cVq*Az3I$@oryR$Ft
zz2oA=vK5tYV6JVZW4d(3VIweLx`lA=nJxq8oFA$2JVp&|>iR*=-rK>qolxg$zN(5%
z;Z3-V#2kL0!CBYN>%Mhn!t8)KRUdj}2jcH>%K>T<QhluFlkfbsE9|^YW<KCO7U>qZ
zuR;i7%?$?n%h>YQ1H1*UvI|!`S9m0m)VQmZ%~@DZ{>BaA^V@bGs^8k4)6U9Ox@jw3
zt`6mox%jkZ_7tPil`;%VeVjRs->yIoe`Iv3^<%p;yM(H3ZuBudLq4@?L9s|lQ7}DQ
zyA?Dew$K5rWnqlgBMvSS5p=!Di$iN<gmL^HvVyw{)`l_Hfi;{%gf)75ZvS&|r6tY#
zxr6Kd91@r@F#KY0IW4%B<aa05Qd$2pF8Cs*CwV3qEXN_i9ESD~vW%PQTpzStI)>_}
z)R}-{nlcoX0>yJI0Am_E0}YnTAcHahp#}LLp@Bp9pbY#cp~1hRU`%uxK<!s8<+dbd
z@H_|pC2<Xmv4o{lnGHc4oCv-3Qqhc;+;()2{mW_ZV&W~T-kj`%!x$5lmGQglpFH{q
zFS9|!!3}a5D8?yQxt3RJx}oel+{d}BUPjQVZfIUkw@6(?MW|2xOQREy**Gd%IQGFz
z0wA763IpN(7FRF!C!gYD<k)_XItJK4$mhBsD^Mw$ra%B<Wj%(CoE~$Tkzi+iyN^o3
ztn{+u!Vd+gwNDzH{oIdFQpG1KMlFYPh=%T&W$^6>T&W(x4twArW0L92_EZ?dl!zHb
z;B;C$DW9WdoJ#yBV6Fvd`AQr(t+ug}4P&#Ry|QL2jy1M~`@LGa2kbFIO08=XT*}D8
zf9FC%$Q@sIk9&kT;J~BL0-84Fh-UyzPlSw(HgZbEdp8BwR|VVj&3XXq&vir&2Kmci
zJ8AM}-giu{{O+9P<_EWu7^g(s3Ut#LUBfj@KufuLDLqs{H!#klFTpjODoX*qm<47%
z5{V5FhSFRK-Dr>vYa;PmX)IFvmq<Wl8R`T1?UD`qm)4~_+$uS)-}2KuVb+|K09+TV
z0%@NN?in43g>HllXbGlxBb+;ivGeaqO%k(wTfSNzNwa2~?44B8enkF+4{;mK?!AcE
za_K4m2iadf0}gJ=<$M#1Z;WnI1M7EAc_qJ{dXAa|FmeO&$@z{tzvq24X}9lViLcAn
zQBhTznIMl)YKQ>s6x9Hu61WW~Nv{u1VmyX^6fp5muH;2*C&`g*eF|tT7JLPBr%aT7
zHH6jf@V||ol#}6uBlMKEDT21@Bh!aUbBRjHQdm^QwWlk<k_)VbR<nCjK;xlS=6s*=
z;&93-z8Yf9gFAbf!f-t6MbXGG=1qB}BciSL7Ue~D3S|*h;Cp%F?-WegbETVfk3PKR
z&%_LFD$tfMc#~#l-we<^HuA4%(ba8|k#&SBmwrq9#K5uq<jn41uVo$D>Tjb{nn1i_
z_S!S|ef%3mwywDINya2T5vP4{O8bUeR-@FTePplpAb&gcHXCPJnIr9~n>JH8*V~h*
zt2Te41}EB(xKR#G9^`>H`<aT$Tghn&F;=?9VZSS5BUmelr)KU2NHWeFwoCIKVIvWf
z9aOQ3hZsaj(v7I(Om0eb`fXJ}D?W{j{+nPhl$WH(!drq$truO1BbaVBE(*lMl?a#i
zlALf`#WCKE<DZEElswMASHoqLy-@3S#}1FVrL5?hSC(HMhn}vrwI1$$ik(r=CtG*T
zozznhO|Ii)0!Ecz6(uKqk|_JN!Xl=+z(U!tMDmjGr?cKcRJqN*7SPJFyhZ{yq{F$(
zrP$WLu|K+0EeIa(AB%0#r*l_a=xI4!!b;kEN=}~5%?T@EuY*Wib+%40H$C86-M+?^
zG5#W+jFwp>@sM+NqB|t`c?-o&XyM~QSWqT69Z`K$>Kcahe$N}1)O{46pmPq@oyG#I
z`T}b2?2sDDe4u(_A?|nBWO(@5-UBoam>Vv%jzU-XnxA~Ua5`hFnnG<<`BznKf2c&#
z#ii1VbVeWOw@^l}%kBgIe(#8@{Dw_#XG(ga@oe=ZNV2~hZL=_RTqL{WnojL;_jt>z
zMgrPAh`z)I(@SXTGOq9BQ!q@t_RmmEgk&x+^G_1n;jsmqgbgC(K3@ILK-io6aG4wU
z*%_Yt-}a&S%oEeX?Ji9W!p%Kwdf8OYR9s6vjD`%<6}(=l<&;Expv1c(g|17QD{pTj
zpy0gp2>(#NTbtfZDY|7i#+J~IK&{`Dj&<~p912M#)DNHT+N(#z(Qun(RkpC@R8(oL
zzjIwhl%a7JVao-7U?q;=O&+9k$1kx5t@-VNfn>dJdR7@2H@h|K7HP@JYAUJ`F>E|f
zSR{(|7g%YA%;4M$u$Dth$<k$3`Ar5o4vM$kuignq^WmTRtE<RIuLk7fG_j;{C&-_u
zt3OqZjnH}lt5kZ9KSVPyU0=0IhO|4sCz3}!&KTLY-8g>}WT|OnO5bw?{TtAEqDa;$
zs49|M7uB_G$#@HwV*DaV@@_Qodz#oT^PGhdQCW>D49?GM9uU#x(%?dLN}4>>tB$1F
zBF6;`p-n0+EKAJAxL`!=8(yLL6Vz-MuJ3_k*iUk;rzjuex0SpGFE55QN07Dyg}je@
zS2JYB%?RXQ<mXOv`Gh*(UWeD^EYkBm>@y~b5g2(OPG3X{KkVEMec5J;X%GdzxGkbS
zb_H-Dmjxf~4EqBe?`Z?PZ(KByhFlPM(ig^GU~((R;U&3tKPxa;3Uz-Ik0C#{W6+Ar
z6m}k@Tr#N%Kt$q=e~Wc!^qRYWe29UFx3TyB8H^tYn}HMfCbw^BV(MEK&zwrCDNh`b
z&xF^eGS{luWBv8>s>|*#fz@9-Y%-Q`wBL_;8#x`6jc+d=cP|eiZ<l-=_{x&(bib2=
zjAJEmSC0haCq-%><*Np_eejO3eRN;3@4g_cU(-!Nltl}@YJ0n1X=i$zfs{|=0Mp}E
z-|koL313G!C4uitx#>MQ|Kq!-dehfTwbW#}y;_>utI_<TC0iPajbii7{vWlP$O-ab
zbYlZqCDERkH<3)X@zM0%Z!wn1r1v=<e&#H^ywb;p-}ue#q1#-IM8EOluF=oeqk#$E
z0mw3K4nMH=Y3-a&$BMo)e`9H2VQgx(``q_(6Ii+aNb9b~AO3xk@p-uq9f6)-o;uH1
z0Hx2aZbRl*)jOZ7Ig|<i?xYv5xs}{nzt93r>{e?aj4WaM+uhePsFjci8RLy_%ymjH
zQdQdK%CNaLuSD>Kh*GDXnr3h7MBTo<dlF+zoql2Aiwhw|^W3ET7{7~Nfi$`3kxO5d
zKS1FM{hse`2f-i8_rs+``fxts1__zWJiph(unFW97D5ft?O|y(1=vwoqLFv28zC>z
zgS}4^Avdr?T)*u|naGiyaJAc68B*J65qwogtS~BmY<zK<3(C9vg%hxj6_A$6L<c>x
zO{Ho#YjVcsxHG*fY*=YJ{W*Km!j$>h2MS98;ftV8!mV#HgCLaP4@%A{O18I)D7dqD
z{0%i66V8BLgxWNc4qBu>);wy|_P(tA3G>G&nNPMK=kG=}3wiP(=Q>8@rXQP4m;u9a
zK7cJa{mM0YH0_}x&YnasGj~jttcKtHL*&Yr@ObFXc>cHOrdUNov!eZLnfoqNL5U2~
za5k7*(B;qJF;ln6;CA2lj2N7Fczz3)q=I>>YI;fcJ0N31D*}>mezv!=%`sWn&nd(h
zOU^=H+xITRhYL<n1)3t@sI_xPVta9b3vFfHJNV9hXra3^Y!<gva~X~Tk)H_7MF>82
zq>{P3%`iAt3Ws7TLr4%9wqq3bN%W!XfyL+=o=P*$Hqb4-m2IoL$wf|+$?g-vLVY=y
z9E*R>Ycg45_#*1dwdK5sYYdvi-C8`ayS(df9bAXmu9oL?r|;(U&1urtY1w+lin&g>
zA1G`S!r8(?MGQZ@ExY`NL@$(RJuzZzJV=-^DvAB2)5gK$wJ7vQabeRddEk=?xLYU4
z%GvVoIyy+}Pwm~Gin~AG>|E?z9LRqYx&B?4r|Au}VpTP>v0_!Sva~akaPa~dutB=H
zxw!!xoSa6;kWM!%XApoBLR3d)Rkw0?@o=-Sa)%W4sk*sXXjp*^SRrM95a4?+kc<Wh
zQql+D`wI)H>0_02aRx!S-2s1U0o9Qqq`yu0BkmuyfdJM&HUR!I!X3b>U}xzLFhKre
zqB^n>03yKyqMiM}u(JP)^>0o|Gmx31i_Jep{B5a{70ApI(gWcC57oT?o&Vo<BeOzo
z!~pCfB7fWXXBdCiFRJq*|6x=A_W=G#{k!uoAO2GvGOLD%Ip~j9b(IwVS@+1!#l|Hf
z@^@r}M7@A!77kV*fVq{8owG3QkFGvifSsi<tq!*m<O#B*m93qukDHZ-kFutPkAnrj
zC9S9kvXH15B%oHzUS>dM>%R&p-2qna7Q&R0>Wb2oe*j(rUQR$K$fyBcPL9s*0$#$j
zf3XNa+JDHbw1B@LAO~SuT_sfj$i>Cc!43rAVBu$BXJ+SQ0$975IaztSxH$mWS$Jq6
zBHb*l1=OXa{}u@OB}{7z0s#eBSv@^HSv)yeT-<C}+4=eTS=l&PIXIXh2xfP0XONi}
zv$H!DMBiVA*y;X@&DovhFPm5_T%1_#bXnP1*jWFmND49>H#;E6&czw<N3of?iw8)U
z*2BZjQh?W-gPVtq!;0C8mzS5Bo!y#)nGJF=vs-XLx;VHjczF2#sSfyG%=~ZqARQ2e
zoDe@iS^|*lwe+yCaucSNQB?%U*t)ob{-GZLaR(nQ>)*rw-?9o4oB!DhkbwT}1|bMW
zK*`0@&e~fFGL1wy*w`Usg$$BIlbuI^lUIP7i;;~(fQ?Ov^*@l1;6qp?AqBLMg4+KN
ztp5f1Uszq-G$AP=BJE~nCHr0#Ame5Rw6(Kv=a2$Ox!HZNa{Gtczj*(K3bFnP)PHf&
z|F+Z2fJ#dL?{0H)`jgjxZ~{^;79LKJ)kaS0|4-KZZwK1a;$H>>J=`4snj)4KtX7Uz
ze{6Gy#F`yqv89E8wTqjR86?5YfIvq(3$s6Y&k9LQS`i^gC&&)uXeIL31Qd4!{iES%
z_h&*0IGQ=z2-A8oTUuG0c{qX~_`jmyX!m#HFFfnNWk6aMe=@_x?H^(^tX%&o?>|}p
z>iWa_FNXgiz<-e-s{$m}tp8ebM34dhOV|INpnos4kOf_s*7MIY3TgfK4U*&E>-axc
z1iJreL7<0hW`Dd0iPE1LMkgikXL_=;v2(EVaPs~&Gilk_Xdy7j4E{eL{+>c~|Bo{b
zB1PN-Wb5Ka2We^kNd-Fke{D-%ZdTUFYyfsP<UdQ`KP~_dH#a9Yz#8xm8ao#kCuH~f
za{-+HNrOBBf(-V*)3`W!Asg9$&^WmMlNSPDhwR}0iQ|CmiT^?4X8XVE;O2sC$^VJt
z{VzC>o0*-Xl^f)b0syQUc0N{+u>n}sU0fjh{+~@3z$)i#?E?4{Daaj&oD{%7NQ$46
zn^#JLLyD7&n@@_JkBy&STuPc#Qk;WFf?J%ITjc+YF#2;#;SMr$1O4Stb{-B+Hsp8j
J-m6F>|6fe=<>UYW

literal 0
HcmV?d00001

diff --git a/doc/related_pages/dg_introduction/jump_terms.tex b/doc/related_pages/dg_introduction/jump_terms.tex
new file mode 100644
index 000000000..17c97e09a
--- /dev/null
+++ b/doc/related_pages/dg_introduction/jump_terms.tex
@@ -0,0 +1,10 @@
+\begin{longtable}{ccc}
+  \toprule
+ & left & right \\ 
+periodic & $L+R$ & $L+R$ \\ 
+Dirichlet & $L+R$ & $L+R$ \\
+Neumann & $R$ & $L$ \\
+\bottomrule
+\caption{Top left and bottom right entries for jump matrix}
+\label{tab:jump_terms}
+\end{longtable}
diff --git a/doc/related_pages/dg_introduction/table_polarisation.tex b/doc/related_pages/dg_introduction/table_polarisation.tex
new file mode 100644
index 000000000..91418fb27
--- /dev/null
+++ b/doc/related_pages/dg_introduction/table_polarisation.tex
@@ -0,0 +1,42 @@
+%\begin{longtable}[htbp]
+{\small
+\begin{longtable}{cccccccccc}
+%\begin{center}
+%\rotatebox{90}{%
+%\begin{tabular}{|c|c|c|c|c|c|c|c|c|c|}
+&  & \multicolumn{2}{c}{Forward} & \multicolumn{2}{c}{Backward}  &  & \multicolumn{2}{c}{Centered} &  \\ 
+\toprule
+\# of cells & $\eps_{res}$ & iterations & $L^2$ error &iterations & $L^2$ error & Order & iterations & $L^2$ error & Order \\ 
+ \multicolumn{10}{c}{ $P=1$ }  \\ 
+$17^2$ & 1.0E-04 & 33 & 1.40E-01 & 33 & 1.40E-01 & - & 13 &1.10E-01 & - \\ 
+$34^2$ & 1.0E-05 & 78 & 7.50E-02 & 78 & 7.50E-02 & 0.90 & 25&6.17E-02 & 0.83 \\ 
+$68^2$ & 1.0E-06 & 175 & 3.87E-02 & 175 & 3.87E-02 & 0.95 & 54&3.29E-02 & 0.91 \\ 
+$136^2$& 1.0E-07 & 396 & 1.97E-02 & 396 & 1.97E-02 & 0.98 & 124&1.70E-02 & 0.95 \\ 
+ \multicolumn{10}{c}{ $P=2$ }  \\ 
+$17^2$ & 1.0E-05 & 102 & 2.46E-03 & 102 & 2.46E-03 & - & 47&4.10E-03 & - \\ 
+$34^2$ & 1.0E-06 & 226 & 5.93E-04 & 226 & 5.93E-04 & 2.05 & 114&1.10E-03 & 1.90 \\ 
+$68^2$ & 1.0E-07 & 485 & 1.46E-04 & 485 & 1.46E-04 & 2.02 & 259&2.86E-04 & 1.94 \\ 
+$136^2$& 1.0E-08 & 1052 & 3.64E-05 & 1052 & 3.64E-05 & 2.01 & 580&7.30E-05 & 1.97 \\ 
+ \multicolumn{10}{c}{ $P=3$ }  \\ 
+$17^2$ & 1.0E-06 & 181 & 4.77E-05 & 181 & 4.77E-05 & - & 113&5.37E-06 & - \\ 
+$34^2$ & 1.0E-07 & 403 & 5.22E-06 & 403 & 5.22E-06 & 3.19 & 259&3.67E-07 & 3.87 \\ 
+$68^2$ & 1.0E-08 & 893 & 5.93E-07 & 892 & 5.93E-07 & 3.14 & 583&2.64E-08 & 3.80 \\ 
+$136^2$& 1.0E-09 & 1946 & 6.97E-08 & 1946 & 6.97E-08 & 3.09 & 1277&1.92E-09 & 3.78 \\ 
+ \multicolumn{10}{c}{ $P=4$ }  \\ 
+$17^2$ & 1.0E-08 & 357 & 4.62E-07 & 357 & 4.62E-07 & - & 221&7.60E-07 & - \\ 
+$34^2$ & 1.0E-09 & 793 & 2.47E-08 & 795 & 2.47E-08 & 4.22 & 498&5.54E-08 & 3.78 \\ 
+$68^2$ & 1.0E-09 & 1637 & 1.48E-09 & 1637 & 1.48E-09 & 4.06 & 1035&3.80E-09 & 3.87 \\ 
+$136^2$& 1.0E-10 & 3505 & 9.13E-11 & 3505 & 9.13E-11 & 4.02 & 2223&2.49E-10 & 3.93 \\ 
+ \multicolumn{10}{c}{ $P=5$ }  \\ 
+$17^2$ & 1.0E-09 & 581 & 1.57E-08 & 580 & 1.57E-08 & - & 354&2.16E-09 & - \\ 
+$34^2$ & 1.0E-10 & 1277 & 3.62E-10 & 1277 & 3.62E-10 & 5.44 & 782&3.51E-11 & 5.95 \\ 
+$68^2$ & 1.0E-11 & 2751 & 8.39E-12 & 2752 & 8.39E-12 & 5.43 & 1697&6.68E-13 & 5.71 \\ 
+$136^2$& 1.0E-12 & 5816 & 2.03E-13 & 5816 & 2.03E-13 & 5.37 & 3597&4.01E-14 & 4.06 \\ 
+%\end{tabular}
+%}
+%\end{center}
+\bottomrule
+\caption{Accuracy for the dG method proposed for various number of cells $N$ and polynomial degrees in each cell $P$.}
+\label{tab:polarisation}
+\end{longtable}
+}
diff --git a/inc/dg/multistep.h b/inc/dg/multistep.h
index cb5246c7a..ad5e28c12 100644
--- a/inc/dg/multistep.h
+++ b/inc/dg/multistep.h
@@ -209,6 +209,9 @@ struct MatrixTraits< detail::Implicit<M, V> >
 * Uses blas1::axpby routines to integrate one step
 * and only one right-hand-side evaluation per step. 
 * Uses a conjugate gradient method for the implicit operator  
+@note To our experience the implicit treatment of diffusive or hyperdiffusive 
+terms can significantly reduce the required number of time steps. This
+far outweighs the increased computational cost of the additional matrix inversions.
 * @ingroup time
 * @copydoc hide_container
 */
@@ -336,6 +339,24 @@ void Karniadakis<container>::operator()( Functor& f, Diffusion& diff, container&
 /**
  * @brief Semi implicit Runge Kutta method after Yoh and Zhong (AIAA 42, 2004)
  *
+The SIRK algorithm reads
+\f[
+	\vec v^{n+1} = \vec v^n + \sum_{i=0}^2 w_i \vec k_i \\
+	\vec k_i = \Delta t\left[ \vec E\left( \vec v^n + \sum_{j=0}^{i-1} b_{ij}\vec k_j\right) 
+	+\vec I\left( \vec v^n + \sum_{j=0}^{i-1}c_{ij}\vec k_j + d_i \vec k_i\right) \right] 
+  \f]
+with rational coefficients
+\f[
+	w_0 = \frac{1}{8} \quad b_{10} = \frac{8}{7} \quad d_0 = \frac{3}{4}  \quad c_{10} = \frac{5589}{6524}  \\
+	w_1 = \frac{1}{8} \quad b_{20} = \frac{71}{252} \quad d_1 = \frac{75}{233}  \quad c_{20} = \frac{7691}{26096} \\
+	w_2 = \frac{3}{4} \quad b_{21} = \frac{7}{36}   \quad d_2 = \frac{65}{168}  \quad c_{21} = -\frac{26335}{78288}   
+\f]
+We solve the implicit substeps by a conjugate gradient method, which works as long 
+as the implicit part remains symmetric and linear. 
+
+@note To our experience the implicit treatment of diffusive or hyperdiffusive 
+terms can significantly reduce the required number of time steps. This
+far outweighs the increased computational cost of the additional matrix inversions.
  * @ingroup time
  * @copydoc hide_container
  */
-- 
GitLab


From 6d5ec75b5cb6f62801017496c105ee18194990a1 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Thu, 12 Oct 2017 17:44:12 +0200
Subject: [PATCH 351/453] added writeup in dg mainpage

---
 doc/Makefile    | 1 +
 inc/dg/dg_doc.h | 8 ++++++++
 2 files changed, 9 insertions(+)

diff --git a/doc/Makefile b/doc/Makefile
index 2584bd9fb..f03b3a69c 100644
--- a/doc/Makefile
+++ b/doc/Makefile
@@ -28,6 +28,7 @@ dg: geometries.tag
 		(cat Doxyfile; \
 		echo "OUTPUT_DIRECTORY = ../../doc/dg"; \
 		echo "HTML_HEADER = ../../doc/header.html"; \
+		echo "HTML_EXTRA_FILES  = ../../doc/related_pages/dg_introduction/dg_introduction.pdf"; \
     	echo "EXTERNAL_GROUPS=NO" ;\
     	echo "EXTERNAL_PAGES=NO" ;\
 		echo "TAGFILES = ../../doc/geometries.tag=../../geometries/html") | doxygen - ; 
diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index 0d6c38d47..623f4da45 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -1,4 +1,12 @@
 #error Documentation only
+/*! @mainpage
+ * This is the FELTOR core dg library. 
+ *
+ * @subsection pdf PDF writeups
+ * 
+ * We have a collection of writeups: 
+ *  - <a href="./dg_introduction.pdf" target="_blank">Introduction to dg methods</a>
+ */
 /*! @namespace dg 
  * @brief This is the namespace for all functions and 
  * classes defined and used by the discontinuous Galerkin solvers.
-- 
GitLab


From c844b3f348c75de4626317df29a73c5c69e1448e Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Fri, 13 Oct 2017 10:24:27 +0200
Subject: [PATCH 352/453] made unified headers and added parallel.tex

---
 .../dg_introduction/dg_introduction.tex       |  82 +--
 doc/related_pages/header.tex                  |  29 ++
 doc/related_pages/hector/hector.tex           |  85 +---
 doc/related_pages/newcommands.tex             |  48 ++
 doc/related_pages/parallel/parallel.tex       | 470 ++++++++++++++++++
 5 files changed, 553 insertions(+), 161 deletions(-)
 create mode 100644 doc/related_pages/header.tex
 create mode 100644 doc/related_pages/newcommands.tex
 create mode 100644 doc/related_pages/parallel/parallel.tex

diff --git a/doc/related_pages/dg_introduction/dg_introduction.tex b/doc/related_pages/dg_introduction/dg_introduction.tex
index d33ad3db6..49d9229ff 100644
--- a/doc/related_pages/dg_introduction/dg_introduction.tex
+++ b/doc/related_pages/dg_introduction/dg_introduction.tex
@@ -1,84 +1,6 @@
-%\documentclass[12pt]{article}
-%\documentclass[12pt]{scrartcl}
-\documentclass{hitec} % contained in texlive-latex-extra
-\settextfraction{0.9} % indent text
-\usepackage{csquotes}
-\usepackage[hidelinks]{hyperref} % doi links are short and usefull?
-\hypersetup{%
-    colorlinks=true,
-    linkcolor=blue,
-    urlcolor=magenta
-}
-\urlstyle{rm}
-\usepackage[english]{babel}
-\usepackage{mathtools} % loads and extends amsmath
-\usepackage{amssymb}
-% packages not used
-%\usepackage{graphicx}
-%\usepackage{amsthm}
-%\usepackage{subfig}
-\usepackage{bm}
-\usepackage{longtable}
-\usepackage{booktabs}
-\usepackage{ragged2e} % maybe use \RaggedRight for tables and literature?
-\usepackage[table]{xcolor} % for alternating colors
-%\rowcolors{2}{gray!25}{white}
-\renewcommand\arraystretch{1.3}
-\usepackage{doi}
-\usepackage[sort,square,numbers]{natbib}
-\bibliographystyle{abbrvnat}
-
 %%%%%%%%%%%%%%%%%%%%%definitions%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\newcommand{\eps}{\varepsilon}
-\renewcommand{\d}{\mathrm{d}}
-\newcommand{\T}{\mathrm{T}}
-\renewcommand{\vec}[1]{{\mathbf{#1}}}
-\newcommand{\dx}{\,\mathrm{d}x}
-%\newcommand{\dA}{\,\mathrm{d}(x,y)}
-%\newcommand{\dV}{\mathrm{d}^3{x}\,}
-\newcommand{\dA}{\,\mathrm{dA}}
-\newcommand{\dV}{\mathrm{dV}\,}
-
-\newcommand{\Eins}{\mathbf{1}}
-
-\newcommand{\ExB}{$\bm{E}\times\bm{B} \,$}
-\newcommand{\GKI}{\int d^6 \bm{Z} \BSP}	
-\newcommand{\GKIV}{\int dv_{\|} d \mu d \theta \BSP}	
-\newcommand{\BSP}{B_{\|}^*}
-\newcommand{\GA}[1]{\langle #1	 \rangle}
-
-\newcommand{\Abar}{\langle A_\parallel \rangle}
-%Vectors
-\newcommand{\bhat}{\bm{\hat{b}}}
-\newcommand{\bbar}{\overline{\bm{b}}}
-\newcommand{\chat}{\bm{\hat{c}}}
-\newcommand{\ahat}{\bm{\hat{a}}}
-\newcommand{\xhat}{\bm{\hat{x}}}
-\newcommand{\yhat}{\bm{\hat{y}}}
-\newcommand{\zhat}{\bm{\hat{z}}}
-
-\newcommand{\Xbar}{\bar{\vec{X}}}
-\newcommand{\phat}{\bm{\hat{\perp}}}
-\newcommand{\that}{\bm{\hat{\theta}}}
-
-\newcommand{\eI}{\bm{\hat{e}}_1}
-\newcommand{\eII}{\bm{\hat{e}}_2}
-\newcommand{\ud}{\mathrm{d}}
-
-%Derivatives etc.
-\newcommand{\pfrac}[2]{\frac{\partial#1}{\partial#2}}
-\newcommand{\ffrac}[2]{\frac{\delta#1}{\delta#2}}
-\newcommand{\fixd}[1]{\Big{\arrowvert}_{#1}}
-\newcommand{\curl}[1]{\nabla \times #1}
-\newcommand{\np}{\nabla_{\perp}}
-\newcommand{\npc}{\nabla_{\perp} \cdot }
-\newcommand{\nc}{\nabla\cdot }
-\newcommand{\GAI}{\Gamma_{1}^{\dagger}}
-\newcommand{\GAII}{\Gamma_{1}^{\dagger -1}}
-\newcommand{\Tp}{\mathcal T^+_{\Delta\varphi}}
-\newcommand{\Tm}{\mathcal T^-_{\Delta\varphi}}
-
+\input{../header.tex}
+\input{../newcommands.tex}
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%DOCUMENT%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 \begin{document}
diff --git a/doc/related_pages/header.tex b/doc/related_pages/header.tex
new file mode 100644
index 000000000..250ba6028
--- /dev/null
+++ b/doc/related_pages/header.tex
@@ -0,0 +1,29 @@
+%\documentclass[12pt]{article}
+%\documentclass[12pt]{scrartcl}
+\documentclass{hitec} % contained in texlive-latex-extra
+\settextfraction{0.9} % indent text
+\usepackage{csquotes}
+\usepackage[hidelinks]{hyperref} % doi links are short and usefull?
+\hypersetup{%
+    colorlinks=true,
+    linkcolor=blue,
+    urlcolor=magenta
+}
+\urlstyle{rm}
+\usepackage[english]{babel}
+\usepackage{mathtools} % loads and extends amsmath
+\usepackage{amssymb}
+% packages not used
+%\usepackage{graphicx}
+%\usepackage{amsthm}
+%\usepackage{subfig}
+\usepackage{bm}
+\usepackage{longtable}
+\usepackage{booktabs}
+\usepackage{ragged2e} % maybe use \RaggedRight for tables and literature?
+\usepackage[table]{xcolor} % for alternating colors
+%\rowcolors{2}{gray!25}{white}
+\renewcommand\arraystretch{1.3}
+\usepackage{doi}
+\usepackage[sort,square,numbers]{natbib}
+\bibliographystyle{abbrvnat}
diff --git a/doc/related_pages/hector/hector.tex b/doc/related_pages/hector/hector.tex
index af3f98224..70ffc0afe 100644
--- a/doc/related_pages/hector/hector.tex
+++ b/doc/related_pages/hector/hector.tex
@@ -1,84 +1,7 @@
-%\documentclass[12pt]{article}
-%\documentclass[12pt]{scrartcl}
-\documentclass{hitec} % contained in texlive-latex-extra
-\settextfraction{0.9} % indent text
-\usepackage{csquotes}
-\usepackage[hidelinks]{hyperref} % doi links are short and usefull?
-\hypersetup{%
-    colorlinks=true,
-    linkcolor=blue,
-    urlcolor=magenta
-}
-\urlstyle{rm}
-\usepackage[english]{babel}
-\usepackage{mathtools} % loads and extends amsmath
-\usepackage{amssymb}
-% packages not used
-%\usepackage{graphicx}
-%\usepackage{amsthm}
-%\usepackage{subfig}
-\usepackage{bm}
-\usepackage{longtable}
-\usepackage{booktabs}
-%\usepackage{ragged2e} % maybe use \RaggedRight for tables and literature?
-%\usepackage[table]{xcolor} % for alternating colors
-%\rowcolors{2}{gray!25}{white}
-%\renewcommand\arraystretch{1.3}
-\usepackage{doi}
-\usepackage[sort,square,numbers]{natbib}
-\bibliographystyle{abbrvnat}
-
-%%%%%%%%%%%%%%%%%%%%%definitions%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\newcommand{\eps}{\varepsilon}
-\renewcommand{\d}{\mathrm{d}}
-\newcommand{\T}{\mathrm{T}}
-\renewcommand{\vec}[1]{{\mathbf{#1}}}
-\newcommand{\dx}{\,\mathrm{d}x}
-%\newcommand{\dA}{\,\mathrm{d}(x,y)}
-%\newcommand{\dV}{\mathrm{d}^3{x}\,}
-\newcommand{\dA}{\,\mathrm{dA}}
-\newcommand{\dV}{\mathrm{dV}\,}
-
-\newcommand{\Eins}{\mathbf{1}}
-
-\newcommand{\ExB}{$\bm{E}\times\bm{B} \,$}
-\newcommand{\GKI}{\int d^6 \bm{Z} \BSP}	
-\newcommand{\GKIV}{\int dv_{\|} d \mu d \theta \BSP}	
-\newcommand{\BSP}{B_{\|}^*}
-\newcommand{\GA}[1]{\langle #1	 \rangle}
-
-\newcommand{\Abar}{\langle A_\parallel \rangle}
-%Vectors
-\newcommand{\bhat}{\bm{\hat{b}}}
-\newcommand{\bbar}{\overline{\bm{b}}}
-\newcommand{\chat}{\bm{\hat{c}}}
-\newcommand{\ahat}{\bm{\hat{a}}}
-\newcommand{\xhat}{\bm{\hat{x}}}
-\newcommand{\yhat}{\bm{\hat{y}}}
-\newcommand{\zhat}{\bm{\hat{z}}}
-
-\newcommand{\Xbar}{\bar{\vec{X}}}
-\newcommand{\phat}{\bm{\hat{\perp}}}
-\newcommand{\that}{\bm{\hat{\theta}}}
-
-\newcommand{\eI}{\bm{\hat{e}}_1}
-\newcommand{\eII}{\bm{\hat{e}}_2}
-\newcommand{\ud}{\mathrm{d}}
-
-%Derivatives etc.
-\newcommand{\pfrac}[2]{\frac{\partial#1}{\partial#2}}
-\newcommand{\ffrac}[2]{\frac{\delta#1}{\delta#2}}
-\newcommand{\fixd}[1]{\Big{\arrowvert}_{#1}}
-\newcommand{\curl}[1]{\nabla \times #1}
-\newcommand{\np}{\nabla_{\perp}}
-\newcommand{\npc}{\nabla_{\perp} \cdot }
-\newcommand{\nc}{\nabla\cdot }
-\newcommand{\GAI}{\Gamma_{1}^{\dagger}}
-\newcommand{\GAII}{\Gamma_{1}^{\dagger -1}}
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%DOCUMENT%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\input{../header.tex}
+\input{../newcommands.tex}
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \begin{document}
 %\preprint{}
 
diff --git a/doc/related_pages/newcommands.tex b/doc/related_pages/newcommands.tex
new file mode 100644
index 000000000..5671293e7
--- /dev/null
+++ b/doc/related_pages/newcommands.tex
@@ -0,0 +1,48 @@
+\newcommand{\eps}{\varepsilon}
+\renewcommand{\d}{\mathrm{d}}
+\newcommand{\T}{\mathrm{T}}
+\renewcommand{\vec}[1]{{\mathbf{#1}}}
+\newcommand{\dx}{\,\mathrm{d}x}
+%\newcommand{\dA}{\,\mathrm{d}(x,y)}
+%\newcommand{\dV}{\mathrm{d}^3{x}\,}
+\newcommand{\dA}{\,\mathrm{dA}}
+\newcommand{\dV}{\mathrm{dV}\,}
+
+\newcommand{\Eins}{\mathbf{1}}
+
+\newcommand{\ExB}{$\bm{E}\times\bm{B} \,$}
+\newcommand{\GKI}{\int d^6 \bm{Z} \BSP}	
+\newcommand{\GKIV}{\int dv_{\|} d \mu d \theta \BSP}	
+\newcommand{\BSP}{B_{\|}^*}
+\newcommand{\GA}[1]{\langle #1	 \rangle}
+
+\newcommand{\Abar}{\langle A_\parallel \rangle}
+%Vectors
+\newcommand{\bhat}{\bm{\hat{b}}}
+\newcommand{\bbar}{\overline{\bm{b}}}
+\newcommand{\chat}{\bm{\hat{c}}}
+\newcommand{\ahat}{\bm{\hat{a}}}
+\newcommand{\xhat}{\bm{\hat{x}}}
+\newcommand{\yhat}{\bm{\hat{y}}}
+\newcommand{\zhat}{\bm{\hat{z}}}
+
+\newcommand{\Xbar}{\bar{\vec{X}}}
+\newcommand{\phat}{\bm{\hat{\perp}}}
+\newcommand{\that}{\bm{\hat{\theta}}}
+
+\newcommand{\eI}{\bm{\hat{e}}_1}
+\newcommand{\eII}{\bm{\hat{e}}_2}
+\newcommand{\ud}{\mathrm{d}}
+
+%Derivatives etc.
+\newcommand{\pfrac}[2]{\frac{\partial#1}{\partial#2}}
+\newcommand{\ffrac}[2]{\frac{\delta#1}{\delta#2}}
+\newcommand{\fixd}[1]{\Big{\arrowvert}_{#1}}
+\newcommand{\curl}[1]{\nabla \times #1}
+\newcommand{\np}{\nabla_{\perp}}
+\newcommand{\npc}{\nabla_{\perp} \cdot }
+\newcommand{\nc}{\nabla\cdot }
+\newcommand{\GAI}{\Gamma_{1}^{\dagger}}
+\newcommand{\GAII}{\Gamma_{1}^{\dagger -1}}
+\newcommand{\Tp}{\mathcal T^+_{\Delta\varphi}}
+\newcommand{\Tm}{\mathcal T^-_{\Delta\varphi}}
diff --git a/doc/related_pages/parallel/parallel.tex b/doc/related_pages/parallel/parallel.tex
new file mode 100644
index 000000000..42319641a
--- /dev/null
+++ b/doc/related_pages/parallel/parallel.tex
@@ -0,0 +1,470 @@
+%%%%%%%%%%%%%%%%%%%%%definitions%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\input{../header.tex}
+\input{../newcommands.tex}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%DOCUMENT%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\begin{document}
+%\preprint{}
+
+\title{The parallel derivative on structured grids}
+\author{M.~Wiesenberger}
+%\email{Matthias.Wiesenberger@uibk.ac.at}
+%\ead{Matthias.Wiesenberger@uibk.ac.at}
+%\affiliation{Institute for Ion Physics and Applied Physics, Association EURATOM-\"OAW,  University of
+%   Innsbruck, A-6020 Innsbruck, Austria}
+%\author{M.~Held}
+%\affiliation{Institute for Ion Physics and Applied Physics, Association EURATOM-\"OAW,  University of
+   %Innsbruck, A-6020 Innsbruck, Austria}
+%\address{Institute for Ion Physics and Applied Physics, Association EURATOM-\"OAW,  Universit\"at 
+%   Innsbruck, A-6020 Innsbruck, Austria}
+ 
+\maketitle
+\section{Semi-Lagrangian schemes} \label{sec:parallel}
+In this section we show how to numerically treat parallel derivatives in a non field
+aligned coordinate system.
+We introduce the method in Section~\ref{sec:parallela} %.In Section~\ref{sec:parallelb} we
+and discuss some problems arising from the boundaries of the computational domain.
+Finally, we propose a field line mapping for variable initialization in Section~\ref{sec:parallelc}.
+
+\subsection{Discretization of parallel derivatives} \label{sec:parallela}
+One idea to discretize $\bhat\cdot\nabla$ is to simply use 
+the dG discretization developed in the last section directly, i.e.~discretize $b^R\partial_R+b^Z\partial_Z+b^\varphi\partial_\varphi$, where $(R,Z,\varphi)$ are cylindrical coordinates. 
+However, a very restrictive CFL-condition due to the high resolution
+in the $R$-$Z$ planes and the fast parallel motion makes 
+this approach impractical. We therefore decided to use a 
+Lagrangian approach for the parallel derivatives.
+
+We begin with the formulation of a field-aligned discretization. If $s$ denotes the 
+field line following coordinate, then the one-dimensional discrete derivative along the field line reads
+\begin{align}
+    \frac{\d f }{\d s} \rightarrow \frac{f_{k+1}-f_{k-1}}{s_{k+1} - s_{k-1}}.
+    \label{eq:proposition}
+\end{align}
+
+From differential geometry we 
+know that to every smooth vector field $\bhat$ there is a unique curve of which the
+tangent in a point is the value of $\bhat$ at that point. It is given by
+the solution of the differential equation
+\begin{align}
+    \frac{\d z^i}{\d s} = \hat b^i(\vec z)
+    \label{eq:integralcurve}
+\end{align}
+where $z^i$ is one of $(R, Z, \varphi)$ and $\hat b^i$ are the contravariant components
+of $\bhat$ in cylindrical coordinates. 
+Moreover, by definition we have
+\begin{align}
+    \frac{\d f(\vec z(s))}{\d s} = \bhat\cdot \nabla f|_{\vec z(s)}
+    \label{}
+\end{align}
+along a field line parameterized by distance $s$, 
+i.e. instead of $\nabla_\parallel f$ we can choose to discretize $\frac{\d f}{\d s}$.
+
+Let us divide the $\varphi$ direction into $N_\varphi$ equidistant planes of distance
+$\Delta \varphi$. Unfortunately, from Eq.~\eqref{eq:integralcurve} we cannot easily determine the 
+distance $\Delta s$ for given $\Delta \varphi$. It is better to integrate
+\begin{align}
+    \frac{\d z^i}{\d t}=\frac{b^i}{b^\varphi} = \frac{B^i}{B^\varphi}
+    \label{}
+\end{align}
+since in this case $\d\varphi/\d t = 1 \Rightarrow t=\varphi$. We get
+\begin{subequations}
+\begin{align}
+    \frac{\d R}{\d\varphi}&= \frac{B^R}{B^\varphi},\\ %\frac{R}{I}\frac{\partial\psi}{\partial Z},\\
+    \frac{\d Z}{\d\varphi}&=\frac{B^Z}{B^\varphi},%-\frac{R}{I}\frac{\partial\psi}{\partial R}.
+\end{align}
+\text{ together with the equation  }
+\begin{align}
+    \frac{\d s}{\d\varphi} &= \frac{1}{|\hat b^\varphi|} = \frac{B}{|B^\varphi|}% = \frac{R_0R^2B}{I}=R_0R\sqrt{1+\frac{|\nabla\psi|^2}{I^2}}
+    \label{eq:fieldlinec}
+\end{align}
+\label{eq:fieldline}
+\end{subequations}
+for the length of the field line $s$. 
+Eqs.~\eqref{eq:fieldline} are integrated from $\varphi=0$ to $\varphi=\pm \Delta \varphi$. 
+We characterize the flow generated by $\bhat/b^\varphi$ by
+\begin{align}
+    T_{\Delta \varphi}^{\pm 1}\vec z := T_{\Delta \varphi}^{\pm 1}[R, Z, \varphi]:= ( R(\pm \Delta\varphi), Z( \pm \Delta\varphi), \varphi\pm\Delta \varphi),
+    \label{}
+\end{align}
+where $(R(\varphi), Z(\varphi), s(\varphi))$ is the solution to Eqs.~\eqref{eq:fieldline} 
+with initial condition 
+\begin{align}
+    (R(0), Z(0), s(0)) = (R, Z, 0).
+    \label{}
+\end{align} 
+Obviously we have $T^{-1}_{\Delta\varphi}\circ T^{+1}_{\Delta\varphi} = \Eins$, but $T^{\pm}_{\Delta\varphi}$ is not unitary since $\bhat/b^\varphi$ is 
+not divergence free. 
+
+The proposed centered discretization~\eqref{eq:proposition} for the parallel derivative then reads
+\begin{align}
+    \nabla_\parallel f \equiv \frac{df}{ds} = \frac{df}{d\varphi}\frac{d\varphi}{ds} 
+    \rightarrow \frac{f\left(T_{\Delta\varphi}^+\vec z\right)-f\left(T_{\Delta\varphi}^-\vec z\right)}{s(+\Delta\varphi) - s(-\Delta\varphi)},
+    \label{eq:paralleldis}
+\end{align}
+which is slightly different from Reference~\cite{Hariri2014}, where
+the relation~\eqref{eq:fieldlinec} was used to replace $\d \varphi/\d s$. 
+Previous derivations now needed to construct an interpolation scheme in order
+to evaluate functions on the transformed coordinates. 
+Since the $R$ and $Z$ coordinates are still discretized in the dG framework we note that in our work
+the interpolation of $f$ on the transformed points $T_{\Delta\varphi}^{\pm 1}\vec z$
+is naturally given by Eq.~\eqref{eq:dgexpansion} (the extension to two dimensions is immediate). 
+Let us for a moment omit the $Z$ coordinate for ease of notation. If $(R_{nj}, \varphi_k)$ are the grid points, 
+we call $(R^+_{nj}, \varphi_{k+1}) := T^+_{\Delta\varphi}[R_{nj}, \varphi_k]$ and $(R_{nj}^-, \varphi_{k-1}) := T^-_{\Delta\varphi}[R_{nj}, \varphi_k]$ the transformed coordinates along
+the field lines. We then have
+\begin{subequations}
+\begin{align}
+    f(T^+_{\Delta\varphi}\vec z) = f( R^+_{nj}, \varphi_{k+1}) = \bar f_{k+1}^{ml}p_{ml}(R^+_{nj}) =: (I^+)_{nj}^{ml}f_{(k+1)ml} , \\
+    f(T^-_{\Delta\varphi}\vec z) = f( R^-_{nj}, \varphi_{k-1}) = \bar f_{k-1}^{ml}p_{ml}(R^-_{nj}) =: (I^-)_{nj}^{ml}f_{(k-1)ml} , 
+\end{align}
+\label{eq:interpolation}
+\end{subequations}
+where the backward transformations of $\bar{ \vec f}$ are hidden in $I$.
+Thus the interpolation of all the necessary points can simply be written as a matrix-vector product, where the interpolation matrices $I^+$  and $I^-$ are independent of time since
+the field lines are constant in time. The order of this interpolation is given by $P$, the number of polynomial coefficients.
+A consistency check is the relation $I^+\circ I^- = \Eins$. 
+
+The discretization~\eqref{eq:paralleldis} can now be written as a matrix vector product
+\begin{align}
+    \nabla_\parallel f \rightarrow S \circ \left[ \Eins^+\otimes I^+ - \Eins^- \otimes I^-  \right] \vec f, 
+    \label{}
+\end{align}
+where $S$ is the diagonal matrix that contains the entries $1/(s(+\Delta\varphi) - s(-\Delta\varphi))$.
+This discretization is not skew-symmetric since the
+field lines are not volume-preserving, or~$(I^+)^\mathrm{T} \neq I^-$.
+In fact, the adjoint of the parallel derivative is
+\begin{align}
+    \nabla_\parallel^\dagger = - \nabla\cdot(\bhat\ . ) \neq -\nabla_\parallel.
+    \label{}
+\end{align}
+Note that with this relation we can define the parallel 
+diffusion operator as
+\begin{align}
+    \Delta_\parallel := -\nabla_\parallel^\dagger \nabla_\parallel = (\nabla\cdot \bhat) \nabla_\parallel + \nabla_\parallel^2 , 
+    \label{}
+\end{align}
+which is indeed the parallel part of the full Laplacian $\Delta = \nabla\cdot( \bhat \nabla_\parallel + \nabla_\perp)$.
+The second order derivative $\nabla_\parallel^2$ can be 
+discretized using 
+\begin{align}
+    \frac{\d^2 f}{\d s^2} \rightarrow  
+    \frac{2f_{k+1} }{(s_{k+1}-s_k)(s_{k+1}-s_{k-1})} -
+    \frac{2f_k}{(s_{k+1}-s_k)(s_k - s_{k-1})} \nonumber\\ + 
+    \frac{2f_{k-1} }{(s_{k}-s_{k-1})(s_{k+1}-s_{k-1})} 
+    \label{}
+\end{align}
+and repeating the procedure of this section.
+
+%We see that the term in brackets is obviously skew-symmetric if $(I^+)^\mathrm{T} = I^- = (I^+)^{-1}$, i.e.~the interpolation matrix is unitary.
+% IT ISNT
+%\subsection{Limiter and Cutting procedures} \label{sec:parallelb}
+The main problem with the above scheme is the question what
+ to do when a field line crosses the simulation boundaries. 
+Boundary conditions are formulated in cylindrical coordinates. 
+One idea is to simply cut the contribution from field lines
+that leave the computational domain. While this works in practice
+it is unclear what numerical and physical side-effects this procedure might have. 
+Another idea is to check to every point $\vec z$ whether $T_{\Delta\varphi}\vec z$
+lies inside our simulation box or not. If not, we have to find where exactly the 
+field line intersects the simulation box.  We have to find
+$\varphi_b$ such that the result of the integration of Eq.~\eqref{eq:fieldline} from 
+$0$ to $\varphi_b$ lies on the boundary. 
+The angle $\varphi_b$ can be found by a bisection algorithm knowing that $0<\varphi_b < \Delta\varphi$. 
+This kind of procedure is known as a shooting method. 
+When all points are found, ghost cells can be constructed in the correct way. 
+
+A poloidal limiter can simply be implemented via a boundary condition in $\varphi$. 
+As long as the form of the limiter is a flux-function we do not have to 
+integrate a field line in order to determine which points lie in the
+limiter-shadow. It is therefore straightforward to implement ghost-cells 
+in that case. 
+
+\subsection{Field aligned initialization} \label{sec:parallelc}
+
+An important aspect of our simulations is a judicious initialization of the 
+fields. We want structures to be field-aligned in the beginning of the simulation with
+a possible modulation along the direction of the field line.
+If a Gaussian shape is used, we call $\sigma_\parallel$ the extension in parallel
+direction and write
+\begin{align}
+    f_0(R,Z,\varphi) = F(R,Z,\varphi) \exp\left( - \frac{(\varphi-\varphi_0)^2}{2\sigma_\parallel^2}\right),
+    \label{eq:parallelInit}
+\end{align}
+where $F$ is a function that is invariant under the field line transformations
+\begin{subequations}
+\begin{align}
+    T_{\Delta\varphi}^+ F(\vec z) &= F( T_{\Delta\varphi}^+\vec z) \overset{!}{=} F(\vec z) \text{ (pull-back), } \\
+    T_{\Delta\varphi}^- F(\vec z) &= F( T_{\Delta\varphi}^-\vec z) \overset{!}{=} F(\vec z) \text{ (push-forward). } 
+\end{align}
+\label{}
+\end{subequations}
+We can use these relations to construct aligned structures
+by active transformations of some given field.
+Our idea is to initialize a two-dimensional field $F(R,Z, \varphi_k)$ in a given plane $k$ and 
+transform this field to all other planes using the recursive relations
+\begin{subequations}
+\begin{align}
+    F( R, Z, \varphi_{k+1}) = T_{\Delta\varphi}^- F( R, Z, \varphi_{k+1}) = F(R^-, Z^-, \varphi_k), \\
+    F( R, Z, \varphi_{k-1}) = T_{\Delta\varphi}^+ F( R, Z, \varphi_{k-1}) = F(R^+, Z^+, \varphi_k),
+\end{align}
+    \label{eq:recursiveInit}
+\end{subequations}
+which is the statement that $F$ in the next plane equals the push-forward  
+and $F$ in the previous plane equals the pull-back of $F$ in the current plane. 
+Note here that Eq.~\eqref{eq:interpolation} applies for the required interpolation
+procedures. 
+
+\section{The parallel derivative}
+The idea of the sandwich method \cite{Held2016} is to bracket the parallel derivative by interpolation and projection matrices:
+\begin{align}
+    \nabla^c_\parallel &= P\nabla_\parallel^f Q \\
+    \nabla^{c\dagger}_\parallel &= P \nabla^{f\dagger}_\parallel Q
+    \label{eq:sandwich}
+\end{align}
+In this way the projection integrals
+\begin{align}
+    \int\dV \nabla_\parallel f p_i(x)p_j(y) 
+    \label{}
+\end{align}
+are computed more precisely.
+The size of the fine grid should therefore be as large as
+possible (reasonable? what resolution is needed?).
+We first notice that the one interpolation matrix can be absorbed
+in the parallel derivative since this also consists of 
+interpolation operations. 
+\begin{align}
+    \nabla^c_\parallel &= P\nabla_\parallel^{fc} \\
+    \nabla^{c\dagger}_\parallel &= \nabla^{fc\dagger}_\parallel Q
+    \label{eq:sandwich}
+\end{align}
+
+
+In order to understand what the adjoint operators do let us denote $\mathcal T^+_{\Delta\varphi}$ as the push-forwward operator. Then we have
+\begin{align}
+    \int f(\vec x) \Tp h(\vec x) \sqrt{g(\vec x)}\d^3x \\
+    =  \int f(\vec x) h(\Tm \vec x)\sqrt{g(\vec x)}\d^3x \\
+    =  \int f(\Tp \vec x') h(\vec x')\sqrt{g(\Tp \vec x')}J^{-1}( \Tp\vec x') \d^3x' \\
+    =  \int \frac{1}{\sqrt{g(\vec x')}}\Tm\left[J^{-1}(\vec x')\sqrt{g(\vec x')}f(\vec x')\right] h(\vec x')\sqrt{g(\vec x')}   \d^3x' \\
+    \equiv  \int (\Tp)^\dagger\left[f(\vec x)\right] h(\vec x)\sqrt{g(\vec x)}   \d^3x
+    \label{}
+\end{align}
+$J$ is the determinant of the Jacobian $\partial(\vec x')/\partial(\vec x)$.
+In the last step we simply replaced the dummy variable $\vec x'$ with $\vec x$ again and identified the relevant terms
+as the adjoint operator:
+\begin{align}
+    (\Tp)^\dagger f(\vec x ) := \frac{1}{\sqrt{g(\vec x)}} \Tm\left[\sqrt{g(\vec x)} J^{-1}(\vec x) f(\vec x) \right]
+    \label{}
+\end{align}
+This means that numerically the adjoint of the push-forward 
+operator should be a valid discretization of its inverse. (how can this be exploited as a test?)
+Note that $\sqrt{g}J^{-1}(\vec x) = \sqrt{g'(\Tm \vec x)}$.
+With this we can write
+\begin{align}
+    (\Tp)^\dagger f(\vec x ) := \sqrt{\frac{g'(\vec x)}{g(\vec x)}} \Tm\left[f(\vec x) \right]
+    \label{}
+\end{align}
+Also note that while $\Tp [fh] = \Tp f \Tp h$ this might not 
+hold on the discrete level. Also the question is how $J$ enters 
+on the discrete level. We have to multiply $\sqrt{g}$ artificially when we form the adjoint. 
+Theoretically $J$ could be hidden somehow when we integrate the fieldlines, so the information could be contained in the discrete version? (Maybe in the back-projection?)
+
+If the streamlines are divergence free, we have $J=1$.
+A numerical test could be ( if we neglect the volume form in the adjoint)
+\begin{align}
+    (\Tp)^\dagger \left[J(\vec x)\Tp f(\vec x)\right] - f(\vec x) = 0
+    \label{}
+\end{align}
+The numerical computation of $J$ might a bit tricky at the boundaries. 
+In a flux-aligned $\zeta, \eta$ it should be feasible but in cylindrical coordinates I don't know how. Maybe we can simply cut the last few cells before the boundary.
+Even easier might be
+\begin{align}
+    \left(\Tp\right)^\dagger J(\vec x ) = 1
+    \label{}
+\end{align}
+
+If we integrate streamlines of the vector field $\vec B/B^\varphi$, then we have
+\begin{align}
+    \frac{\d J}{\d \varphi} = J(\vec x ) \nabla\cdot\left( \vec B/B^\varphi\right)
+    \label{}
+\end{align}
+along these streamlines.
+Also we have that $\d s = B/B^\varphi \d \varphi $ and the interpolation/projection of $\triangle s$ can probably be neglected. (We want to neglect it because it's memory intensive to store all combinations)
+Also this means that when we transpose $\nabla_\parallel$ we get a 
+multiplication by $B^\varphi/B$ in the beginning.
+In any case this means that we want to have 
+\begin{align}
+\left(\Tp\right)^\dagger B^\varphi  =  B^\varphi
+\end{align}
+\section{Boundary conditions}
+The question is what to do when a fieldline intersects with the boundary
+of the simulation domain before reaching the next plane.
+The problem with simply putting a value exactly where the fieldline 
+reaches the boundary is first that a true interpolation at that 
+position requires a 3d interpolation (we 
+are between planes) instead of a 2d interpolation.
+Secondly, for Dirichlet boundaries the small 
+distance seriously deteriorates the CFL condition.
+A simple solution is to set the perpendicular field zero on the 
+boundary such that the field becomes purely 
+toroidal on the boundary. The fieldlines then have a kink on the 
+boundary. On the other hand we can implement boundary conditions consistent with 
+the perpendicular ones since the fieldlines never leave the domain. 
+We simply interpolate the quantity to derive on the inner side of the
+domain boundary (Neumann conditions = "No boundary condition") or 
+set the value to zero (Dirichlet condition).
+
+
+\section{Geometry}
+When we are on a different geometry than $(R,Z)$ the question is how to integrate the field lines. There are two possibilities. 
+First, interpolate $R(\zeta_i, \eta_i), Z(\zeta_i, \eta_i)$ for 
+all $i$, then integrate in $(R,Z)$ space and finally use
+Newton iteration to find $\zeta(R^\pm_i, Z^\pm_i), \eta(R^\pm_i, Z^\pm_i)$. 
+The downside here is that it is difficult to tell when and where the fieldline leaves the simulation domain and even worse in MPI the next points might belong to another process. 
+
+The second possibiliy is to integrate entirely in the 
+transformed coordinate system $\zeta, \eta$. 
+The magnetic field can be easily transformed since we have the
+Jacobian of the coordinate transformation
+\begin{align}
+    B^\zeta(\zeta, \eta) &= \left(\frac{\partial \zeta}{\partial R} B^{R} + \frac{\partial \zeta}{\partial Z}B^Z\right)_{R(\zeta, \eta), Z(\zeta, \eta)} \\
+    B^\eta(\zeta, \eta) &= \left(\frac{\partial \eta}{\partial R} B^{R} + \frac{\partial \eta}{\partial Z}B^Z\right)_{R(\zeta, \eta), Z(\zeta, \eta)} \\
+    B^\varphi(\zeta, \eta) &= B^\varphi({R(\zeta, \eta), Z(\zeta, \eta)})
+    \label{eq:field_trafo}
+\end{align}
+The advantage is that we can do this for any coordinate
+system. Only for cylindrical coordinates, we integrate directly in physical space. The equations to integrate
+are
+\begin{subequations}
+\begin{align}
+\frac{\d \zeta}{\d\varphi} &= \frac{B^\zeta}{B^\varphi}\\
+\frac{\d \eta}{\d\varphi} &= \frac{B^\eta}{B^\varphi}\\
+\frac{\d s}{\d\varphi} &= \frac{|B|}{|B^\varphi|}
+\end{align}
+\label{eq:fieldlines}
+\end{subequations}
+The downside here is that when integrating fieldlines we
+have to interpolate the magnetic field at arbitrary points. 
+However, the error should vanish with $3rd$ order in the 
+perpendicular plane. (check this). In order to mitigate this error
+we could maybe transform the B-Field on a finer grid/higher order polynomials for more accurate
+integration. Also, we need interpolation in the 
+first algorithm as well. 
+
+Note that the matrix-matrix multiplications in Eq.~\eqref{eq:sandwich} can
+be precomputed and stored. The memory requirements 
+in the final computations are 
+therefore the same  as in the old version. (Not entirely, since
+the diagonal $1/\Delta s$ matrix does not commute with $Q$ or $P$).
+
+Finally remember that the adjoint of a matrix in the modified geometry 
+involves the volume element. This means that after you've adjoined the 
+parallel derivative the normal way simply bracket the result 
+by $1/\sqrt{g}$ and $\sqrt{g}$. 
+\section{Algorithm}
+Given are the components $v^i(R,Z)$ for $i\in\{R,Z,\varphi\}$ and a compuational grid (in the following the ``coarse grid``)
+\begin{itemize}
+  \item generate a fine grid by multiplying the cell numbers of the given coarse grid (only topologcially, metric and Jacobian are not needed)
+  \item integrate the fieldlines for the fine grid:
+    \begin{itemize}
+      \item evaluate the starting points on the coarse grid in computational space 
+      \item For a curvilinear grid set up a (higher order) grid for the 
+        interpolation of the vector components $v^i$ and push forward the vector components
+        to the curvilinear coordinate system
+      \item Integrate the fieldline equations 
+\begin{subequations}
+\begin{align}
+\frac{\d \zeta}{\d\varphi} &= \frac{v^\zeta}{v^\varphi}\\
+\frac{\d \eta}{\d\varphi} &= \frac{v^\eta}{v^\varphi}\\
+\frac{\d s}{\d\varphi} &= \frac{1}{|v^\varphi|}
+\end{align}
+\label{eq:fieldlines_converted}
+\end{subequations}
+    with the given starting points and $s(0)=0$ from $\varphi=0$ until $\varphi = \pm\Delta \varphi$.
+      \item create an interpolation matrix that interpolates from the coarse grid 
+        to the fine grid
+      \item use the interpolation matrix to generate the plus/minus points for the fine grid
+    \end{itemize}
+  \item create the interpolation matrices that interpolate from the given coarse grid 
+    to the plus/minus points 
+  \item create a projection matrix that projects from the fine grid to the coarse grid
+  \item compute the matrix-matrix multiplications $P\cdot I^\pm$ as well as the transpose
+  \item project the $s$ vectors to the coarse grid
+\end{itemize}
+\section{MPI implementation}
+Let us also note the mpi-implementation, which is not entirely
+trivial due to the matrix-matrix multiplications involed in Eq.~\eqref{eq:sandwich}.
+\subsection{Row and column distributed sparse matrices}
+In Feltor each mpi process gets an equally sized chunk of a 
+vector.
+Contrary to a vector
+a matrix can be distributed in two ways, row-wise and column wise. 
+In a row-distributed matrix each process gets the complete 
+rows of the matrix that correspond to the indices in the 
+vector it holds. 
+In a column-distributed matrix each process gets the complete 
+columns of the matrix corresponding to the indices in the 
+vector it holds. 
+When we implement a matrix-vector multiplication the order 
+of communication and computation depends on the distribution 
+of the matrix.
+For the row-distributed matrix each process first has to gather all elements of the input vector it needs to be able to compute the elements of the output. This requires MPI communication.
+Formally, the gather operation can be written as a matrix $G$
+of $1'$s and $0'$s where the output vector is of equal or larger size than the input vector.
+After the elements have been gathered the local matrix-vector
+multiplications can be executed.
+\begin{align}
+M = R\cdot G
+\end{align}
+where $R$ is the row-distributed matrix with modified indices 
+and $G$ is the gather matrix, in which the MPI-communication takes place.
+
+In a column distributed matrix the local matrix-vector multiplication can be executed first because each processor already
+has all vector elements it needs. 
+However the resuling elements have to be communicated back to 
+the process they belong to. Furthermore, a process has to sum
+all elements it receives from other processes on the same
+index. This is a scatter and reduce operation and
+it can be written as a scatter matrix $S$. The transpose
+of the scatter matrix is a gather matrix and vice-versa.
+\begin{align}
+M = S\cdot C
+\end{align}
+where $S$ is the scatter matrix and $C$ is the column distributed
+matrix with modified indices. 
+
+It turns out that a row-distributed matrix can be transposed
+by transposition of the local matrices and the gather matrix.
+The result is then a column distributed matrix.
+The transpose of a column distributed matrix is a row-distributed matrix and vice-versa.
+
+\subsection{Matrix-Matrix multiplication}
+We note that we normally construct $\nabla_\parallel^{fc}$ as a column 
+distributed
+matrix. The advantage is then that the gather operation is bijective, i.e. the transpose of the gather matrix is its inverse. 
+This advantage is lost in the present problem. 
+It turns out that it is advantageous to construct $\nabla_\parallel^{fc}$
+as s row-distributed matrix with global indices. 
+This is because a column distributed matrix can be easily (without mpi-communication) multiplied
+with a row distributed matrix especially if the indices are global indices. 
+Each process just multiplies its local matrices.
+\begin{align}
+M = C\cdot R
+\end{align}
+This is not true the other way round. 
+The result is then a row distributed matrix with global indices. 
+From the global indices the gather map/matrix and the local
+indices can be constructed.
+We note here that we even don't need to construct the gather matrix
+for $\nabla_\parallel^{fc}$, only the one for $\nabla_\parallel^c$ is
+needed.
+
+
+
+%..................................................................
+\bibliography{../references}
+%..................................................................
+
+
+\end{document}
+
-- 
GitLab


From bb4449cee76781287c93d48af3210756b68a3c1b Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Fri, 13 Oct 2017 12:12:15 +0200
Subject: [PATCH 353/453] small updates to parallel.tex

---
 doc/related_pages/newcommands.tex       |  3 ++-
 doc/related_pages/parallel/parallel.tex | 34 ++++++++++++-------------
 2 files changed, 19 insertions(+), 18 deletions(-)

diff --git a/doc/related_pages/newcommands.tex b/doc/related_pages/newcommands.tex
index 5671293e7..ce64c7952 100644
--- a/doc/related_pages/newcommands.tex
+++ b/doc/related_pages/newcommands.tex
@@ -1,6 +1,5 @@
 \newcommand{\eps}{\varepsilon}
 \renewcommand{\d}{\mathrm{d}}
-\newcommand{\T}{\mathrm{T}}
 \renewcommand{\vec}[1]{{\mathbf{#1}}}
 \newcommand{\dx}{\,\mathrm{d}x}
 %\newcommand{\dA}{\,\mathrm{d}(x,y)}
@@ -44,5 +43,7 @@
 \newcommand{\nc}{\nabla\cdot }
 \newcommand{\GAI}{\Gamma_{1}^{\dagger}}
 \newcommand{\GAII}{\Gamma_{1}^{\dagger -1}}
+\newcommand{\T}{\mathrm{T}}
 \newcommand{\Tp}{\mathcal T^+_{\Delta\varphi}}
 \newcommand{\Tm}{\mathcal T^-_{\Delta\varphi}}
+\newcommand{\Tpm}{\mathcal T^\pm_{\Delta\varphi}}
diff --git a/doc/related_pages/parallel/parallel.tex b/doc/related_pages/parallel/parallel.tex
index 42319641a..7f42268ee 100644
--- a/doc/related_pages/parallel/parallel.tex
+++ b/doc/related_pages/parallel/parallel.tex
@@ -65,18 +65,18 @@ Let us divide the $\varphi$ direction into $N_\varphi$ equidistant planes of dis
 $\Delta \varphi$. Unfortunately, from Eq.~\eqref{eq:integralcurve} we cannot easily determine the 
 distance $\Delta s$ for given $\Delta \varphi$. It is better to integrate
 \begin{align}
-    \frac{\d z^i}{\d t}=\frac{b^i}{b^\varphi} = \frac{B^i}{B^\varphi}
+    \frac{\d z^i}{\d t} = \frac{\hat b^i}{\hat b^\varphi}
     \label{}
 \end{align}
 since in this case $\d\varphi/\d t = 1 \Rightarrow t=\varphi$. We get
 \begin{subequations}
 \begin{align}
-    \frac{\d R}{\d\varphi}&= \frac{B^R}{B^\varphi},\\ %\frac{R}{I}\frac{\partial\psi}{\partial Z},\\
-    \frac{\d Z}{\d\varphi}&=\frac{B^Z}{B^\varphi},%-\frac{R}{I}\frac{\partial\psi}{\partial R}.
+    \frac{\d R}{\d\varphi}&= \frac{\hat b^R}{\hat b^\varphi},\\ %\frac{R}{I}\frac{\partial\psi}{\partial Z},\\
+    \frac{\d Z}{\d\varphi}&=\frac{\hat b^Z}{\hat b^\varphi},%-\frac{R}{I}\frac{\partial\psi}{\partial R}.
 \end{align}
 \text{ together with the equation  }
 \begin{align}
-    \frac{\d s}{\d\varphi} &= \frac{1}{|\hat b^\varphi|} = \frac{B}{|B^\varphi|}% = \frac{R_0R^2B}{I}=R_0R\sqrt{1+\frac{|\nabla\psi|^2}{I^2}}
+    \frac{\d s}{\d\varphi} &= \frac{1}{|\hat b^\varphi|} %= \frac{B}{|B^\varphi|}% = \frac{R_0R^2B}{I}=R_0R\sqrt{1+\frac{|\nabla\psi|^2}{I^2}}
     \label{eq:fieldlinec}
 \end{align}
 \label{eq:fieldline}
@@ -85,7 +85,7 @@ for the length of the field line $s$.
 Eqs.~\eqref{eq:fieldline} are integrated from $\varphi=0$ to $\varphi=\pm \Delta \varphi$. 
 We characterize the flow generated by $\bhat/b^\varphi$ by
 \begin{align}
-    T_{\Delta \varphi}^{\pm 1}\vec z := T_{\Delta \varphi}^{\pm 1}[R, Z, \varphi]:= ( R(\pm \Delta\varphi), Z( \pm \Delta\varphi), \varphi\pm\Delta \varphi),
+    \Tpm\vec z := \Tpm[R, Z, \varphi]:= ( R(\pm \Delta\varphi), Z( \pm \Delta\varphi), \varphi\pm\Delta \varphi),
     \label{}
 \end{align}
 where $(R(\varphi), Z(\varphi), s(\varphi))$ is the solution to Eqs.~\eqref{eq:fieldline} 
@@ -94,7 +94,7 @@ with initial condition
     (R(0), Z(0), s(0)) = (R, Z, 0).
     \label{}
 \end{align} 
-Obviously we have $T^{-1}_{\Delta\varphi}\circ T^{+1}_{\Delta\varphi} = \Eins$, but $T^{\pm}_{\Delta\varphi}$ is not unitary since $\bhat/b^\varphi$ is 
+Obviously we have $\Tm\circ\Tp = \Eins$, but $\Tpm$ is not unitary since $\bhat/b^\varphi$ is 
 not divergence free. 
 
 The proposed centered discretization~\eqref{eq:proposition} for the parallel derivative then reads
@@ -108,15 +108,15 @@ the relation~\eqref{eq:fieldlinec} was used to replace $\d \varphi/\d s$.
 Previous derivations now needed to construct an interpolation scheme in order
 to evaluate functions on the transformed coordinates. 
 Since the $R$ and $Z$ coordinates are still discretized in the dG framework we note that in our work
-the interpolation of $f$ on the transformed points $T_{\Delta\varphi}^{\pm 1}\vec z$
-is naturally given by Eq.~\eqref{eq:dgexpansion} (the extension to two dimensions is immediate). 
+the interpolation of $f$ on the transformed points $\Tpm\vec z$
+is naturally given by interpolating the base polynomials. 
 Let us for a moment omit the $Z$ coordinate for ease of notation. If $(R_{nj}, \varphi_k)$ are the grid points, 
-we call $(R^+_{nj}, \varphi_{k+1}) := T^+_{\Delta\varphi}[R_{nj}, \varphi_k]$ and $(R_{nj}^-, \varphi_{k-1}) := T^-_{\Delta\varphi}[R_{nj}, \varphi_k]$ the transformed coordinates along
+we call $(R^+_{nj}, \varphi_{k+1}) := \Tp[R_{nj}, \varphi_k]$ and $(R_{nj}^-, \varphi_{k-1}) := \Tm[R_{nj}, \varphi_k]$ the transformed coordinates along
 the field lines. We then have
 \begin{subequations}
 \begin{align}
-    f(T^+_{\Delta\varphi}\vec z) = f( R^+_{nj}, \varphi_{k+1}) = \bar f_{k+1}^{ml}p_{ml}(R^+_{nj}) =: (I^+)_{nj}^{ml}f_{(k+1)ml} , \\
-    f(T^-_{\Delta\varphi}\vec z) = f( R^-_{nj}, \varphi_{k-1}) = \bar f_{k-1}^{ml}p_{ml}(R^-_{nj}) =: (I^-)_{nj}^{ml}f_{(k-1)ml} , 
+    f(\Tp\vec z) = f( R^+_{nj}, \varphi_{k+1}) = \bar f_{k+1}^{ml}p_{ml}(R^+_{nj}) =: (I^+)_{nj}^{ml}f_{(k+1)ml} , \\
+    f(\Tm\vec z) = f( R^-_{nj}, \varphi_{k-1}) = \bar f_{k-1}^{ml}p_{ml}(R^-_{nj}) =: (I^-)_{nj}^{ml}f_{(k-1)ml} , 
 \end{align}
 \label{eq:interpolation}
 \end{subequations}
@@ -165,7 +165,7 @@ Boundary conditions are formulated in cylindrical coordinates.
 One idea is to simply cut the contribution from field lines
 that leave the computational domain. While this works in practice
 it is unclear what numerical and physical side-effects this procedure might have. 
-Another idea is to check to every point $\vec z$ whether $T_{\Delta\varphi}\vec z$
+Another idea is to check to every point $\vec z$ whether $\Tpm \vec z$
 lies inside our simulation box or not. If not, we have to find where exactly the 
 field line intersects the simulation box.  We have to find
 $\varphi_b$ such that the result of the integration of Eq.~\eqref{eq:fieldline} from 
@@ -194,8 +194,8 @@ direction and write
 where $F$ is a function that is invariant under the field line transformations
 \begin{subequations}
 \begin{align}
-    T_{\Delta\varphi}^+ F(\vec z) &= F( T_{\Delta\varphi}^+\vec z) \overset{!}{=} F(\vec z) \text{ (pull-back), } \\
-    T_{\Delta\varphi}^- F(\vec z) &= F( T_{\Delta\varphi}^-\vec z) \overset{!}{=} F(\vec z) \text{ (push-forward). } 
+    \Tp F(\vec z) &= F( \Tp \vec z) \overset{!}{=} F(\vec z) \text{ (pull-back), } \\
+    \Tm F(\vec z) &= F( \Tm \vec z) \overset{!}{=} F(\vec z) \text{ (push-forward). } 
 \end{align}
 \label{}
 \end{subequations}
@@ -205,8 +205,8 @@ Our idea is to initialize a two-dimensional field $F(R,Z, \varphi_k)$ in a given
 transform this field to all other planes using the recursive relations
 \begin{subequations}
 \begin{align}
-    F( R, Z, \varphi_{k+1}) = T_{\Delta\varphi}^- F( R, Z, \varphi_{k+1}) = F(R^-, Z^-, \varphi_k), \\
-    F( R, Z, \varphi_{k-1}) = T_{\Delta\varphi}^+ F( R, Z, \varphi_{k-1}) = F(R^+, Z^+, \varphi_k),
+    F( R, Z, \varphi_{k+1}) = \Tm F( R, Z, \varphi_{k+1}) = F(R^-, Z^-, \varphi_k), \\
+    F( R, Z, \varphi_{k-1}) = \Tp F( R, Z, \varphi_{k-1}) = F(R^+, Z^+, \varphi_k),
 \end{align}
     \label{eq:recursiveInit}
 \end{subequations}
@@ -240,7 +240,7 @@ interpolation operations.
 \end{align}
 
 
-In order to understand what the adjoint operators do let us denote $\mathcal T^+_{\Delta\varphi}$ as the push-forwward operator. Then we have
+In order to understand what the adjoint operators do let us denote $\Tp$ as the push-forwward operator. Then we have
 \begin{align}
     \int f(\vec x) \Tp h(\vec x) \sqrt{g(\vec x)}\d^3x \\
     =  \int f(\vec x) h(\Tm \vec x)\sqrt{g(\vec x)}\d^3x \\
-- 
GitLab


From 5778bd88a352458a714bce9ae30ede79abe3eabc Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Fri, 13 Oct 2017 18:11:10 +0200
Subject: [PATCH 354/453] clean up arakawa.h

---
 inc/dg/arakawa.h                  | 4 ----
 inc/dg/backend/projection_mpit.cu | 1 +
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/inc/dg/arakawa.h b/inc/dg/arakawa.h
index 46823dec6..573558549 100644
--- a/inc/dg/arakawa.h
+++ b/inc/dg/arakawa.h
@@ -123,13 +123,9 @@ void ArakawaX< Geometry, Matrix, container>::operator()( const container& lhs, c
     blas2::symv( bdyf, rhs, dyrhs);
 
     blas1::pointwiseDot( 1./3., dxlhs, dyrhs, -1./3., dylhs, dxrhs, 0., result);
-    //blas1::pointwiseDot( 1./3.,   lhs, dyrhs, -1./3., dylhs,   rhs, 0., helper_);
-    //blas1::pointwiseDot( 1./3., dxlhs,   rhs, -1./3.,   lhs, dxrhs, 0., dylhs);
     blas1::pointwiseDot( 1./3.,   lhs, dyrhs, -1./3., dylhs,   rhs, 0., dylhs);
     blas1::pointwiseDot( 1./3., dxlhs,   rhs, -1./3.,   lhs, dxrhs, 0., dxrhs);
 
-    //blas2::symv( 1., bdxf, helper_, 1., result);
-    //blas2::symv( 1., bdyf, dylhs, 1., result);
     blas2::symv( 1., bdxf, dylhs, 1., result);
     blas2::symv( 1., bdyf, dxrhs, 1., result);
     tensor::pointwiseDot( perp_vol_inv_, result, result);
diff --git a/inc/dg/backend/projection_mpit.cu b/inc/dg/backend/projection_mpit.cu
index 000b77280..b189d4589 100644
--- a/inc/dg/backend/projection_mpit.cu
+++ b/inc/dg/backend/projection_mpit.cu
@@ -4,6 +4,7 @@
 #include "mpi_projection.h"
 #include "mpi_init.h"
 #include "mpi_evaluation.h"
+#includ2 "transpose.h"
 
 
 double shift = 0.2;
-- 
GitLab


From 71311640812ab34ff8e81529e1d00f77b1c43e92 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sun, 15 Oct 2017 18:27:17 +0200
Subject: [PATCH 355/453] updated the parallel derivative writeup

---
 README.md                               |   2 +-
 doc/related_pages/parallel/parallel.tex | 356 ++++++++++++------------
 inc/dg/backend/mpi_matrix.h             |   2 +-
 inc/dg/dg_doc.h                         |  75 ++++-
 4 files changed, 242 insertions(+), 193 deletions(-)

diff --git a/README.md b/README.md
index 7e2e99f95..1afb84104 100644
--- a/README.md
+++ b/README.md
@@ -116,7 +116,7 @@ Moreover, we maintain tex files in every src folder for technical documentation,
 The [developer oriented documentation](http://feltor-dev.github.io/feltor/inc/dg/html/modules.html) of the dG library was generated with [Doxygen](http://www.doxygen.org). 
 You can generate a local version from source code.
 This depends on the `doxygen`, `libjs-mathjax` and `graphviz` packages.
-Type `make doc` in the folder `path/to/feltor/inc/doc` and open `index.html` (a symbolic link to `dg/html/modules.html`) with your favorite browser. 
+Type `make doc` in the folder `path/to/feltor/doc` and open `index.html` (a symbolic link to `dg/html/modules.html`) with your favorite browser. 
 
 ## 3. Contributions and Acknowledgements
 For instructions on how to contribute read the [wiki page](https://github.com/feltor-dev/feltor/wiki/Contributions).
diff --git a/doc/related_pages/parallel/parallel.tex b/doc/related_pages/parallel/parallel.tex
index 7f42268ee..d222e6569 100644
--- a/doc/related_pages/parallel/parallel.tex
+++ b/doc/related_pages/parallel/parallel.tex
@@ -29,8 +29,13 @@ and discuss some problems arising from the boundaries of the computational domai
 Finally, we propose a field line mapping for variable initialization in Section~\ref{sec:parallelc}.
 
 \subsection{Discretization of parallel derivatives} \label{sec:parallela}
-One idea to discretize $\bhat\cdot\nabla$ is to simply use 
-the dG discretization developed in the last section directly, i.e.~discretize $b^R\partial_R+b^Z\partial_Z+b^\varphi\partial_\varphi$, where $(R,Z,\varphi)$ are cylindrical coordinates. 
+Given is a vector field $\bhat(R,Z)$ in cylindrical coordinates $R,Z,\varphi$ independent of $\varphi$ and we need to 
+discretize the derivative $\bhat \cdot\nabla$. 
+This $\bhat$ might be the magnetic unit vector field but the algorithm works 
+for any vector field $\vec v$ with $v^\varphi\neq 0$, in especially $\vec v$ does not need
+to have unit length.
+One idea to discretize $\bhat\cdot\nabla$ might be to simply use 
+the dG discretization directly, i.e.~discretize $b^R\partial_R+b^Z\partial_Z+b^\varphi\partial_\varphi$.
 However, a very restrictive CFL-condition due to the high resolution
 in the $R$-$Z$ planes and the fast parallel motion makes 
 this approach impractical. We therefore decided to use a 
@@ -42,10 +47,11 @@ field line following coordinate, then the one-dimensional discrete derivative al
     \frac{\d f }{\d s} \rightarrow \frac{f_{k+1}-f_{k-1}}{s_{k+1} - s_{k-1}}.
     \label{eq:proposition}
 \end{align}
+Note here that $s$ does NOT denote the distance 
+(especially since we do not require the existence of a metric at this point).
 
-From differential geometry we 
-know that to every smooth vector field $\bhat$ there is a unique curve of which the
-tangent in a point is the value of $\bhat$ at that point. It is given by
+To every smooth vector field $\bhat(\vec x)$ there is a unique curve of which the
+tangent in a point $p$ is the value of $\bhat(p)$ at that point. It is given by
 the solution of the differential equation
 \begin{align}
     \frac{\d z^i}{\d s} = \hat b^i(\vec z)
@@ -58,12 +64,12 @@ Moreover, by definition we have
     \frac{\d f(\vec z(s))}{\d s} = \bhat\cdot \nabla f|_{\vec z(s)}
     \label{}
 \end{align}
-along a field line parameterized by distance $s$, 
+along a field line parameterized by $s$, 
 i.e. instead of $\nabla_\parallel f$ we can choose to discretize $\frac{\d f}{\d s}$.
 
-Let us divide the $\varphi$ direction into $N_\varphi$ equidistant planes of distance
-$\Delta \varphi$. Unfortunately, from Eq.~\eqref{eq:integralcurve} we cannot easily determine the 
-distance $\Delta s$ for given $\Delta \varphi$. It is better to integrate
+Let us divide the $\varphi$ direction into $N_\varphi$ equidistant planes of 
+$\Delta \varphi$. Unfortunately, from Eq.~\eqref{eq:integralcurve} we cannot easily determine
+$\Delta s$ for given $\Delta \varphi$. It is better to use a transformation $\d t = \hat b^\varphi \d s$
 \begin{align}
     \frac{\d z^i}{\d t} = \frac{\hat b^i}{\hat b^\varphi}
     \label{}
@@ -81,7 +87,7 @@ since in this case $\d\varphi/\d t = 1 \Rightarrow t=\varphi$. We get
 \end{align}
 \label{eq:fieldline}
 \end{subequations}
-for the length of the field line $s$. 
+for $s$. 
 Eqs.~\eqref{eq:fieldline} are integrated from $\varphi=0$ to $\varphi=\pm \Delta \varphi$. 
 We characterize the flow generated by $\bhat/b^\varphi$ by
 \begin{align}
@@ -105,13 +111,13 @@ The proposed centered discretization~\eqref{eq:proposition} for the parallel der
 \end{align}
 which is slightly different from Reference~\cite{Hariri2014}, where
 the relation~\eqref{eq:fieldlinec} was used to replace $\d \varphi/\d s$. 
-Previous derivations now needed to construct an interpolation scheme in order
-to evaluate functions on the transformed coordinates. 
 Since the $R$ and $Z$ coordinates are still discretized in the dG framework we note that in our work
 the interpolation of $f$ on the transformed points $\Tpm\vec z$
 is naturally given by interpolating the base polynomials. 
-Let us for a moment omit the $Z$ coordinate for ease of notation. If $(R_{nj}, \varphi_k)$ are the grid points, 
-we call $(R^+_{nj}, \varphi_{k+1}) := \Tp[R_{nj}, \varphi_k]$ and $(R_{nj}^-, \varphi_{k-1}) := \Tm[R_{nj}, \varphi_k]$ the transformed coordinates along
+Let us for a moment omit the $Z$ coordinate for ease of notation. 
+If $(R_{nj}, \varphi_k)$ are the grid points, 
+we call $(R^+_{nj}, \varphi_{k+1}) := \Tp[R_{nj}, \varphi_k]$ and 
+$(R_{nj}^-, \varphi_{k-1}) := \Tm[R_{nj}, \varphi_k]$ the transformed coordinates along
 the field lines. We then have
 \begin{subequations}
 \begin{align}
@@ -155,91 +161,106 @@ discretized using
     \label{}
 \end{align}
 and repeating the procedure of this section.
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\subsection{Change of coordinates}
+In principle the above considerations hold in any
+coordinate system $\eta,\zeta,\varphi$.
+The only question is how to integrate the field lines in the 
+$\eta, \zeta,\varphi$ system 
+since we assumed that our vector field $\bhat$ was given 
+analytically in 
+cylindrical coordinates. There are two possibilities. 
+First, interpolate $R(\zeta_i, \eta_i), Z(\zeta_i, \eta_i)$ for 
+all $i$, then integrate $\bhat$ in $(R,Z)$ space and finally use
+Newton iteration to find $\zeta(R^\pm_i, Z^\pm_i), \eta(R^\pm_i, Z^\pm_i)$. 
+The downside here is that it is difficult to tell when and where the fieldline leaves the simulation domain.
 
-%We see that the term in brackets is obviously skew-symmetric if $(I^+)^\mathrm{T} = I^- = (I^+)^{-1}$, i.e.~the interpolation matrix is unitary.
-% IT ISNT
-%\subsection{Limiter and Cutting procedures} \label{sec:parallelb}
-The main problem with the above scheme is the question what
- to do when a field line crosses the simulation boundaries. 
-Boundary conditions are formulated in cylindrical coordinates. 
-One idea is to simply cut the contribution from field lines
-that leave the computational domain. While this works in practice
-it is unclear what numerical and physical side-effects this procedure might have. 
-Another idea is to check to every point $\vec z$ whether $\Tpm \vec z$
-lies inside our simulation box or not. If not, we have to find where exactly the 
-field line intersects the simulation box.  We have to find
+The second possibiliy (the one currently implemented) 
+is to integrate entirely in the 
+transformed coordinate system $\zeta, \eta, \varphi$. 
+The magnetic field can be easily transformed since we have the
+Jacobian of the coordinate transformation
+\begin{align}
+    b^\zeta(\zeta, \eta) &= \left(\frac{\partial \zeta}{\partial R} b^{R} + \frac{\partial \zeta}{\partial Z}b^Z\right)_{R(\zeta, \eta), Z(\zeta, \eta)} \\
+    b^\eta(\zeta, \eta) &= \left(\frac{\partial \eta}{\partial R} b^{R} + \frac{\partial \eta}{\partial Z}b^Z\right)_{R(\zeta, \eta), Z(\zeta, \eta)} \\
+    b^\varphi(\zeta, \eta) &= b^\varphi({R(\zeta, \eta), Z(\zeta, \eta)})
+    \label{eq:field_trafo}
+\end{align}
+The fieldline equations are still
+\begin{subequations}
+\begin{align}
+\frac{\d \zeta}{\d\varphi} &= \frac{b^\zeta}{b^\varphi}\\
+\frac{\d \eta}{\d\varphi} &= \frac{b^\eta}{b^\varphi}\\
+\frac{\d s}{\d\varphi} &= \frac{1}{|b^\varphi|}
+\end{align}
+\label{eq:fieldlines}
+\end{subequations}
+The downside here is that when integrating fieldlines we
+have to interpolate the magnetic field at arbitrary points. 
+However, the error should vanish with order $P$ in the 
+perpendicular plane. In order to mitigate this error
+we transform the B-Field on a finer grid/higher order polynomials for more accurate
+integration.
+Apart from the problem of how to get the transformed vector field 
+the remaining algorithm for $\bhat\cdot\nabla$ is entirely unchanged.
+
+\subsection{Boundary conditions}
+The question is what to do when a fieldline intersects with the boundary
+of the simulation domain before reaching the next plane.
+Boundary conditions are formulated by either setting a value 
+on the boundary of the domain (Dirichlet) or by fixing 
+the derivative perpendicularly to the boundary (Neumann), or 
+a combination of both ( Robin). 
+
+The problem with a Dirichlet boundary condition 
+is that we need to find the exact place where the fieldline 
+reaches the boundary. 
+We have to find
 $\varphi_b$ such that the result of the integration of Eq.~\eqref{eq:fieldline} from 
 $0$ to $\varphi_b$ lies on the boundary. 
 The angle $\varphi_b$ can be found by a bisection algorithm knowing that $0<\varphi_b < \Delta\varphi$. 
 This kind of procedure is known as a shooting method. 
-When all points are found, ghost cells can be constructed in the correct way. 
+Secondly, for Dirichlet boundaries the small 
+distance of a point to the wall seriously deteriorates the CFL condition. (To ease the CFL condition 
+was the reason to devise the algorithm in the first place)
+
+The problem with Neumann boundaries is that they are usually given 
+perpendicularly to the boundary and that the fieldlines are not necessarily
+perpendicular to the boundary. 
+
+\subsection{Avoiding boundary conditions}
+An obvious way to avoid boundary conditions is to
+align the simulation domain to the vector field such that the fieldlines
+never intersect the boundary. This is possible in FELTOR (read the
+geometry section).
+
+When computing in non-aligned coordinate systems
+one idea to avoid boundary conditions 
+is to simply cut the contribution from field lines
+that leave the computational domain. While this works in practice
+it is unclear what numerical and physical side-effects this procedure might have. 
+
+Another solution would be to change the 
+vector field $\bhat$ and only retain the toroidal part of $\bhat$ on the 
+boundary ( $\hat b^R|_{\partial\Omega} = \hat b^Z|_{\partial\Omega} =0$). The fieldlines then have a kink on the boundary $\partial\Omega$. 
+On the other hand we can implement boundary conditions consistent with 
+the perpendicular ones since the fieldlines never leave the domain. 
+We simply interpolate the quantity to derive on the inner side of the
+domain boundary (Neumann conditions = "No boundary condition") or 
+set the value to zero (Dirichlet condition).
 
+\subsection{Poloidal limiters}
 A poloidal limiter can simply be implemented via a boundary condition in $\varphi$. 
-As long as the form of the limiter is a flux-function we do not have to 
+As long as the form of the limiter is aligned with a flux-function we do not have to 
 integrate a field line in order to determine which points lie in the
 limiter-shadow. It is therefore straightforward to implement ghost-cells 
 in that case. 
 
-\subsection{Field aligned initialization} \label{sec:parallelc}
-
-An important aspect of our simulations is a judicious initialization of the 
-fields. We want structures to be field-aligned in the beginning of the simulation with
-a possible modulation along the direction of the field line.
-If a Gaussian shape is used, we call $\sigma_\parallel$ the extension in parallel
-direction and write
-\begin{align}
-    f_0(R,Z,\varphi) = F(R,Z,\varphi) \exp\left( - \frac{(\varphi-\varphi_0)^2}{2\sigma_\parallel^2}\right),
-    \label{eq:parallelInit}
-\end{align}
-where $F$ is a function that is invariant under the field line transformations
-\begin{subequations}
-\begin{align}
-    \Tp F(\vec z) &= F( \Tp \vec z) \overset{!}{=} F(\vec z) \text{ (pull-back), } \\
-    \Tm F(\vec z) &= F( \Tm \vec z) \overset{!}{=} F(\vec z) \text{ (push-forward). } 
-\end{align}
-\label{}
-\end{subequations}
-We can use these relations to construct aligned structures
-by active transformations of some given field.
-Our idea is to initialize a two-dimensional field $F(R,Z, \varphi_k)$ in a given plane $k$ and 
-transform this field to all other planes using the recursive relations
-\begin{subequations}
-\begin{align}
-    F( R, Z, \varphi_{k+1}) = \Tm F( R, Z, \varphi_{k+1}) = F(R^-, Z^-, \varphi_k), \\
-    F( R, Z, \varphi_{k-1}) = \Tp F( R, Z, \varphi_{k-1}) = F(R^+, Z^+, \varphi_k),
-\end{align}
-    \label{eq:recursiveInit}
-\end{subequations}
-which is the statement that $F$ in the next plane equals the push-forward  
-and $F$ in the previous plane equals the pull-back of $F$ in the current plane. 
-Note here that Eq.~\eqref{eq:interpolation} applies for the required interpolation
-procedures. 
-
-\section{The parallel derivative}
-The idea of the sandwich method \cite{Held2016} is to bracket the parallel derivative by interpolation and projection matrices:
-\begin{align}
-    \nabla^c_\parallel &= P\nabla_\parallel^f Q \\
-    \nabla^{c\dagger}_\parallel &= P \nabla^{f\dagger}_\parallel Q
-    \label{eq:sandwich}
-\end{align}
-In this way the projection integrals
-\begin{align}
-    \int\dV \nabla_\parallel f p_i(x)p_j(y) 
-    \label{}
-\end{align}
-are computed more precisely.
-The size of the fine grid should therefore be as large as
-possible (reasonable? what resolution is needed?).
-We first notice that the one interpolation matrix can be absorbed
-in the parallel derivative since this also consists of 
-interpolation operations. 
-\begin{align}
-    \nabla^c_\parallel &= P\nabla_\parallel^{fc} \\
-    \nabla^{c\dagger}_\parallel &= \nabla^{fc\dagger}_\parallel Q
-    \label{eq:sandwich}
-\end{align}
-
 
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\section{The adjoint methods}
+The idea is to discretize the operation $\nabla\cdot( \bhat .)$ by
+taking the adjoint of the discretization for $\nabla_\parallel$. 
 In order to understand what the adjoint operators do let us denote $\Tp$ as the push-forwward operator. Then we have
 \begin{align}
     \int f(\vec x) \Tp h(\vec x) \sqrt{g(\vec x)}\d^3x \\
@@ -249,7 +270,7 @@ In order to understand what the adjoint operators do let us denote $\Tp$ as the
     \equiv  \int (\Tp)^\dagger\left[f(\vec x)\right] h(\vec x)\sqrt{g(\vec x)}   \d^3x
     \label{}
 \end{align}
-$J$ is the determinant of the Jacobian $\partial(\vec x')/\partial(\vec x)$.
+$J$ is the determinant of the Jacobian $\partial(\vec x')/\partial(\vec x)$ with $\vec x' = \Tm \vec x$.
 In the last step we simply replaced the dummy variable $\vec x'$ with $\vec x$ again and identified the relevant terms
 as the adjoint operator:
 \begin{align}
@@ -257,7 +278,7 @@ as the adjoint operator:
     \label{}
 \end{align}
 This means that numerically the adjoint of the push-forward 
-operator should be a valid discretization of its inverse. (how can this be exploited as a test?)
+operator should be a valid discretization of its inverse.
 Note that $\sqrt{g}J^{-1}(\vec x) = \sqrt{g'(\Tm \vec x)}$.
 With this we can write
 \begin{align}
@@ -296,61 +317,35 @@ In any case this means that we want to have
 \begin{align}
 \left(\Tp\right)^\dagger B^\varphi  =  B^\varphi
 \end{align}
-\section{Boundary conditions}
-The question is what to do when a fieldline intersects with the boundary
-of the simulation domain before reaching the next plane.
-The problem with simply putting a value exactly where the fieldline 
-reaches the boundary is first that a true interpolation at that 
-position requires a 3d interpolation (we 
-are between planes) instead of a 2d interpolation.
-Secondly, for Dirichlet boundaries the small 
-distance seriously deteriorates the CFL condition.
-A simple solution is to set the perpendicular field zero on the 
-boundary such that the field becomes purely 
-toroidal on the boundary. The fieldlines then have a kink on the 
-boundary. On the other hand we can implement boundary conditions consistent with 
-the perpendicular ones since the fieldlines never leave the domain. 
-We simply interpolate the quantity to derive on the inner side of the
-domain boundary (Neumann conditions = "No boundary condition") or 
-set the value to zero (Dirichlet condition).
 
 
-\section{Geometry}
-When we are on a different geometry than $(R,Z)$ the question is how to integrate the field lines. There are two possibilities. 
-First, interpolate $R(\zeta_i, \eta_i), Z(\zeta_i, \eta_i)$ for 
-all $i$, then integrate in $(R,Z)$ space and finally use
-Newton iteration to find $\zeta(R^\pm_i, Z^\pm_i), \eta(R^\pm_i, Z^\pm_i)$. 
-The downside here is that it is difficult to tell when and where the fieldline leaves the simulation domain and even worse in MPI the next points might belong to another process. 
-
-The second possibiliy is to integrate entirely in the 
-transformed coordinate system $\zeta, \eta$. 
-The magnetic field can be easily transformed since we have the
-Jacobian of the coordinate transformation
+\subsection{A grid refinement approach}
+While the idea of the last section sounds appealing the problem 
+is that the discretization does not converge.
+The idea of the sandwich method \cite{Stegmeir2016} is 
+to bracket the parallel derivative by interpolation and 
+projection matrices:
 \begin{align}
-    B^\zeta(\zeta, \eta) &= \left(\frac{\partial \zeta}{\partial R} B^{R} + \frac{\partial \zeta}{\partial Z}B^Z\right)_{R(\zeta, \eta), Z(\zeta, \eta)} \\
-    B^\eta(\zeta, \eta) &= \left(\frac{\partial \eta}{\partial R} B^{R} + \frac{\partial \eta}{\partial Z}B^Z\right)_{R(\zeta, \eta), Z(\zeta, \eta)} \\
-    B^\varphi(\zeta, \eta) &= B^\varphi({R(\zeta, \eta), Z(\zeta, \eta)})
-    \label{eq:field_trafo}
+    \nabla^c_\parallel &= P\nabla_\parallel^f Q \\
+    \nabla^{c\dagger}_\parallel &= P \nabla^{f\dagger}_\parallel Q
+    \label{eq:sandwich}
 \end{align}
-The advantage is that we can do this for any coordinate
-system. Only for cylindrical coordinates, we integrate directly in physical space. The equations to integrate
-are
-\begin{subequations}
+In this way the projection integrals
 \begin{align}
-\frac{\d \zeta}{\d\varphi} &= \frac{B^\zeta}{B^\varphi}\\
-\frac{\d \eta}{\d\varphi} &= \frac{B^\eta}{B^\varphi}\\
-\frac{\d s}{\d\varphi} &= \frac{|B|}{|B^\varphi|}
+    \int\dV \nabla_\parallel f p_i(x)p_j(y) 
+    \label{}
+\end{align}
+are computed more precisely.
+The size of the fine grid should therefore be as large as
+possible.
+We first notice that the one interpolation matrix can be absorbed
+in the parallel derivative since this also consists of 
+interpolation operations. 
+\begin{align}
+    \nabla^c_\parallel &= P\nabla_\parallel^{fc} \\
+    \nabla^{c\dagger}_\parallel &= \nabla^{fc\dagger}_\parallel Q
+    \label{eq:sandwich}
 \end{align}
-\label{eq:fieldlines}
-\end{subequations}
-The downside here is that when integrating fieldlines we
-have to interpolate the magnetic field at arbitrary points. 
-However, the error should vanish with $3rd$ order in the 
-perpendicular plane. (check this). In order to mitigate this error
-we could maybe transform the B-Field on a finer grid/higher order polynomials for more accurate
-integration. Also, we need interpolation in the 
-first algorithm as well. 
-
 Note that the matrix-matrix multiplications in Eq.~\eqref{eq:sandwich} can
 be precomputed and stored. The memory requirements 
 in the final computations are 
@@ -361,6 +356,9 @@ Finally remember that the adjoint of a matrix in the modified geometry
 involves the volume element. This means that after you've adjoined the 
 parallel derivative the normal way simply bracket the result 
 by $1/\sqrt{g}$ and $\sqrt{g}$. 
+
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \section{Algorithm}
 Given are the components $v^i(R,Z)$ for $i\in\{R,Z,\varphi\}$ and a compuational grid (in the following the ``coarse grid``)
 \begin{itemize}
@@ -391,54 +389,8 @@ Given are the components $v^i(R,Z)$ for $i\in\{R,Z,\varphi\}$ and a compuational
   \item compute the matrix-matrix multiplications $P\cdot I^\pm$ as well as the transpose
   \item project the $s$ vectors to the coarse grid
 \end{itemize}
-\section{MPI implementation}
-Let us also note the mpi-implementation, which is not entirely
-trivial due to the matrix-matrix multiplications involed in Eq.~\eqref{eq:sandwich}.
-\subsection{Row and column distributed sparse matrices}
-In Feltor each mpi process gets an equally sized chunk of a 
-vector.
-Contrary to a vector
-a matrix can be distributed in two ways, row-wise and column wise. 
-In a row-distributed matrix each process gets the complete 
-rows of the matrix that correspond to the indices in the 
-vector it holds. 
-In a column-distributed matrix each process gets the complete 
-columns of the matrix corresponding to the indices in the 
-vector it holds. 
-When we implement a matrix-vector multiplication the order 
-of communication and computation depends on the distribution 
-of the matrix.
-For the row-distributed matrix each process first has to gather all elements of the input vector it needs to be able to compute the elements of the output. This requires MPI communication.
-Formally, the gather operation can be written as a matrix $G$
-of $1'$s and $0'$s where the output vector is of equal or larger size than the input vector.
-After the elements have been gathered the local matrix-vector
-multiplications can be executed.
-\begin{align}
-M = R\cdot G
-\end{align}
-where $R$ is the row-distributed matrix with modified indices 
-and $G$ is the gather matrix, in which the MPI-communication takes place.
-
-In a column distributed matrix the local matrix-vector multiplication can be executed first because each processor already
-has all vector elements it needs. 
-However the resuling elements have to be communicated back to 
-the process they belong to. Furthermore, a process has to sum
-all elements it receives from other processes on the same
-index. This is a scatter and reduce operation and
-it can be written as a scatter matrix $S$. The transpose
-of the scatter matrix is a gather matrix and vice-versa.
-\begin{align}
-M = S\cdot C
-\end{align}
-where $S$ is the scatter matrix and $C$ is the column distributed
-matrix with modified indices. 
 
-It turns out that a row-distributed matrix can be transposed
-by transposition of the local matrices and the gather matrix.
-The result is then a column distributed matrix.
-The transpose of a column distributed matrix is a row-distributed matrix and vice-versa.
-
-\subsection{Matrix-Matrix multiplication}
+\subsection{Notes on the MPI implmentation}
 We note that we normally construct $\nabla_\parallel^{fc}$ as a column 
 distributed
 matrix. The advantage is then that the gather operation is bijective, i.e. the transpose of the gather matrix is its inverse. 
@@ -458,6 +410,40 @@ indices can be constructed.
 We note here that we even don't need to construct the gather matrix
 for $\nabla_\parallel^{fc}$, only the one for $\nabla_\parallel^c$ is
 needed.
+\section{Field aligned initialization} \label{sec:parallelc}
+
+An important aspect of our simulations is a judicious initialization of the 
+fields. We want structures to be field-aligned in the beginning of the simulation with
+a possible modulation along the direction of the field line.
+If a Gaussian shape is used, we call $\sigma_\parallel$ the extension in parallel
+direction and write
+\begin{align}
+    f_0(R,Z,\varphi) = F(R,Z,\varphi) \exp\left( - \frac{(\varphi-\varphi_0)^2}{2\sigma_\parallel^2}\right),
+    \label{eq:parallelInit}
+\end{align}
+where $F$ is a function that is invariant under the field line transformations
+\begin{subequations}
+\begin{align}
+    \Tp F(\vec z) &= F( \Tp \vec z) \overset{!}{=} F(\vec z) \text{ (pull-back), } \\
+    \Tm F(\vec z) &= F( \Tm \vec z) \overset{!}{=} F(\vec z) \text{ (push-forward). } 
+\end{align}
+\label{}
+\end{subequations}
+We can use these relations to construct aligned structures
+by active transformations of some given field.
+Our idea is to initialize a two-dimensional field $F(R,Z, \varphi_k)$ in a given plane $k$ and 
+transform this field to all other planes using the recursive relations
+\begin{subequations}
+\begin{align}
+    F( R, Z, \varphi_{k+1}) = \Tm F( R, Z, \varphi_{k+1}) = F(R^-, Z^-, \varphi_k), \\
+    F( R, Z, \varphi_{k-1}) = \Tp F( R, Z, \varphi_{k-1}) = F(R^+, Z^+, \varphi_k),
+\end{align}
+    \label{eq:recursiveInit}
+\end{subequations}
+which is the statement that $F$ in the next plane equals the push-forward  
+and $F$ in the previous plane equals the pull-back of $F$ in the current plane. 
+Note here that Eq.~\eqref{eq:interpolation} applies for the required interpolation
+procedures. 
 
 
 
diff --git a/inc/dg/backend/mpi_matrix.h b/inc/dg/backend/mpi_matrix.h
index d2f0f2558..c77a5ff5f 100644
--- a/inc/dg/backend/mpi_matrix.h
+++ b/inc/dg/backend/mpi_matrix.h
@@ -190,7 +190,7 @@ enum dist_type
 *
 * The idea of this mpi matrix is to separate communication and computation in order to reuse existing optimized matrix formats for the computation. 
 * It can be expected that this works particularly well for cases in which the communication to computation ratio is low. 
-* This class assumes that the matrix and vector elements are distributed rowwise among mpi processes.
+* In this class the matrix elements can be distributed rowwise or columnwise among mpi processes.
 * @tparam LocalMatrix The class of the matrix for local computations. 
  symv needs to be callable on the container class of the MPI_Vector
 * @tparam Collective models aCommunicator The Communication class needs to scatter and gather values across processes. 
diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index 623f4da45..f15764a51 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -1,10 +1,7 @@
 #error Documentation only
 /*! @mainpage
- * This is the FELTOR core dg library. 
- *
  * @subsection pdf PDF writeups
- * 
- * We have a collection of writeups: 
+ * DON'T PANIC!
  *  - <a href="./dg_introduction.pdf" target="_blank">Introduction to dg methods</a>
  */
 /*! @namespace dg 
@@ -33,8 +30,10 @@
  *     @}
  *     @defgroup sparsematrix Sparse matrix formats
  *     @defgroup mpi_structures MPI backend functionality
- *             The general idea is to separate global communication from local parallelization and thus 
- *             readily reuse the existing, optimized library for the local part
+ *             In this section the blas functions are implemented for the MPI+X hardware architectures, where X 
+ *             is e.g. CPU, GPU, accelerator cards...
+ *             The general idea to achieve this is to separate global communication from local computations and thus 
+ *             readily reuse the existing, optimized library for the local part.
  *     @defgroup typedefs Typedefs
  *          Useful type definitions for easy programming
  * @}
@@ -181,3 +180,67 @@
  @note you can make your own SymmetricOp by providing the member function void symv(const container&, container&);
   and specializing MatrixTraits with the SelfMadeMatrixTag as the matrix_category
   */
+
+/*!@addtogroup mpi_structures
+@{
+@page mpi_matrix MPI Vectors and the blas1 functions
+
+In Feltor each mpi process gets an equally sized chunk of a vector.
+The corresponding structure in FELTOR is the dg::MPI_Vector, which is 
+nothing but a wrapper around any container type object and a MPI_Comm. 
+With this the dg::blas1 functions can readily implemented by just redirecting to the
+implementation for the container type. The only functions that need
+communication are the dg::blas1::dot functions (MPI_Allreduce).
+
+@page mpi_vector Row and column distributed matrices
+
+Contrary to a vector
+a matrix can be distributed in two ways, row-wise and column wise. 
+The structure dg::MPIDistMat is a wrapper around a LocalMatrix type object 
+and an instance of dg::aCommunicator
+In a row-distributed matrix each process gets the complete 
+rows of the matrix that correspond to the indices in the 
+vector it holds. 
+In a column-distributed matrix each process gets the complete 
+columns of the matrix corresponding to the indices in the 
+vector it holds. 
+When we implement a matrix-vector multiplication the order 
+of communication and computation depends on the distribution 
+of the matrix.
+For the row-distributed matrix each process first has to gather 
+all elements of the input vector it needs to be able to compute the elements of the output. In general this requires MPI communication.
+(read the documentation of dg::aCommunicator for more info of how global scatter/gather operations work).
+Formally, the gather operation can be written as a matrix \f$G\f$
+of \f$1'\f$s and \f$0'\f$s.
+After the elements have been gathered into a buffer the local matrix-vector
+multiplications can be executed.
+\f[
+M = R\cdot G
+\f]
+where \f$R\f$ is the row-distributed matrix with modified indices 
+and \f$G\f$ is the gather matrix, in which the MPI-communication takes place.
+The dg::RowColDistMat goes one step further and separates the matrix \f$ R\f$ into 
+a part that can be computed entirely on the local process and a part that needs communication.
+
+\section column 
+
+In a column distributed matrix the local matrix-vector multiplication can be executed first because each processor already
+has all vector elements it needs. 
+However the resuling elements have to be communicated back to 
+the process they belong to. Furthermore, a process has to sum
+all elements it receives from other processes on the same
+index. This is a scatter and reduce operation and
+it can be written as a scatter matrix \f$S\f$ (s.a. dg::aCommunicator). The transpose
+of the scatter matrix is a gather matrix and vice-versa.
+\f[
+M = S\cdot C
+\f]
+where \f$S\f$ is the scatter matrix and \f$C\f$ is the column distributed
+matrix with modified indices. 
+
+It turns out that a row-distributed matrix can be transposed
+by transposition of the local matrices and the gather matrix (s.a. dg::transpose).
+The result is then a column distributed matrix.
+The transpose of a column distributed matrix is a row-distributed matrix and vice-versa.
+@}
+*/
-- 
GitLab


From 5772be586153a886c08514e85c9b6c012a881452 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sun, 15 Oct 2017 18:31:10 +0200
Subject: [PATCH 356/453] added const to new Functors

---
 inc/dg/functors.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/inc/dg/functors.h b/inc/dg/functors.h
index ae4942269..e15c77a3c 100644
--- a/inc/dg/functors.h
+++ b/inc/dg/functors.h
@@ -444,7 +444,7 @@ struct CosXCosY
      
      * @return \f$ f(x,y)\f$
      */
-    double operator()( double x, double y){ return bamp_+amp_*cos(x*kx_)*cos(y*ky_);}
+    double operator()( double x, double y)const{ return bamp_+amp_*cos(x*kx_)*cos(y*ky_);}
   private:
     double amp_,bamp_,kx_,ky_;
 };
@@ -471,7 +471,7 @@ struct SinXCosY
      
      * @return \f$ f(x,y)\f$
      */
-    double operator()( double x, double y){ return bamp_+amp_*sin(x*kx_)*cos(y*ky_);}
+    double operator()( double x, double y)const{ return bamp_+amp_*sin(x*kx_)*cos(y*ky_);}
   private:
     double amp_,bamp_,kx_,ky_;
 };
-- 
GitLab


From f297913aa94ac6232358ed5994aaaf920769c18c Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 16 Oct 2017 13:23:41 +0200
Subject: [PATCH 357/453] added const to operator() and inv_weights() to
 feltorShw.cuh

---
 src/feltorShw/feltor.cuh | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/feltorShw/feltor.cuh b/src/feltorShw/feltor.cuh
index 63b41f5e9..dca5531d1 100644
--- a/src/feltorShw/feltor.cuh
+++ b/src/feltorShw/feltor.cuh
@@ -35,7 +35,7 @@ struct Rolkar
         LaplacianM_perp_phi ( g,p.bc_x_phi,g.bcy(), dg::normed, dg::centered)
     {
     }
-    void operator()( std::vector<container>& x, std::vector<container>& y)
+    void operator()( const std::vector<container>& x, std::vector<container>& y)
     {
         /* x[0] := N_e - (bgamp+profamp)
            x[1] := N_i - (bgamp+profamp)
@@ -51,6 +51,7 @@ struct Rolkar
     }
     dg::Elliptic<Geometry, Matrix, container>& laplacianM() {return LaplacianM_perp_phi;}
     const container& weights(){return LaplacianM_perp.weights();}
+    const container& inv_weights(){return LaplacianM_perp.inv_weights();}
     const container& precond(){return LaplacianM_perp.precond();}
   private:
     const eule::Parameters p;
@@ -80,7 +81,7 @@ struct Feltor
     const std::vector<container>& potential( ) const { return phi;}
     void initializene( const container& y, container& target);
 
-    void operator()( std::vector<container>& y, std::vector<container>& yp);
+    void operator()( const std::vector<container>& y, std::vector<container>& yp);
 
     double mass( ) {return mass_;}
     double mass_diffusion( ) {return diff_;}
@@ -247,7 +248,7 @@ void Feltor<Grid, Matrix, container>::initializene( const container& src, contai
 }
 
 template<class Grid, class Matrix, class container>
-void Feltor<Grid, Matrix, container>::operator()( std::vector<container>& y, std::vector<container>& yp)
+void Feltor<Grid, Matrix, container>::operator()( const std::vector<container>& y, std::vector<container>& yp)
 {
 
     dg::Timer t;
-- 
GitLab


From 154c87cfff9f89a43cfd25b4fc04f443f167f195 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Mon, 16 Oct 2017 17:54:35 +0200
Subject: [PATCH 358/453] added interpolation and projection to dg writeup

---
 doc/Makefile                                  |  2 +-
 .../dg_introduction/dg_introduction.tex       | 96 ++++++++++++++++++-
 inc/dg/dg_doc.h                               |  2 +-
 3 files changed, 97 insertions(+), 3 deletions(-)

diff --git a/doc/Makefile b/doc/Makefile
index f03b3a69c..5a991e4aa 100644
--- a/doc/Makefile
+++ b/doc/Makefile
@@ -23,7 +23,7 @@ geometries.tag:
 		echo "GENERATE_HTML = NO"; \
 		echo "GENERATE_TAGFILE = ../../doc/geometries.tag" ) | doxygen - ;
 
-dg: geometries.tag
+dg: geometries.tag dg_introduction.pdf
 	cd ../inc/dg; \
 		(cat Doxyfile; \
 		echo "OUTPUT_DIRECTORY = ../../doc/dg"; \
diff --git a/doc/related_pages/dg_introduction/dg_introduction.tex b/doc/related_pages/dg_introduction/dg_introduction.tex
index 49d9229ff..c27fe65fc 100644
--- a/doc/related_pages/dg_introduction/dg_introduction.tex
+++ b/doc/related_pages/dg_introduction/dg_introduction.tex
@@ -366,7 +366,8 @@ for our implementations and thus use $\vec f$ rather than $\bar{\vec f}$ to repr
 \end{align}
 to represent the corresponding products.
 
-\subsection{Discretization of elliptic equations} \label{sec:elliptic}
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\section{Discretization of elliptic equations} \label{sec:elliptic}
 We are now ready to discretize the one-dimensional general elliptic equation 
 \begin{align}
     -\frac{\partial}{\partial x}\left( \chi(x)\frac{\partial \phi}{\partial x}(x)\right) = \rho(x)
@@ -488,6 +489,99 @@ the equality in the results for forward and
 backward discretization is broken. 
 To the knowledge of the author this superconvergence has not yet been observed before. 
 
+\section{Interpolation and Projection}
+Eq.~\eqref{eq:dgexpansion} provides a natural way to interpolate any
+dg expanded function $f_h(x)$ on any point in the interval $[a,b]$. 
+The interpolation has the same order $P$ as the expansion and 
+can be formulated as a matrix vector multiplication
+\begin{align}
+f_h(x_0) = \sum_{n=1}^N\sum_{k=0}^{P-1} p_{nk}(x_0) \bar f^{nk} =: \sum_{n=1}^N\sum_{k=0}^{P-1}I_{nk} \bar f^{nk}
+\label{eq:basic_interpolation}
+\end{align}
+where the sparse interpolation matrix $I$ has as many rows as there are 
+points to interpolate. $I$ has $P^d$ entries per line, where $d$ is the dimensionality of the grid. 
+
+
+\subsection{Interpolation and Projection}
+A special case emerges when the list of points to interpolate is made up
+by the Gaussian abscissas of another grid. 
+Suppose we want to divide each cell $C_n$ of the original grid into $M$ equidistant subcells.
+We use the letter $c$ to denote the coarse grid and the letter $f$ to denote
+the fine grid. 
+For ease of discussion we assume that the number of polynomial coefficients
+in the fine and the coarse grid are the same. 
+We denote $q_{ml}(x)$ the polynomials on the fine grid and $y^P_{mj}$ the 
+corresponding Gaussian abscissas. 
+If a vector $\vec f$ is given on the coarse grid, we can simply interpolate
+it onto the fine grid analogous to Eq.~\eqref{eq:basic_interpolation} via
+\begin{subequations}
+\begin{align}
+  f^F_{mj} &= f_h(y^P_{mj}) = p_{nk}(y^P_{mj})F^{ki} f^C_{ni} \\
+  f^F_{mj} &=: {Q_{mj}}^{ni} f^C_{ni}
+  \label{eq:interpolation}
+\end{align}
+\end{subequations}
+where we denote the special interpolation matrix with $Q$ and implicitly assume the sum of repeated indices.
+No information is lost in this process if the cells $C_n$ are divided by an 
+integer number $M$ and the number of polynomials is the same, i.e. 
+$f^F$ and $f^C$ represent exactly the same expansion $f_h(x)$. 
+\begin{align}
+  f^C(x) = \bar f_C^{nk}p_{nk}(x) = f^F(x) = \bar f_F^{ml}q_{ml}(x) 
+  \label{eq:no_loss}
+\end{align}
+
+Vice versa, given a expansion $f^F(x)$ on the fine grid, we can the projection integrals from the fine grid to the coarse grid. 
+It can be shown that
+\begin{align}
+  \bar f_C^{nk} &:= T_C^{ks}\int \dx f^F(x)p_{ns}(x) = T_C^{ks}  W_m^{ij}p_{ns}(y_{mj}) f^F_{mi} \\
+  f^C_{nt} &= B_{tk}\bar f_C^{nk} = V^c_{tk}  {{(Q^T)}^{nk}}_{mj}W_m^{ji} f^F_{mi} =: {P_{nt}}^{mi} f^F_{mi}
+  %\bar f_C^{nk} = T_C^{ks}\int \dx f^F(x)p_{ns}(x) = T_C^{ks} \bar f_F^{ml} \int \dx q_{ml}(x)p_{ns}(x) \\
+  %= T_C^{ks} F^{lo}f^F_{mo} \int \dx q_{ml}(x)p_{ns}(x)  \\
+  %= T_C^{ks} F^{lo}f^F_{mo} W_F^{ij}q_{ml}(y_{mi})p_{ns}(y_{mj})  \\
+  %= T_C^{ks} F^{lo}f^F_{mo} W_m^{ij}B_{il}p_{ns}(y_{mj})  \\
+  %= T_C^{ks} f^F_{mi} W_m^{ij}p_{ns}(y_{mj})  \\
+  %f^C_{nt} = B_{tk}\bar f_C^{nk} = B_{tk} T_C^{ks} W_m^{ij} p_{ns}(y_{mj}) f^F_{mi}\\
+  %= V^c_{tk} F^{sk} W_m^{ij} p_{ns}(y_{mj}) f^F_{mi}\\
+  %= V^c_{tk}  {Q_{mj}}^{nk}W_m^{ji} f^F_{mi}\\
+  %= V^c_{tk}  {{(Q^T)}^{nk}}_{mj}W_m^{ji} f^F_{mi}\\
+  %=: {P_{nt}}^{mi} f^F_{mi}
+  \label{eq:basic_projection}
+\end{align}
+from where we directly conclude that 
+\begin{align}
+  P = Q^\dagger = V^C Q^T W_F
+  \label{eq:projection_adjoint}
+\end{align}
+i.e. the projection matrix is the adjoint of the interpolation matrix.
+We can also proof that
+\begin{align}
+  P\circ Q = 1_C
+  \label{}
+\end{align}
+which is a reformulation of Eq.~\eqref{eq:no_loss}.
+Note that $Q\circ P \neq 1$.
+The projection is not loss-free but it conserves the integral
+value of the function on the fine grid and the error in the 
+$L_2$-norm between the vectors on fine and coarse grid are
+minimal (proof?).
+\subsection{Grid transformations}
+The above transformation Eq.~\eqref{eq:projection_adjoint} is
+valid only if the number of cells in the fine grid is an integer multiple of the number of cells in the coarse grid. 
+If this is not the case, the adjoint of the interpolation matrix
+does not(!) give a valid projection matrix (tried this 
+out in the code).
+However, what we can always do is to find the least common multiple grid (c) of two given grids (a) and (b). Then
+we can interpolate loss-free from the given grid (a) to the 
+common grid (c) and then project back from (c) to (b).
+In this way we get the forward and backward transformation matrices
+\begin{align}
+    \mathcal T &= P_{c2b}Q_{a2c}\\
+    \mathcal T^\dagger &= P_{c2a} Q_{b2c}
+    \label{}
+\end{align}
+If the least common grid reduces to either grid (a) or (b) the
+transformation matrix reduces to either $P_{a2b}$ or $Q_{a2b}$ as expected.
+
 
 \bibliography{../references}
 %..................................................................
diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index f15764a51..9989d2dd9 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -1,6 +1,6 @@
 #error Documentation only
 /*! @mainpage
- * @subsection pdf PDF writeups
+ * @section pdf PDF writeups
  * DON'T PANIC!
  *  - <a href="./dg_introduction.pdf" target="_blank">Introduction to dg methods</a>
  */
-- 
GitLab


From b038dba1a40d1d07799d00d270f9964a0a10b826 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Mon, 16 Oct 2017 18:40:35 +0200
Subject: [PATCH 359/453] update parallel.tex

---
 .../dg_introduction/dg_introduction.tex       |  2 +-
 doc/related_pages/parallel/parallel.tex       | 94 +++++++++----------
 inc/geometries/geometries_doc.h               |  1 -
 3 files changed, 46 insertions(+), 51 deletions(-)

diff --git a/doc/related_pages/dg_introduction/dg_introduction.tex b/doc/related_pages/dg_introduction/dg_introduction.tex
index c27fe65fc..73d08946a 100644
--- a/doc/related_pages/dg_introduction/dg_introduction.tex
+++ b/doc/related_pages/dg_introduction/dg_introduction.tex
@@ -562,7 +562,7 @@ which is a reformulation of Eq.~\eqref{eq:no_loss}.
 Note that $Q\circ P \neq 1$.
 The projection is not loss-free but it conserves the integral
 value of the function on the fine grid and the error in the 
-$L_2$-norm between the vectors on fine and coarse grid are
+$L_2$-norm between the vectors on fine and coarse grid is 
 minimal (proof?).
 \subsection{Grid transformations}
 The above transformation Eq.~\eqref{eq:projection_adjoint} is
diff --git a/doc/related_pages/parallel/parallel.tex b/doc/related_pages/parallel/parallel.tex
index d222e6569..460be8663 100644
--- a/doc/related_pages/parallel/parallel.tex
+++ b/doc/related_pages/parallel/parallel.tex
@@ -29,17 +29,11 @@ and discuss some problems arising from the boundaries of the computational domai
 Finally, we propose a field line mapping for variable initialization in Section~\ref{sec:parallelc}.
 
 \subsection{Discretization of parallel derivatives} \label{sec:parallela}
-Given is a vector field $\bhat(R,Z)$ in cylindrical coordinates $R,Z,\varphi$ independent of $\varphi$ and we need to 
-discretize the derivative $\bhat \cdot\nabla$. 
-This $\bhat$ might be the magnetic unit vector field but the algorithm works 
+Given is a vector field $\vec v(R,Z)$ in cylindrical coordinates $R,Z,\varphi$ independent of $\varphi$ and we want to 
+discretize the derivative $\vec v \cdot\nabla f \equiv \nabla_\parallel f$. 
+This $\vec v$ might be the magnetic unit vector field but the algorithm works 
 for any vector field $\vec v$ with $v^\varphi\neq 0$, in especially $\vec v$ does not need
 to have unit length.
-One idea to discretize $\bhat\cdot\nabla$ might be to simply use 
-the dG discretization directly, i.e.~discretize $b^R\partial_R+b^Z\partial_Z+b^\varphi\partial_\varphi$.
-However, a very restrictive CFL-condition due to the high resolution
-in the $R$-$Z$ planes and the fast parallel motion makes 
-this approach impractical. We therefore decided to use a 
-Lagrangian approach for the parallel derivatives.
 
 We begin with the formulation of a field-aligned discretization. If $s$ denotes the 
 field line following coordinate, then the one-dimensional discrete derivative along the field line reads
@@ -50,18 +44,18 @@ field line following coordinate, then the one-dimensional discrete derivative al
 Note here that $s$ does NOT denote the distance 
 (especially since we do not require the existence of a metric at this point).
 
-To every smooth vector field $\bhat(\vec x)$ there is a unique curve of which the
-tangent in a point $p$ is the value of $\bhat(p)$ at that point. It is given by
+To every smooth vector field $\vec v(\vec x)$ there is a unique curve of which the
+tangent in a point $p$ is the value of $\vec v(p)$ at that point. It is given by
 the solution of the differential equation
 \begin{align}
-    \frac{\d z^i}{\d s} = \hat b^i(\vec z)
+    \frac{\d z^i}{\d s} = v^i(\vec z)
     \label{eq:integralcurve}
 \end{align}
-where $z^i$ is one of $(R, Z, \varphi)$ and $\hat b^i$ are the contravariant components
-of $\bhat$ in cylindrical coordinates. 
+where $z^i$ is one of $(R, Z, \varphi)$ and $v^i$ are the contravariant components
+of $\vec v$ in cylindrical coordinates. 
 Moreover, by definition we have
 \begin{align}
-    \frac{\d f(\vec z(s))}{\d s} = \bhat\cdot \nabla f|_{\vec z(s)}
+    \frac{\d f(\vec z(s))}{\d s} = \vec v\cdot \nabla f|_{\vec z(s)}
     \label{}
 \end{align}
 along a field line parameterized by $s$, 
@@ -69,27 +63,27 @@ i.e. instead of $\nabla_\parallel f$ we can choose to discretize $\frac{\d f}{\d
 
 Let us divide the $\varphi$ direction into $N_\varphi$ equidistant planes of 
 $\Delta \varphi$. Unfortunately, from Eq.~\eqref{eq:integralcurve} we cannot easily determine
-$\Delta s$ for given $\Delta \varphi$. It is better to use a transformation $\d t = \hat b^\varphi \d s$
+$\Delta s$ for given $\Delta \varphi$. It is better to use the transformation $\d t = v^\varphi \d s$
 \begin{align}
-    \frac{\d z^i}{\d t} = \frac{\hat b^i}{\hat b^\varphi}
+    \frac{\d z^i}{\d t} = \frac{v^i}{v^\varphi}
     \label{}
 \end{align}
 since in this case $\d\varphi/\d t = 1 \Rightarrow t=\varphi$. We get
 \begin{subequations}
 \begin{align}
-    \frac{\d R}{\d\varphi}&= \frac{\hat b^R}{\hat b^\varphi},\\ %\frac{R}{I}\frac{\partial\psi}{\partial Z},\\
-    \frac{\d Z}{\d\varphi}&=\frac{\hat b^Z}{\hat b^\varphi},%-\frac{R}{I}\frac{\partial\psi}{\partial R}.
+    \frac{\d R}{\d\varphi}&= \frac{v^R}{v^\varphi},\\ %\frac{R}{I}\frac{\partial\psi}{\partial Z},\\
+    \frac{\d Z}{\d\varphi}&=\frac{v^Z}{v^\varphi},%-\frac{R}{I}\frac{\partial\psi}{\partial R}.
 \end{align}
 \text{ together with the equation  }
 \begin{align}
-    \frac{\d s}{\d\varphi} &= \frac{1}{|\hat b^\varphi|} %= \frac{B}{|B^\varphi|}% = \frac{R_0R^2B}{I}=R_0R\sqrt{1+\frac{|\nabla\psi|^2}{I^2}}
+    \frac{\d s}{\d\varphi} &= \frac{1}{|v^\varphi|} %= \frac{B}{|B^\varphi|}% = \frac{R_0R^2B}{I}=R_0R\sqrt{1+\frac{|\nabla\psi|^2}{I^2}}
     \label{eq:fieldlinec}
 \end{align}
 \label{eq:fieldline}
 \end{subequations}
 for $s$. 
 Eqs.~\eqref{eq:fieldline} are integrated from $\varphi=0$ to $\varphi=\pm \Delta \varphi$. 
-We characterize the flow generated by $\bhat/b^\varphi$ by
+We characterize the flow generated by $\vec v/v^\varphi$ by
 \begin{align}
     \Tpm\vec z := \Tpm[R, Z, \varphi]:= ( R(\pm \Delta\varphi), Z( \pm \Delta\varphi), \varphi\pm\Delta \varphi),
     \label{}
@@ -100,7 +94,7 @@ with initial condition
     (R(0), Z(0), s(0)) = (R, Z, 0).
     \label{}
 \end{align} 
-Obviously we have $\Tm\circ\Tp = \Eins$, but $\Tpm$ is not unitary since $\bhat/b^\varphi$ is 
+Obviously we have $\Tm\circ\Tp = \Eins$, but $\Tpm$ is not unitary since $\vec v/v^\varphi$ is 
 not divergence free. 
 
 The proposed centered discretization~\eqref{eq:proposition} for the parallel derivative then reads
@@ -141,16 +135,17 @@ This discretization is not skew-symmetric since the
 field lines are not volume-preserving, or~$(I^+)^\mathrm{T} \neq I^-$.
 In fact, the adjoint of the parallel derivative is
 \begin{align}
-    \nabla_\parallel^\dagger = - \nabla\cdot(\bhat\ . ) \neq -\nabla_\parallel.
+    \nabla_\parallel^\dagger f = - \nabla\cdot(\vec v\ f ) \neq -\nabla_\parallel f.
     \label{}
 \end{align}
 Note that with this relation we can define the parallel 
 diffusion operator as
 \begin{align}
-    \Delta_\parallel := -\nabla_\parallel^\dagger \nabla_\parallel = (\nabla\cdot \bhat) \nabla_\parallel + \nabla_\parallel^2 , 
+    \Delta_\parallel := -\nabla_\parallel^\dagger \nabla_\parallel = (\nabla\cdot \vec{ \hat v}) \nabla_\parallel + \nabla_\parallel^2 , 
     \label{}
 \end{align}
-which is indeed the parallel part of the full Laplacian $\Delta = \nabla\cdot( \bhat \nabla_\parallel + \nabla_\perp)$.
+which is indeed the parallel part of the full Laplacian $\Delta = \nabla\cdot( \vec{ \hat v} \nabla_\parallel + \nabla_\perp)$.
+$\vec{ \hat v} $ is the unit vector $\vec v/ |\vec v|$.
 The second order derivative $\nabla_\parallel^2$ can be 
 discretized using 
 \begin{align}
@@ -164,14 +159,15 @@ and repeating the procedure of this section.
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \subsection{Change of coordinates}
 In principle the above considerations hold in any
-coordinate system $\eta,\zeta,\varphi$.
+coordinate system $\eta,\zeta,\varphi$, since the directional derivative is 
+an intrinsic operation.
 The only question is how to integrate the field lines in the 
 $\eta, \zeta,\varphi$ system 
-since we assumed that our vector field $\bhat$ was given 
+since we assumed that our vector field $\vec v(\vec x)$ was given 
 analytically in 
 cylindrical coordinates. There are two possibilities. 
 First, interpolate $R(\zeta_i, \eta_i), Z(\zeta_i, \eta_i)$ for 
-all $i$, then integrate $\bhat$ in $(R,Z)$ space and finally use
+all $i$, then integrate $\vec v$ in $(R,Z)$ space and finally use
 Newton iteration to find $\zeta(R^\pm_i, Z^\pm_i), \eta(R^\pm_i, Z^\pm_i)$. 
 The downside here is that it is difficult to tell when and where the fieldline leaves the simulation domain.
 
@@ -181,28 +177,28 @@ transformed coordinate system $\zeta, \eta, \varphi$.
 The magnetic field can be easily transformed since we have the
 Jacobian of the coordinate transformation
 \begin{align}
-    b^\zeta(\zeta, \eta) &= \left(\frac{\partial \zeta}{\partial R} b^{R} + \frac{\partial \zeta}{\partial Z}b^Z\right)_{R(\zeta, \eta), Z(\zeta, \eta)} \\
-    b^\eta(\zeta, \eta) &= \left(\frac{\partial \eta}{\partial R} b^{R} + \frac{\partial \eta}{\partial Z}b^Z\right)_{R(\zeta, \eta), Z(\zeta, \eta)} \\
-    b^\varphi(\zeta, \eta) &= b^\varphi({R(\zeta, \eta), Z(\zeta, \eta)})
+    v^\zeta(\zeta, \eta) &= \left(\frac{\partial \zeta}{\partial R} v^{R} + \frac{\partial \zeta}{\partial Z}v^Z\right)_{R(\zeta, \eta), Z(\zeta, \eta)} \\
+    v^\eta(\zeta, \eta) &= \left(\frac{\partial \eta}{\partial R} v^{R} + \frac{\partial \eta}{\partial Z}v^Z\right)_{R(\zeta, \eta), Z(\zeta, \eta)} \\
+    v^\varphi(\zeta, \eta) &= v^\varphi({R(\zeta, \eta), Z(\zeta, \eta)})
     \label{eq:field_trafo}
 \end{align}
-The fieldline equations are still
+The fieldline equations~\eqref{eq:fieldline} are still
 \begin{subequations}
 \begin{align}
-\frac{\d \zeta}{\d\varphi} &= \frac{b^\zeta}{b^\varphi}\\
-\frac{\d \eta}{\d\varphi} &= \frac{b^\eta}{b^\varphi}\\
-\frac{\d s}{\d\varphi} &= \frac{1}{|b^\varphi|}
+\frac{\d \zeta}{\d\varphi} &= \frac{v^\zeta}{v^\varphi}\\
+\frac{\d \eta}{\d\varphi} &= \frac{v^\eta}{v^\varphi}\\
+\frac{\d s}{\d\varphi} &= \frac{1}{|v^\varphi|}
 \end{align}
 \label{eq:fieldlines}
 \end{subequations}
-The downside here is that when integrating fieldlines we
-have to interpolate the magnetic field at arbitrary points. 
-However, the error should vanish with order $P$ in the 
+The issue here is that when integrating fieldlines we
+have to interpolate the vector field $\vec v$ at arbitrary points. 
+However, the interpolation error vanishes with order $P$ in the 
 perpendicular plane. In order to mitigate this error
-we transform the B-Field on a finer grid/higher order polynomials for more accurate
+we transform $\vec v$ on a finer grid/higher order polynomials for more accurate
 integration.
-Apart from the problem of how to get the transformed vector field 
-the remaining algorithm for $\bhat\cdot\nabla$ is entirely unchanged.
+Apart from the issue of how to get the transformed vector field 
+the remaining algorithm for $\vec v\cdot\nabla$ is entirely unchanged.
 
 \subsection{Boundary conditions}
 The question is what to do when a fieldline intersects with the boundary
@@ -241,8 +237,8 @@ that leave the computational domain. While this works in practice
 it is unclear what numerical and physical side-effects this procedure might have. 
 
 Another solution would be to change the 
-vector field $\bhat$ and only retain the toroidal part of $\bhat$ on the 
-boundary ( $\hat b^R|_{\partial\Omega} = \hat b^Z|_{\partial\Omega} =0$). The fieldlines then have a kink on the boundary $\partial\Omega$. 
+vector field $\vec v$ and only retain the toroidal part of $\vec v$ on the 
+boundary ( $v^R|_{\partial\Omega} = v^Z|_{\partial\Omega} =0$). The fieldlines then have a kink on the boundary $\partial\Omega$. 
 On the other hand we can implement boundary conditions consistent with 
 the perpendicular ones since the fieldlines never leave the domain. 
 We simply interpolate the quantity to derive on the inner side of the
@@ -259,13 +255,13 @@ in that case.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \section{The adjoint methods}
-The idea is to discretize the operation $\nabla\cdot( \bhat .)$ by
+The idea is to discretize the operation $\nabla\cdot( \vec v .)$ by
 taking the adjoint of the discretization for $\nabla_\parallel$. 
 In order to understand what the adjoint operators do let us denote $\Tp$ as the push-forwward operator. Then we have
 \begin{align}
-    \int f(\vec x) \Tp h(\vec x) \sqrt{g(\vec x)}\d^3x \\
-    =  \int f(\vec x) h(\Tm \vec x)\sqrt{g(\vec x)}\d^3x \\
-    =  \int f(\Tp \vec x') h(\vec x')\sqrt{g(\Tp \vec x')}J^{-1}( \Tp\vec x') \d^3x' \\
+    \int f(\vec x) \Tp h(\vec x) \sqrt{g(\vec x)}\d^3x% \\
+    %=  \int f(\vec x) h(\Tm \vec x)\sqrt{g(\vec x)}\d^3x \\
+    %=  \int f(\Tp \vec x') h(\vec x')\sqrt{g(\Tp \vec x')}J^{-1}( \Tp\vec x') \d^3x' \\
     =  \int \frac{1}{\sqrt{g(\vec x')}}\Tm\left[J^{-1}(\vec x')\sqrt{g(\vec x')}f(\vec x')\right] h(\vec x')\sqrt{g(\vec x')}   \d^3x' \\
     \equiv  \int (\Tp)^\dagger\left[f(\vec x)\right] h(\vec x)\sqrt{g(\vec x)}   \d^3x
     \label{}
@@ -285,7 +281,7 @@ With this we can write
     (\Tp)^\dagger f(\vec x ) := \sqrt{\frac{g'(\vec x)}{g(\vec x)}} \Tm\left[f(\vec x) \right]
     \label{}
 \end{align}
-Also note that while $\Tp [fh] = \Tp f \Tp h$ this might not 
+Also note that $\Tp [fh] = \Tp f \Tp h$ might not 
 hold on the discrete level. Also the question is how $J$ enters 
 on the discrete level. We have to multiply $\sqrt{g}$ artificially when we form the adjoint. 
 Theoretically $J$ could be hidden somehow when we integrate the fieldlines, so the information could be contained in the discrete version? (Maybe in the back-projection?)
@@ -332,7 +328,7 @@ projection matrices:
 \end{align}
 In this way the projection integrals
 \begin{align}
-    \int\dV \nabla_\parallel f p_i(x)p_j(y) 
+    \int\dV (\nabla_\parallel f) p_i(x)p_j(y) 
     \label{}
 \end{align}
 are computed more precisely.
diff --git a/inc/geometries/geometries_doc.h b/inc/geometries/geometries_doc.h
index d903fab8c..40f04f395 100644
--- a/inc/geometries/geometries_doc.h
+++ b/inc/geometries/geometries_doc.h
@@ -30,7 +30,6 @@
  *   and profiles are added to the dg::geo namespace
  * - there are some miscellaneous additions like a flux surface average class
  * and one used to integrate the field lines for parallel derivatives all in the dg::geo namespace.
- * @subsection pdf PDF writeups
  * 
  * We have a collection of writeups: 
  *  - The <a href="./parallel.pdf" target="_blank">parallel derivative</a>
-- 
GitLab


From 7106aa4b9cc0644e40e13a30168d7ad5fe8660c4 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Mon, 16 Oct 2017 21:38:55 +0200
Subject: [PATCH 360/453] adaptions in ds.h

---
 inc/geometries/curvilinear.h      |   4 +-
 inc/geometries/ds.h               | 161 ++++++++++++++----------------
 inc/geometries/ds_b.cu            |  15 ++-
 inc/geometries/fieldaligned.h     |  31 ++++--
 inc/geometries/mpi_curvilinear.h  |   4 +-
 inc/geometries/mpi_fieldaligned.h |   5 +-
 6 files changed, 110 insertions(+), 110 deletions(-)

diff --git a/inc/geometries/curvilinear.h b/inc/geometries/curvilinear.h
index 04b78ab04..e25ce5f47 100644
--- a/inc/geometries/curvilinear.h
+++ b/inc/geometries/curvilinear.h
@@ -72,8 +72,8 @@ void square( const dg::SparseTensor<thrust::host_vector<double> >& jac, const th
 
 
 /**
- * @brief A three-dimensional grid based on curvilinear coordinates
- * 
+ * @brief A 2x1 curvilinear product space grid
+
  * The base coordinate system is the cylindrical coordinate system R,Z,phi
  */
 struct CurvilinearProductGrid3d : public dg::aProductGeometry3d
diff --git a/inc/geometries/ds.h b/inc/geometries/ds.h
index bd941e902..5bb474056 100644
--- a/inc/geometries/ds.h
+++ b/inc/geometries/ds.h
@@ -17,129 +17,115 @@
 
 //TODO: use buffers to make symv const
 namespace dg{
+namespace geo{
 
 /**
 * @brief Class for the evaluation of a parallel derivative
 *
-* This class discretizes the operators \f$ \nabla_\parallel = 
-\mathbf{b}\cdot \nabla = b_R\partial_R + b_Z\partial_Z + b_\phi\partial_\phi \f$, \f$\nabla_\parallel^\dagger\f$ and \f$\Delta_\parallel=\nabla_\parallel^\dagger\cdot\nabla_\parallel\f$ in
-arbitrary coordinates
+* This class discretizes the operators 
+\f$ \nabla_\parallel = \mathbf{b}\cdot \nabla = b_R\partial_R + b_Z\partial_Z + b_\phi\partial_\phi \f$, 
+\f$\nabla_\parallel^\dagger\f$ and 
+\f$\Delta_\parallel=\nabla_\parallel^\dagger\cdot\nabla_\parallel\f$
+in arbitrary coordinates
 * @ingroup fieldaligned
-* @tparam IMatrix The type of the interpolation matrix
-* @tparam Matrix The matrix class of the jump matrix
-* @tparam container The container-class on which the interpolation matrix operates on (does not need to be dg::HVec)
+* @tparam ProductGeometry must be either aProductGeometry3d or aProductMPIGeometry3d or any derivative 
+* @tparam IMatrix The type of the interpolation matrix 
+    -dg::IHMatrix, or dg::IDMatrix, dg::MIHMatrix, or dg::MIDMatrix
+* @tparam Matrix The matrix class of the jump matrix   
+    -dg::HMatrix, or dg::DMatrix, dg::MHMatrix, or dg::MDMatrix
+* @tparam container The container-class on which the interpolation matrix operates on
+    -dg::HVec, or dg::DVec, dg::MHVec, or dg::MDVec
 */
-template< class Geometry, class IMatrix, class Matrix, class container >
+template< class ProductGeometry, class IMatrix, class Matrix, class container >
 struct DS
 {
-    /**
-    * @brief Construct from a field and a grid
-    *
-    * @param mag Take the magnetic field as vector field
-    * @param g  the boundary conditions are also taken from here
-    * @param no norm or not_normed affects the behaviour of the symv function
-    * @param dir the direction affects both the operator() and the symv function
-    * @param dependsOnX performance indicator for the fine 2 coarse operations (elements of vec depend on first coordinate yes or no) also determines if a jump matrix is added in the x-direction
-    * @param dependsOnY performance indicator for the fine 2 coarse operations (elements of vec depend on second coordinate yes or no)
-    * @param mx Multiplication factor in x
-    * @param my Multiplication factor in y
-    */
-    DS(const dg::geo::TokamakMagneticField& mag, const Geometry& g, dg::norm no=dg::normed, dg::direction dir = dg::centered, bool dependsOnX = true, bool dependsOnY=true, unsigned mx=1, unsigned my=1);
+    DS(){}
+    DS(const dg::geo::TokamakMagneticField& mag, const ProductGeometry& g, unsigned mx=1, unsigned my=1, double eps = 1e-5, dg::norm no=dg::normed, dg::direction dir = dg::centered, bool dependsOnX = true, bool dependsOnY=true)
+    {
+        dg::geo::BinaryVectorLvl0 vec( dg::geo::BHatR(mag), dg::geo::BHatZ(mag), dg::geo::BHatP(mag));
+        m_fa.construct( vec, grid, mx, my, eps, FullLimiter(), grid.bcx(), grid.bcy());
+        construct( m_fa, g, no, dir, dependsOnX, dependsOnY);
+    }
+    DS(const dg::geo::BinaryVectorLvl0& vec, const ProductGeometry& g, unsigned mx=1, unsigned my=1, double eps = 1e-5, dg::norm no=dg::normed, dg::direction dir = dg::centered, bool dependsOnX = true, bool dependsOnY=true)
+    {
+        m_fa.construct( vec, grid, mx, my, eps, FullLimiter(), grid.bcx(), grid.bcy());
+        construct( m_fa, g, no, dir, dependsOnX, dependsOnY);
+    }
+    DS(const dg::geo::Fieldaligned<ProductGeometry, I, M, container>& fa, const ProductGeometry& g, dg::norm no=dg::normed, dg::direction dir = dg::centered, bool dependsOnX = true, bool dependsOnY=true)
+    {
+        construct( fa, g, no, dir, dependsOnX, dependsOnY);
+    }
+    void construct(const dg::geo::Fieldaligned<ProductGeometry, I, M, container>& fa, const ProductGeometry& g, dg::norm no=dg::normed, dg::direction dir = dg::centered, bool dependsOnX = true, bool dependsOnY=true);
 
     /**
     * @brief Apply the forward derivative on a 3d vector
     *
     * forward derivative \f$ g_i = \alpha \frac{1}{h_z^+}(f_{i+1} - f_{i}) + \beta g_i\f$
+    * @param alpha Scalar
     * @param f The vector to derive
-    * @param dsf contains result on output (write only)
+    * @param beta Scalar
+    * @param g contains result on output (write only)
     */
-    void forward( double alpha, const container& f, double beta, container& dsf){
-        do_forward( alpha, f, beta, dsf);
+    void forward( double alpha, const container& f, double beta, container& g){
+        do_forward( alpha, f, beta, g);
     }
     /**
     * @brief Apply the backward derivative on a 3d vector
     *
-    * backward derivative \f$ \frac{1}{2h_z^-}(f_{i} - f_{i-1})\f$
+    * backward derivative \f$ g_i = \alpha \frac{1}{2h_z^-}(f_{i} - f_{i-1}) + \beta g_i \f$
+    * @param alpha Scalar
     * @param f The vector to derive
-    * @param dsf contains result on output (write only)
+    * @param beta Scalar
+    * @param g contains result on output (write only)
     */
-    void backward( double alpha, const container& f, double beta, container& dsf){
-        do_backward( alpha, f, beta, dsf);
+    void backward( double alpha, const container& f, double beta, container& g){
+        do_backward( alpha, f, beta, g);
     }
     /**
     * @brief Apply the centered derivative on a 3d vector
     *
-    * centered derivative \f$ \frac{1}{2h_z}(f_{i+1} - f_{i-1})\f$
+    * centered derivative \f$ g_i = \alpha \frac{1}{2h_z}(f_{i+1} - f_{i-1}) + \beta g_i\f$
+    * @param alpha Scalar
     * @param f The vector to derive
-    * @param dsf contains result on output (write only)
+    * @param beta Scalar
+    * @param g contains result on output (write only)
     */
-    void centered( double alpha, const container& f, double beta, container& dsf){
-        do_centered( alpha, f, beta, dsf);
+    void centered( double alpha, const container& f, double beta, container& g){
+        do_centered( alpha, f, beta, g);
     }
 
     /**
     * @brief Apply the negative adjoint derivative on a 3d vector
     *
+    * @param alpha Scalar
     * @param f The vector to derive
-    * @param dsf contains result on output (write only)
+    * @param beta Scalar
+    * @param g contains result on output (write only)
     */
-    void forwardAdj( double alpha, const container& f, double beta, container& dsf){
-        do_forwardAdj( alpha, f, beta, dsf, dg::normed);
+    void forwardAdj( double alpha, const container& f, double beta, container& g){
+        do_forwardAdj( alpha, f, beta, g, dg::normed);
     }
     /**
     * @brief Apply the negative adjoint derivative on a 3d vector
     *
+    * @param alpha Scalar
     * @param f The vector to derive
-    * @param dsf contains result on output (write only)
+    * @param beta Scalar
+    * @param g contains result on output (write only)
     */
-    void backwardAdj( double alpha, const container& f, double beta, container& dsf){
-        do_backwardAdj( alpha, f, beta, dsf, dg::normed);
+    void backwardAdj( double alpha, const container& f, double beta, container& g){
+        do_backwardAdj( alpha, f, beta, g, dg::normed);
     }
     /**
     * @brief Apply the negative adjoint derivative on a 3d vector
     *
+    * @param alpha Scalar
     * @param f The vector to derive
-    * @param dsf contains result on output (write only)
-    */
-    void centeredAdj(double alpha, const container& f, double beta, container& dsf){
-        do_centeredAdj( alpha, f, beta, dsf, dg::normed);
-    }
-    /**
-    * @brief Apply the negative adjoint derivative on a 3d vector with the direct method
-    *
-    * forward derivative \f$ \frac{1}{h_z^+}(f_{i+1} - f_{i})\f$
-    * @param f The vector to derive
-    * @param dsf contains result on output (write only)
+    * @param beta Scalar
+    * @param g contains result on output (write only)
     */
-    void forwardAdjDir( double alpha, const container& f, double beta, container& dsf)
-    {
-        dg::blas1::pointwiseDivide( f, m_B, m_temp);
-        forward( m_temp, m_temp);
-        dg::blas1::pointwiseDot( alpha, m_temp, m_B, beta, dsf);
-    }
-    /**
-    * @brief Apply the negative adjoint derivative on a 3d vector with the direct method
-    *
-    * @param f The vector to derive
-    * @param dsf contains result on output (write only)
-    */
-    void backwardAdjDir( double alpha, const container& f, double beta, container& dsf)
-    {
-        dg::blas1::pointwiseDivide( f, m_B, m_temp);
-        backward( 1., m_temp, 0., m_temp);
-        dg::blas1::pointwiseDot( alpha, m_temp, m_B, beta, dsf);
-    }
-    /**
-    * @brief Apply the negative adjoint derivative on a 3d vector with the direct method
-    *
-    * @param f The vector to derive
-    * @param dsf contains result on output (write only)
-    */
-    void centeredAdjDir( double alpha, const container& f, double beta, container& dsf)
-    {
-        dg::blas1::pointwiseDivide( f, m_B, m_temp);
-        backward( 1., m_temp, m_temp);
-        dg::blas1::pointwiseDot( alpha, m_temp, m_B, beta, dsf);
+    void centeredAdj(double alpha, const container& f, double beta, container& g){
+        do_centeredAdj( alpha, f, beta, g, dg::normed);
     }
 
     /**
@@ -147,9 +133,9 @@ struct DS
     *
     * dependent on dir redirects to either forward(), backward() or centered()
     * @param f The vector to derive
-    * @param dsf contains result on output (write only)
+    * @param g contains result on output (write only)
     */
-    void operator()( const container& f, container& dsf);
+    void operator()( const container& f, container& g);
 
 
     /**
@@ -212,7 +198,7 @@ struct DS
     *
     * @return acces to fieldaligned object
     */
-    const FieldAligned<Geometry, IMatrix, container>& fieldaligned() const{return f_;}
+    const dg::geo::FieldAligned<ProductGeometry, IMatrix, container>& fieldaligned() const{return f_;}
     private:
     void do_forward(double alpha, const container& f, double beta, container& dsf);
     void do_backward(double alpha, const container& f, double beta, container& dsf);
@@ -222,13 +208,11 @@ struct DS
     void do_centeredAdj(double alpha, const container& f, double beta, container& dsf, dg::norm no);
     void do_symv(const container& f, container& dsf);
 
-    FieldAligned<Geometry,IMatrix,container> m_fa;
+    FieldAligned<ProductGeometry,IMatrix,container> m_fa;
     Matrix m_jumpX, m_jumpY;
     container m_temp;
     container m_tempP, m_temp0, m_tempM;
     container m_vol3d, m_inv3d, m_weights_wo_vol;
-    container m_B;
-    //container R_;
     dg::norm m_no;
     dg::direction m_dir;
     bool m_apply_jumpX, m_apply_jumpY;
@@ -238,17 +222,17 @@ struct DS
 ////////////////////////////////////DEFINITIONS////////////////////////////////////////
 
 template<class Geometry, class I, class M, class container>
-DS<Geometry, I, M,container>::DS(const dg::geo::TokamakMagneticField& mag, const Geometry& grid, dg::norm no, dg::direction dir, bool jumpX, bool jumpY, unsigned mx, unsigned my):
-        m_fa( dg::geo::BinaryVectorLvl0( dg::geo::BHatR(mag), dg::geo::BHatZ(mag), dg::geo::BHatP(mag)), grid, mx, my, 1e-5, FullLimiter(), grid.bcx(), grid.bcy()),
-        m_no(no), m_dir(dir), m_apply_jumpX(jumpX), m_apply_jumpY(jumpY)
+void DS<Geometry, I, M,container>::construct(const Fieldaligned<Geometry, I, M, container>& fa, const Geometry& grid, dg::norm no, dg::direction dir, bool jumpX, bool jumpY)
 {
-    dg::blas1::transfer( dg::pullback( dg::geo::Bmodule(mag), grid), m_B);
-    m_temp = m_B, m_tempP = m_B, m_temp0 = m_b, m_tempM = m_b;
+    m_fa=fa;
+    m_no=no, m_dir=dir, m_apply_jumpX=jumpX, m_apply_jumpY=jumpY;
+
     dg::blas1::transfer( dg::create::volume(     grid), m_vol3d); 
     dg::blas1::transfer( dg::create::weights(    grid), m_weights_wo_vol); 
     dg::blas1::transfer( dg::create::inv_volume( grid), m_inv3d); 
     dg::blas2::transfer( dg::create::jumpX( grid), jumpX);
     dg::blas2::transfer( dg::create::jumpY( grid), jumpY);
+    m_temp = m_vol3d, m_tempP = m_temp, m_temp0 = m_temp, m_tempM = m_temp;
 }
 
 template<class G, class I, class M, class container>
@@ -358,5 +342,6 @@ struct MatrixTraits< DS<G,I,M, V> >
 
 ///@endcond
 
+}//namespace geo
 }//namespace dg
 
diff --git a/inc/geometries/ds_b.cu b/inc/geometries/ds_b.cu
index 74b392dea..c3ff50744 100644
--- a/inc/geometries/ds_b.cu
+++ b/inc/geometries/ds_b.cu
@@ -2,12 +2,12 @@
 
 #include <cusp/print.h>
 
-#include "blas.h"
+#include "dg/blas.h"
+#include "dg/functors.h"
+#include "dg/geometry/geometry.h"
 #include "ds.h"
-#include "functors.h"
 #include "solovev.h"
 #include "flux.h"
-#include "geometry.h"
 
 #include "backend/functions.h"
 #include "backend/timer.cuh"
@@ -50,10 +50,9 @@ int main()
     const dg::DVec vol3d = dg::create::volume( g3d);
     dg::Timer t;
     t.tic();
-    dg::geo::BinaryVectorLvl0 bhat( bhatR, bhatZ, bhatP);
-    dg::FieldAligned dsFA( bhat, g3d, 1e-10, dg::DefaultLimiter(), dg::NEU);
+    dg::geo::FieldAligned<dg::aProductGeometry,dg::IDMatrix, dg::DMatrix>  dsFA( bhat, g3d, 2,2,1e-10, dg::NoLimiter(), dg::NEU, dg::NEU);
 
-    dg::DS<dg::aGeometry3d, dg::IDMatrix, dg::DMatrix, dg::DVec> ds ( dsFA, g3d, field, dg::not_normed, dg::centered);
+    dg::geo::DS<dg::aProductGeometry3d, dg::IDMatrix, dg::DMatrix, dg::DVec> ds( dsFA, g3d, dg::not_normed, dg::centered);
     t.toc();
     std::cout << "Creation of parallel Derivative took     "<<t.diff()<<"s\n";
 
@@ -94,8 +93,8 @@ int main()
 
     dg::geo::FluxGenerator flux( mag.get_psip(), mag.get_ipol(), psi_0, psi_1, gp.R_0, 0., 1);
     dg::geo::CurvilinearProductGrid3d g3d(flux, n, Nx, Ny,Nz, dg::DIR);
-    //dg::FieldAligned<dg::aGeometry3d, dg::IHMatrix, dg::HVec> fieldaligned( bhat, g3d, 1, 4, gp.rk4eps, dg::NoLimiter() ); 
-    dg::DS<dg::aGeometry3d, dg::IHMatrix, dg::HMatrix, dg::HVec> ds( mag, g3d, dg::normed, dg::centered, false, true, mx, my);
+    //dg::geo::FieldAligned<dg::aGeometry3d, dg::IHMatrix, dg::HVec> fieldaligned( bhat, g3d, 1, 4, gp.rk4eps, dg::NoLimiter() ); 
+    dg::geo::DS<dg::aProductGeometry3d, dg::IHMatrix, dg::HMatrix, dg::HVec> ds( mag, g3d, mx, my, 1e-8, dg::normed, dg::centered, false, true);
 
     
     t.toc();
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index 6b79c0346..26bea0b9b 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -20,6 +20,7 @@
 #include "curvilinear.h"
 
 namespace dg{
+namespace geo{
 
 ///@brief Enum for the use in Fieldaligned
 ///@ingroup fieldaligned
@@ -353,11 +354,13 @@ aGeometry2d* clone_3d_to_perp( const aGeometry3d* grid_ptr)
 \mathbf{b}\cdot \nabla = b_R\partial_R + b_Z\partial_Z + b_\phi\partial_\phi \f$, \f$\nabla_\parallel^\dagger\f$ and \f$\Delta_\parallel=\nabla_\parallel^\dagger\cdot\nabla_\parallel\f$ in
 cylindrical coordinates
 * @ingroup fieldaligned
-* @tparam Geometry The Geometry class 
-* @tparam IMatrix The matrix class of the interpolation matrix
-* @tparam container The container-class on which the interpolation matrix operates on (does not need to be dg::HVec)
+* @tparam ProductGeometry must be either aProductGeometry3d or aProductMPIGeometry3d or any derivative 
+* @tparam IMatrix The type of the interpolation matrix 
+    -dg::IHMatrix, or dg::IDMatrix, dg::MIHMatrix, or dg::MIDMatrix
+* @tparam container The container-class on which the interpolation matrix operates on
+    -dg::HVec, or dg::DVec, dg::MHVec, or dg::MDVec
 */
-template<class Geometry, class IMatrix, class container >
+template<class ProductGeometry, class IMatrix, class container >
 struct FieldAligned
 {
 
@@ -365,6 +368,12 @@ struct FieldAligned
     ///@brief do not allocate memory
     FieldAligned(){}
 
+    ///@copydoc construct()
+    template <class Limiter>
+    FieldAligned(const dg::geo::BinaryVectorLvl0& vec, const ProductGeometry& grid, unsigned multiplyX, unsigned multiplyY, double eps = 1e-5, Limiter limit = FullLimiter(), dg::bc globalbcx = dg::DIR, dg::bc globalbcy = dg::DIR, double deltaPhi = -1);
+    {
+        construct( vec, grid, multiplyX, multiplyY, eps, limit, globalbcx, globalbcy, deltaPhi);
+    }
     /**
     * @brief Construct from a field and a grid
     *
@@ -386,7 +395,7 @@ struct FieldAligned
         If there is no limiter the boundary condition is periodic.
     */
     template <class Limiter>
-    FieldAligned(const dg::geo::BinaryVectorLvl0& vec, const Geometry& grid, unsigned multiplyX, unsigned multiplyY, double eps = 1e-4, Limiter limit = FullLimiter(), dg::bc globalbcx = dg::DIR, dg::bc globalbcy = dg::DIR, double deltaPhi = -1);
+    void construct(const dg::geo::BinaryVectorLvl0& vec, const ProductGeometry& grid, unsigned multiplyX, unsigned multiplyY, double eps = 1e-5, Limiter limit = FullLimiter(), dg::bc globalbcx = dg::DIR, dg::bc globalbcy = dg::DIR, double deltaPhi = -1);
 
     /**
     * @brief Set boundary conditions in the limiter region
@@ -495,7 +504,7 @@ struct FieldAligned
     container m_left, m_right;
     container m_limiter;
     std::vector<container> m_temp;
-    dg::Handle<Geometry> m_g;
+    dg::Handle<ProductGeometry> m_g;
 };
 
 ///@cond 
@@ -505,15 +514,16 @@ struct FieldAligned
 
 template<class Geometry, class IMatrix, class container>
 template <class Limiter>
-FieldAligned<Geometry, IMatrix, container>::FieldAligned(const dg::geo::BinaryVectorLvl0& vec, const Geometry& grid, unsigned mx, unsigned my, double eps, Limiter limit, dg::bc globalbcx, dg::bc globalbcy, double deltaPhi):
-        m_hz_inv( dg::evaluate( dg::zero, grid)), m_hp_inv( m_hz), m_hm_inv( m_hz), 
-        m_Nz(grid.Nz()), m_bcz(grid.bcz()), m_g(grid)
+void FieldAligned<Geometry, IMatrix, container>::construct(const dg::geo::BinaryVectorLvl0& vec, const Geometry& grid, unsigned mx, unsigned my, double eps, Limiter limit, dg::bc globalbcx, dg::bc globalbcy, double deltaPhi):
 {
+    dg::blas1::transfer( dg::evaluate( dg::zero, grid), m_hz_inv), m_hp_inv= m_hz_inv, m_hm_inv= m_hz_inv;
+    m_Nz=grid.Nz(), m_bcz=grid.bcz(); 
+    m_g=grid;
     dg::split( m_hz_inv, m_temp);
     if( deltaPhi <=0) deltaPhi = grid.hz();
     else assert( grid.Nz() == 1 || grid.hz()==deltaPhi);
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-    const aGeometry2d* grid2d_ptr = detail::clone_3d_to_perp(&grid);
+    const aGeometry2d* grid2d_ptr = grid->perp_grid();
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     //Resize vector to 2D grid size
     m_perp_size = grid2d_ptr->size();
@@ -709,5 +719,6 @@ void FieldAligned<G, I, container>::eMinus( enum whichMatrix which, const contai
 ///@endcond 
 
 
+}//namespace geo
 }//namespace dg
 
diff --git a/inc/geometries/mpi_curvilinear.h b/inc/geometries/mpi_curvilinear.h
index f73cebf62..921602dcc 100644
--- a/inc/geometries/mpi_curvilinear.h
+++ b/inc/geometries/mpi_curvilinear.h
@@ -89,7 +89,9 @@ struct CurvilinearMPIGrid2d : public dg::aMPIGeometry2d
 };
 
 /**
- * This is s 2x1 curvilinear product space MPI grid
+ * @brief A 2x1 curvilinear product space MPI grid
+
+ * The base coordinate system is the cylindrical coordinate system R,Z,phi
  */
 struct CurvilinearProductMPIGrid3d : public dg::aProductMPIGeometry3d
 {
diff --git a/inc/geometries/mpi_fieldaligned.h b/inc/geometries/mpi_fieldaligned.h
index 9caa62d3d..d414657f5 100644
--- a/inc/geometries/mpi_fieldaligned.h
+++ b/inc/geometries/mpi_fieldaligned.h
@@ -13,6 +13,7 @@
 #include "dg/runge_kutta.h"
 
 namespace dg{
+namespace geo{
  
 ///@cond
 namespace detail{
@@ -144,7 +145,7 @@ FieldAligned<MPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vector<L
     if( deltaPhi <=0) deltaPhi = grid.hz();
     else assert( grid.Nz() == 1 || grid.hz()==deltaPhi);
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-    const aMPIGeometry2d* grid2d_ptr = detail::clone_MPI3d_to_global_perp(&grid);
+    const aMPIGeometry2d* grid2d_ptr = grid->perp_grid();
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     unsigned localsize = grid2d_ptr->size();
     MPI_Vector<dg::HVec> temp = dg::pullback( limit, *grid2d_ptr);
@@ -389,5 +390,7 @@ void FieldAligned<G,RowDistMat<M,C>, MPI_Vector<container> >::eMinus( enum which
 
 
 ///@endcond
+
+}//namespace geo
 }//namespace dg
 
-- 
GitLab


From 31e724fcf5ba6c81c257323b563569c25d5c6ce2 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Tue, 17 Oct 2017 09:53:42 +0200
Subject: [PATCH 361/453] added missing references.bib

---
 doc/related_pages/references.bib | 245 +++++++++++++++++++++++++++++++
 1 file changed, 245 insertions(+)
 create mode 100644 doc/related_pages/references.bib

diff --git a/doc/related_pages/references.bib b/doc/related_pages/references.bib
new file mode 100644
index 000000000..7563ee299
--- /dev/null
+++ b/doc/related_pages/references.bib
@@ -0,0 +1,245 @@
+% This file was created with JabRef 2.10.
+% Encoding: UTF-8
+
+
+@Book{AS,
+  Title                    = {{Handbook of mathematical functions}},
+  Author                   = {Abramowitz, M. and Stegun, I.},
+  Publisher                = {Dover Publishing Inc. New York},
+  Year                     = {1972},
+  Edition                  = {10th},
+
+  Owner                    = {matthias},
+  Timestamp                = {2014.03.11},
+  Url                      = {http://people.math.sfu.ca/~cbm/aands/}
+}
+
+@Article{Cockburn2001runge,
+  Title                    = {{Runge--Kutta discontinuous Galerkin methods for convection-dominated problems}},
+  Author                   = {Cockburn, B. and Shu, C. W.},
+  Journal                  = {J. Sci. Comput.},
+  Year                     = {2001},
+  Number                   = {3},
+  Pages                    = {173--261},
+  Volume                   = {16},
+
+  Doi                      = {10.1023/a:1012873910884},
+  Owner                    = {matthias},
+  Timestamp                = {2013.12.04}
+}
+
+@Article{Cockburn1998,
+  Title                    = {The local discontinuous Galerkin method for time-dependent convection-diffusion systems},
+  Author                   = {Cockburn, B. and Shu, C. W.},
+  Journal                  = {Siam Journal On Numerical Analysis},
+  Year                     = {1998},
+
+  Month                    = nov,
+  Number                   = {6},
+  Pages                    = {2440--2463},
+  Volume                   = {35},
+
+  __markedentry            = {[matthias:]},
+  Af                       = {Cockburn, BEOLEOLShu, CW},
+  Doi                      = {10.1137/S0036142997316712},
+  Ei                       = {1095-7170},
+  Oi                       = {Shu, Chi-Wang/0000-0001-7720-9564; Cockburn,EOLEOLBernardo/0000-0001-6085-3441},
+  Owner                    = {matthias},
+  Ri                       = {Shu, Chi-Wang/A-3216-2013; Cockburn, Bernardo/M-9617-2013},
+  Sn                       = {0036-1429},
+  Timestamp                = {2017.10.12},
+  Ut                       = {WOS:000077016000016}
+}
+
+@Book{haeseleer,
+  Title                    = {{Flux Coordinates and Magnetic Field Structure}},
+  Author                   = {D'haeseleer, W.D. and Hitchon, W.N.G. and Callen, J.D. and Shohet, J.L.},
+  Publisher                = {Springer-Verlag},
+  Year                     = {1991},
+  Series                   = {Springer Series in Computational Physics},
+
+  Owner                    = {matthias},
+  Timestamp                = {2017.10.11}
+}
+
+@Article{Einkemmer2014,
+  Title                    = {A conservative discontinuous Galerkin scheme for the 2D incompressible Navier--Stokes equations},
+  Author                   = {Einkemmer, L. and Wiesenberger, M.},
+  Journal                  = {Comput. Phys. Commun.},
+  Year                     = {2014},
+
+  Month                    = nov,
+  Number                   = {11},
+  Pages                    = {2865-2873},
+  Volume                   = {185},
+
+  Adsnote                  = {Provided by the SAO/NASA Astrophysics Data System},
+  Doi                      = {10.1016/j.cpc.2014.07.007},
+  Keywords                 = {Physics - Computational Physics; Computer Science - Numerical Analysis; Mathematics - Numerical Analysis; Physics - Fluid Dynamics; Physics - Plasma Physics},
+  Owner                    = {matthias},
+  Timestamp                = {2017.10.11}
+}
+
+@Book{Frankel,
+  Title                    = {{The geometry of physics: an introduction}},
+  Author                   = {Frankel, Theodore},
+  Publisher                = {Cambridge University Press},
+  Year                     = {2004},
+  Edition                  = {Second},
+
+  Owner                    = {Jens Madsen},
+  Timestamp                = {2008.05.14}
+}
+
+@Article{Glasser2006,
+  Title                    = {Some computational aspects on generating numerical grids},
+  Author                   = {Glasser, A. H. and Liseikin, V. D. and Vaseva, I. A. and Likhanova, Y. V.},
+  Journal                  = {Russ. J. Numer. Anal. Math. Modelling},
+  Year                     = {2006},
+  Number                   = {6},
+  Pages                    = {481--505},
+  Volume                   = {21},
+
+  Af                       = {Glasser, A. H.EOLEOLLiseikin, V. D.EOLEOLVaseva, I. A.EOLEOLLikhanova, Yu. V.},
+  C1                       = {Los Alamos Natl Lab, Los Alamos, NM 87545 USA.EOLEOLRussian Acad Sci, Siberian Branch, Inst Computat Technol, Novosibirsk 630090, Russia.},
+  Doi                      = {10.1515/rnam.2006.21.6.481},
+  Ga                       = {118IP},
+  J9                       = {RUSS J NUMER ANAL M},
+  Ji                       = {Russ. J. Numer. Anal. Math. Model},
+  Keywords                 = {EQUIDISTRIBUTION},
+  La                       = {English},
+  Nr                       = {22},
+  Owner                    = {matthias},
+  Pa                       = {BRILL ACADEMIC PUBLISHERS, PO BOX 9000, 2300 PA LEIDEN, NETHERLANDS},
+  Pg                       = {25},
+  Pi                       = {LEIDEN},
+  Publisher                = {Vsp Bv},
+  Rp                       = {Glasser, AH (reprint author), Los Alamos Natl Lab, POB 1663, Los Alamos, NM 87545 USA.},
+  Sc                       = {Engineering; Mathematics},
+  Sn                       = {0927-6467},
+  Tc                       = {1},
+  Timestamp                = {2016.10.14},
+  U1                       = {0},
+  U2                       = {0},
+  Ut                       = {WOS:000242936200001},
+  Wc                       = {Engineering, Multidisciplinary; Mathematics, Applied},
+  Z9                       = {1}
+}
+
+@Article{Hariri2014,
+  Title                    = {The flux-coordinate independent approach applied to X-point geometries},
+  Author                   = {Hariri, F. and Hill, P. and Ottaviani, M. and Sarazin, Y.},
+  Journal                  = {Physics of Plasmas},
+  Year                     = {2014},
+
+  Month                    = aug,
+  Number                   = {8},
+  Pages                    = {082509},
+  Volume                   = {21},
+
+  AF                       = {Hariri, F.EOLEOLHill, P.EOLEOLOttaviani, M.EOLEOLSarazin, Y.},
+  EI                       = {1089-7674},
+  SN                       = {1070-664X},
+  UT                       = {WOS:000342760600037},
+  __markedentry            = {[matthias:6]},
+  Doi                      = {10.1063/1.4892405},
+  Owner                    = {matthias},
+  Timestamp                = {2017.10.13}
+}
+
+@Article{Held2016,
+  Title                    = {Three discontinuous Galerkin schemes for the anisotropic heat conduction equation on non-aligned grids},
+  Author                   = {Held, M. and Wiesenberger, M. and Stegmeir, A.},
+  Journal                  = {Computer Physics Communications},
+  Year                     = {2016},
+
+  Month                    = feb,
+  Pages                    = {29--39},
+  Volume                   = {199},
+  Af                       = {Held, M.EOLEOLWiesenberger, M.EOLEOLStegmeir, A.},
+  Doi                      = {10.1016/j.cpc.2015.10.009},
+  Ei                       = {1879-2944},
+  Oi                       = {Wiesenberger, Matthias/0000-0002-5921-0163},
+  Owner                    = {matthias},
+  Sn                       = {0010-4655},
+  Timestamp                = {2017.10.11},
+  Ut                       = {WOS:000367113200005}
+}
+
+@Book{Liseikin,
+  Title                    = {A Computational Differential Geometry Approach to Grid Generation},
+  Author                   = {Liseikin, Vladimir D.},
+  Publisher                = {Springer-Verlag},
+  Year                     = {2007},
+  Edition                  = {Second},
+
+  Owner                    = {matthias},
+  Timestamp                = {2016.10.06}
+}
+
+@Article{Vaseva2009,
+  Title                    = {An elliptic method for construction of adaptive spatial grids},
+  Author                   = {Vaseva, I. A. and Liseikin, V. D. and Likhanova, Y. V. and Morokov, Y. N.},
+  Journal                  = {Russ. J. Numer. Anal. Math. Modelling},
+  Year                     = {2009},
+  Number                   = {1},
+  Pages                    = {65--78},
+  Volume                   = {24},
+
+  Af                       = {Vaseva, I. A.EOLEOLLiseikin, V. D.EOLEOLLikhanova, Yu. V.EOLEOLMorokov, Yu. N.},
+  C1                       = {[Vaseva, I. A.; Liseikin, V. D.; Likhanova, Yu. V.; Morokov, Yu. N.] Russian Acad Sci, Siberian Branch, Inst Computat Technol, Novosibirsk 630090, Russia.},
+  Doi                      = {10.1515/RJNAMM.2009.006},
+  Fu                       = {Russian Foundation [06-01-08009, 07-0100336-a]; Siberian Division ofEOLEOLRussian Academy of Sciences [1.8]},
+  Fx                       = {The work was supported by the Russian Foundation for Basic ResearchEOLEOL(06-01-08009,07-0100336-a) and the Grant of complex integration projectEOLEOLSiberian Division of Russian Academy of Sciences, 2006 (1.8).},
+  Ga                       = {431FB},
+  J9                       = {RUSS J NUMER ANAL M},
+  Ji                       = {Russ. J. Numer. Anal. Math. Model},
+  La                       = {English},
+  Nr                       = {12},
+  Owner                    = {matthias},
+  Pa                       = {GENTHINER STRASSE 13, D-10785 BERLIN, GERMANY},
+  Pg                       = {14},
+  Pi                       = {BERLIN},
+  Publisher                = {Walter De Gruyter \& Co},
+  Rp                       = {Vaseva, IA (reprint author), Russian Acad Sci, Siberian Branch, Inst Computat Technol, Novosibirsk 630090, Russia.},
+  Sc                       = {Engineering; Mathematics},
+  Sn                       = {0927-6467},
+  Tc                       = {1},
+  Timestamp                = {2016.10.14},
+  U1                       = {0},
+  U2                       = {1},
+  Ut                       = {WOS:000265045600006},
+  Wc                       = {Engineering, Multidisciplinary; Mathematics, Applied},
+  Z9                       = {1}
+}
+
+@PhdThesis{WiesenbergerPhD,
+  Title                    = {Gyrofluid computations of filament dynamics in tokamak scrape-off layers},
+  Author                   = {M.~Wiesenberger},
+  Year                     = {2014},
+
+  Owner                    = {matthias},
+  Timestamp                = {2017.10.12},
+  Url                      = {http://resolver.obvsg.at/urn:nbn:at:at-ubi:1-1799}
+}
+
+@Article{Wiesenberger2017,
+  Title                    = {Streamline integration as a method for two-dimensional elliptic grid generation},
+  Author                   = {Wiesenberger, M. and Held, M. and Einkemmer, L.},
+  Journal                  = {Journal of Computational Physics},
+  Year                     = {2017},
+
+  Month                    = jul,
+  Pages                    = {435--450},
+  Volume                   = {340},
+
+  Af                       = {Wiesenberger, M.EOLEOLHeld, M.EOLEOLEinkemmer, L.},
+  Doi                      = {10.1016/j.jcp.2017.03.056},
+  Ei                       = {1090-2716},
+  Oi                       = {Wiesenberger, Matthias/0000-0002-5921-0163},
+  Owner                    = {matthias},
+  Sn                       = {0021-9991},
+  Timestamp                = {2017.10.11},
+  Ut                       = {WOS:000401137900020}
+}
+
-- 
GitLab


From 69855ac93e830402b718fe02fb9673b899d6268f Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Tue, 17 Oct 2017 12:01:46 +0200
Subject: [PATCH 362/453] update interfaces of DS and FieldAligned

---
 inc/geometries/ds.h               | 140 ++++++++++++++----------------
 inc/geometries/fieldaligned.h     |  41 +++++----
 inc/geometries/mpi_fieldaligned.h |  33 +++----
 3 files changed, 107 insertions(+), 107 deletions(-)

diff --git a/inc/geometries/ds.h b/inc/geometries/ds.h
index 5bb474056..0aa5db9a9 100644
--- a/inc/geometries/ds.h
+++ b/inc/geometries/ds.h
@@ -20,110 +20,106 @@ namespace dg{
 namespace geo{
 
 /**
-* @brief Class for the evaluation of a parallel derivative
+* @brief Class for the evaluation of parallel derivatives
 *
 * This class discretizes the operators 
-\f$ \nabla_\parallel = \mathbf{b}\cdot \nabla = b_R\partial_R + b_Z\partial_Z + b_\phi\partial_\phi \f$, 
+\f$ \nabla_\parallel = \mathbf{v}\cdot \nabla = v^\zeta\partial_\zeta + v^\eta\partial_\eta + v^\varphi\partial_\varphi \f$, 
 \f$\nabla_\parallel^\dagger\f$ and 
 \f$\Delta_\parallel=\nabla_\parallel^\dagger\cdot\nabla_\parallel\f$
 in arbitrary coordinates
 * @ingroup fieldaligned
-* @tparam ProductGeometry must be either aProductGeometry3d or aProductMPIGeometry3d or any derivative 
+* @tparam ProductGeometry must be either dg::aProductGeometry3d or dg::aProductMPIGeometry3d or any derivative 
 * @tparam IMatrix The type of the interpolation matrix 
-    -dg::IHMatrix, or dg::IDMatrix, dg::MIHMatrix, or dg::MIDMatrix
+    - dg::IHMatrix, or dg::IDMatrix, dg::MIHMatrix, or dg::MIDMatrix
 * @tparam Matrix The matrix class of the jump matrix   
-    -dg::HMatrix, or dg::DMatrix, dg::MHMatrix, or dg::MDMatrix
+    - dg::HMatrix, or dg::DMatrix, dg::MHMatrix, or dg::MDMatrix
 * @tparam container The container-class on which the interpolation matrix operates on
-    -dg::HVec, or dg::DVec, dg::MHVec, or dg::MDVec
+    - dg::HVec, or dg::DVec, dg::MHVec, or dg::MDVec
+* @sa The pdf <a href="./parallel.pdf" target="_blank">parallel derivative</a> writeup 
 */
 template< class ProductGeometry, class IMatrix, class Matrix, class container >
 struct DS
 {
+    ///@brief No memory allocation; all member calls except construct are invalid
     DS(){}
-    DS(const dg::geo::TokamakMagneticField& mag, const ProductGeometry& g, unsigned mx=1, unsigned my=1, double eps = 1e-5, dg::norm no=dg::normed, dg::direction dir = dg::centered, bool dependsOnX = true, bool dependsOnY=true)
+    
+    /**
+     * @brief Create the magnetic unit vector field and construct
+     * @copydetails DS(const dg::geo::BinaryVectorLvl0&,const ProductGeometry&,unsigned,unsigned,bool,bool,double,dg::norm,dg::direction)
+     */
+    DS(const dg::geo::TokamakMagneticField& vec, const ProductGeometry& grid, unsigned multiplyX=1, unsigned multiplyY=1, bool dependsOnX = true, bool dependsOnY=true, double eps = 1e-5, dg::norm no=dg::normed, dg::direction dir = dg::centered)
     {
         dg::geo::BinaryVectorLvl0 vec( dg::geo::BHatR(mag), dg::geo::BHatZ(mag), dg::geo::BHatP(mag));
-        m_fa.construct( vec, grid, mx, my, eps, FullLimiter(), grid.bcx(), grid.bcy());
-        construct( m_fa, g, no, dir, dependsOnX, dependsOnY);
+        m_fa.construct( vec, grid, mx, my, dependsOnX, dependsOnY, eps, grid.bcx(), grid.bcy(), FullLimiter());
+        construct( m_fa, no, dir);
     }
-    DS(const dg::geo::BinaryVectorLvl0& vec, const ProductGeometry& g, unsigned mx=1, unsigned my=1, double eps = 1e-5, dg::norm no=dg::normed, dg::direction dir = dg::centered, bool dependsOnX = true, bool dependsOnY=true)
+    /**
+     * @brief Create a FieldAligned object and construct
+     *
+     * @param vec The vector field to integrate
+     * @param grid The grid on which to operate defines the parallel boundary condition in case there is a limiter.
+     * @param multiplyX defines the resolution in X of the fine grid relative to grid
+     * @param multiplyY defines the resolution in Y of the fine grid relative to grid
+     * @param dependsOnX indicates, whether the given vector field vec depends on the first coordinate
+     * @param dependsOnY indicates, whether the given vector field vec depends on the second coordinate
+     * @param eps Desired accuracy of the fieldline integrator
+     * @param no indicate if the symv function should be symmetric (not_normed) or not
+     * @param dir indicate the direction in the bracket operator and in symv
+     */
+    DS(const dg::geo::BinaryVectorLvl0& vec, const ProductGeometry& grid, unsigned multiplyX=1, unsigned multiplyY=1, bool dependsOnX = true, bool dependsOnY=true, double eps = 1e-5, dg::norm no=dg::normed, dg::direction dir = dg::centered)
     {
-        m_fa.construct( vec, grid, mx, my, eps, FullLimiter(), grid.bcx(), grid.bcy());
-        construct( m_fa, g, no, dir, dependsOnX, dependsOnY);
+        m_fa.construct( vec, grid, multiplyX, multiplyY, dependsOnX, dependsOnY, eps, grid.bcx(), grid.bcy(), FullLimiter());
+        construct( m_fa, no, dir);
     }
-    DS(const dg::geo::Fieldaligned<ProductGeometry, I, M, container>& fa, const ProductGeometry& g, dg::norm no=dg::normed, dg::direction dir = dg::centered, bool dependsOnX = true, bool dependsOnY=true)
+    ///@copydoc construct
+    DS(const dg::geo::Fieldaligned<ProductGeometry, IMatrix, container>& fa, dg::norm no=dg::normed, dg::direction dir = dg::centered)
     {
-        construct( fa, g, no, dir, dependsOnX, dependsOnY);
+        construct( fa, no, dir);
     }
-    void construct(const dg::geo::Fieldaligned<ProductGeometry, I, M, container>& fa, const ProductGeometry& g, dg::norm no=dg::normed, dg::direction dir = dg::centered, bool dependsOnX = true, bool dependsOnY=true);
+    /**
+     * @brief Re-construct from a given Fieldaligned object
+     *
+     * @param fa this object will be used in all further member calls
+     * @param no indicate if the symv function should be symmetric (not_normed) or not
+     * @param dir indicate the direction in the bracket operator and in symv
+     */
+    void construct(const dg::geo::Fieldaligned<ProductGeometry, IMatrix, container>& fa, dg::norm no=dg::normed, dg::direction dir = dg::centered);
 
     /**
-    * @brief Apply the forward derivative on a 3d vector
+    * @brief forward derivative \f$ g_i = \alpha \frac{1}{h_z^+}(f_{i+1} - f_{i}) + \beta g_i\f$
     *
-    * forward derivative \f$ g_i = \alpha \frac{1}{h_z^+}(f_{i+1} - f_{i}) + \beta g_i\f$
     * @param alpha Scalar
     * @param f The vector to derive
     * @param beta Scalar
     * @param g contains result on output (write only)
+    * @note the vector sizes need to equal the grid size in the constructor
     */
     void forward( double alpha, const container& f, double beta, container& g){
         do_forward( alpha, f, beta, g);
     }
-    /**
-    * @brief Apply the backward derivative on a 3d vector
-    *
-    * backward derivative \f$ g_i = \alpha \frac{1}{2h_z^-}(f_{i} - f_{i-1}) + \beta g_i \f$
-    * @param alpha Scalar
-    * @param f The vector to derive
-    * @param beta Scalar
-    * @param g contains result on output (write only)
-    */
+    ///@brief backward derivative \f$ g_i = \alpha \frac{1}{2h_z^-}(f_{i} - f_{i-1}) + \beta g_i \f$
+    ///@copydetails forward
     void backward( double alpha, const container& f, double beta, container& g){
         do_backward( alpha, f, beta, g);
     }
-    /**
-    * @brief Apply the centered derivative on a 3d vector
-    *
-    * centered derivative \f$ g_i = \alpha \frac{1}{2h_z}(f_{i+1} - f_{i-1}) + \beta g_i\f$
-    * @param alpha Scalar
-    * @param f The vector to derive
-    * @param beta Scalar
-    * @param g contains result on output (write only)
-    */
+    ///@brief centered derivative \f$ g_i = \alpha \frac{1}{2h_z}(f_{i+1} - f_{i-1}) + \beta g_i\f$
+    ///@copydetails forward
     void centered( double alpha, const container& f, double beta, container& g){
         do_centered( alpha, f, beta, g);
     }
 
-    /**
-    * @brief Apply the negative adjoint derivative on a 3d vector
-    *
-    * @param alpha Scalar
-    * @param f The vector to derive
-    * @param beta Scalar
-    * @param g contains result on output (write only)
-    */
+    ///@brief Apply the negative forward adjoint derivative on a 3d vector
+    ///@copydetails forward
     void forwardAdj( double alpha, const container& f, double beta, container& g){
         do_forwardAdj( alpha, f, beta, g, dg::normed);
     }
-    /**
-    * @brief Apply the negative adjoint derivative on a 3d vector
-    *
-    * @param alpha Scalar
-    * @param f The vector to derive
-    * @param beta Scalar
-    * @param g contains result on output (write only)
-    */
+    ///@brief Apply the negative backward adjoint derivative on a 3d vector
+    ///@copydetails forward
     void backwardAdj( double alpha, const container& f, double beta, container& g){
         do_backwardAdj( alpha, f, beta, g, dg::normed);
     }
-    /**
-    * @brief Apply the negative adjoint derivative on a 3d vector
-    *
-    * @param alpha Scalar
-    * @param f The vector to derive
-    * @param beta Scalar
-    * @param g contains result on output (write only)
-    */
+    ///@brief Apply the negative centered adjoint derivative on a 3d vector
+    ///@copydetails forward
     void centeredAdj(double alpha, const container& f, double beta, container& g){
         do_centeredAdj( alpha, f, beta, g, dg::normed);
     }
@@ -139,12 +135,12 @@ struct DS
 
 
     /**
-     * @brief Discretizes the parallel Laplacian as a symmetric matrix
+     * @brief Discretizes \f$ \nabla\cdot ( \vec v \vec v \cdot \nabla . )\f$ as a symmetric matrix
      *
      * if direction is centered then centered followed by centeredAdj and adding jump terms
      * @param f The vector to derive
      * @param dsTdsf contains result on output (write only)
-     * @note if apply_jumpX is false then no jumpy terms will be added in the x-direction
+     * @note if dependsOnX is false then no jump terms will be added in the x-direction and similar in y
      */
     void symv( const container& f, container& dsTdsf){ do_symv( f, dsTdsf);}
 
@@ -208,30 +204,29 @@ struct DS
     void do_centeredAdj(double alpha, const container& f, double beta, container& dsf, dg::norm no);
     void do_symv(const container& f, container& dsf);
 
-    FieldAligned<ProductGeometry,IMatrix,container> m_fa;
-    Matrix m_jumpX, m_jumpY;
+    FieldAligned<ProductGeometry, IMatrix, container> m_fa;
     container m_temp;
     container m_tempP, m_temp0, m_tempM;
     container m_vol3d, m_inv3d, m_weights_wo_vol;
     dg::norm m_no;
     dg::direction m_dir;
-    bool m_apply_jumpX, m_apply_jumpY;
+    Matrix m_jumpX, m_jumpY;
 };
 
 ///@cond
 ////////////////////////////////////DEFINITIONS////////////////////////////////////////
 
 template<class Geometry, class I, class M, class container>
-void DS<Geometry, I, M,container>::construct(const Fieldaligned<Geometry, I, M, container>& fa, const Geometry& grid, dg::norm no, dg::direction dir, bool jumpX, bool jumpY)
+void DS<Geometry, I, M,container>::construct(const Fieldaligned<Geometry, I, M, container>& fa, dg::norm no, dg::direction dir, bool jumpX, bool jumpY)
 {
     m_fa=fa;
     m_no=no, m_dir=dir, m_apply_jumpX=jumpX, m_apply_jumpY=jumpY;
 
-    dg::blas1::transfer( dg::create::volume(     grid), m_vol3d); 
-    dg::blas1::transfer( dg::create::weights(    grid), m_weights_wo_vol); 
-    dg::blas1::transfer( dg::create::inv_volume( grid), m_inv3d); 
-    dg::blas2::transfer( dg::create::jumpX( grid), jumpX);
-    dg::blas2::transfer( dg::create::jumpY( grid), jumpY);
+    dg::blas1::transfer( dg::create::volume(     fa.grid()), m_vol3d); 
+    dg::blas1::transfer( dg::create::weights(    fa.grid()), m_weights_wo_vol); 
+    dg::blas1::transfer( dg::create::inv_volume( fa.grid()), m_inv3d); 
+    dg::blas2::transfer( dg::create::jumpX( fa.grid()), jumpX);
+    dg::blas2::transfer( dg::create::jumpY( fa.grid()), jumpY);
     m_temp = m_vol3d, m_tempP = m_temp, m_temp0 = m_temp, m_tempM = m_temp;
 }
 
@@ -323,9 +318,9 @@ void DS<G,I,M,container>::do_symv( const container& f, container& dsTdsf)
     }
     dg::blas1::pointwiseDivide( dsTdsf, m_weights_wo_vol, dsTdsf);
     //     add jump term 
-    if(m_apply_jumpX)
+    if(m_fa.dependsOnX())
         dg::blas2::symv( -1., jumpX, f, 1., dsTdsf);
-    if(m_apply_jumpY)
+    if(m_fa.dependsOnY())
         dg::blas2::symv( -1., jumpY, f, 1., dsTdsf);
     dg::blas1::pointwiseDot( m_weights_wo_vol, dsTdsf, dsTdsf); //make it symmetric
     if( m_no == dg::normed)
@@ -344,4 +339,3 @@ struct MatrixTraits< DS<G,I,M, V> >
 
 }//namespace geo
 }//namespace dg
-
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index 26bea0b9b..5d6f9dd87 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -348,31 +348,29 @@ aGeometry2d* clone_3d_to_perp( const aGeometry3d* grid_ptr)
 
 //////////////////////////////FieldAlignedCLASS////////////////////////////////////////////
 /**
-* @brief Class for the evaluation of a parallel derivative
+* @brief Create and manage interpolation matrices from fieldline integration
 *
-* This class discretizes the operators \f$ \nabla_\parallel = 
-\mathbf{b}\cdot \nabla = b_R\partial_R + b_Z\partial_Z + b_\phi\partial_\phi \f$, \f$\nabla_\parallel^\dagger\f$ and \f$\Delta_\parallel=\nabla_\parallel^\dagger\cdot\nabla_\parallel\f$ in
-cylindrical coordinates
 * @ingroup fieldaligned
 * @tparam ProductGeometry must be either aProductGeometry3d or aProductMPIGeometry3d or any derivative 
 * @tparam IMatrix The type of the interpolation matrix 
-    -dg::IHMatrix, or dg::IDMatrix, dg::MIHMatrix, or dg::MIDMatrix
+    - dg::IHMatrix, or dg::IDMatrix, dg::MIHMatrix, or dg::MIDMatrix
 * @tparam container The container-class on which the interpolation matrix operates on
-    -dg::HVec, or dg::DVec, dg::MHVec, or dg::MDVec
+    - dg::HVec, or dg::DVec, dg::MHVec, or dg::MDVec
+* @sa The pdf <a href="./parallel.pdf" target="_blank">parallel derivative</a> writeup 
 */
 template<class ProductGeometry, class IMatrix, class container >
 struct FieldAligned
 {
 
-    typedef IMatrix InterpolationMatrix;
-    ///@brief do not allocate memory
+    typedef IMatrix InterpolationMatrix; //!< typdef to reveal the interpolation matrix
+    ///@brief do not allocate memory; no member call except construct is valid
     FieldAligned(){}
 
     ///@copydoc construct()
     template <class Limiter>
-    FieldAligned(const dg::geo::BinaryVectorLvl0& vec, const ProductGeometry& grid, unsigned multiplyX, unsigned multiplyY, double eps = 1e-5, Limiter limit = FullLimiter(), dg::bc globalbcx = dg::DIR, dg::bc globalbcy = dg::DIR, double deltaPhi = -1);
+    FieldAligned(const dg::geo::BinaryVectorLvl0& vec, const ProductGeometry& grid, unsigned multiplyX, unsigned multiplyY, bool dependsOnX, bool dependsOnY, double eps = 1e-5, dg::bc globalbcx = dg::NEU, dg::bc globalbcy = dg::NEU, Limiter limit = FullLimiter(), double deltaPhi = -1)
     {
-        construct( vec, grid, multiplyX, multiplyY, eps, limit, globalbcx, globalbcy, deltaPhi);
+        construct( vec, grid, multiplyX, multiplyY, dependsOnX, dependsOnY, eps, limit, globalbcx, globalbcy, limit, deltaPhi);
     }
     /**
     * @brief Construct from a field and a grid
@@ -384,18 +382,23 @@ struct FieldAligned
     * @param grid The grid on which to operate defines the parallel boundary condition in case there is a limiter.
     * @param multiplyX defines the resolution in X of the fine grid relative to grid
     * @param multiplyY defines the resolution in Y of the fine grid relative to grid
-    * @param eps Desired accuracy of runge kutta
-    * @param limit Instance of the limiter class (Default is a limiter everywhere, 
-        note that if grid.bcz() is periodic it doesn't matter if there is a limiter or not)
+    * @param dependsOnX indicates, whether the given vector field vec depends on the first coordinate
+    * @param dependsOnY indicates, whether the given vector field vec depends on the second coordinate
+    * @param eps Desired accuracy of the fieldline integrator
     * @param globalbcx Defines the interpolation behaviour when a fieldline intersects the boundary box in the perpendicular direction
     * @param globalbcy Defines the interpolation behaviour when a fieldline intersects the boundary box in the perpendicular direction
-    * @param deltaPhi Is either <0 (then it's ignored), may differ from hz() only if Nz() == 1
+    * @param limit Instance of the limiter class (Default is a limiter everywhere, 
+        note that if grid.bcz() is periodic it doesn't matter if there is a limiter or not)
+    * @param deltaPhi Is either <0 (then it's ignored), or may differ from grid.hz() if grid.Nz() == 1
     * @note If there is a limiter, the boundary condition on the first/last plane is set 
         by the grid.bcz() variable and can be changed by the set_boundaries function. 
-        If there is no limiter the boundary condition is periodic.
+        If there is no limiter, the boundary condition is periodic.
     */
     template <class Limiter>
-    void construct(const dg::geo::BinaryVectorLvl0& vec, const ProductGeometry& grid, unsigned multiplyX, unsigned multiplyY, double eps = 1e-5, Limiter limit = FullLimiter(), dg::bc globalbcx = dg::DIR, dg::bc globalbcy = dg::DIR, double deltaPhi = -1);
+    void construct(const dg::geo::BinaryVectorLvl0& vec, const ProductGeometry& grid, unsigned multiplyX, unsigned multiplyY, bool dependsOnX, bool dependsOnY, double eps = 1e-5, dg::bc globalbcx = dg::NEU, dg::bc globalbcy = dg::NEU, Limiter limit = FullLimiter(), double deltaPhi = -1);
+
+    bool dependsOnX()const{return m_dependsOnX;}
+    bool dependsOnY()const{return m_dependsOnY;}
 
     /**
     * @brief Set boundary conditions in the limiter region
@@ -494,6 +497,7 @@ struct FieldAligned
     ///@brief hm is the distance between the current and minus planes
     ///@return three-dimensional vector
     const container& hm_inv()const {return hm_inv;}
+    const ProductGeoemtry& grid()const{return m_g.get();}
     private:
     void ePlus( enum whichMatrix which, const container& in, container& out);
     void eMinus(enum whichMatrix which, const container& in, container& out);
@@ -505,6 +509,7 @@ struct FieldAligned
     container m_limiter;
     std::vector<container> m_temp;
     dg::Handle<ProductGeometry> m_g;
+    bool m_dependsOnX, m_dependsOnY;
 };
 
 ///@cond 
@@ -514,8 +519,9 @@ struct FieldAligned
 
 template<class Geometry, class IMatrix, class container>
 template <class Limiter>
-void FieldAligned<Geometry, IMatrix, container>::construct(const dg::geo::BinaryVectorLvl0& vec, const Geometry& grid, unsigned mx, unsigned my, double eps, Limiter limit, dg::bc globalbcx, dg::bc globalbcy, double deltaPhi):
+void FieldAligned<Geometry, IMatrix, container>::construct(const dg::geo::BinaryVectorLvl0& vec, const Geometry& grid, unsigned mx, unsigned my, bool bx, bool by, double eps, dg::bc globalbcx, dg::bc globalbcy, Limiter limit, double deltaPhi):
 {
+    m_dependsOnX=bx, m_dependsOnY=by;
     dg::blas1::transfer( dg::evaluate( dg::zero, grid), m_hz_inv), m_hp_inv= m_hz_inv, m_hm_inv= m_hz_inv;
     m_Nz=grid.Nz(), m_bcz=grid.bcz(); 
     m_g=grid;
@@ -721,4 +727,3 @@ void FieldAligned<G, I, container>::eMinus( enum whichMatrix which, const contai
 
 }//namespace geo
 }//namespace dg
-
diff --git a/inc/geometries/mpi_fieldaligned.h b/inc/geometries/mpi_fieldaligned.h
index d414657f5..e9171e509 100644
--- a/inc/geometries/mpi_fieldaligned.h
+++ b/inc/geometries/mpi_fieldaligned.h
@@ -86,7 +86,12 @@ struct FieldAligned< Geometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vec
 {
     FieldAligned(){}
     template <class Limiter>
-    FieldAligned(const dg::geo::BinaryVectorLvl0& vec, const Geometry& grid, unsigned multiplyX, unsigned multiplyY, double eps = 1e-4, Limiter limit = FullLimiter(), dg::bc globalbcx = dg::DIR, dg::bc globalbcy = dg::DIR, double deltaPhi = -1);
+    FieldAligned(const dg::geo::BinaryVectorLvl0& vec, const Geometry& grid, unsigned multiplyX, unsigned multiplyY, double eps = 1e-5, dg::bc globalbcx = dg::NEU, dg::bc globalbcy = dg::NEU, Limiter limit = FullLimiter(), double deltaPhi = -1)
+    {
+        construct( vec, grid, multiplyX, multiplyY, eps, limit, globalbcx, globalbcy, limit, deltaPhi);
+    }
+    template <class Limiter>
+    void construct(const dg::geo::BinaryVectorLvl0& vec, const Geometry& grid, unsigned multiplyX, unsigned multiplyY, double eps = 1e-5, dg::bc globalbcx = dg::NEU, dg::bc globalbcy = dg::NEU, Limiter limit = FullLimiter(), double deltaPhi = -1);
 
     void set_boundaries( dg::bc bcz, double left, double right)
     {
@@ -130,18 +135,21 @@ struct FieldAligned< Geometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vec
     dg::bc m_bcz;
     MPI_Vector<LocalContainer> m_left, m_right;
     MPI_Vector<LocalContainer> m_limiter;
-    std::vector<LocalContainer> tempXYplus_, tempXYminus_, m_temp; 
+    std::vector<LocalContainer> m_temp; 
     MPIDistMat<LocalIMatrix, CommunicatorXY> m_plus, m_minus, m_plusT, m_minusT;
 };
 //////////////////////////////////////DEFINITIONS/////////////////////////////////////
 template<class MPIGeometry, class LocalIMatrix, class CommunicatorXY, class LocalContainer>
 template <class Limiter>
-FieldAligned<MPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vector<LocalContainer> >::FieldAligned(
-    const dg::geo::BinaryVectorLvl0& vec, const MPIGeometry& grid, unsigned mx, unsigned my, double eps, Limiter limit, dg::bc globalbcx, dg::bc globalbcy, double deltaPhi):
-    m_hz( dg::evaluate( dg::zero, grid)), m_hp( m_hz), m_hm( m_hz), 
-    m_g(grid), m_bcz(grid.bcz()), 
-    tempXYplus_(g_.Nz()), tempXYminus_(g_.Nz()), temp_(g_.Nz())
+void FieldAligned<MPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vector<LocalContainer> >::construct(
+    const dg::geo::BinaryVectorLvl0& vec, const MPIGeometry& grid, unsigned mx, unsigned my, double eps, dg::bc globalbcx, dg::bc globalbcy, Limiter limit, double deltaPhi)
 {
+    temp_(g_.Nz())
+
+    dg::blas1::transfer( dg::evaluate( dg::zero, grid), m_hz_inv), m_hp_inv= m_hz_inv, m_hm_inv= m_hz_inv;
+    m_Nz=grid.Nz(), m_bcz=grid.bcz(); 
+    m_g=grid;
+    dg::split( m_hz_inv, m_temp);
     if( deltaPhi <=0) deltaPhi = grid.hz();
     else assert( grid.Nz() == 1 || grid.hz()==deltaPhi);
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -201,26 +209,20 @@ FieldAligned<MPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vector<L
     dg::blas1::axpby(  1., hp_, +1., hm_, hz_);
     delete grid2d_ptr;
 
-    CommunicatorXY cp( pids, grid2d_ptr.communicator());
-    commXYplus_ = cp;
     thrust::host_vector<double> pX, pY;
     dg::blas1::transfer( cp.global_gather( yp[0]), pX);
     dg::blas1::transfer( cp.global_gather( yp[1]), pY);
 
     //construt interpolation matrix
-    plus = dg::create::interpolation( pX, pY, grid2d_ptr.local(), globalbcz); //inner points hopefully never lie exactly on local boundary
+    plus = dg::create::interpolation( pX, pY, grid2d_ptr.local(), globalbcz);
     cusp::transpose( plus, plusT);
 
-    CommunicatorXY cm( pids, grid2d_ptr.communicator());
-    commXYminus_ = cm;
     dg::blas1::transfer( cm.global_gather( ym[0]), pX);
     dg::blas1::transfer( cm.global_gather( ym[1]), pY);
-    minus = dg::create::interpolation( pX, pY, grid2d_ptr.local(), globalbcz); //inner points hopefully never lie exactly on local boundary
+    minus = dg::create::interpolation( pX, pY, grid2d_ptr.local(), globalbcz);
     cusp::transpose( minus, minusT);
     for( unsigned i=0; i<g_.Nz(); i++)
     {
-        tempXYplus_[i].resize( commXYplus_.size());
-        tempXYminus_[i].resize( commXYminus_.size());
         temp_[i].resize( localsize);
     }
     delete grid2d_ptr;
@@ -393,4 +395,3 @@ void FieldAligned<G,RowDistMat<M,C>, MPI_Vector<container> >::eMinus( enum which
 
 }//namespace geo
 }//namespace dg
-
-- 
GitLab


From 1e5edaf02fe72419b83a03c7d2ae41fb2df259ed Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Tue, 17 Oct 2017 13:27:11 +0200
Subject: [PATCH 363/453] restructure the new flux function documentation

---
 inc/geometries/ds_b.cu          |  2 +-
 inc/geometries/geometries_doc.h |  6 ++++++
 inc/geometries/guenther.h       | 10 +++++++++-
 inc/geometries/solovev.h        |  8 +++++++-
 inc/geometries/taylor.h         | 10 ++++++++--
 inc/geometries/toroidal.h       | 18 ++++++++++++++++++
 6 files changed, 49 insertions(+), 5 deletions(-)

diff --git a/inc/geometries/ds_b.cu b/inc/geometries/ds_b.cu
index c3ff50744..450f69d5c 100644
--- a/inc/geometries/ds_b.cu
+++ b/inc/geometries/ds_b.cu
@@ -52,7 +52,7 @@ int main()
     t.tic();
     dg::geo::FieldAligned<dg::aProductGeometry,dg::IDMatrix, dg::DMatrix>  dsFA( bhat, g3d, 2,2,1e-10, dg::NoLimiter(), dg::NEU, dg::NEU);
 
-    dg::geo::DS<dg::aProductGeometry3d, dg::IDMatrix, dg::DMatrix, dg::DVec> ds( dsFA, g3d, dg::not_normed, dg::centered);
+    dg::geo::DS<dg::aProductGeometry3d, dg::IDMatrix, dg::DMatrix, dg::DVec> ds( dsFA, dg::not_normed, dg::centered);
     t.toc();
     std::cout << "Creation of parallel Derivative took     "<<t.diff()<<"s\n";
 
diff --git a/inc/geometries/geometries_doc.h b/inc/geometries/geometries_doc.h
index 40f04f395..6e7030bec 100644
--- a/inc/geometries/geometries_doc.h
+++ b/inc/geometries/geometries_doc.h
@@ -11,6 +11,12 @@
  All functors in this section model two or three-dimensional functions, i.e. they all overload the operator() like aBinaryOperator
  * @{
       @defgroup geom 3.1 new flux functions and derivatives
+      @{
+        @defgroup solovev The solovev magnetic field
+        @defgroup taylor The Taylor state magnetic field
+        @defgroup guenther The Guenther magnetic field
+        @defgroup toroidal The purely toroidal magnetic field
+      @}
       @defgroup magnetic 3.2 magnetic field and associated functors
       @defgroup profiles 3.3 miscellaneous functors based on flux functions
  * @}
diff --git a/inc/geometries/guenther.h b/inc/geometries/guenther.h
index 8b97c62a8..5bc0390a9 100644
--- a/inc/geometries/guenther.h
+++ b/inc/geometries/guenther.h
@@ -24,7 +24,7 @@ namespace geo
  */
 namespace guenther
 {
-///@addtogroup geom
+///@addtogroup guenther
 ///@{
 
    
@@ -506,6 +506,14 @@ struct Divb
 };
 ///@endcond
 } //namespace guenther
+
+/**
+ * @brief Create a Guenther Magnetic field
+ * @param R_0 the major radius
+ * @param I_0 the current
+ * @return A magnetic field object
+ * @ingroup geom
+ */
 TokamakMagneticField createGuentherField( double R_0, double I_0)
 {
     return TokamakMagneticField( R_0, guenther::createPsip(R_0), guenther::createIpol(I_0));
diff --git a/inc/geometries/solovev.h b/inc/geometries/solovev.h
index 6016370a8..67064a127 100644
--- a/inc/geometries/solovev.h
+++ b/inc/geometries/solovev.h
@@ -27,7 +27,7 @@ namespace geo
  */
 namespace solovev 
 {
-///@addtogroup geom
+///@addtogroup solovev
 ///@{
 
 /**
@@ -554,6 +554,12 @@ struct PsipRZ: public aCloneableBinaryFunctor<PsipRZ>
 
 } //namespace solovev
 
+/**
+ * @brief Create a Solovev Magnetic field
+ * @param gp Solovev parameters
+ * @return A magnetic field object
+ * @ingroup geom
+ */
 TokamakMagneticField createSolovevField( solovev::GeomParameters gp)
 {
     return TokamakMagneticField( gp.R_0, solovev::createPsip(gp), solovev::createIpol(gp));
diff --git a/inc/geometries/taylor.h b/inc/geometries/taylor.h
index f0361a618..692a7197f 100644
--- a/inc/geometries/taylor.h
+++ b/inc/geometries/taylor.h
@@ -31,7 +31,7 @@ namespace geo
  */
 namespace taylor
 {
-///@addtogroup geom
+///@addtogroup taylor
 ///@{
 typedef dg::geo::solovev::GeomParameters GeomParameters; //!< bring GeomParameters into the taylor namespace 
 
@@ -311,7 +311,13 @@ dg::geo::TokamakMagneticField createMagField( solovev::GeomParameters gp)
 ///@}
 
 } //namespace taylor
-dg::geo::TokamakMagneticField createTaylorField( solovev::GeomParameters gp)
+/**
+ * @brief Create a Taylor Magnetic field
+ * @param gp Solovev parameters
+ * @return A magnetic field object
+ * @ingroup geom
+ */
+TokamakMagneticField createTaylorField( solovev::GeomParameters gp)
 {
     return TokamakMagneticField( gp.R_0, dg::geo::taylor::createPsip(gp), dg::geo::taylor::createIpol(gp));
 }
diff --git a/inc/geometries/toroidal.h b/inc/geometries/toroidal.h
index d82765d38..239c89ae9 100644
--- a/inc/geometries/toroidal.h
+++ b/inc/geometries/toroidal.h
@@ -6,11 +6,21 @@ namespace dg{
 namespace geo{
 namespace toroidal{
 
+///@addtogroup toroidal
+///@{
+/**
+ * @brief constant \f$\psi_p = 1\f$
+ * @return 
+ */
 BinaryFunctorsLvl2 createPsip( )
 {
     BinaryFunctorsLvl2 psip( new Constant(1), new Constant(0), new Constant(0),new Constant(0), new Constant(0), new Constant(0));
     return psip;
 }
+/**
+ * @brief constant \f$ I = 1\f$
+ * @return 
+ */
 BinaryFunctorsLvl1 createIpol( )
 {
     BinaryFunctorsLvl1 ipol( new Constant(1), new Constant(0), new Constant(0))
@@ -27,9 +37,17 @@ TokamakMagneticField createMagField( double R0)
 {
     return TokamakMagneticField( R0, createPsip(), createIpol());
 }
+///@}
 
 }//namespace toroidal
 
+/**
+ * @brief Create a Toroidal Magnetic field
+ * @param R0 the major radius
+ * @return A magnetic field object
+ * @ingroup geom
+ * @note The solovev field can also be made to model a todoidal slab field
+ */
 TokamakMagneticField createToroidalField( double R0)
 {
     return TokamakMagneticField( R0, toiroidal::createPsip(), toiroidal::createIpol());
-- 
GitLab


From 1e4fb72fbddc8ae5c28a680d9f4f76524979ff03 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Tue, 17 Oct 2017 17:42:01 +0200
Subject: [PATCH 364/453] about to debug ds again

---
 inc/dg/backend/split_and_join.h      |  22 ++--
 inc/dg/backend/transpose.h           |   2 -
 inc/dg/blas1.h                       |  39 +++---
 inc/dg/blas2.h                       |  20 +--
 inc/dg/dg_doc.h                      |   1 +
 inc/dg/multigrid.h                   |   2 +-
 inc/geometries/curvilinear.h         | 174 +++++++++++++------------
 inc/geometries/ds.h                  |  84 +++++-------
 inc/geometries/ds_b.cu               |  21 ++-
 inc/geometries/fieldaligned.h        | 185 +++++++++++++--------------
 inc/geometries/geometries_doc.h      |   3 +-
 inc/geometries/guenther.h            |   2 +-
 inc/geometries/guenther_parameters.h |   5 +-
 inc/geometries/mpi_fieldaligned.h    |  51 ++------
 inc/geometries/solovev.h             |  38 +++---
 inc/geometries/solovev_parameters.h  |   4 +-
 inc/geometries/taylor.h              |  28 ++--
 inc/geometries/toroidal.h            |  80 +++++++++++-
 18 files changed, 387 insertions(+), 374 deletions(-)

diff --git a/inc/dg/backend/split_and_join.h b/inc/dg/backend/split_and_join.h
index cf7d02242..effd7bd6f 100644
--- a/inc/dg/backend/split_and_join.h
+++ b/inc/dg/backend/split_and_join.h
@@ -12,13 +12,14 @@ namespace dg
 ///@{
 /** @brief  Split a vector into planes 
 *
-* @tparam thrust_vector either thrust::host_vector or thrust::device_vector
+* @tparam thrust_vector1 either thrust::host_vector or thrust::device_vector
+* @tparam thrust_vector2 either thrust::host_vector or thrust::device_vector
 * @param in contiguous 3d vector (must be of size grid.size())
 * @param out contains grid.Nz() 2d vectors of 2d size on output (gets resized if necessary)
 * @param grid provide dimensions in 3rd and first two dimensions
 */
-template<class thrust_vector>
-void split( const thrust_vector& in, std::vector<thrust_vector>& out, const aTopology3d& grid)
+template<class thrust_vector1, class thrust_vector2>
+void split( const thrust_vector1& in, std::vector<thrust_vector2>& out, const aTopology3d& grid)
 {
     unsigned size2d=grid.n()*grid.n()*grid.Nx()*grid.Ny();
     out.resize( grid.Nz());
@@ -30,8 +31,8 @@ void split( const thrust_vector& in, std::vector<thrust_vector>& out, const aTop
 ///@copydetails dg::split()
 ///@note every plane in out holds a 2d Cartesian MPI_Communicator 
 ///@note two seperately split vectors have congruent (not identical) MPI_Communicators (Note here the MPI concept of congruent vs. identical communicators)
-template <class thrust_vector>
-void split( const MPI_Vector<thrust_vector>& in, std::vector<MPI_Vector<thrust_vector> >& out, const aMPITopology3d& grid)
+template<class thrust_vector1, class thrust_vector2>
+void split( const MPI_Vector<thrust_vector1>& in, std::vector<MPI_Vector<thrust_vector2> >& out, const aMPITopology3d& grid)
 {
     int result;
     MPI_Comm_compare( in.communicator(), grid.communicator(), &result);
@@ -52,14 +53,15 @@ void split( const MPI_Vector<thrust_vector>& in, std::vector<MPI_Vector<thrust_v
 /**
 * @brief Revert split operation
 *
-* @tparam thrust_vector either thrust::host_vector or thrust::device_vector
+* @tparam thrust_vector1 either thrust::host_vector or thrust::device_vector
+* @tparam thrust_vector2 either thrust::host_vector or thrust::device_vector
 * @param in grid.Nz() 2d vectors of 2d size 
 * @param out contiguous 3d vector (gets resized if necessary) 
 * @param grid provide dimensions in 3rd and first two dimensions
 * @note split followed by join restores the original vector
 */
-template<class thrust_vector>
-void join( const std::vector<thrust_vector>& in, thrust_vector& out, const aTopology3d& grid)
+template<class thrust_vector1, class thrust_vector2>
+void join( const std::vector<thrust_vector1>& in, thrust_vector2& out, const aTopology3d& grid)
 {
     unsigned size2d=grid.n()*grid.n()*grid.Nx()*grid.Ny();
     out.resize( size2d*grid.Nz());
@@ -70,8 +72,8 @@ void join( const std::vector<thrust_vector>& in, thrust_vector& out, const aTopo
 #ifdef MPI_VERSION
 ///@brief MPI Version of join
 ///@copydetails dg::join()
-template<class thrust_vector>
-void join( const std::vector<MPI_Vector<thrust_vector> >& in, MPI_Vector<thrust_vector >& out, const aMPITopology3d& grid)
+template<class thrust_vector1, class thrust_vector2>
+void join( const std::vector<MPI_Vector<thrust_vector1> >& in, MPI_Vector<thrust_vector2 >& out, const aMPITopology3d& grid)
 {
     unsigned size2d=grid.n()*grid.n()*grid.Nx()*grid.Ny();
     out.data().resize( size2d*grid.Nz());
diff --git a/inc/dg/backend/transpose.h b/inc/dg/backend/transpose.h
index 5fcf5d07f..ea1edaa73 100644
--- a/inc/dg/backend/transpose.h
+++ b/inc/dg/backend/transpose.h
@@ -1,7 +1,5 @@
 #pragma once
 
-//some thoughts on how to transpose a MPI matrix 
-
 namespace dg
 {
 
diff --git a/inc/dg/blas1.h b/inc/dg/blas1.h
index 99a58da77..83f27ba10 100644
--- a/inc/dg/blas1.h
+++ b/inc/dg/blas1.h
@@ -31,7 +31,7 @@ namespace blas1
 ///@{
 
 /**
- * @brief Generic way to copy and/or convert a container type to a different container type (e.g. from CPU to GPU, or double to float, etc.)
+ * @brief y=x; Generic way to copy and/or convert a container type to a different container type (e.g. from CPU to GPU, or double to float, etc.)
  *
  * @copydoc hide_container
  * @tparam other_container another container type
@@ -47,7 +47,7 @@ inline void transfer( const container& x, other_container& y)
 
 
 /**
- * @brief Invoke assignment operator
+ * @brief y=x; Invoke assignment operator
  *
  * Same as y=x
  * @tparam Assignable any assignable type
@@ -57,7 +57,7 @@ inline void transfer( const container& x, other_container& y)
 template<class Assignable>
 inline void copy( const Assignable& x, Assignable& y){y=x;}
 
-/*! @brief Euclidean dot product between two containers
+/*! @brief \f$ x^T y\f$; Euclidean dot product between two containers
  *
  * This routine computes \f[ x^T y = \sum_{i=0}^{N-1} x_i y_i \f]  i iterates over @b all elements inside the container. Specifically for a std::vector<container_type> i includes both the inner and the outer loop. If the container sizes
  * do not match, the result is undefined.
@@ -75,7 +75,7 @@ inline typename VectorTraits<container>::value_type dot( const container& x, con
     return dg::blas1::detail::doDot( x, y, typename dg::VectorTraits<container>::vector_category() );
 }
 
-/*! @brief Modified BLAS 1 routine axpy
+/*! @brief \f$ y = \alpha x + \beta y\f$
  *
  * This routine computes \f[ y_i =  \alpha x_i + \beta y_i \f]  i iterates over @b all elements inside the container. Specifically for a std::vector<container_type> i includes both the inner and the outer loop. If the container sizes
  * do not match, the result is undefined.
@@ -92,26 +92,26 @@ inline void axpby( typename VectorTraits<container>::value_type alpha, const con
     return;
 }
 
-/*! @brief Modified BLAS 1 routine axpy
+/*! @brief \f$ z = \alpha x + \beta y\f$
  *
  * This routine computes \f[ z_i =  \alpha x_i + \beta y_i \f]  i iterates over @b all elements inside the container. Specifically for a std::vector<container_type> i includes both the inner and the outer loop. If the container sizes
  * do not match, the result is undefined.
 
  * @copydoc hide_container
  * @param alpha Scalar  
- * @param x container x may alias result
+ * @param x container x may alias z
  * @param beta Scalar
- * @param y container y may alias result
- * @param result container contains solution on output
+ * @param y container y may alias z
+ * @param z container z contains solution on output
  */
 template< class container>
-inline void axpby( typename VectorTraits<container>::value_type alpha, const container& x, typename VectorTraits<container>::value_type beta, const container& y, container& result)
+inline void axpby( typename VectorTraits<container>::value_type alpha, const container& x, typename VectorTraits<container>::value_type beta, const container& y, container& z)
 {
-    dg::blas1::detail::doAxpby( alpha, x, beta, y, result, typename dg::VectorTraits<container>::vector_category() );
+    dg::blas1::detail::doAxpby( alpha, x, beta, y, z, typename dg::VectorTraits<container>::vector_category() );
     return;
 }
 
-/*! @brief Modified BLAS 1 routine axpy
+/*! @brief \f$ z = \alpha x + \beta y + \gamma z\f$
  *
  * This routine computes \f[ z_i =  \alpha x_i + \beta y_i + \gamma z_i \f]  i iterates over @b all elements inside the container. Specifically for a std::vector<container_type> i includes both the inner and the outer loop. If the container sizes
  * do not match, the result is undefined.
@@ -131,7 +131,7 @@ inline void axpbypgz( typename VectorTraits<container>::value_type alpha, const
     return;
 }
 
-/*! @brief "new" BLAS 1 routine transform
+/*! @brief \f$ y = op(x)\f$
  *
  * This routine computes \f[ y_i = op(x_i) \f]
  * This is strictly speaking not a BLAS routine since f can be a nonlinear function.
@@ -148,7 +148,7 @@ inline void transform( const container& x, container& y, UnaryOp op)
     return;
 }
 
-/*! @brief BLAS 1 routine scal
+/*! @brief \f$ x = \alpha x\f$
  *
  * This routine computes \f[ \alpha x_i \f]
  * @copydoc hide_container
@@ -162,7 +162,7 @@ inline void scal( container& x, typename VectorTraits<container>::value_type alp
     return;
 }
 
-/*! @brief pointwise add a scalar
+/*! @brief \f$ x = x + \alpha \f$
  *
  * This routine computes \f[ x_i + \alpha \f] 
  * @copydoc hide_container
@@ -176,8 +176,7 @@ inline void plus( container& x, typename VectorTraits<container>::value_type alp
     return;
 }
 
-/**
-* @brief A 'new' BLAS 1 routine. 
+/*! @brief \f$ y = x_1 x_2 \f$
 *
 * Multiplies two vectors element by element: \f[ y_i = x_{1i}x_{2i}\f]  i iterates over @b all elements inside the container. Specifically for a std::vector<container_type> i includes both the inner and the outer loop. If the container sizes
  * do not match, the result is undefined.
@@ -195,7 +194,7 @@ inline void pointwiseDot( const container& x1, const container& x2, container& y
 }
 
 /**
-* @brief A 'new' BLAS 1 routine. 
+* @brief \f$ y = \alpha x_1 x_2 + \beta y\f$ 
 *
 * Multiplies two vectors element by element: \f[ y_i = \alpha x_{1i}x_{2i} + \beta y_i\f]  i iterates over @b all elements inside the container. Specifically for a std::vector<container_type> i includes both the inner and the outer loop. If the container sizes
  * do not match, the result is undefined.
@@ -214,7 +213,7 @@ inline void pointwiseDot( typename VectorTraits<container>::value_type alpha, co
 }
 
 /**
-* @brief A 'new' BLAS 1 routine. 
+* @brief \f$ y = x_1/ x_2\f$ 
 *
 * Divides two vectors element by element: \f[ y_i = x_{1i}/x_{2i}\f]  i iterates over @b all elements inside the container. Specifically for a std::vector<container_type> i includes both the inner and the outer loop. If the container sizes
  * do not match, the result is undefined.
@@ -232,7 +231,9 @@ inline void pointwiseDivide( const container& x1, const container& x2, container
 }
 
 /**
-* @brief A 'new' fused multiply-add BLAS 1 routine. 
+* @brief \f$ z = \alpha x_1x_2 + \beta x_2y_2 + \gamma z\f$
+*
+* A 'new' fused multiply-add BLAS 1 routine. 
 *
 * Multiplies and adds vectors element by element: \f[ z_i = \alpha x_{1i}y_{1i} + \beta x_{2i}y_{2i} + \gamma z_i \f]  i iterates over @b all elements inside the container. Specifically for a std::vector<container_type> i includes both the inner and the outer loop. If the container sizes
  * do not match, the result is undefined.
diff --git a/inc/dg/blas2.h b/inc/dg/blas2.h
index ab65d23de..bbc51053b 100644
--- a/inc/dg/blas2.h
+++ b/inc/dg/blas2.h
@@ -30,7 +30,7 @@ namespace blas2{
 ///@{
 
 /**
- * @brief Generic way to copy and/or convert a Matrix type to a different Matrix type (e.g. from CPU to GPU, or double to float, etc.)
+ * @brief \f$ y = x\f$; Generic way to copy and/or convert a Matrix type to a different Matrix type (e.g. from CPU to GPU, or double to float, etc.)
  *
  * @copydoc hide_matrix
  * @tparam AnotherMatrix Another Matrix type
@@ -44,12 +44,12 @@ inline void transfer( const Matrix& x, AnotherMatrix& y)
     dg::blas2::detail::doTransfer( x,y, typename dg::MatrixTraits<Matrix>::matrix_category(), typename dg::MatrixTraits<AnotherMatrix>::matrix_category());
 }
 
-/*! @brief General dot produt
+/*! @brief \f$ x^T M y\f$; General dot produt
  *
  * This routine computes the scalar product defined by the symmetric positive definite 
  * matrix M \f[ x^T M y = \sum_{i,j=0}^{N-1} x_i M_{ij} y_j \f]
  * ( Note that if M is not diagonal it is generally more efficient to 
- * precalculate \f$ My\f$ and then call the blas1::dot() routine!
+ * precalculate \f$ My\f$ and then call the dg::blas1::dot() routine!
  * @tparam DiagonalMatrix Right now DiagonalMatrix has to be the same as container, except if the container is a std::vector<container_type>, then the DiagonalMatrix has to be the container_type.
  * In the latter case the Matrix is applied to all containers in the std::vector and the sum is returned. 
  * @copydoc hide_container
@@ -68,7 +68,7 @@ inline typename MatrixTraits<DiagonalMatrix>::value_type dot( const container& x
                        typename dg::VectorTraits<container>::vector_category() );
 }
 
-/*! @brief General dot produt
+/*! @brief \f$ x^T M x\f$; General dot produt
  *
  * This routine is equivalent to the call blas2::dot( x, m, x):
  * \f[ x^T M x = \sum_{i,j=0}^{N-1} x_i M_{ij} x_j \f]
@@ -89,7 +89,7 @@ inline typename MatrixTraits<DiagonalMatrix>::value_type dot( const DiagonalMatr
                        typename dg::VectorTraits<container>::vector_category() );
 }
 
-/*! @brief Matrix Vector product
+/*! @brief \f$ y = \alpha M x + \beta y\f$ 
  *
  * This routine computes \f[ y = \alpha M x + \beta y \f]
  * where \f$ M\f$ is a matrix.
@@ -119,7 +119,7 @@ inline void symv( typename MatrixTraits<Matrix>::value_type alpha,
     return;
 }
 
-/*! @brief Matrix Vector product
+/*! @brief \f$ y = M x\f$ 
  *
  * This routine computes \f[ y = M x \f]
  * where \f$ M\f$ is a matrix. 
@@ -144,8 +144,8 @@ inline void symv( Matrix& M,
     return;
 }
 
-/**
- * @brief General Matrix-Vector product
+/*! @brief \f$ y = M x\f$;
+ * General Matrix-Vector product
  *
  * Does exactly the same as symv. 
  * @copydoc hide_matrix
@@ -167,8 +167,8 @@ inline void gemv( Matrix& M,
                        typename dg::VectorTraits<same_or_another_container>::vector_category() );
     return;
 }
-/**
- * @brief General Matrix-Vector product
+/*! @brief \f$ y = \alpha M x + \beta y \f$;
+ * General Matrix-Vector product
  *
  * Does exactly the same as symv. 
  * @copydoc hide_matrix
diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index 9989d2dd9..3b70ffffe 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -90,6 +90,7 @@
  * @{
  *     @defgroup arakawa Discretization of Poisson bracket
  *     @defgroup matrixoperators Elliptic and Helmholtz operators
+ *     @defgroup multigrid Multigrid matrix inversion
  * @}
  * @defgroup misc Level 00: Miscellaneous additions
  * @{
diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index cbde669f9..9df4c3ac0 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -16,7 +16,7 @@ namespace dg
 *
 * We use conjugate gradien (CG) at each stage and refine the grids in the first two dimensions (2d / x and y) 
 * @copydoc hide_geometry_matrix_container
-* @ingroup numerical1
+* @ingroup multigrid
 */
 template< class Geometry, class Matrix, class container> 
 struct MultigridCG2d
diff --git a/inc/geometries/curvilinear.h b/inc/geometries/curvilinear.h
index e25ce5f47..a5c260780 100644
--- a/inc/geometries/curvilinear.h
+++ b/inc/geometries/curvilinear.h
@@ -9,40 +9,42 @@ namespace dg
 {
 namespace geo
 {
-    /*!@class hide_grid_parameters3d
-     * @brief Construct a 3D grid
-     *
-     * the coordinates of the computational space are called x,y,z
-     * @param generator generates the perpendicular grid
-     * @param n number of %Gaussian nodes in x and y
-     *  (1<=n<=20, note that the library is optimized for n=3 )
-     * @attention # of polynomial coefficients in z direction is always 1
-     * @param Nx number of cells in x
-     * @param Ny number of cells in y 
-     * @param Nz  number of cells z
-     * @param bcx boundary condition in x
-     * @param bcy boundary condition in y
-     * @param bcz boundary condition in z
-     */
-    /*!@class hide_grid_parameters2d
-     * @brief Construct a 2D grid
-     *
-     * the coordinates of the computational space are called x,y,z
-     * @param generator generates the grid
-     * @param n number of %Gaussian nodes in x and y
-     *  (1<=n<=20, note that the library is optimized for n=3 )
-     * @param Nx number of cells in x
-     * @param Ny number of cells in y 
-     * @param bcx boundary condition in x
-     * @param bcy boundary condition in y
-     */
+
+/*!@class hide_grid_parameters3d
+ * @brief Construct a 3D grid
+ *
+ * the coordinates of the computational space are called x,y,z
+ * @param generator generates the perpendicular grid
+ * @param n number of %Gaussian nodes in x and y
+ *  (1<=n<=20, note that the library is optimized for n=3 )
+ * @attention # of polynomial coefficients in z direction is always 1
+ * @param Nx number of cells in x
+ * @param Ny number of cells in y 
+ * @param Nz  number of cells z
+ * @param bcx boundary condition in x
+ * @param bcy boundary condition in y
+ * @param bcz boundary condition in z
+ */
+
+/*!@class hide_grid_parameters2d
+ * @brief Construct a 2D grid
+ *
+ * the coordinates of the computational space are called x,y,z
+ * @param generator generates the grid
+ * @param n number of %Gaussian nodes in x and y
+ *  (1<=n<=20, note that the library is optimized for n=3 )
+ * @param Nx number of cells in x
+ * @param Ny number of cells in y 
+ * @param bcx boundary condition in x
+ * @param bcy boundary condition in y
+ */
 
 
 ///@addtogroup grids
 ///@{
 
 ///@cond
-struct CurvilinearGrid2d; 
+struct CurvilinearProductGrid3d;
 namespace detail
 {
 void square( const dg::SparseTensor<thrust::host_vector<double> >& jac, const thrust::host_vector<double>& R, dg::SparseTensor<thrust::host_vector<double> >& metric, bool orthogonal)
@@ -70,6 +72,46 @@ void square( const dg::SparseTensor<thrust::host_vector<double> >& jac, const th
 //when we make a 3d grid with eta and phi swapped the metric structure and the transformation changes 
 //In practise it can only be orthogonal due to the projection tensor in the elliptic operator
 
+/**
+ * @brief A two-dimensional grid based on curvilinear coordinates
+ */
+struct CurvilinearGrid2d : public dg::aGeometry2d
+{
+    ///@copydoc hide_grid_parameters2d
+    CurvilinearGrid2d( const aGenerator2d& generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx=dg::DIR, bc bcy=dg::PER):
+        dg::aGeometry2d( 0, generator.width(), 0., generator.height(), n, Nx, Ny, bcx, dg::PER), handle_(generator)
+    {
+        construct( n,Nx,Ny);
+    }
+
+    /**
+     * @brief Explicitly convert 3d product grid to the perpendicular grid
+     * @param g 3d product grid
+     */
+    explicit CurvilinearGrid2d( CurvilinearProductGrid3d g);
+
+    ///read access to the generator 
+    const aGenerator2d& generator() const{return handle_.get();}
+    virtual CurvilinearGrid2d* clone()const{return new CurvilinearGrid2d(*this);}
+    private:
+    virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny)
+    {
+        dg::aTopology2d::do_set( new_n, new_Nx, new_Ny);
+        construct( new_n, new_Nx, new_Ny);
+    }
+    void construct( unsigned n, unsigned Nx, unsigned Ny);
+    virtual SparseTensor<thrust::host_vector<double> > do_compute_jacobian( ) const {
+        return jac_;
+    }
+    virtual SparseTensor<thrust::host_vector<double> > do_compute_metric( ) const {
+        return metric_;
+    }
+    virtual std::vector<thrust::host_vector<double> > do_compute_map()const{return map_;}
+    dg::SparseTensor<thrust::host_vector<double> > jac_, metric_;
+    std::vector<thrust::host_vector<double> > map_;
+    dg::Handle<aGenerator2d> handle_;
+};
+
 
 /**
  * @brief A 2x1 curvilinear product space grid
@@ -90,18 +132,12 @@ struct CurvilinearProductGrid3d : public dg::aProductGeometry3d
         constructParallel(Nz);
     }
 
-    /*!
-     * @brief The grid made up by the first two dimensions
-     *
-     * This is possible because the 3d grid is a product grid of a 2d perpendicular grid and a 1d parallel grid
-     * @return A newly constructed perpendicular grid
-     */
-    perpendicular_grid perp_grid() const;// { return perpendicular_grid(*this);}
 
     ///@copydoc CurvilinearGrid2d::generator()const
     const aGenerator2d & generator() const{return handle_.get();}
     virtual CurvilinearProductGrid3d* clone()const{return new CurvilinearProductGrid3d(*this);}
     private:
+    virtual CurvilinearGrid2d* do_perp_grid() const;
     virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny,unsigned new_Nz){
         dg::aTopology3d::do_set( new_n, new_Nx, new_Ny,new_Nz);
         if( !( new_n == n() && new_Nx == Nx() && new_Ny == Ny() ) )
@@ -154,61 +190,23 @@ struct CurvilinearProductGrid3d : public dg::aProductGeometry3d
     dg::Handle<aGenerator2d> handle_;
 };
 
-/**
- * @brief A two-dimensional grid based on curvilinear coordinates
- */
-struct CurvilinearGrid2d : public dg::aGeometry2d
-{
-    ///@copydoc hide_grid_parameters2d
-    CurvilinearGrid2d( const aGenerator2d& generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx=dg::DIR, bc bcy=dg::PER):
-        dg::aGeometry2d( 0, generator.width(), 0., generator.height(), n, Nx, Ny, bcx, dg::PER), handle_(generator)
-    {
-        construct( n,Nx,Ny);
-    }
-
-    /**
-     * @brief Explicitly convert 3d product grid to the perpendicular grid
-     * @param g 3d product grid
-     */
-    explicit CurvilinearGrid2d( CurvilinearProductGrid3d g):
-        dg::aGeometry2d( g.x0(), g.x1(), g.y0(), g.y1(), g.n(), g.Nx(), g.Ny(), g.bcx(), g.bcy() ), handle_(g.generator())
-    {
-        g.set( n(), Nx(), Ny(), 1); //shouldn't trigger 2d grid generator
-        map_=g.map();
-        jac_=g.jacobian().perp();
-        metric_=g.metric().perp();
-        map_.pop_back();
-    }
-
-    ///read access to the generator 
-    const aGenerator2d& generator() const{return handle_.get();}
-    virtual CurvilinearGrid2d* clone()const{return new CurvilinearGrid2d(*this);}
-    private:
-    virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny)
-    {
-        dg::aTopology2d::do_set( new_n, new_Nx, new_Ny);
-        construct( new_n, new_Nx, new_Ny);
-    }
-    void construct( unsigned n, unsigned Nx, unsigned Ny)
-    {
-        CurvilinearProductGrid3d g( handle_.get(), n,Nx,Ny,1,bcx());
-        *this = CurvilinearGrid2d(g);
-    }
-    virtual SparseTensor<thrust::host_vector<double> > do_compute_jacobian( ) const {
-        return jac_;
-    }
-    virtual SparseTensor<thrust::host_vector<double> > do_compute_metric( ) const {
-        return metric_;
-    }
-    virtual std::vector<thrust::host_vector<double> > do_compute_map()const{return map_;}
-    dg::SparseTensor<thrust::host_vector<double> > jac_, metric_;
-    std::vector<thrust::host_vector<double> > map_;
-    dg::Handle<aGenerator2d> handle_;
-};
-
 ///@}
 ///@cond
-CurvilinearProductGrid3d::perpendicular_grid CurvilinearProductGrid3d::perp_grid() const { return CurvilinearProductGrid3d::perpendicular_grid(*this);}
+CurvilinearGrid2d::CurvilinearGrid2d( CurvilinearProductGrid3d g):
+    dg::aGeometry2d( g.x0(), g.x1(), g.y0(), g.y1(), g.n(), g.Nx(), g.Ny(), g.bcx(), g.bcy() ), handle_(g.generator())
+{
+    g.set( n(), Nx(), Ny(), 1); //shouldn't trigger 2d grid generator
+    map_=g.map();
+    jac_=g.jacobian().perp();
+    metric_=g.metric().perp();
+    map_.pop_back();
+}
+void CurvilinearGrid2d::construct( unsigned n, unsigned Nx, unsigned Ny)
+{
+    CurvilinearProductGrid3d g( handle_.get(), n,Nx,Ny,1,bcx());
+    *this = CurvilinearGrid2d(g);
+}
+CurvilinearProductGrid3d::perpendicular_grid* CurvilinearProductGrid3d::do_perp_grid() const { return new CurvilinearProductGrid3d::perpendicular_grid(*this);}
 ///@endcond
 
 }//namespace geo
diff --git a/inc/geometries/ds.h b/inc/geometries/ds.h
index 0aa5db9a9..0af543b7d 100644
--- a/inc/geometries/ds.h
+++ b/inc/geometries/ds.h
@@ -5,7 +5,7 @@
 #include "dg/backend/derivatives.h"
 #include "fieldaligned.h"
 #ifdef MPI_VERSION
-#include "backend/mpi_derivatives.h"
+#include "dg/backend/mpi_derivatives.h"
 #include "mpi_fieldaligned.h"
 #endif //MPI_VERSION
 #include "magnetic_field.h"
@@ -49,12 +49,12 @@ struct DS
      */
     DS(const dg::geo::TokamakMagneticField& vec, const ProductGeometry& grid, unsigned multiplyX=1, unsigned multiplyY=1, bool dependsOnX = true, bool dependsOnY=true, double eps = 1e-5, dg::norm no=dg::normed, dg::direction dir = dg::centered)
     {
-        dg::geo::BinaryVectorLvl0 vec( dg::geo::BHatR(mag), dg::geo::BHatZ(mag), dg::geo::BHatP(mag));
-        m_fa.construct( vec, grid, mx, my, dependsOnX, dependsOnY, eps, grid.bcx(), grid.bcy(), FullLimiter());
+        dg::geo::BinaryVectorLvl0 bhat( (dg::geo::BHatR)(vec), (dg::geo::BHatZ)(vec), (dg::geo::BHatP)(vec));
+        m_fa.construct( bhat, grid, multiplyX, multiplyY, dependsOnX, dependsOnY, eps, grid.bcx(), grid.bcy(), FullLimiter());
         construct( m_fa, no, dir);
     }
     /**
-     * @brief Create a FieldAligned object and construct
+     * @brief Create a Fieldaligned object and construct
      *
      * @param vec The vector field to integrate
      * @param grid The grid on which to operate defines the parallel boundary condition in case there is a limiter.
@@ -144,57 +144,32 @@ struct DS
      */
     void symv( const container& f, container& dsTdsf){ do_symv( f, dsTdsf);}
 
-    /**
-    * @brief Set boundary conditions in the limiter region
-    *
-    * if Dirichlet boundaries are used the left value is the left function
-    value, if Neumann boundaries are used the left value is the left derivative value
-    * @param bcz boundary condition
-    * @param left left boundary value
-    * @param right right boundary value
-    */
+    ///@copydoc FieldAligned::set_boundaries(dg::bc,double,double)
     void set_boundaries( dg::bc bcz, double left, double right)
     {
-        f_.set_boundaries( bcz, left, right);
+        m_fa.set_boundaries( bcz, left, right);
     }
-    /**
-     * @brief Set boundary conditions in the limiter region
-     *
-     * if Dirichlet boundaries are used the left value is the left function
-     value, if Neumann boundaries are used the left value is the left derivative value
-     * @param bcz boundary condition
-     * @param left left boundary value
-     * @param right right boundary value
-    */
+    ///@copydoc FieldAligned::set_boundaries(dg::bc,const container&,const container&)
     void set_boundaries( dg::bc bcz, const container& left, const container& right)
     {
-        f_.set_boundaries( bcz, left, right);
+        m_fa.set_boundaries( bcz, left, right);
     }
-    /**
-     * @brief Set boundary conditions in the limiter region
-     *
-     * if Dirichlet boundaries are used the left value is the left function
-     value, if Neumann boundaries are used the left value is the left derivative value
-     * @param bcz boundary condition
-     * @param global 3D vector containing boundary values
-     * @param scal_left left scaling factor
-     * @param scal_right right scaling factor
-     */
+    ///@copydoc FieldAligned::set_boundaries(dg::bc,const container&,double,double)
     void set_boundaries( dg::bc bcz, const container& global, double scal_left, double scal_right)
     {
-        f_.set_boundaries( bcz, global, scal_left, scal_right);
+        m_fa.set_boundaries( bcz, global, scal_left, scal_right);
     }
 
-    const container& weights()const {return vol3d;}
-    const container& inv_weights()const {return inv3d;}
-    const container& precond()const {return inv3d;}
+    const container& weights()const {return m_vol3d;}
+    const container& inv_weights()const {return m_inv3d;}
+    const container& precond()const {return m_inv3d;}
 
     /**
     * @brief access the underlying Fielaligned object for evaluate
     *
     * @return acces to fieldaligned object
     */
-    const dg::geo::FieldAligned<ProductGeometry, IMatrix, container>& fieldaligned() const{return f_;}
+    const Fieldaligned<ProductGeometry, IMatrix, container>& fieldaligned() const{return m_fa;}
     private:
     void do_forward(double alpha, const container& f, double beta, container& dsf);
     void do_backward(double alpha, const container& f, double beta, container& dsf);
@@ -204,7 +179,7 @@ struct DS
     void do_centeredAdj(double alpha, const container& f, double beta, container& dsf, dg::norm no);
     void do_symv(const container& f, container& dsf);
 
-    FieldAligned<ProductGeometry, IMatrix, container> m_fa;
+    Fieldaligned<ProductGeometry, IMatrix, container> m_fa;
     container m_temp;
     container m_tempP, m_temp0, m_tempM;
     container m_vol3d, m_inv3d, m_weights_wo_vol;
@@ -217,24 +192,24 @@ struct DS
 ////////////////////////////////////DEFINITIONS////////////////////////////////////////
 
 template<class Geometry, class I, class M, class container>
-void DS<Geometry, I, M,container>::construct(const Fieldaligned<Geometry, I, M, container>& fa, dg::norm no, dg::direction dir, bool jumpX, bool jumpY)
+void DS<Geometry, I, M,container>::construct(const Fieldaligned<Geometry, I, container>& fa, dg::norm no, dg::direction dir)
 {
     m_fa=fa;
-    m_no=no, m_dir=dir, m_apply_jumpX=jumpX, m_apply_jumpY=jumpY;
+    m_no=no, m_dir=dir;
 
     dg::blas1::transfer( dg::create::volume(     fa.grid()), m_vol3d); 
     dg::blas1::transfer( dg::create::weights(    fa.grid()), m_weights_wo_vol); 
     dg::blas1::transfer( dg::create::inv_volume( fa.grid()), m_inv3d); 
-    dg::blas2::transfer( dg::create::jumpX( fa.grid()), jumpX);
-    dg::blas2::transfer( dg::create::jumpY( fa.grid()), jumpY);
+    dg::blas2::transfer( dg::create::jumpX( fa.grid()), m_jumpX);
+    dg::blas2::transfer( dg::create::jumpY( fa.grid()), m_jumpY);
     m_temp = m_vol3d, m_tempP = m_temp, m_temp0 = m_temp, m_tempM = m_temp;
 }
 
 template<class G, class I, class M, class container>
 inline void DS<G,I,M,container>::operator()( const container& f, container& dsf) { 
-    if( dir_ == dg::centered)
+    if( m_dir == dg::centered)
         return centered( 1., f, 0., dsf);
-    else if( dir_ == dg::forward)
+    else if( m_dir == dg::forward)
         return forward( 1., f, 0., dsf);
     else
         return backward( 1., f, 0., dsf);
@@ -288,7 +263,7 @@ void DS<G,I,M,container>::do_backwardAdj( double alpha, const container& f, doub
         dg::blas1::pointwiseDot( alpha, m_inv3d, m_temp0, beta, dsf); 
 }
 template<class G, class I, class M, class container>
-void DS<G, I,M,container>::do_centeredAdj( double alpha, container& f, double beta, container& dsf, dg::norm no)
+void DS<G, I,M,container>::do_centeredAdj( double alpha, const container& f, double beta, container& dsf, dg::norm no)
 {               
     //adjoint discretisation
     dg::blas1::pointwiseDot( m_vol3d, f, m_temp0);
@@ -319,23 +294,24 @@ void DS<G,I,M,container>::do_symv( const container& f, container& dsTdsf)
     dg::blas1::pointwiseDivide( dsTdsf, m_weights_wo_vol, dsTdsf);
     //     add jump term 
     if(m_fa.dependsOnX())
-        dg::blas2::symv( -1., jumpX, f, 1., dsTdsf);
+        dg::blas2::symv( -1., m_jumpX, f, 1., dsTdsf);
     if(m_fa.dependsOnY())
-        dg::blas2::symv( -1., jumpY, f, 1., dsTdsf);
+        dg::blas2::symv( -1., m_jumpY, f, 1., dsTdsf);
     dg::blas1::pointwiseDot( m_weights_wo_vol, dsTdsf, dsTdsf); //make it symmetric
     if( m_no == dg::normed)
         dg::blas1::pointwiseDot( m_inv3d, dsTdsf, dsTdsf); //make it symmetric
 }
+///@endcond
+
+
+}//namespace geo
 
-//enables the use of the dg::blas2::symv function 
+///@cond
 template< class G, class I, class M, class V>
-struct MatrixTraits< DS<G,I,M, V> >
+struct MatrixTraits< geo::DS<G,I,M, V> >
 {
     typedef double value_type;
     typedef SelfMadeMatrixTag matrix_category;
 };
-
 ///@endcond
-
-}//namespace geo
 }//namespace dg
diff --git a/inc/geometries/ds_b.cu b/inc/geometries/ds_b.cu
index 450f69d5c..5413ce2f1 100644
--- a/inc/geometries/ds_b.cu
+++ b/inc/geometries/ds_b.cu
@@ -2,16 +2,18 @@
 
 #include <cusp/print.h>
 
+#include "dg/backend/functions.h"
+#include "dg/backend/timer.cuh"
 #include "dg/blas.h"
 #include "dg/functors.h"
 #include "dg/geometry/geometry.h"
 #include "ds.h"
 #include "solovev.h"
 #include "flux.h"
+#include "toroidal.h"
 
-#include "backend/functions.h"
-#include "backend/timer.cuh"
 
+const double R_0 = 10;
 double func(double R, double Z, double phi)
 {
     double r2 = (R-R_0)*(R-R_0)+Z*Z;
@@ -23,7 +25,7 @@ double deri(double R, double Z, double phi)
     return I_0/R/sqrt(I_0*I_0 + r2)* r2*cos(phi);
 }
 
-int main()
+int main(int argc, char * argv[])
 {
     std::cout << "First test the cylindrical version\n";
     std::cout << "Type n, Nx, Ny, Nz\n";
@@ -40,17 +42,14 @@ int main()
         std::ifstream is(argv[1]);
         reader.parse(is,js,false);
     }
-    dg::geo::solovev::GeomParameters gp(js);
-    dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField( gp);
-    dg::geo::BHatR bhatR(mag);
-    dg::geo::BHatZ bhatZ(mag);
-    dg::geo::BHatP bhatP(mag);
-    dg::geo::BinaryVectorLvl0 bhat( bhatR, bhatZ, bhatP);
+    dg::geo::solovev::Parameters gp(js);
+    dg::geo::TokamakMagneticField mag = dg::geo::createCircularField( gp);
+    dg::geo::BinaryVectorLvl0 bhat( (dg::geo::BHatR)(mag), (dg::geo::BHatZ)(mag), (dg::geo::BHatP)(mag));
     dg::CylindricalGrid3d g3d( gp.R_0 - 1, gp.R_0+1, -1, 1, 0, 2.*M_PI, n, Nx, Ny, Nz, dg::NEU, dg::NEU, dg::PER);
     const dg::DVec vol3d = dg::create::volume( g3d);
     dg::Timer t;
     t.tic();
-    dg::geo::FieldAligned<dg::aProductGeometry,dg::IDMatrix, dg::DMatrix>  dsFA( bhat, g3d, 2,2,1e-10, dg::NoLimiter(), dg::NEU, dg::NEU);
+    dg::geo::Fieldaligned<dg::aProductGeometry3d,dg::IDMatrix,dg::DMatrix>  dsFA( bhat, g3d, 2,2,true,true,1e-10, dg::geo::NoLimiter(), dg::NEU, dg::NEU);
 
     dg::geo::DS<dg::aProductGeometry3d, dg::IDMatrix, dg::DMatrix, dg::DVec> ds( dsFA, dg::not_normed, dg::centered);
     t.toc();
@@ -93,7 +92,7 @@ int main()
 
     dg::geo::FluxGenerator flux( mag.get_psip(), mag.get_ipol(), psi_0, psi_1, gp.R_0, 0., 1);
     dg::geo::CurvilinearProductGrid3d g3d(flux, n, Nx, Ny,Nz, dg::DIR);
-    //dg::geo::FieldAligned<dg::aGeometry3d, dg::IHMatrix, dg::HVec> fieldaligned( bhat, g3d, 1, 4, gp.rk4eps, dg::NoLimiter() ); 
+    //dg::geo::Fieldaligned<dg::aGeometry3d, dg::IHMatrix, dg::HVec> fieldaligned( bhat, g3d, 1, 4, gp.rk4eps, dg::NoLimiter() ); 
     dg::geo::DS<dg::aProductGeometry3d, dg::IHMatrix, dg::HMatrix, dg::HVec> ds( mag, g3d, mx, my, 1e-8, dg::normed, dg::centered, false, true);
 
     
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index 5d6f9dd87..7e0e9c950 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -98,7 +98,7 @@ struct DSFieldCylindrical
 struct DSField
 {
     //z component of v may not vanish
-    DSField( const dg::geo::BinaryVectorLvl0& v, const aGeometry2d& g): g_(g)
+    DSField( const dg::geo::BinaryVectorLvl0& v, const dg::aGeometry2d& g): g_(g)
     {
         thrust::host_vector<double> v_zeta, v_eta;
         dg::pushForwardPerp( v.x(), v.y(), v_zeta, v_eta, g);
@@ -146,30 +146,30 @@ struct DSField
     }
     private:
     thrust::host_vector<double> dzetadphi_, detadphi_, dsdphi_;
-    dg::Handle<aGeometry2d> g_;
+    dg::Handle<dg::aGeometry2d> g_;
 
 };
 
-void clip_to_boundary( double& x, double& y, const aTopology2d* grid)
+void clip_to_boundary( double& x, double& y, const dg::aTopology2d* grid)
 {
     if (!(x > grid->x0())) { x=grid->x0();}
     if (!(x < grid->x1())) { x=grid->x1();}
     if (!(y > grid->y0())) { y=grid->y0();}
     if (!(y < grid->y1())) { y=grid->y1();}
 }
-void clip_to_boundary( thrust::host_vector<double>& x, const aTopology2d* grid)
+void clip_to_boundary( thrust::host_vector<double>& x, const dg::aTopology2d* grid)
 {
     clip_to_boundary(x[0], x[1], grid);
 }
 
-void interpolate_and_clip( const dg::IHMatrix& interpolate, const aTopolgy2d* boundary_ptr,
+void interpolate_and_clip( const dg::IHMatrix& interpolate, const dg::aTopology2d* g2dFine, const dg::aTopology2d* boundary_ptr, //2 different grid on account of the MPI implementation
         const std::vector<thrust::host_vector<double> >& yp_coarse,
         const std::vector<thrust::host_vector<double> >& ym_coarse,
-        std::vector<thrust::host_vector<double> >& yp_,
-        std::vector<thrust::host_vector<double> >& ym_
+        std::vector<thrust::host_vector<double> >& yp_fine,
+        std::vector<thrust::host_vector<double> >& ym_fine
         )
 {
-    std::vector<thrust::host_vector<double> > yp( 3, dg::evaluate(dg::zero, g2dFine)), ym(yp); 
+    std::vector<thrust::host_vector<double> > yp( 3, dg::evaluate(dg::zero, *g2dFine)), ym(yp); 
     for( unsigned i=0; i<3; i++)
     {
         dg::blas2::symv( interpolate, yp_coarse[i], yp[i]);
@@ -182,7 +182,7 @@ void interpolate_and_clip( const dg::IHMatrix& interpolate, const aTopolgy2d* bo
         detail::clip_to_boundary( yp[0][i], yp[1][i], boundary_ptr);
         detail::clip_to_boundary( ym[0][i], ym[1][i], boundary_ptr);
     }
-    yp_=yp, ym_=ym;
+    yp_fine=yp, ym_fine=ym;
 }
 
 /**
@@ -278,8 +278,8 @@ void boxintegrator( const Field& field, const Topology& grid,
     }
 }
 
-//used in constructor of FieldAligned
-void integrate_all_fieldlines2d( const dg::geo::BinaryVectorLvl0& vec, const aGeometry2d* g2dField_ptr, const aTopology2d* evaluate_ptr, std::vector<thrust::host_vector<double> >& yp_result, std::vector<thrust::host_vector<double> >& ym_result , double deltaPhi, double eps)
+//used in constructor of Fieldaligned
+void integrate_all_fieldlines2d( const dg::geo::BinaryVectorLvl0& vec, const dg::aGeometry2d* g2dField_ptr, const dg::aTopology2d* evaluate_ptr, std::vector<thrust::host_vector<double> >& yp_result, std::vector<thrust::host_vector<double> >& ym_result , double deltaPhi, double eps)
 {
     //g2dField contains the global geometry 
     //evaluate_ptr contains the points to actually integrate
@@ -288,13 +288,13 @@ void integrate_all_fieldlines2d( const dg::geo::BinaryVectorLvl0& vec, const aGe
     y[2] = dg::evaluate( dg::zero, *evaluate_ptr); //s
     std::vector<thrust::host_vector<double> > yp( 3, y[0]), ym(yp); 
     //construct field on high polynomial grid, then integrate it
-    Timer t;
+    dg::Timer t;
     t.tic();
-    dg::detail::DSField field( vec, *g2dField_ptr);
+    dg::geo::detail::DSField field( vec, *g2dField_ptr);
     t.toc();
     std::cout << "Generation of interpolate grid took "<<t.diff()<<"s\n";
     //field in case of cartesian grid
-    dg::detail::DSFieldCylindrical cyl_field(vec);
+    dg::geo::detail::DSFieldCylindrical cyl_field(vec);
     for( unsigned i=0; i<g2dField_ptr->size(); i++)
     {
         thrust::host_vector<double> coords(3), coordsP(3), coordsM(3);
@@ -316,37 +316,11 @@ void integrate_all_fieldlines2d( const dg::geo::BinaryVectorLvl0& vec, const aGe
     ym_result=ym;
 }
 
-aGeometry2d* clone_3d_to_perp( const aGeometry3d* grid_ptr)
-{
-    //%%%%%%%%%%%downcast grid since we don't have a virtual function perp_grid%%%%%%%%%%%%%
-    const dg::CartesianGrid3d* grid_cart = dynamic_cast<const dg::CartesianGrid3d*>(grid_ptr);
-    const dg::CylindricalGrid3d* grid_cyl = dynamic_cast<const dg::CylindricalGrid3d*>(grid_ptr);
-    const dg::geo::CurvilinearProductGrid3d*  grid_curvi = dynamic_cast<const dg::geo::CurvilinearProductGrid3d*>(grid_ptr);
-    aGeometry2d* g2d_ptr;
-    if( grid_cart) 
-    {
-        dg::CartesianGrid2d cart = grid_cart->perp_grid();
-        g2d_ptr = cart.clone();
-    }
-    else if( grid_cyl) 
-    {
-        dg::CartesianGrid2d cart = grid_cyl->perp_grid();
-        g2d_ptr = cart.clone();
-    }
-    else if( grid_curvi) 
-    {
-        dg::geo::CurvilinearGrid2d curv = grid_curvi->perp_grid();
-        g2d_ptr = curv.clone();
-    }
-    else
-        throw dg::Error( dg::Message(_ping_)<<"Grid class not recognized!");
-    return g2d_ptr;
-}
 }//namespace detail
 ///@endcond
 
 
-//////////////////////////////FieldAlignedCLASS////////////////////////////////////////////
+//////////////////////////////FieldalignedCLASS////////////////////////////////////////////
 /**
 * @brief Create and manage interpolation matrices from fieldline integration
 *
@@ -359,16 +333,16 @@ aGeometry2d* clone_3d_to_perp( const aGeometry3d* grid_ptr)
 * @sa The pdf <a href="./parallel.pdf" target="_blank">parallel derivative</a> writeup 
 */
 template<class ProductGeometry, class IMatrix, class container >
-struct FieldAligned
+struct Fieldaligned
 {
 
     typedef IMatrix InterpolationMatrix; //!< typdef to reveal the interpolation matrix
     ///@brief do not allocate memory; no member call except construct is valid
-    FieldAligned(){}
+    Fieldaligned(){}
 
     ///@copydoc construct()
     template <class Limiter>
-    FieldAligned(const dg::geo::BinaryVectorLvl0& vec, const ProductGeometry& grid, unsigned multiplyX, unsigned multiplyY, bool dependsOnX, bool dependsOnY, double eps = 1e-5, dg::bc globalbcx = dg::NEU, dg::bc globalbcy = dg::NEU, Limiter limit = FullLimiter(), double deltaPhi = -1)
+    Fieldaligned(const dg::geo::BinaryVectorLvl0& vec, const ProductGeometry& grid, unsigned multiplyX, unsigned multiplyY, bool dependsOnX, bool dependsOnY, double eps = 1e-5, dg::bc globalbcx = dg::NEU, dg::bc globalbcy = dg::NEU, Limiter limit = FullLimiter(), double deltaPhi = -1)
     {
         construct( vec, grid, multiplyX, multiplyY, dependsOnX, dependsOnY, eps, limit, globalbcx, globalbcy, limit, deltaPhi);
     }
@@ -406,8 +380,8 @@ struct FieldAligned
     * if Dirichlet boundaries are used the left value is the left function
     value, if Neumann boundaries are used the left value is the left derivative value
     * @param bcz boundary condition
-    * @param left left boundary value
-    * @param right right boundary value
+    * @param left constant left boundary value
+    * @param right constant right boundary value
     */
     void set_boundaries( dg::bc bcz, double left, double right)
     {
@@ -417,7 +391,15 @@ struct FieldAligned
         m_right = dg::evaluate( dg::CONSTANT(right),g2d);
     }
 
-    ///@copydoc set_boundaries()
+    /**
+    * @brief Set boundary conditions in the limiter region
+    *
+    * if Dirichlet boundaries are used the left value is the left function
+    value, if Neumann boundaries are used the left value is the left derivative value
+    * @param bcz boundary condition
+    * @param left spatially variable left boundary value
+    * @param right spatially variable right boundary value
+    */
     void set_boundaries( dg::bc bcz, const container& left, const container& right)
     {
         m_bcz = bcz;
@@ -449,13 +431,13 @@ struct FieldAligned
      * Evaluates the given functor on a 2d plane and then follows fieldlines to
      * get the values in the 3rd dimension. Uses the grid given in the constructor.
      * @tparam BinaryOp Binary Functor
-     * @param f Functor to evaluate
-     * @param plane The number of the plane to start
+     * @param binary Functor to evaluate
+     * @param p0 The number of the plane to start
      *
      * @return Returns an instance of container
      */
     template< class BinaryOp>
-    container evaluate( const BinaryOp& f, unsigned plane=0) const
+    container evaluate( const BinaryOp& binary, unsigned p0=0) const
     {
         return evaluate( binary, dg::CONSTANT(1), p0, 0);
     }
@@ -469,8 +451,8 @@ struct FieldAligned
      * The fieldlines are assumed to be periodic.
      * @tparam BinaryOp Binary Functor
      * @tparam UnaryOp Unary Functor
-     * @param f Functor to evaluate in x-y
-     * @param g Functor to evaluate in z
+     * @param binary Functor to evaluate in x-y
+     * @param unary Functor to evaluate in z
      * @param p0 The number of the plane to start
      * @param rounds The number of rounds to follow a fieldline
      * @note g is evaluated such that p0 corresponds to z=0, p0+1 corresponds to z=hz, p0-1 to z=-hz, ...
@@ -478,7 +460,7 @@ struct FieldAligned
      * @return Returns an instance of container
      */
     template< class BinaryOp, class UnaryOp>
-    container evaluate( const BinaryOp& f, const UnaryOp& g, unsigned p0, unsigned rounds) const;
+    container evaluate( const BinaryOp& binary, const UnaryOp& unary, unsigned p0, unsigned rounds) const;
 
     /**
     * @brief Applies the interpolation 
@@ -490,24 +472,26 @@ struct FieldAligned
 
     ///@brief hz is the distance between the plus and minus planes
     ///@return three-dimensional vector
-    const container& hz_inv()const {return hz_inv;}
+    const container& hz_inv()const {return m_hz_inv;}
     ///@brief hp is the distance between the plus and current planes
     ///@return three-dimensional vector
-    const container& hp_inv()const {return hp_inv;}
+    const container& hp_inv()const {return m_hp_inv;}
     ///@brief hm is the distance between the current and minus planes
     ///@return three-dimensional vector
-    const container& hm_inv()const {return hm_inv;}
-    const ProductGeoemtry& grid()const{return m_g.get();}
+    const container& hm_inv()const {return m_hm_inv;}
+    const ProductGeometry& grid()const{return m_g.get();}
     private:
     void ePlus( enum whichMatrix which, const container& in, container& out);
     void eMinus(enum whichMatrix which, const container& in, container& out);
-    IMatrix m_plus, m_minus, m_plusT, m_minusT; //interpolation matrices
-    container m_hz_inv, m_hp_inv, m_hm_inv, m_ghostM, m_ghostP;
-    unsigned m_Nz, m_perp_size;
+    IMatrix m_plus, m_minus, m_plusT, m_minusT; //2d interpolation matrices
+    container m_hz_inv, m_hp_inv, m_hm_inv; //3d size
+    container m_hp, m_hm; //2d size
+    container m_left, m_right;      //perp_size
+    container m_limiter;            //perp_size
+    container m_ghostM, m_ghostP;   //perp_size
+    unsigned m_Nz, m_perp_size; 
     dg::bc m_bcz;
-    container m_left, m_right;
-    container m_limiter;
-    std::vector<container> m_temp;
+    std::vector<container> m_f, m_temp; //split 3d vectors
     dg::Handle<ProductGeometry> m_g;
     bool m_dependsOnX, m_dependsOnY;
 };
@@ -519,23 +503,24 @@ struct FieldAligned
 
 template<class Geometry, class IMatrix, class container>
 template <class Limiter>
-void FieldAligned<Geometry, IMatrix, container>::construct(const dg::geo::BinaryVectorLvl0& vec, const Geometry& grid, unsigned mx, unsigned my, bool bx, bool by, double eps, dg::bc globalbcx, dg::bc globalbcy, Limiter limit, double deltaPhi):
+void Fieldaligned<Geometry, IMatrix, container>::construct(const dg::geo::BinaryVectorLvl0& vec, const Geometry& grid, unsigned mx, unsigned my, bool bx, bool by, double eps, dg::bc globalbcx, dg::bc globalbcy, Limiter limit, double deltaPhi)
 {
     m_dependsOnX=bx, m_dependsOnY=by;
-    dg::blas1::transfer( dg::evaluate( dg::zero, grid), m_hz_inv), m_hp_inv= m_hz_inv, m_hm_inv= m_hz_inv;
     m_Nz=grid.Nz(), m_bcz=grid.bcz(); 
     m_g=grid;
+    dg::blas1::transfer( dg::evaluate( dg::zero, grid), m_hz_inv), m_hp_inv= m_hz_inv, m_hm_inv= m_hz_inv;
     dg::split( m_hz_inv, m_temp);
+    dg::split( m_hz_inv, m_f);
     if( deltaPhi <=0) deltaPhi = grid.hz();
     else assert( grid.Nz() == 1 || grid.hz()==deltaPhi);
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-    const aGeometry2d* grid2d_ptr = grid->perp_grid();
+    const dg::aGeometry2d* grid2d_ptr = grid->perp_grid();
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     //Resize vector to 2D grid size
     m_perp_size = grid2d_ptr->size();
     m_limiter = dg::pullback( limit, *grid2d_ptr);
     m_right = m_left = dg::evaluate( zero, *grid2d_ptr);
-    ghostM.resize( m_perp_size); ghostP.resize( m_perp_size);
+    m_ghostM.resize( m_perp_size); m_ghostP.resize( m_perp_size);
     //%%%%%%%%%%%%%%%%%%%%%%%%%%Set starting points and integrate field lines%%%%%%%%%%%%%%
     std::cout << "Start fieldline integration!\n";
     dg::Timer t;
@@ -549,7 +534,7 @@ void FieldAligned<Geometry, IMatrix, container>::construct(const dg::geo::Binary
     dg::Grid2d g2dFine((dg::Grid2d(*grid2d_ptr)));//FINE GRID
     g2dFine.multiplyCellNumbers((double)mx, (double)my);
     dg::IHMatrix interpolate = dg::create::interpolation( g2dFine, *g2dField_ptr);  //INTERPOLATE TO FINE GRID
-    interpolate_and_clip( interpolate, grid2d_ptr, yp_coarse, ym_coarse, yp, ym);
+    dg::geo::detail::interpolate_and_clip( interpolate, &g2dFine, &g2dFine, yp_coarse, ym_coarse, yp, ym);
     delete g2dField_ptr;
     t.toc(); 
     std::cout << "Fieldline integration took "<<t.diff()<<"s\n";
@@ -566,43 +551,47 @@ void FieldAligned<Geometry, IMatrix, container>::construct(const dg::geo::Binary
     t.toc();
     std::cout<< "Multiplication of P*I took: "<<t.diff()<<"s\n";
     //%Transposed matrices work only for csr_matrix due to bad matrix form for ell_matrix!!!
-    dg::transpose( plus, plusT);
-    dg::transpose( minus, minusT);     
+    plusT = dg::transpose( plus);
+    minusT = dg::transpose( minus);     
     dg::blas2::transfer( plus, m_plus);
     dg::blas2::transfer( plusT, m_plusT);
     dg::blas2::transfer( minus, m_minus);
     dg::blas2::transfer( minusT, m_minusT);
     //%%%%%%%%%%%%%%%%%%%%%%%project h and copy into h vectors%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-    thrust::host_vector<double> hp( m_perp_size), hm(hp);
+    thrust::host_vector<double> hp( m_perp_size), hm(hp), hz(hp);
     dg::blas2::symv( projection, yp[2], hp);
     dg::blas2::symv( projection, ym[2], hm);
-    for( unsigned i=0; i<grid.Nz(); i++)
-    {
-        thrust::copy( hp.begin(), hp.end(), hp_.begin() + i*perp_size_);
-        thrust::copy( hm.begin(), hm.end(), hm_.begin() + i*perp_size_);
-    }
-    dg::blas1::scal( hm_, -1.);
-    dg::blas1::axpby(  1., hp_, +1., hm_, hz_);
+    dg::blas1::scal( hm, -1.);
+    dg::blas1::axpby(  1., hp, +1., hm, hz);
+    dg::blas1::transfer( hp, m_hp);
+    dg::blas1::transfer( hm, m_hm);
+    dg::blas1::transform( hp, hp, dg::INVERT<double>());
+    dg::blas1::transform( hm, hm, dg::INVERT<double>());
+    dg::blas1::transform( hz, hz, dg::INVERT<double>());
+    dg::join( std::vector<thrust::host_vector<double> >( m_Nz, hp), m_hp_inv);
+    dg::join( std::vector<thrust::host_vector<double> >( m_Nz, hm), m_hm_inv);
+    dg::join( std::vector<thrust::host_vector<double> >( m_Nz, hz), m_hz_inv);
+    
     delete grid2d_ptr;
 }
 
 template<class G, class I, class container>
 template< class BinaryOp, class UnaryOp>
-container FieldAligned<G, I,container>::evaluate( const BinaryOp& binary, const UnaryOp& unary, unsigned p0, unsigned rounds) const
+container Fieldaligned<G, I,container>::evaluate( const BinaryOp& binary, const UnaryOp& unary, unsigned p0, unsigned rounds) const
 {
     //idea: simply apply I+/I- enough times on the init2d vector to get the result in each plane
     //unary function is always such that the p0 plane is at x=0
     assert( p0 < m_g.get().Nz());
-    const aGeometry2d* g2d_ptr = m_g.get().perp_grid();
+    const dg::aGeometry2d* g2d_ptr = m_g.get().perp_grid();
     container init2d = dg::pullback( binary, *g2d_ptr); 
     delete g2d_ptr;
 
     container temp(init2d), tempP(init2d), tempM(init2d);
     container vec3d = dg::evaluate( dg::zero, m_g.get());
     std::vector<container>  plus2d, minus2d, result; 
-    dg::split( vec3d, plus2d);
-    dg::split( vec3d, minus2d);
-    dg::split( vec3d, result);
+    dg::split( vec3d, plus2d, m_g.get());
+    dg::split( vec3d, minus2d, m_g.get());
+    dg::split( vec3d, result, m_g.get());
     unsigned turns = rounds; 
     if( turns ==0) turns++;
     //first apply Interpolation many times, scale and store results
@@ -614,9 +603,9 @@ container FieldAligned<G, I,container>::evaluate( const BinaryOp& binary, const
             unsigned rep = i0 + r*m_Nz;
             for(unsigned k=0; k<rep; k++)
             {
-                dg::blas2::symv( plus, tempP, temp);
+                dg::blas2::symv( m_plus, tempP, temp);
                 temp.swap( tempP);
-                dg::blas2::symv( minus, tempM, temp);
+                dg::blas2::symv( m_minus, tempM, temp);
                 temp.swap( tempM);
             }
             dg::blas1::scal( tempP, unary(  (double)rep*m_g.get().hz() ) );
@@ -657,22 +646,22 @@ container FieldAligned<G, I,container>::evaluate( const BinaryOp& binary, const
 
 
 template<class G, class I, class container>
-void FieldAligned<G, I, container >::operator()(enum whichMatrix which, const container& f, container& fe)
+void Fieldaligned<G, I, container >::operator()(enum whichMatrix which, const container& f, container& fe)
 {
     if(which == einsPlus  || which == einsMinusT) ePlus(  which, f, fe);
     if(which == einsMinus || which == einsPlusT ) eMinus( which, f, fe);
 }
 
 template< class G, class I, class container>
-void FieldAligned<G, I, container>::ePlus( enum whichMatrix which, const container& f, container& fpe)
+void Fieldaligned<G, I, container>::ePlus( enum whichMatrix which, const container& f, container& fpe)
 {
-    dg::spit( f, m_f);
+    dg::split( f, m_f, m_g.get());
     //1. compute 2d interpolation in every plane and store in m_temp
     for( unsigned i0=0; i0<m_Nz; i0++)
     {
         unsigned ip = (i0==m_Nz-1) ? 0:i0+1;
-        if(which == einsPlus)           dg::blas2::symv( plus,   m_f[ip], m_temp[i0]);
-        else if(which == einsMinusT)    dg::blas2::symv( minusT, m_f[ip], m_temp[i0]);
+        if(which == einsPlus)           dg::blas2::symv( m_plus,   m_f[ip], m_temp[i0]);
+        else if(which == einsMinusT)    dg::blas2::symv( m_minusT, m_f[ip], m_temp[i0]);
     }
     //2. apply right boundary conditions in last plane
     unsigned i0=m_Nz-1;
@@ -682,26 +671,26 @@ void FieldAligned<G, I, container>::ePlus( enum whichMatrix which, const contain
             dg::blas1::axpby( 2, m_right, -1., m_f[i0], m_ghostP);
         if( m_bcz == dg::NEU || m_bcz == dg::DIR_NEU)
         {
-            dg::blas1::pointwiseDivide( m_right, m_hp_inv[i0], m_ghostP);
+            dg::blas1::pointwiseDot( m_right, m_hp, m_ghostP);
             dg::blas1::axpby( 1., m_ghostP, 1., m_f[i0], m_ghostP);
         }
         //interlay ghostcells with periodic cells: L*g + (1-L)*fpe
         dg::blas1::axpby( 1., m_ghostP, -1., m_temp[i0], m_ghostP);
         dg::blas1::pointwiseDot( 1., m_limiter, m_ghostP, 1., m_temp[i0]);
     }
-    dg::join( m_temp, fpe);
+    dg::join( m_temp, fpe, m_g.get());
 }
 
 template< class G, class I, class container>
-void FieldAligned<G, I, container>::eMinus( enum whichMatrix which, const container& f, container& fme)
+void Fieldaligned<G, I, container>::eMinus( enum whichMatrix which, const container& f, container& fme)
 {
-    dg::split( f, m_f);
+    dg::split( f, m_f, m_g.get());
     //1. compute 2d interpolation in every plane and store in m_temp
     for( unsigned i0=0; i0<m_Nz; i0++)
     {
         unsigned im = (i0==0) ? m_Nz-1:i0-1;
-        if(which == einsPlusT)          dg::blas2::symv( plusT, m_f[im], m_temp[i0]);
-        else if (which == einsMinus)    dg::blas2::symv( minus, m_f[im], m_temp[i0]);
+        if(which == einsPlusT)          dg::blas2::symv( m_plusT, m_f[im], m_temp[i0]);
+        else if (which == einsMinus)    dg::blas2::symv( m_minus, m_f[im], m_temp[i0]);
     }
     //2. apply left boundary conditions in first plane
     unsigned i0=0;
@@ -711,14 +700,14 @@ void FieldAligned<G, I, container>::eMinus( enum whichMatrix which, const contai
             dg::blas1::axpby( 2., m_left,  -1., m_f[i0], m_ghostM);
         if( m_bcz == dg::NEU || m_bcz == dg::NEU_DIR)
         {
-            dg::blas1::pointwiseDivide( m_left, m_hm_inv[i0], m_ghostM);
+            dg::blas1::pointwiseDot( m_left, m_hm, m_ghostM);
             dg::blas1::axpby( -1., m_ghostM, 1., m_f[i0], m_ghostM);
         }
         //interlay ghostcells with periodic cells: L*g + (1-L)*fme
         dg::blas1::axpby( 1., m_ghostM, -1., m_temp[i0], m_ghostM);
         dg::blas1::pointwiseDot( 1., m_limiter, m_ghostM, 1., m_temp[i0]);
     }
-    dg::join( m_temp, fme);
+    dg::join( m_temp, fme, m_g.get());
 }
 
 
diff --git a/inc/geometries/geometries_doc.h b/inc/geometries/geometries_doc.h
index 6e7030bec..69ce6d7ab 100644
--- a/inc/geometries/geometries_doc.h
+++ b/inc/geometries/geometries_doc.h
@@ -15,7 +15,8 @@
         @defgroup solovev The solovev magnetic field
         @defgroup taylor The Taylor state magnetic field
         @defgroup guenther The Guenther magnetic field
-        @defgroup toroidal The purely toroidal magnetic field
+        @defgroup toroidal The Purely Toroidal magnetic field
+        @defgroup circular The Circular magnetic field
       @}
       @defgroup magnetic 3.2 magnetic field and associated functors
       @defgroup profiles 3.3 miscellaneous functors based on flux functions
diff --git a/inc/geometries/guenther.h b/inc/geometries/guenther.h
index 5bc0390a9..83209f6a6 100644
--- a/inc/geometries/guenther.h
+++ b/inc/geometries/guenther.h
@@ -247,7 +247,7 @@ struct GradLnB
 struct Field
 {
      Field( double R0, double I0):  R_0(R0), I_0(I0){}
-     Field( GeomParameters gp ):  R_0(gp.R_0), I_0(gp.I_0){}
+     Field( Parameters gp ):  R_0(gp.R_0), I_0(gp.I_0){}
     void operator()( const std::vector<thrust::host_vector<double> >& y, std::vector<thrust::host_vector<double> >& yp) const
     {
         for( unsigned i=0; i<y[0].size(); i++)
diff --git a/inc/geometries/guenther_parameters.h b/inc/geometries/guenther_parameters.h
index 563cb1bf8..8b3d71db5 100644
--- a/inc/geometries/guenther_parameters.h
+++ b/inc/geometries/guenther_parameters.h
@@ -13,9 +13,8 @@ namespace guenther
 {
 /**
  * @brief Constructs and display geometric parameters for the guenther field
- * @ingroup geom
  */    
-struct GeomParameters
+struct Parameters
 {
     double I_0, //!< the current
            R_0, //!< central tokamak radius
@@ -29,7 +28,7 @@ struct GeomParameters
            psipmaxcut, //!< for cutting
            psipmaxlim; //!< for limiter
     std::vector<double> c;  //!< coefficients for the solovev equilibrium
-    GeomParameters( const Json::Value& js) {
+    Parameters( const Json::Value& js) {
         I_0  = js["I_0"].asDouble();
         R_0  = js["R_0"].asDouble();
         a  = R_0*js["inverseaspectratio"].asDouble();
diff --git a/inc/geometries/mpi_fieldaligned.h b/inc/geometries/mpi_fieldaligned.h
index e9171e509..eef6dd90e 100644
--- a/inc/geometries/mpi_fieldaligned.h
+++ b/inc/geometries/mpi_fieldaligned.h
@@ -52,41 +52,14 @@ void sendBackward( InputIterator begin, InputIterator end, OutputIterator result
                     source, 3, //source
                     comm, &status);
 }
-
-aMPIGeometry2d* clone_MPI3d_to_perp( const aMPIGeometry3d* grid_ptr)
-{
-    //%%%%%%%%%%%downcast grid since we don't have a virtual function perp_grid%%%%%%%%%%%%%
-    const dg::CartesianMPIGrid3d* grid_cart = dynamic_cast<const dg::CartesianMPIGrid3d*>(grid_ptr);
-    const dg::CylindricalMPIGrid3d* grid_cyl = dynamic_cast<const dg::CylindricalMPIGrid3d*>(grid_ptr);
-    const dg::geo::CurvilinearProductMPIGrid3d*  grid_curvi = dynamic_cast<const dg::geo::CurvilinearProductMPIGrid3d*>(grid_ptr);
-    aMPIGeometry2d* g2d_ptr;
-    if( grid_cart) 
-    {
-        dg::CartesianMPIGrid2d cart = grid_cart->perp_grid();
-        g2d_ptr = cart.clone();
-    }
-    else if( grid_cyl) 
-    {
-        dg::CartesianMPIGrid2d cart = grid_cyl->perp_grid();
-        g2d_ptr = cart.clone();
-    }
-    else if( grid_curvi) 
-    {
-        dg::geo::CurvilinearMPIGrid2d curv = grid_curvi->perp_grid();
-        g2d_ptr = curv.clone();
-    }
-    else
-        throw dg::Error( dg::Message(_ping_)<<"Grid class not recognized!");
-    return g2d_ptr;
-}
 }//namespace detail
 
 template <class Geometry, class LocalIMatrix, class CommunicatorXY, class LocalContainer>
-struct FieldAligned< Geometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vector<LocalContainer> > 
+struct Fieldaligned< Geometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vector<LocalContainer> > 
 {
-    FieldAligned(){}
+    Fieldaligned(){}
     template <class Limiter>
-    FieldAligned(const dg::geo::BinaryVectorLvl0& vec, const Geometry& grid, unsigned multiplyX, unsigned multiplyY, double eps = 1e-5, dg::bc globalbcx = dg::NEU, dg::bc globalbcy = dg::NEU, Limiter limit = FullLimiter(), double deltaPhi = -1)
+    Fieldaligned(const dg::geo::BinaryVectorLvl0& vec, const Geometry& grid, unsigned multiplyX, unsigned multiplyY, double eps = 1e-5, dg::bc globalbcx = dg::NEU, dg::bc globalbcy = dg::NEU, Limiter limit = FullLimiter(), double deltaPhi = -1)
     {
         construct( vec, grid, multiplyX, multiplyY, eps, limit, globalbcx, globalbcy, limit, deltaPhi);
     }
@@ -141,7 +114,7 @@ struct FieldAligned< Geometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vec
 //////////////////////////////////////DEFINITIONS/////////////////////////////////////
 template<class MPIGeometry, class LocalIMatrix, class CommunicatorXY, class LocalContainer>
 template <class Limiter>
-void FieldAligned<MPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vector<LocalContainer> >::construct(
+void Fieldaligned<MPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vector<LocalContainer> >::construct(
     const dg::geo::BinaryVectorLvl0& vec, const MPIGeometry& grid, unsigned mx, unsigned my, double eps, dg::bc globalbcx, dg::bc globalbcy, Limiter limit, double deltaPhi)
 {
     temp_(g_.Nz())
@@ -172,7 +145,7 @@ void FieldAligned<MPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vec
     dg::MPIGrid2d g2dFine((dg::MPIGrid2d(*grid2d_ptr)));//FINE GRID
     g2dFine.multiplyCellNumbers((double)mx, (double)my);
     dg::IHMatrix interpolate = dg::create::interpolation( g2dFine.local(), local_g2dField);  //INTERPOLATE TO FINE GRID
-    interpolate_and_clip( interpolate, global_g2dField_ptr, yp_coarse, ym_coarse, yp, ym);
+    dg::geo::detail::interpolate_and_clip( interpolate, g2dFine.local(), global_g2dField_ptr, yp_coarse, ym_coarse, yp, ym);
     delete g2dField_ptr;
     delete global_g2dField_ptr;
     //%%%%%%%%%%%%%%%%%%Create interpolation and projection%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -190,8 +163,8 @@ void FieldAligned<MPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vec
     //%Transposed matrices work only for csr_matrix due to bad matrix form for ell_matrix!!!
     dg::convert( plus, m_plus, *grid2d_ptr);
     dg::convert( minus, m_minus, *grid2d_ptr);
-    dg::transpose( m_plus, m_plusT);
-    dg::transpose( m_minus, m_minusT);     
+    m_plusT = dg::transpose( m_plus);
+    m_minusT = dg::transpose( m_minus);
     dg::blas2::transfer( plus, m_plus);
     dg::blas2::transfer( plusT, m_plusT);
     dg::blas2::transfer( minus, m_minus);
@@ -230,14 +203,14 @@ void FieldAligned<MPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vec
 
 template<class G, class M, class C, class container>
 template< class BinaryOp>
-MPI_Vector<container> FieldAligned<G,RowDistMat<M,C>,MPI_Vector<container> >::evaluate( BinaryOp binary, unsigned p0) const
+MPI_Vector<container> Fieldaligned<G,RowDistMat<M,C>,MPI_Vector<container> >::evaluate( BinaryOp binary, unsigned p0) const
 {
     return evaluate( binary, dg::CONSTANT(1), p0, 0);
 }
 
 template<class G, class M, class C, class container>
 template< class BinaryOp, class UnaryOp>
-MPI_Vector<container> FieldAligned<G,RowDistMat<M,C>, MPI_Vector<container> >::evaluate( BinaryOp binary, UnaryOp unary, unsigned p0, unsigned rounds) const
+MPI_Vector<container> Fieldaligned<G,RowDistMat<M,C>, MPI_Vector<container> >::evaluate( BinaryOp binary, UnaryOp unary, unsigned p0, unsigned rounds) const
 {
     //idea: simply apply I+/I- enough times on the init2d vector to get the result in each plane
     //unary function is always such that the p0 plane is at x=0
@@ -306,14 +279,14 @@ MPI_Vector<container> FieldAligned<G,RowDistMat<M,C>, MPI_Vector<container> >::e
 }
 
 template<class G, class M, class C, class container>
-void FieldAligned<G, RowDistMatr<M,C>, MPI_Vector<container> >::operator()(enum whichMatrix which, const MPI_Vector<container>& f, MPI_Vector<container>& fe)
+void Fieldaligned<G, RowDistMatr<M,C>, MPI_Vector<container> >::operator()(enum whichMatrix which, const MPI_Vector<container>& f, MPI_Vector<container>& fe)
 {
     if(which == einsPlus || which == einsMinusT) ePlus( which, f, fe);
     if(which == einsMinus || which == einsPlusT) eMinus( which, f, fe);
 }
 
 template<class G, class M, class C, class container>
-void FieldAligned<G,RowDistMat<M,C>, MPI_Vector<container> >::ePlus( enum whichMatrix which, const MPI_Vector<container>& f, MPI_Vector<container>& fpe ) 
+void Fieldaligned<G,RowDistMat<M,C>, MPI_Vector<container> >::ePlus( enum whichMatrix which, const MPI_Vector<container>& f, MPI_Vector<container>& fpe ) 
 {
     dg::split( f, m_f);
 
@@ -352,7 +325,7 @@ void FieldAligned<G,RowDistMat<M,C>, MPI_Vector<container> >::ePlus( enum whichM
 }
 
 template<class G, class M, class C, class container>
-void FieldAligned<G,RowDistMat<M,C>, MPI_Vector<container> >::eMinus( enum whichMatrix which, const MPI_Vector<container>& f, MPI_Vector<container>& fme ) 
+void Fieldaligned<G,RowDistMat<M,C>, MPI_Vector<container> >::eMinus( enum whichMatrix which, const MPI_Vector<container>& f, MPI_Vector<container>& fme ) 
 {
     dg::split( f, m_f);
 
diff --git a/inc/geometries/solovev.h b/inc/geometries/solovev.h
index 67064a127..c25d6d454 100644
--- a/inc/geometries/solovev.h
+++ b/inc/geometries/solovev.h
@@ -64,7 +64,7 @@ struct Psip: public aCloneableBinaryFunctor<Psip>
      *
      * @param gp geometric parameters
      */
-    Psip( GeomParameters gp): R_0_(gp.R_0), A_(gp.A), c_(gp.c) {}
+    Psip( Parameters gp): R_0_(gp.R_0), A_(gp.A), c_(gp.c) {}
   private:
     double do_compute(double R, double Z) const
     {
@@ -138,7 +138,7 @@ struct Psip: public aCloneableBinaryFunctor<Psip>
 struct PsipR: public aCloneableBinaryFunctor<PsipR>
 {
     ///@copydoc Psip::Psip()
-    PsipR( GeomParameters gp): R_0_(gp.R_0), A_(gp.A), c_(gp.c) {}
+    PsipR( Parameters gp): R_0_(gp.R_0), A_(gp.A), c_(gp.c) {}
   private:
     double do_compute(double R, double Z) const
     {    
@@ -179,7 +179,7 @@ struct PsipR: public aCloneableBinaryFunctor<PsipR>
 struct PsipRR: public aCloneableBinaryFunctor<PsipRR>
 {
     ///@copydoc Psip::Psip()
-    PsipRR( GeomParameters gp ): R_0_(gp.R_0), A_(gp.A), c_(gp.c) {}
+    PsipRR( Parameters gp ): R_0_(gp.R_0), A_(gp.A), c_(gp.c) {}
   private:
     double do_compute(double R, double Z) const
     {    
@@ -215,7 +215,7 @@ struct PsipRR: public aCloneableBinaryFunctor<PsipRR>
 struct PsipZ: public aCloneableBinaryFunctor<PsipZ>
 {
     ///@copydoc Psip::Psip()
-    PsipZ( GeomParameters gp ): R_0_(gp.R_0), A_(gp.A), c_(gp.c) { }
+    PsipZ( Parameters gp ): R_0_(gp.R_0), A_(gp.A), c_(gp.c) { }
   private:
     double do_compute(double R, double Z) const
     {
@@ -253,7 +253,7 @@ struct PsipZ: public aCloneableBinaryFunctor<PsipZ>
 struct PsipZZ: public aCloneableBinaryFunctor<PsipZZ>
 {
     ///@copydoc Psip::Psip()
-    PsipZZ( GeomParameters gp): R_0_(gp.R_0), A_(gp.A), c_(gp.c) { }
+    PsipZZ( Parameters gp): R_0_(gp.R_0), A_(gp.A), c_(gp.c) { }
   private:
     double do_compute(double R, double Z) const
     {
@@ -283,7 +283,7 @@ struct PsipZZ: public aCloneableBinaryFunctor<PsipZZ>
 struct PsipRZ: public aCloneableBinaryFunctor<PsipRZ>
 {
     ///@copydoc Psip::Psip()
-    PsipRZ( GeomParameters gp ): R_0_(gp.R_0), A_(gp.A), c_(gp.c) { }
+    PsipRZ( Parameters gp ): R_0_(gp.R_0), A_(gp.A), c_(gp.c) { }
   private:
     double do_compute(double R, double Z) const
     {
@@ -309,7 +309,7 @@ struct PsipRZ: public aCloneableBinaryFunctor<PsipRZ>
 struct Ipol: public aCloneableBinaryFunctor<Ipol>
 {
     ///@copydoc Psip::Psip()
-    Ipol(  GeomParameters gp ):  R_0_(gp.R_0), A_(gp.A), qampl_(gp.qampl), psip_(gp) { }
+    Ipol(  Parameters gp ):  R_0_(gp.R_0), A_(gp.A), qampl_(gp.qampl), psip_(gp) { }
   private:
     double do_compute(double R, double Z) const
     {    
@@ -325,7 +325,7 @@ struct Ipol: public aCloneableBinaryFunctor<Ipol>
 struct IpolR: public aCloneableBinaryFunctor<IpolR>
 {
     ///@copydoc Psip::Psip()
-    IpolR(  GeomParameters gp ):  R_0_(gp.R_0), A_(gp.A), qampl_(gp.qampl), psip_(gp), psipR_(gp) { }
+    IpolR(  Parameters gp ):  R_0_(gp.R_0), A_(gp.A), qampl_(gp.qampl), psip_(gp), psipR_(gp) { }
   private:
     double do_compute(double R, double Z) const
     {    
@@ -341,7 +341,7 @@ struct IpolR: public aCloneableBinaryFunctor<IpolR>
 struct IpolZ: public aCloneableBinaryFunctor<IpolZ>
 {
     ///@copydoc Psip::Psip()
-    IpolZ(  GeomParameters gp ):  R_0_(gp.R_0), A_(gp.A), qampl_(gp.qampl), psip_(gp), psipZ_(gp) { }
+    IpolZ(  Parameters gp ):  R_0_(gp.R_0), A_(gp.A), qampl_(gp.qampl), psip_(gp), psipZ_(gp) { }
   private:
     double do_compute(double R, double Z) const
     {    
@@ -352,18 +352,18 @@ struct IpolZ: public aCloneableBinaryFunctor<IpolZ>
     PsipZ psipZ_;
 };
 
-BinaryFunctorsLvl2 createPsip( GeomParameters gp)
+BinaryFunctorsLvl2 createPsip( Parameters gp)
 {
     BinaryFunctorsLvl2 psip( new Psip(gp), new PsipR(gp), new PsipZ(gp),new PsipRR(gp), new PsipRZ(gp), new PsipZZ(gp));
     return psip;
 }
-BinaryFunctorsLvl1 createIpol( GeomParameters gp)
+BinaryFunctorsLvl1 createIpol( Parameters gp)
 {
     BinaryFunctorsLvl1 ipol( new Ipol(gp), new IpolR(gp), new IpolZ(gp));
     return ipol;
 }
 
-TokamakMagneticField createMagField( GeomParameters gp)
+TokamakMagneticField createMagField( Parameters gp)
 {
     return TokamakMagneticField( gp.R_0, createPsip(gp), createIpol(gp));
 }
@@ -375,7 +375,7 @@ namespace mod
 
 struct Psip: public aCloneableBinaryFunctor<Psip>
 {
-    Psip( GeomParameters gp): R_X( gp.R_0-1.1*gp.triangularity*gp.a), Z_X(-1.1*gp.elongation*gp.a),
+    Psip( Parameters gp): R_X( gp.R_0-1.1*gp.triangularity*gp.a), Z_X(-1.1*gp.elongation*gp.a),
         psip_(gp), psipRR_(gp), psipRZ_(gp), psipZZ_(gp), cauchy_( R_X, Z_X, 50, 50,1.)
     {
         psipZZ_X_ = psipZZ_(R_X, Z_X);
@@ -401,7 +401,7 @@ struct Psip: public aCloneableBinaryFunctor<Psip>
 };
 struct PsipR: public aCloneableBinaryFunctor<PsipR>
 {
-    PsipR( GeomParameters gp): R_X( gp.R_0-1.1*gp.triangularity*gp.a), Z_X(-1.1*gp.elongation*gp.a),
+    PsipR( Parameters gp): R_X( gp.R_0-1.1*gp.triangularity*gp.a), Z_X(-1.1*gp.elongation*gp.a),
         psip_(gp), psipR_(gp), psipRR_(gp), psipRZ_(gp), psipZZ_(gp), cauchy_( R_X, Z_X, 50, 50,1.)
     {
         psipZZ_X_ = psipZZ_(R_X, Z_X);
@@ -429,7 +429,7 @@ struct PsipR: public aCloneableBinaryFunctor<PsipR>
 };
 struct PsipZ: public aCloneableBinaryFunctor<PsipZ>
 {
-    PsipZ( GeomParameters gp): R_X( gp.R_0-1.1*gp.triangularity*gp.a), Z_X(-1.1*gp.elongation*gp.a),
+    PsipZ( Parameters gp): R_X( gp.R_0-1.1*gp.triangularity*gp.a), Z_X(-1.1*gp.elongation*gp.a),
         psip_(gp), psipZ_(gp), psipRR_(gp), psipRZ_(gp), psipZZ_(gp), cauchy_( R_X, Z_X, 50, 50, 1)
     {
         psipZZ_X_ = psipZZ_(R_X, Z_X);
@@ -458,7 +458,7 @@ struct PsipZ: public aCloneableBinaryFunctor<PsipZ>
 
 struct PsipZZ: public aCloneableBinaryFunctor<PsipZZ>
 {
-    PsipZZ( GeomParameters gp): R_X( gp.R_0-1.1*gp.triangularity*gp.a), Z_X(-1.1*gp.elongation*gp.a),
+    PsipZZ( Parameters gp): R_X( gp.R_0-1.1*gp.triangularity*gp.a), Z_X(-1.1*gp.elongation*gp.a),
         psip_(gp), psipZ_(gp), psipRR_(gp), psipRZ_(gp), psipZZ_(gp), cauchy_( R_X, Z_X, 50, 50, 1)
     {
         psipZZ_X_ = psipZZ_(R_X, Z_X);
@@ -487,7 +487,7 @@ struct PsipZZ: public aCloneableBinaryFunctor<PsipZZ>
 };
 struct PsipRR: public aCloneableBinaryFunctor<PsipRR>
 {
-    PsipRR( GeomParameters gp): R_X( gp.R_0-1.1*gp.triangularity*gp.a), Z_X(-1.1*gp.elongation*gp.a),
+    PsipRR( Parameters gp): R_X( gp.R_0-1.1*gp.triangularity*gp.a), Z_X(-1.1*gp.elongation*gp.a),
         psip_(gp), psipR_(gp), psipRR_(gp), psipRZ_(gp), psipZZ_(gp), cauchy_( R_X, Z_X, 50, 50, 1)
     {
         psipZZ_X_ = psipZZ_(R_X, Z_X);
@@ -516,7 +516,7 @@ struct PsipRR: public aCloneableBinaryFunctor<PsipRR>
 };
 struct PsipRZ: public aCloneableBinaryFunctor<PsipRZ>
 {
-    PsipRZ( GeomParameters gp): R_X( gp.R_0-1.1*gp.triangularity*gp.a), Z_X(-1.1*gp.elongation*gp.a),
+    PsipRZ( Parameters gp): R_X( gp.R_0-1.1*gp.triangularity*gp.a), Z_X(-1.1*gp.elongation*gp.a),
         psip_(gp), psipR_(gp), psipZ_(gp), psipRR_(gp), psipRZ_(gp), psipZZ_(gp), cauchy_( R_X, Z_X, 50, 50, 1)
     {
         psipZZ_X_ = psipZZ_(R_X, Z_X);
@@ -560,7 +560,7 @@ struct PsipRZ: public aCloneableBinaryFunctor<PsipRZ>
  * @return A magnetic field object
  * @ingroup geom
  */
-TokamakMagneticField createSolovevField( solovev::GeomParameters gp)
+TokamakMagneticField createSolovevField( solovev::Parameters gp)
 {
     return TokamakMagneticField( gp.R_0, solovev::createPsip(gp), solovev::createIpol(gp));
 }
diff --git a/inc/geometries/solovev_parameters.h b/inc/geometries/solovev_parameters.h
index 4ea4b9097..36d107f7e 100644
--- a/inc/geometries/solovev_parameters.h
+++ b/inc/geometries/solovev_parameters.h
@@ -16,7 +16,7 @@ namespace solovev
  * @brief Constructs and display geometric parameters for the solovev and taylor fields
  * @ingroup geom
  */    
-struct GeomParameters
+struct Parameters
 {
     double A, //!< A
            R_0, //!< central tokamak radius
@@ -32,7 +32,7 @@ struct GeomParameters
            qampl; //scales grad-shafranov q factor
     std::vector<double> c;  //!< coefficients for the solovev equilibrium
     std::string equilibrium;
-    GeomParameters( const Json::Value& js) {
+    Parameters( const Json::Value& js) {
         A  = js.get("A", 0).asDouble();
         c.resize(13);//there are only 12 originially c[12] is to make fieldlines straight
         for (unsigned i=0;i<12;i++) c[i] = js["c"][i].asDouble();
diff --git a/inc/geometries/taylor.h b/inc/geometries/taylor.h
index 692a7197f..1d0a7ca87 100644
--- a/inc/geometries/taylor.h
+++ b/inc/geometries/taylor.h
@@ -33,7 +33,7 @@ namespace taylor
 {
 ///@addtogroup taylor
 ///@{
-typedef dg::geo::solovev::GeomParameters GeomParameters; //!< bring GeomParameters into the taylor namespace 
+typedef dg::geo::solovev::Parameters Parameters; //!< bring Parameters into the taylor namespace 
 
 /**
  * @brief \f[ \psi \f]
@@ -47,7 +47,7 @@ struct Psip : public aCloneableBinaryFunctor<Psip>
      *
      * @param gp geometric parameters
      */
-    Psip( solovev::GeomParameters gp): R0_(gp.R_0), c_(gp.c) {
+    Psip( solovev::Parameters gp): R0_(gp.R_0), c_(gp.c) {
         cs_ = sqrt( c_[11]*c_[11]-c_[10]*c_[10]);
     }
   private:
@@ -83,7 +83,7 @@ struct Psip : public aCloneableBinaryFunctor<Psip>
 struct PsipR: public aCloneableBinaryFunctor<PsipR>
 {
     ///@copydoc Psip::Psip()
-    PsipR( solovev::GeomParameters gp): R0_(gp.R_0), c_(gp.c) {
+    PsipR( solovev::Parameters gp): R0_(gp.R_0), c_(gp.c) {
         cs_=sqrt(c_[11]*c_[11]-c_[10]*c_[10]);
     
     }
@@ -121,7 +121,7 @@ struct PsipR: public aCloneableBinaryFunctor<PsipR>
 struct PsipRR: public aCloneableBinaryFunctor<PsipRR>
 {
     ///@copydoc Psip::Psip()
-    PsipRR( solovev::GeomParameters gp ): R0_(gp.R_0), c_(gp.c) {
+    PsipRR( solovev::Parameters gp ): R0_(gp.R_0), c_(gp.c) {
         cs_ = sqrt( c_[11]*c_[11]-c_[10]*c_[10]);
     }
   private:
@@ -154,7 +154,7 @@ struct PsipRR: public aCloneableBinaryFunctor<PsipRR>
 struct PsipZ: public aCloneableBinaryFunctor<PsipZ>
 {
     ///@copydoc Psip::Psip()
-    PsipZ( solovev::GeomParameters gp ): R0_(gp.R_0), c_(gp.c) { 
+    PsipZ( solovev::Parameters gp ): R0_(gp.R_0), c_(gp.c) { 
         cs_ = sqrt( c_[11]*c_[11]-c_[10]*c_[10]);
     }
     private:
@@ -185,7 +185,7 @@ struct PsipZ: public aCloneableBinaryFunctor<PsipZ>
 struct PsipZZ: public aCloneableBinaryFunctor<PsipZZ>
 {
     ///@copydoc Psip::Psip()
-    PsipZZ( solovev::GeomParameters gp): R0_(gp.R_0), c_(gp.c) { 
+    PsipZZ( solovev::Parameters gp): R0_(gp.R_0), c_(gp.c) { 
         cs_ = sqrt( c_[11]*c_[11]-c_[10]*c_[10]);
     }
     private:
@@ -214,7 +214,7 @@ struct PsipZZ: public aCloneableBinaryFunctor<PsipZZ>
 struct PsipRZ: public aCloneableBinaryFunctor<PsipRZ>
 {
     ///@copydoc Psip::Psip()
-    PsipRZ( solovev::GeomParameters gp ): R0_(gp.R_0), c_(gp.c) { 
+    PsipRZ( solovev::Parameters gp ): R0_(gp.R_0), c_(gp.c) { 
         cs_ = sqrt( c_[11]*c_[11]-c_[10]*c_[10]);
     }
   private:
@@ -252,7 +252,7 @@ struct PsipRZ: public aCloneableBinaryFunctor<PsipRZ>
 struct Ipol: public aCloneableBinaryFunctor<Ipol>
 {
     ///@copydoc Psip::Psip()
-    Ipol(  solovev::GeomParameters gp ): c12_(gp.c[11]), psip_(gp) { }
+    Ipol(  solovev::Parameters gp ): c12_(gp.c[11]), psip_(gp) { }
   private:
     double do_compute(double R, double Z) const
     {    
@@ -268,7 +268,7 @@ struct Ipol: public aCloneableBinaryFunctor<Ipol>
 struct IpolR: public aCloneableBinaryFunctor<IpolR>
 {
     ///@copydoc Psip::Psip()
-    IpolR(  solovev::GeomParameters gp ): c12_(gp.c[11]), psipR_(gp) { }
+    IpolR(  solovev::Parameters gp ): c12_(gp.c[11]), psipR_(gp) { }
   private:
     double do_compute(double R, double Z) const
     {    
@@ -283,7 +283,7 @@ struct IpolR: public aCloneableBinaryFunctor<IpolR>
 struct IpolZ: public aCloneableBinaryFunctor<IpolZ>
 {
     ///@copydoc Psip::Psip()
-    IpolZ(  solovev::GeomParameters gp ): c12_(gp.c[11]), psipZ_(gp) { }
+    IpolZ(  solovev::Parameters gp ): c12_(gp.c[11]), psipZ_(gp) { }
   private:
     double do_compute(double R, double Z) const
     {    
@@ -293,17 +293,17 @@ struct IpolZ: public aCloneableBinaryFunctor<IpolZ>
     PsipZ psipZ_;
 };
 
-BinaryFunctorsLvl2 createPsip( solovev::GeomParameters gp)
+BinaryFunctorsLvl2 createPsip( solovev::Parameters gp)
 {
     BinaryFunctorsLvl2 psip( new Psip(gp), new PsipR(gp), new PsipZ(gp),new PsipRR(gp), new PsipRZ(gp), new PsipZZ(gp));
     return psip;
 }
-BinaryFunctorsLvl1 createIpol( solovev::GeomParameters gp)
+BinaryFunctorsLvl1 createIpol( solovev::Parameters gp)
 {
     BinaryFunctorsLvl1 ipol( new Ipol(gp), new IpolR(gp), new IpolZ(gp));
     return ipol;
 }
-dg::geo::TokamakMagneticField createMagField( solovev::GeomParameters gp)
+dg::geo::TokamakMagneticField createMagField( solovev::Parameters gp)
 {
     return TokamakMagneticField( gp.R_0, dg::geo::taylor::createPsip(gp), dg::geo::taylor::createIpol(gp));
 }
@@ -317,7 +317,7 @@ dg::geo::TokamakMagneticField createMagField( solovev::GeomParameters gp)
  * @return A magnetic field object
  * @ingroup geom
  */
-TokamakMagneticField createTaylorField( solovev::GeomParameters gp)
+TokamakMagneticField createTaylorField( solovev::Parameters gp)
 {
     return TokamakMagneticField( gp.R_0, dg::geo::taylor::createPsip(gp), dg::geo::taylor::createIpol(gp));
 }
diff --git a/inc/geometries/toroidal.h b/inc/geometries/toroidal.h
index 239c89ae9..05e5daa93 100644
--- a/inc/geometries/toroidal.h
+++ b/inc/geometries/toroidal.h
@@ -23,7 +23,7 @@ BinaryFunctorsLvl2 createPsip( )
  */
 BinaryFunctorsLvl1 createIpol( )
 {
-    BinaryFunctorsLvl1 ipol( new Constant(1), new Constant(0), new Constant(0))
+    BinaryFunctorsLvl1 ipol( new Constant(1), new Constant(0), new Constant(0));
     return ipol;
 }
 
@@ -40,6 +40,71 @@ TokamakMagneticField createMagField( double R0)
 ///@}
 
 }//namespace toroidal
+namespace circular{
+
+///@addtogroup circular
+///@{
+/**
+ * @brief \f[ \psi_p = \frac{1}{2}\left((R-R_0)^2 + Z^2 \right) \f]
+ * gives circular flux surfaces
+ */
+struct Psip : public aCloneableBinaryFunctor<Psip>
+{ /**
+     * @brief Construct from major radius
+     * @param R0 the major radius
+     */
+    Psip( double R0): m_R0(R0) { }
+  private:
+    double do_compute(double R, double Z) const
+    {    
+        return 0.5*((R-m_R0)*(R-m_R0) + Z*Z);
+    }
+    double m_R0;
+};
+/// @brief \f[ R-R_0 \f]
+struct PsipR : public aCloneableBinaryFunctor<PsipR>
+{ /**
+     * @brief Construct from major radius
+     * @param R0 the major radius
+     */
+    PsipR( double R0): m_R0(R0) { }
+  private:
+    double do_compute(double R, double Z) const
+    {    
+        return R-m_R0;
+    }
+    double m_R0;
+};
+///@brief \f[ Z \f]
+struct PsipZ : public aCloneableBinaryFunctor<PsipZ>
+{ 
+  private:
+    double do_compute(double R, double Z) const
+    {    
+        return Z;
+    }
+};
+
+/**
+ * @brief circular \f$\psi_p = \frac{1}{2}\left((R-R_0)^2 + Z^2 \right)\f$
+ * @return 
+ */
+BinaryFunctorsLvl2 createPsip( double R0 )
+{
+    BinaryFunctorsLvl2 psip( new Psip(R0), new PsipR(R0), new PsipZ(),new Constant(1), new Constant(0), new Constant(1));
+    return psip;
+}
+/**
+ * @brief constant \f$ I = 1\f$
+ * @return 
+ */
+BinaryFunctorsLvl1 createIpol( )
+{
+    BinaryFunctorsLvl1 ipol( new Constant(1), new Constant(0), new Constant(0));
+    return ipol;
+}
+///@}
+}//namespace circular
 
 /**
  * @brief Create a Toroidal Magnetic field
@@ -50,7 +115,18 @@ TokamakMagneticField createMagField( double R0)
  */
 TokamakMagneticField createToroidalField( double R0)
 {
-    return TokamakMagneticField( R0, toiroidal::createPsip(), toiroidal::createIpol());
+    return TokamakMagneticField( R0, toroidal::createPsip(), toroidal::createIpol());
+}
+/**
+ * @brief Create a Magnetic field with circular flux surfaces
+ * @param R0 the major radius
+ * @return A magnetic field object
+ * @ingroup geom
+ */
+TokamakMagneticField createCircularField( double R0)
+{
+    return TokamakMagneticField( R0, circular::createPsip(R0), circular::createIpol());
 }
+
 }//namespace geo
 }//namespace dg
-- 
GitLab


From 72be83ee526100b0a382bc2ee27d42cbbb8e0343 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Tue, 17 Oct 2017 22:28:20 +0200
Subject: [PATCH 365/453] ds_t compiles and runs but output questionable

---
 inc/dg/geometry/base_geometry.h           |  2 +
 inc/dg/geometry/mpi_base.h                |  2 +
 inc/geometries/ds.h                       | 13 +++--
 inc/geometries/ds_curvilinear_t.cu        | 69 +++++++++++++++++++++++
 inc/geometries/{ds_mpib.cu => ds_mpit.cu} |  0
 inc/geometries/{ds_b.cu => ds_t.cu}       | 61 +++-----------------
 inc/geometries/fieldaligned.h             | 23 ++++----
 inc/geometries/toroidal.h                 | 13 +++--
 8 files changed, 105 insertions(+), 78 deletions(-)
 create mode 100644 inc/geometries/ds_curvilinear_t.cu
 rename inc/geometries/{ds_mpib.cu => ds_mpit.cu} (100%)
 rename inc/geometries/{ds_b.cu => ds_t.cu} (52%)

diff --git a/inc/dg/geometry/base_geometry.h b/inc/dg/geometry/base_geometry.h
index e2bd930a5..ed6818940 100644
--- a/inc/dg/geometry/base_geometry.h
+++ b/inc/dg/geometry/base_geometry.h
@@ -179,6 +179,8 @@ struct aProductGeometry3d : public aGeometry3d
     }
     ///allow deletion through base class pointer
     virtual ~aProductGeometry3d(){}
+    ///Geometries are cloneable
+    virtual aProductGeometry3d* clone()const=0;
     protected:
     ///@copydoc aTopology3d::aTopology3d(const aTopology3d&)
     aProductGeometry3d( const aProductGeometry3d& src):aGeometry3d(src){}
diff --git a/inc/dg/geometry/mpi_base.h b/inc/dg/geometry/mpi_base.h
index 850a3371c..2f3580316 100644
--- a/inc/dg/geometry/mpi_base.h
+++ b/inc/dg/geometry/mpi_base.h
@@ -125,6 +125,8 @@ struct aProductMPIGeometry3d : public aMPIGeometry3d
     }
     ///allow deletion through base class pointer
     virtual ~aProductMPIGeometry3d(){}
+    ///Geometries are cloneable
+    virtual aProductMPIGeometry3d* clone()const=0;
     protected:
     ///@copydoc aMPITopology3d::aMPITopology3d()
     aProductMPIGeometry3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz, MPI_Comm comm):
diff --git a/inc/geometries/ds.h b/inc/geometries/ds.h
index 0af543b7d..45274fdc4 100644
--- a/inc/geometries/ds.h
+++ b/inc/geometries/ds.h
@@ -65,6 +65,8 @@ struct DS
      * @param eps Desired accuracy of the fieldline integrator
      * @param no indicate if the symv function should be symmetric (not_normed) or not
      * @param dir indicate the direction in the bracket operator and in symv
+     *@note globalbcx and globalbcy  as well as bcz are taken from grid with full limter 
+     * @sa Fieldaligned
      */
     DS(const dg::geo::BinaryVectorLvl0& vec, const ProductGeometry& grid, unsigned multiplyX=1, unsigned multiplyY=1, bool dependsOnX = true, bool dependsOnY=true, double eps = 1e-5, dg::norm no=dg::normed, dg::direction dir = dg::centered)
     {
@@ -131,7 +133,8 @@ struct DS
     * @param f The vector to derive
     * @param g contains result on output (write only)
     */
-    void operator()( const container& f, container& g);
+    void operator()( const container& f, container& g){operator()(1., f, 0., g);}
+    void operator()(double alpha, const container& f, double beta, container& g);
 
 
     /**
@@ -206,13 +209,13 @@ void DS<Geometry, I, M,container>::construct(const Fieldaligned<Geometry, I, con
 }
 
 template<class G, class I, class M, class container>
-inline void DS<G,I,M,container>::operator()( const container& f, container& dsf) { 
+inline void DS<G,I,M,container>::operator()( double alpha, const container& f, double beta, container& dsf) { 
     if( m_dir == dg::centered)
-        return centered( 1., f, 0., dsf);
+        return centered( alpha, f, beta, dsf);
     else if( m_dir == dg::forward)
-        return forward( 1., f, 0., dsf);
+        return forward( alpha, f, beta, dsf);
     else
-        return backward( 1., f, 0., dsf);
+        return backward( alpha, f, beta, dsf);
 }
 
 template<class G, class I, class M, class container>
diff --git a/inc/geometries/ds_curvilinear_t.cu b/inc/geometries/ds_curvilinear_t.cu
new file mode 100644
index 000000000..4fa8ddf92
--- /dev/null
+++ b/inc/geometries/ds_curvilinear_t.cu
@@ -0,0 +1,69 @@
+#include <iostream>
+
+#include <cusp/print.h>
+
+#include "dg/backend/functions.h"
+#include "dg/backend/timer.cuh"
+#include "dg/blas.h"
+#include "dg/functors.h"
+#include "dg/geometry/geometry.h"
+#include "ds.h"
+#include "solovev.h"
+#include "flux.h"
+#include "toroidal.h"
+
+
+int main(int argc, char * argv[])
+{
+    std::cout << "Start DS test on flux grid!"<<std::endl;
+    Json::Reader reader;
+    Json::Value js;
+    if( argc==1) {
+        std::ifstream is("geometry_params_Xpoint.js");
+        reader.parse(is,js,false);
+    }
+    else {
+        std::ifstream is(argv[1]);
+        reader.parse(is,js,false);
+    }
+    dg::geo::solovev::Parameters gp(js);
+    mag = dg::geo::createSolovevField( gp);
+    t.tic();
+    std::cout << "Type n(3), Nx(8), Ny(80), Nz(20)\n";
+    std::cin >> n>> Nx>>Ny>>Nz;   
+    std::cout << "Type multipleX and multipleY!\n";
+    unsigned mx, my;
+    std::cin >> mx >> my;
+
+    double psi_0 = -20, psi_1 = -4;
+    dg::geo::FluxGenerator flux( mag.get_psip(), mag.get_ipol(), psi_0, psi_1, gp.R_0, 0., 1);
+    dg::geo::CurvilinearProductGrid3d g3d(flux, n, Nx, Ny,Nz, dg::DIR);
+    //dg::geo::Fieldaligned<dg::aGeometry3d, dg::IHMatrix, dg::HVec> fieldaligned( bhat, g3d, 1, 4, gp.rk4eps, dg::NoLimiter() ); 
+    dg::geo::DS<dg::aProductGeometry3d, dg::IHMatrix, dg::HMatrix, dg::HVec> ds( mag, g3d, mx, my, 1e-8, dg::normed, dg::centered, false, true);
+
+    
+    t.toc();
+    std::cout << "Construction took "<<t.diff()<<"s\n";
+    dg::HVec B = dg::pullback( dg::geo::InvB(mag), g3d), divB(B);
+    dg::HVec lnB = dg::pullback( dg::geo::LnB(mag), g3d), gradB(B);
+    dg::HVec gradLnB = dg::pullback( dg::geo::GradLnB(mag), g3d);
+    dg::blas1::pointwiseDivide( ones3d, B, B);
+    dg::HVec function = dg::pullback( dg::geo::FuncNeu(mag), g3d), derivative(function);
+    ds( function, derivative);
+
+    ds.centeredAdj( B, divB);
+    double norm =  sqrt( dg::blas2::dot(divB, vol3d, divB));
+    std::cout << "Divergence of B is "<<norm<<"\n";
+
+    ds.centered( lnB, gradB);
+    std::cout << "num. norm of gradLnB is "<<sqrt( dg::blas2::dot( gradB,vol3d, gradB))<<"\n";
+    norm = sqrt( dg::blas2::dot( gradLnB, vol3d, gradLnB) );
+    std::cout << "ana. norm of gradLnB is "<<norm<<"\n";
+    dg::blas1::axpby( 1., gradB, -1., gradLnB, gradLnB);
+    X = divB;
+    err = nc_put_var_double( ncid, varID[4], periodify(X, g2d_periodic).data());
+    double norm2 = sqrt(dg::blas2::dot(gradLnB, vol3d, gradLnB));
+    std::cout << "rel. error of lnB is    "<<norm2/norm<<"\n";
+    
+    return 0;
+}
diff --git a/inc/geometries/ds_mpib.cu b/inc/geometries/ds_mpit.cu
similarity index 100%
rename from inc/geometries/ds_mpib.cu
rename to inc/geometries/ds_mpit.cu
diff --git a/inc/geometries/ds_b.cu b/inc/geometries/ds_t.cu
similarity index 52%
rename from inc/geometries/ds_b.cu
rename to inc/geometries/ds_t.cu
index 5413ce2f1..45574af05 100644
--- a/inc/geometries/ds_b.cu
+++ b/inc/geometries/ds_t.cu
@@ -14,6 +14,7 @@
 
 
 const double R_0 = 10;
+const double I_0 = 2;
 double func(double R, double Z, double phi)
 {
     double r2 = (R-R_0)*(R-R_0)+Z*Z;
@@ -32,24 +33,13 @@ int main(int argc, char * argv[])
     unsigned n, Nx, Ny, Nz;
     std::cin >> n>> Nx>>Ny>>Nz;
     std::cout << "You typed "<<n<<" "<<Nx<<" "<<Ny<<" "<<Nz<<std::endl;
-    Json::Reader reader;
-    Json::Value js;
-    if( argc==1) {
-        std::ifstream is("geometry_params_Xpoint.js");
-        reader.parse(is,js,false);
-    }
-    else {
-        std::ifstream is(argv[1]);
-        reader.parse(is,js,false);
-    }
-    dg::geo::solovev::Parameters gp(js);
-    dg::geo::TokamakMagneticField mag = dg::geo::createCircularField( gp);
-    dg::geo::BinaryVectorLvl0 bhat( (dg::geo::BHatR)(mag), (dg::geo::BHatZ)(mag), (dg::geo::BHatP)(mag));
-    dg::CylindricalGrid3d g3d( gp.R_0 - 1, gp.R_0+1, -1, 1, 0, 2.*M_PI, n, Nx, Ny, Nz, dg::NEU, dg::NEU, dg::PER);
+    dg::CylindricalGrid3d g3d( R_0 - 1, R_0+1, -1, 1, 0, 2.*M_PI, n, Nx, Ny, Nz, dg::NEU, dg::NEU, dg::PER);
     const dg::DVec vol3d = dg::create::volume( g3d);
     dg::Timer t;
     t.tic();
-    dg::geo::Fieldaligned<dg::aProductGeometry3d,dg::IDMatrix,dg::DMatrix>  dsFA( bhat, g3d, 2,2,true,true,1e-10, dg::geo::NoLimiter(), dg::NEU, dg::NEU);
+    dg::geo::TokamakMagneticField mag = dg::geo::createCircularField( R_0, I_0);
+    dg::geo::BinaryVectorLvl0 bhat( (dg::geo::BHatR)(mag), (dg::geo::BHatZ)(mag), (dg::geo::BHatP)(mag));
+    dg::geo::Fieldaligned<dg::aProductGeometry3d,dg::IDMatrix,dg::DVec>  dsFA( bhat, g3d, 2,2,true,true,1e-10, dg::NEU, dg::NEU, dg::geo::NoLimiter());
 
     dg::geo::DS<dg::aProductGeometry3d, dg::IDMatrix, dg::DMatrix, dg::DVec> ds( dsFA, dg::not_normed, dg::centered);
     t.toc();
@@ -75,49 +65,12 @@ int main(int argc, char * argv[])
     ds( function, derivative);
     norm = dg::blas2::dot(vol3d, derivative);
     std::cout << "Norm Centered Derivative "<<sqrt( norm)<<" (compare with that of ds_mpib)\n";
-    ds.forward( function, derivative);
+    ds.forward( 1., function, 0., derivative);
     norm = dg::blas2::dot(vol3d, derivative);
     std::cout << "Norm Forward  Derivative "<<sqrt( norm)<<" (compare with that of ds_mpib)\n";
-    ds.backward( function, derivative);
+    ds.backward( 1., function, 0., derivative);
     norm = dg::blas2::dot(vol3d, derivative);
     std::cout << "Norm Backward Derivative "<<sqrt( norm)<<" (compare with that of ds_mpib)\n";
 
-    /////////////////////////TEST 3d flux grid//////////////////////////////////////
-    std::cout << "Start DS test on flux grid!"<<std::endl;
-    t.tic();
-    //dg::geo::BinaryVectorLvl0 bhat( dg::geo::BHatR(c), dg::geo::BHatZ(c), dg::geo::BHatP(c));
-    unsigned mx, my;
-    std::cout << "Type multipleX and multipleY!\n";
-    std::cin >> mx >> my;
-
-    dg::geo::FluxGenerator flux( mag.get_psip(), mag.get_ipol(), psi_0, psi_1, gp.R_0, 0., 1);
-    dg::geo::CurvilinearProductGrid3d g3d(flux, n, Nx, Ny,Nz, dg::DIR);
-    //dg::geo::Fieldaligned<dg::aGeometry3d, dg::IHMatrix, dg::HVec> fieldaligned( bhat, g3d, 1, 4, gp.rk4eps, dg::NoLimiter() ); 
-    dg::geo::DS<dg::aProductGeometry3d, dg::IHMatrix, dg::HMatrix, dg::HVec> ds( mag, g3d, mx, my, 1e-8, dg::normed, dg::centered, false, true);
-
-    
-    t.toc();
-    std::cout << "Construction took "<<t.diff()<<"s\n";
-    dg::HVec B = dg::pullback( dg::geo::InvB(mag), g3d), divB(B);
-    dg::HVec lnB = dg::pullback( dg::geo::LnB(mag), g3d), gradB(B);
-    dg::HVec gradLnB = dg::pullback( dg::geo::GradLnB(mag), g3d);
-    dg::blas1::pointwiseDivide( ones3d, B, B);
-    dg::HVec function = dg::pullback( dg::geo::FuncNeu(mag), g3d), derivative(function);
-    ds( function, derivative);
-
-    ds.centeredAdj( B, divB);
-    double norm =  sqrt( dg::blas2::dot(divB, vol3d, divB));
-    std::cout << "Divergence of B is "<<norm<<"\n";
-
-    ds.centered( lnB, gradB);
-    std::cout << "num. norm of gradLnB is "<<sqrt( dg::blas2::dot( gradB,vol3d, gradB))<<"\n";
-    norm = sqrt( dg::blas2::dot( gradLnB, vol3d, gradLnB) );
-    std::cout << "ana. norm of gradLnB is "<<norm<<"\n";
-    dg::blas1::axpby( 1., gradB, -1., gradLnB, gradLnB);
-    X = divB;
-    err = nc_put_var_double( ncid, varID[4], periodify(X, g2d_periodic).data());
-    double norm2 = sqrt(dg::blas2::dot(gradLnB, vol3d, gradLnB));
-    std::cout << "rel. error of lnB is    "<<norm2/norm<<"\n";
-    
     return 0;
 }
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index 7e0e9c950..cb848b7cd 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -249,9 +249,6 @@ void boxintegrator( const Field& field, const Topology& grid,
     grid.shift_topologic( coords0[0], coords0[1], R, Z);
     if ( !grid.contains( R, Z))   //point still outside domain
     {
-#ifdef DG_DEBUG
-        std::cerr << "point "<<coords1[0]<<" "<<coords1[1]<<" is somewhere else!\n";
-#endif //DG_DEBUG
         double deltaPhi = phi1;
         BoxIntegrator<Field, Topology> boxy( field, grid, eps);//stores references to field and grid
         boxy.set_coords( coords0); //nimm alte koordinaten
@@ -344,7 +341,7 @@ struct Fieldaligned
     template <class Limiter>
     Fieldaligned(const dg::geo::BinaryVectorLvl0& vec, const ProductGeometry& grid, unsigned multiplyX, unsigned multiplyY, bool dependsOnX, bool dependsOnY, double eps = 1e-5, dg::bc globalbcx = dg::NEU, dg::bc globalbcy = dg::NEU, Limiter limit = FullLimiter(), double deltaPhi = -1)
     {
-        construct( vec, grid, multiplyX, multiplyY, dependsOnX, dependsOnY, eps, limit, globalbcx, globalbcy, limit, deltaPhi);
+        construct( vec, grid, multiplyX, multiplyY, dependsOnX, dependsOnY, eps, globalbcx, globalbcy, limit, deltaPhi);
     }
     /**
     * @brief Construct from a field and a grid
@@ -507,14 +504,14 @@ void Fieldaligned<Geometry, IMatrix, container>::construct(const dg::geo::Binary
 {
     m_dependsOnX=bx, m_dependsOnY=by;
     m_Nz=grid.Nz(), m_bcz=grid.bcz(); 
-    m_g=grid;
+    m_g.reset(grid);
     dg::blas1::transfer( dg::evaluate( dg::zero, grid), m_hz_inv), m_hp_inv= m_hz_inv, m_hm_inv= m_hz_inv;
-    dg::split( m_hz_inv, m_temp);
-    dg::split( m_hz_inv, m_f);
+    dg::split( m_hz_inv, m_temp, grid);
+    dg::split( m_hz_inv, m_f, grid);
     if( deltaPhi <=0) deltaPhi = grid.hz();
     else assert( grid.Nz() == 1 || grid.hz()==deltaPhi);
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-    const dg::aGeometry2d* grid2d_ptr = grid->perp_grid();
+    const dg::aGeometry2d* grid2d_ptr = grid.perp_grid();
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     //Resize vector to 2D grid size
     m_perp_size = grid2d_ptr->size();
@@ -522,13 +519,13 @@ void Fieldaligned<Geometry, IMatrix, container>::construct(const dg::geo::Binary
     m_right = m_left = dg::evaluate( zero, *grid2d_ptr);
     m_ghostM.resize( m_perp_size); m_ghostP.resize( m_perp_size);
     //%%%%%%%%%%%%%%%%%%%%%%%%%%Set starting points and integrate field lines%%%%%%%%%%%%%%
-    std::cout << "Start fieldline integration!\n";
     dg::Timer t;
     std::vector<thrust::host_vector<double> > yp_coarse( 3), ym_coarse(yp_coarse), yp, ym; 
-    t.tic();
     
     dg::aGeometry2d* g2dField_ptr = grid2d_ptr->clone();//INTEGRATE HIGH ORDER GRID
     g2dField_ptr->set( 7, g2dField_ptr->Nx(), g2dField_ptr->Ny());
+    std::cout << "Start fieldline integration!\n";
+    t.tic();
     detail::integrate_all_fieldlines2d( vec, g2dField_ptr, g2dField_ptr, yp_coarse, ym_coarse, deltaPhi, eps);
 
     dg::Grid2d g2dFine((dg::Grid2d(*grid2d_ptr)));//FINE GRID
@@ -568,9 +565,9 @@ void Fieldaligned<Geometry, IMatrix, container>::construct(const dg::geo::Binary
     dg::blas1::transform( hp, hp, dg::INVERT<double>());
     dg::blas1::transform( hm, hm, dg::INVERT<double>());
     dg::blas1::transform( hz, hz, dg::INVERT<double>());
-    dg::join( std::vector<thrust::host_vector<double> >( m_Nz, hp), m_hp_inv);
-    dg::join( std::vector<thrust::host_vector<double> >( m_Nz, hm), m_hm_inv);
-    dg::join( std::vector<thrust::host_vector<double> >( m_Nz, hz), m_hz_inv);
+    dg::join( std::vector<thrust::host_vector<double> >( m_Nz, hp), m_hp_inv, grid);
+    dg::join( std::vector<thrust::host_vector<double> >( m_Nz, hm), m_hm_inv, grid);
+    dg::join( std::vector<thrust::host_vector<double> >( m_Nz, hz), m_hz_inv, grid);
     
     delete grid2d_ptr;
 }
diff --git a/inc/geometries/toroidal.h b/inc/geometries/toroidal.h
index 05e5daa93..0bf27cb8b 100644
--- a/inc/geometries/toroidal.h
+++ b/inc/geometries/toroidal.h
@@ -95,12 +95,12 @@ BinaryFunctorsLvl2 createPsip( double R0 )
     return psip;
 }
 /**
- * @brief constant \f$ I = 1\f$
+ * @brief constant \f$ I = I_0\f$
  * @return 
  */
-BinaryFunctorsLvl1 createIpol( )
+BinaryFunctorsLvl1 createIpol( double I0 )
 {
-    BinaryFunctorsLvl1 ipol( new Constant(1), new Constant(0), new Constant(0));
+    BinaryFunctorsLvl1 ipol( new Constant(I0), new Constant(0), new Constant(0));
     return ipol;
 }
 ///@}
@@ -118,14 +118,15 @@ TokamakMagneticField createToroidalField( double R0)
     return TokamakMagneticField( R0, toroidal::createPsip(), toroidal::createIpol());
 }
 /**
- * @brief Create a Magnetic field with circular flux surfaces
+ * @brief Create a Magnetic field with circular flux surfaces and constant current
  * @param R0 the major radius
+ * @param I0 the current
  * @return A magnetic field object
  * @ingroup geom
  */
-TokamakMagneticField createCircularField( double R0)
+TokamakMagneticField createCircularField( double R0, double I0)
 {
-    return TokamakMagneticField( R0, circular::createPsip(R0), circular::createIpol());
+    return TokamakMagneticField( R0, circular::createPsip(R0), circular::createIpol(I0));
 }
 
 }//namespace geo
-- 
GitLab


From 260e928709cd8860b7b2e15bf8e122035eef6fc2 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Tue, 17 Oct 2017 22:51:13 +0200
Subject: [PATCH 366/453] ds_curvilinear_t compiles and shows acceptable
 results

---
 inc/geometries/Makefile            |  2 +-
 inc/geometries/ds.h                | 12 ++++++------
 inc/geometries/ds_curvilinear_t.cu | 18 ++++++++++--------
 3 files changed, 17 insertions(+), 15 deletions(-)

diff --git a/inc/geometries/Makefile b/inc/geometries/Makefile
index 9c03a18af..cfa561925 100644
--- a/inc/geometries/Makefile
+++ b/inc/geometries/Makefile
@@ -17,7 +17,7 @@ ds_geom_t: ds_geom_t.cu solovev.h
 geometry_diag: geometry_diag.cu solovev.h 
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(LIBS) $(INCLUDE) $(JSONLIB) -g
 
-%_t: %_t.cu %.h
+%_t: %_t.cu 
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -DDG_DEBUG -g 
 
 %_mpit: %_mpit.cu 
diff --git a/inc/geometries/ds.h b/inc/geometries/ds.h
index 45274fdc4..8f0136bab 100644
--- a/inc/geometries/ds.h
+++ b/inc/geometries/ds.h
@@ -283,15 +283,15 @@ void DS<G,I,M,container>::do_symv( const container& f, container& dsTdsf)
 {
     if(m_dir == dg::centered)
     {
-        do_centered( f, m_tempP);
-        do_centeredAdj( m_tempP, dsTdsf, dg::not_normed);
+        do_centered( 1., f, 0., m_tempP);
+        do_centeredAdj( 1., m_tempP, 0., dsTdsf, dg::not_normed);
     }
     else 
     {
-        do_forward( f, m_tempP);
-        do_forwardAdj( m_tempP, m_temp0, dg::not_normed);
-        do_backward( f, m_tempM);
-        do_backwardAdj( m_tempM, dsTdsf, dg::not_normed);
+        do_forward( 1., f, 0., m_tempP);
+        do_forwardAdj( 1., m_tempP, 0., m_temp0, dg::not_normed);
+        do_backward( 1., f, 0., m_tempM);
+        do_backwardAdj( 1., m_tempM, 0., dsTdsf, dg::not_normed);
         dg::blas1::axpby(0.5,m_temp0,0.5,dsTdsf);
     }
     dg::blas1::pointwiseDivide( dsTdsf, m_weights_wo_vol, dsTdsf);
diff --git a/inc/geometries/ds_curvilinear_t.cu b/inc/geometries/ds_curvilinear_t.cu
index 4fa8ddf92..bfb74b4e8 100644
--- a/inc/geometries/ds_curvilinear_t.cu
+++ b/inc/geometries/ds_curvilinear_t.cu
@@ -7,6 +7,7 @@
 #include "dg/blas.h"
 #include "dg/functors.h"
 #include "dg/geometry/geometry.h"
+#include "testfunctors.h"
 #include "ds.h"
 #include "solovev.h"
 #include "flux.h"
@@ -27,11 +28,13 @@ int main(int argc, char * argv[])
         reader.parse(is,js,false);
     }
     dg::geo::solovev::Parameters gp(js);
-    mag = dg::geo::createSolovevField( gp);
+    dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField( gp);
+    dg::Timer t;
     t.tic();
     std::cout << "Type n(3), Nx(8), Ny(80), Nz(20)\n";
+    unsigned n,Nx,Ny,Nz;
     std::cin >> n>> Nx>>Ny>>Nz;   
-    std::cout << "Type multipleX and multipleY!\n";
+    std::cout << "Type multipleX (1) and multipleY (10)!\n";
     unsigned mx, my;
     std::cin >> mx >> my;
 
@@ -39,29 +42,28 @@ int main(int argc, char * argv[])
     dg::geo::FluxGenerator flux( mag.get_psip(), mag.get_ipol(), psi_0, psi_1, gp.R_0, 0., 1);
     dg::geo::CurvilinearProductGrid3d g3d(flux, n, Nx, Ny,Nz, dg::DIR);
     //dg::geo::Fieldaligned<dg::aGeometry3d, dg::IHMatrix, dg::HVec> fieldaligned( bhat, g3d, 1, 4, gp.rk4eps, dg::NoLimiter() ); 
-    dg::geo::DS<dg::aProductGeometry3d, dg::IHMatrix, dg::HMatrix, dg::HVec> ds( mag, g3d, mx, my, 1e-8, dg::normed, dg::centered, false, true);
-
+    dg::geo::DS<dg::aProductGeometry3d, dg::IHMatrix, dg::HMatrix, dg::HVec> ds( mag, g3d, mx, my, false, true, 1e-8, dg::normed, dg::centered);
     
     t.toc();
     std::cout << "Construction took "<<t.diff()<<"s\n";
     dg::HVec B = dg::pullback( dg::geo::InvB(mag), g3d), divB(B);
     dg::HVec lnB = dg::pullback( dg::geo::LnB(mag), g3d), gradB(B);
     dg::HVec gradLnB = dg::pullback( dg::geo::GradLnB(mag), g3d);
+    dg::HVec ones3d = dg::evaluate( dg::one, g3d);
+    dg::HVec vol3d = dg::create::volume( g3d);
     dg::blas1::pointwiseDivide( ones3d, B, B);
     dg::HVec function = dg::pullback( dg::geo::FuncNeu(mag), g3d), derivative(function);
     ds( function, derivative);
 
-    ds.centeredAdj( B, divB);
+    ds.centeredAdj( 1., B, 0., divB);
     double norm =  sqrt( dg::blas2::dot(divB, vol3d, divB));
     std::cout << "Divergence of B is "<<norm<<"\n";
 
-    ds.centered( lnB, gradB);
+    ds.centered( 1., lnB, 0., gradB);
     std::cout << "num. norm of gradLnB is "<<sqrt( dg::blas2::dot( gradB,vol3d, gradB))<<"\n";
     norm = sqrt( dg::blas2::dot( gradLnB, vol3d, gradLnB) );
     std::cout << "ana. norm of gradLnB is "<<norm<<"\n";
     dg::blas1::axpby( 1., gradB, -1., gradLnB, gradLnB);
-    X = divB;
-    err = nc_put_var_double( ncid, varID[4], periodify(X, g2d_periodic).data());
     double norm2 = sqrt(dg::blas2::dot(gradLnB, vol3d, gradLnB));
     std::cout << "rel. error of lnB is    "<<norm2/norm<<"\n";
     
-- 
GitLab


From a6041b29883befa93f561a1ad72927d8607b2360 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Wed, 18 Oct 2017 12:07:32 +0200
Subject: [PATCH 367/453] made an example code snippet. Looks good and maybe we
 should try to make this for more of our functions

---
 inc/dg/backend/evaluation.cuh |  2 +-
 inc/dg/backend/grid.h         |  1 +
 inc/dg/backend/weights.cuh    |  1 +
 inc/dg/blas2.h                |  1 +
 inc/dg/dg_doc.h               | 13 +++++++++++++
 5 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/inc/dg/backend/evaluation.cuh b/inc/dg/backend/evaluation.cuh
index a9e90fc53..680cb8252 100644
--- a/inc/dg/backend/evaluation.cuh
+++ b/inc/dg/backend/evaluation.cuh
@@ -54,7 +54,7 @@ thrust::host_vector<double> evaluate( double (f)(double), const Grid1d& g)
  * @param g The 2d grid on which to evaluate f
  *
  * @return  A dG Host Vector with values
- * @note if you don't like to copy f then just pass a (const) reference then the type should adapt
+ * @copydoc hide_code_evaluate2d
  */
 template< class BinaryOp>
 thrust::host_vector<double> evaluate( const BinaryOp& f, const aTopology2d& g)
diff --git a/inc/dg/backend/grid.h b/inc/dg/backend/grid.h
index fc356f300..03059f43d 100644
--- a/inc/dg/backend/grid.h
+++ b/inc/dg/backend/grid.h
@@ -706,6 +706,7 @@ struct aTopology3d
 /**
  * @brief The simplest implementation of aTopology2d
  * @ingroup grid
+ * @copydoc hide_code_evaluate2d
  */
 struct Grid2d : public aTopology2d
 {
diff --git a/inc/dg/backend/weights.cuh b/inc/dg/backend/weights.cuh
index da1e4355b..0385001db 100644
--- a/inc/dg/backend/weights.cuh
+++ b/inc/dg/backend/weights.cuh
@@ -85,6 +85,7 @@ int get_j( unsigned n, unsigned Nx, int idx) { return idx%n;}
 * @param g The grid 
 *
 * @return Host Vector
+* @copydoc hide_code_evaluate2d
 */
 thrust::host_vector<double> weights( const aTopology2d& g)
 {
diff --git a/inc/dg/blas2.h b/inc/dg/blas2.h
index bbc51053b..7acd5e181 100644
--- a/inc/dg/blas2.h
+++ b/inc/dg/blas2.h
@@ -59,6 +59,7 @@ inline void transfer( const Matrix& x, AnotherMatrix& y)
  * @return Generalized scalar product
  * @note This routine is always executed synchronously due to the 
     implicit memcpy of the result.
+ * @copydoc hide_code_evaluate2d
  */
 template< class DiagonalMatrix, class container>
 inline typename MatrixTraits<DiagonalMatrix>::value_type dot( const container& x, const DiagonalMatrix& m, const container& y)
diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index 3b70ffffe..b9ba73085 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -182,6 +182,19 @@
   and specializing MatrixTraits with the SelfMadeMatrixTag as the matrix_category
   */
 
+/** @class hide_code_evaluate2d
+ * This code snippet demonstrates how to integrate a function discretized with dG
+@code{.cpp}
+double function(double x, double y){return exp(x)*exp(y);}
+//...
+dg::Grid2d g2d( 0, 2, 0, 2, 3, 20, 20);
+const dg::HVec w2d = dg::create::weights( g2d);
+const dg::HVec h_x = dg::evaluate( function, g2d);
+double norm = dg::blas2::dot( h_x, w2d, h_x); // norm is now: (exp(4)-exp(0))/2
+@endcode
+*/
+
+
 /*!@addtogroup mpi_structures
 @{
 @page mpi_matrix MPI Vectors and the blas1 functions
-- 
GitLab


From 407a85685b24d89e5672d094ae24ba8b9c156d36 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Wed, 18 Oct 2017 19:35:54 +0200
Subject: [PATCH 368/453] ds_curvilinear_mpit compiles

---
 inc/dg/backend/mpi_grid.h             |   5 +
 inc/dg/backend/mpi_init.h             |   1 +
 inc/dg/backend/mpi_matrix.h           |  10 -
 inc/dg/backend/mpi_projection.h       |   7 +-
 inc/dg/backend/timer.cuh              |   3 +-
 inc/dg/backend/transpose.h            |  13 ++
 inc/geometries/Makefile               |   2 +-
 inc/geometries/ds.h                   |   6 +-
 inc/geometries/ds_curvilinear_mpit.cu |  74 ++++++++
 inc/geometries/fieldaligned.h         |  27 ++-
 inc/geometries/mpi_curvilinear.h      |   2 +-
 inc/geometries/mpi_fieldaligned.h     | 251 ++++++++++++--------------
 12 files changed, 233 insertions(+), 168 deletions(-)
 create mode 100644 inc/geometries/ds_curvilinear_mpit.cu

diff --git a/inc/dg/backend/mpi_grid.h b/inc/dg/backend/mpi_grid.h
index 0346d2f57..54fcc2404 100644
--- a/inc/dg/backend/mpi_grid.h
+++ b/inc/dg/backend/mpi_grid.h
@@ -735,6 +735,8 @@ struct MPIGrid2d: public aMPITopology2d
     MPIGrid2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx, bc bcy, MPI_Comm comm):
         aMPITopology2d( x0,x1,y0,y1,n,Nx,Ny,bcx,bcy,comm)
     { }
+    ///allow explicit type conversion from any other topology
+    explicit MPIGrid2d( const aMPITopology2d& src): aMPITopology2d(src){}
     private:
     virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny){
         aMPITopology2d::do_set(new_n,new_Nx,new_Ny);
@@ -759,6 +761,9 @@ struct MPIGrid3d : public aMPITopology3d
     MPIGrid3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz, MPI_Comm comm):
         aMPITopology3d( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, bcx, bcy, bcz, comm)
     { }
+    ///allow explicit type conversion from any other topology
+    ///@param src source
+    explicit MPIGrid3d( const aMPITopology3d& src): aMPITopology3d(src){ }
     private:
     virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz){
         aMPITopology3d::do_set(new_n,new_Nx,new_Ny,new_Nz);
diff --git a/inc/dg/backend/mpi_init.h b/inc/dg/backend/mpi_init.h
index 655669de0..7bcc6dcf4 100644
--- a/inc/dg/backend/mpi_init.h
+++ b/inc/dg/backend/mpi_init.h
@@ -1,6 +1,7 @@
 #pragma once
 
 #include <iostream>
+#include "../enums.h"
 
 /*!@file
 @brief convenience mpi init functions
diff --git a/inc/dg/backend/mpi_matrix.h b/inc/dg/backend/mpi_matrix.h
index c77a5ff5f..ded961022 100644
--- a/inc/dg/backend/mpi_matrix.h
+++ b/inc/dg/backend/mpi_matrix.h
@@ -337,16 +337,6 @@ struct MatrixTraits<const MPIDistMat<L, C> >
     typedef typename MatrixTraits<L>::value_type value_type;//!< value type
     typedef MPIMatrixTag matrix_category; //!< 
 };
-namespace detail
-{
-template <class Matrix, class Collective>
-MPIDistMat<Matrix, Collective> doTranspose( const MPIDistMat<Matrix, Collective>& src, MPIMatrixTag)
-{
-    Matrix tr = doTranspose( src.matrix(), typename MatrixTraits<Matrix>::matrix_category());
-    MPIDistMat<Matrix, Collective> out( tr, src.collective());
-    return out;
-}
-} //namespace detail
 ///@endcond
 
 
diff --git a/inc/dg/backend/mpi_projection.h b/inc/dg/backend/mpi_projection.h
index 996aff44d..912761472 100644
--- a/inc/dg/backend/mpi_projection.h
+++ b/inc/dg/backend/mpi_projection.h
@@ -137,11 +137,10 @@ dg::MIHMatrix projection( const aMPITopology3d& g_new, const aMPITopology3d& g_o
  *
  * @copydetails interpolation(const thrust::host_vector<double>&,const thrust::host_vector<double>&,const aTopology2d&,dg::bc,dg::bc)
  */
-dg::MIHMatrix interpolation( const dg::HVec& x, const dg::HVec& y, const aMPITopology2d& grid, dg::bc bcx = dg::NEU, dg::bc bcy = dg::NEU)
+dg::MIHMatrix interpolation( const dg::HVec& x, const dg::HVec& y, const aMPITopology2d& g, dg::bc bcx = dg::NEU, dg::bc bcy = dg::NEU)
 {
-    return convert(  
-            dg::create::interpolation( x, y, grid.global(), bcx, bcy), 
-            grid);
+    dg::IHMatrix mat = dg::create::interpolation( x,y, g.global(), bcx, bcy);
+    return convert(  mat, g);
 }
 
 
diff --git a/inc/dg/backend/timer.cuh b/inc/dg/backend/timer.cuh
index 4780c0884..509eab514 100644
--- a/inc/dg/backend/timer.cuh
+++ b/inc/dg/backend/timer.cuh
@@ -1,6 +1,7 @@
 #ifndef _DG_TIMER_
 #define _DG_TIMER_
-//the <thrust/device_vector.h> header must be included before this file for the THRUST_DEVICE_SYSTEM macros to work
+#include "thrust/device_vector.h"
+//the <thrust/device_vector.h> header must be included for the THRUST_DEVICE_SYSTEM macros to work
 namespace dg
 {
 #if (THRUST_DEVICE_SYSTEM!=THRUST_DEVICE_SYSTEM_CUDA) //if we don't use a GPU
diff --git a/inc/dg/backend/transpose.h b/inc/dg/backend/transpose.h
index ea1edaa73..852edf30d 100644
--- a/inc/dg/backend/transpose.h
+++ b/inc/dg/backend/transpose.h
@@ -1,4 +1,8 @@
 #pragma once
+#include <cusp/transpose.h>
+#ifdef MPI_VERSION
+#include "mpi_matrix.h"
+#endif //MPI_VERSION
 
 namespace dg
 {
@@ -13,6 +17,15 @@ Matrix doTranspose( const Matrix& src, CuspMatrixTag)
     cusp::transpose( src, out);
     return out;
 }
+#ifdef MPI_VERSION
+template <class Matrix, class Collective>
+MPIDistMat<Matrix, Collective> doTranspose( const MPIDistMat<Matrix, Collective>& src, MPIMatrixTag)
+{
+    Matrix tr = doTranspose( src.matrix(), typename MatrixTraits<Matrix>::matrix_category());
+    MPIDistMat<Matrix, Collective> out( tr, src.collective());
+    return out;
+}
+#endif// MPI_VERSION
 }//namespace detail
 ///@endcond
 
diff --git a/inc/geometries/Makefile b/inc/geometries/Makefile
index cfa561925..ecbcfadc0 100644
--- a/inc/geometries/Makefile
+++ b/inc/geometries/Makefile
@@ -21,7 +21,7 @@ geometry_diag: geometry_diag.cu solovev.h
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -DDG_DEBUG -g 
 
 %_mpit: %_mpit.cu 
-	$(MPICC) $(INCLUDE) $(MPICFLAGS)  $< -o $@ -g $(LIBS) $(JSONLIB)
+	$(MPICC) $(OPT) $(INCLUDE) $(MPICFLAGS)  $< -o $@ -g $(LIBS) $(JSONLIB) -DDG_DEBUG
 
 %_mpib: %_mpib.cu
 	$(MPICC) $(OPT) $(MPICFLAGS) $< -o $@ -g $(INCLUDE) $(LIBS) $(JSONLIB)
diff --git a/inc/geometries/ds.h b/inc/geometries/ds.h
index 8f0136bab..bb1ab5870 100644
--- a/inc/geometries/ds.h
+++ b/inc/geometries/ds.h
@@ -147,17 +147,17 @@ struct DS
      */
     void symv( const container& f, container& dsTdsf){ do_symv( f, dsTdsf);}
 
-    ///@copydoc FieldAligned::set_boundaries(dg::bc,double,double)
+    ///@copydoc Fieldaligned::set_boundaries(dg::bc,double,double)
     void set_boundaries( dg::bc bcz, double left, double right)
     {
         m_fa.set_boundaries( bcz, left, right);
     }
-    ///@copydoc FieldAligned::set_boundaries(dg::bc,const container&,const container&)
+    ///@copydoc Fieldaligned::set_boundaries(dg::bc,const container&,const container&)
     void set_boundaries( dg::bc bcz, const container& left, const container& right)
     {
         m_fa.set_boundaries( bcz, left, right);
     }
-    ///@copydoc FieldAligned::set_boundaries(dg::bc,const container&,double,double)
+    ///@copydoc Fieldaligned::set_boundaries(dg::bc,const container&,double,double)
     void set_boundaries( dg::bc bcz, const container& global, double scal_left, double scal_right)
     {
         m_fa.set_boundaries( bcz, global, scal_left, scal_right);
diff --git a/inc/geometries/ds_curvilinear_mpit.cu b/inc/geometries/ds_curvilinear_mpit.cu
new file mode 100644
index 000000000..6e8f40962
--- /dev/null
+++ b/inc/geometries/ds_curvilinear_mpit.cu
@@ -0,0 +1,74 @@
+#include <iostream>
+
+#include "mpi.h"
+
+#include "dg/backend/timer.cuh"
+#include "dg/backend/mpi_init.h"
+#include "dg/backend/functions.h"
+#include "dg/blas.h"
+#include "dg/functors.h"
+#include "dg/geometry/geometry.h"
+#include "testfunctors.h"
+#include "ds.h"
+#include "solovev.h"
+#include "flux.h"
+#include "toroidal.h"
+#include "mpi_curvilinear.h"
+
+
+int main(int argc, char * argv[])
+{
+    MPI_Init( &argc, &argv);
+    int rank;
+    unsigned n, Nx, Ny, Nz; 
+    MPI_Comm comm;
+    dg::mpi_init3d( dg::DIR, dg::PER, dg::PER, n, Nx, Ny, Nz, comm);
+    MPI_Comm_rank( MPI_COMM_WORLD, &rank);
+    Json::Reader reader;
+    Json::Value js;
+    if( argc==1)
+    {
+        std::ifstream is("geometry_params_Xpoint.js");
+        reader.parse(is,js,false);
+    }
+    else
+    {
+        std::ifstream is(argv[1]);
+        reader.parse(is,js,false);
+    }
+    dg::geo::solovev::Parameters gp(js);
+    if(rank==0)std::cout << "Start DS test on flux grid!"<<std::endl;
+    dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField( gp);
+    dg::Timer t;
+    t.tic();
+    unsigned mx=1, my=10;
+    double psi_0 = -20, psi_1 = -4;
+    dg::geo::FluxGenerator flux( mag.get_psip(), mag.get_ipol(), psi_0, psi_1, gp.R_0, 0., 1);
+    dg::geo::CurvilinearProductMPIGrid3d g3d(flux, n, Nx, Ny,Nz, dg::DIR, dg::PER, dg::PER, comm);
+    dg::geo::DS<dg::aProductMPIGeometry3d, dg::MIHMatrix, dg::MHMatrix, dg::MHVec> ds( mag, g3d, mx, my, false, true, 1e-8, dg::normed, dg::centered);
+    
+    t.toc();
+    if(rank==0)std::cout << "Construction took "<<t.diff()<<"s\n";
+    dg::MHVec B = dg::pullback( dg::geo::InvB(mag), g3d), divB(B);
+    dg::MHVec lnB = dg::pullback( dg::geo::LnB(mag), g3d), gradB(B);
+    dg::MHVec gradLnB = dg::pullback( dg::geo::GradLnB(mag), g3d);
+    dg::MHVec ones3d = dg::evaluate( dg::one, g3d);
+    dg::MHVec vol3d = dg::create::volume( g3d);
+    dg::blas1::pointwiseDivide( ones3d, B, B);
+    dg::MHVec function = dg::pullback( dg::geo::FuncNeu(mag), g3d), derivative(function);
+    ds( function, derivative);
+
+    ds.centeredAdj( 1., B, 0., divB);
+    double norm =  sqrt( dg::blas2::dot(divB, vol3d, divB));
+    if(rank==0)std::cout << "Divergence of B is "<<norm<<"\n";
+
+    ds.centered( 1., lnB, 0., gradB);
+    if(rank==0)std::cout << "num. norm of gradLnB is "<<sqrt( dg::blas2::dot( gradB,vol3d, gradB))<<"\n";
+    norm = sqrt( dg::blas2::dot( gradLnB, vol3d, gradLnB) );
+    if(rank==0)std::cout << "ana. norm of gradLnB is "<<norm<<"\n";
+    dg::blas1::axpby( 1., gradB, -1., gradLnB, gradLnB);
+    double norm2 = sqrt(dg::blas2::dot(gradLnB, vol3d, gradLnB));
+    if(rank==0)std::cout << "rel. error of lnB is    "<<norm2/norm<<"\n";
+    MPI_Finalize();
+    return 0;
+}
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index cb848b7cd..7a7f646fc 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -162,14 +162,14 @@ void clip_to_boundary( thrust::host_vector<double>& x, const dg::aTopology2d* gr
     clip_to_boundary(x[0], x[1], grid);
 }
 
-void interpolate_and_clip( const dg::IHMatrix& interpolate, const dg::aTopology2d* g2dFine, const dg::aTopology2d* boundary_ptr, //2 different grid on account of the MPI implementation
+void interpolate_and_clip( const dg::IHMatrix& interpolate, const dg::aTopology2d& g2dFine, const dg::aTopology2d& boundary_ptr, //2 different grid on account of the MPI implementation
         const std::vector<thrust::host_vector<double> >& yp_coarse,
         const std::vector<thrust::host_vector<double> >& ym_coarse,
         std::vector<thrust::host_vector<double> >& yp_fine,
         std::vector<thrust::host_vector<double> >& ym_fine
         )
 {
-    std::vector<thrust::host_vector<double> > yp( 3, dg::evaluate(dg::zero, *g2dFine)), ym(yp); 
+    std::vector<thrust::host_vector<double> > yp( 3, dg::evaluate(dg::zero, g2dFine)), ym(yp); 
     for( unsigned i=0; i<3; i++)
     {
         dg::blas2::symv( interpolate, yp_coarse[i], yp[i]);
@@ -177,10 +177,10 @@ void interpolate_and_clip( const dg::IHMatrix& interpolate, const dg::aTopology2
     }
     for( unsigned i=0; i<yp[0].size(); i++)
     {
-        boundary_ptr->shift_topologic( yp[0][i], yp[1][i], yp[0][i], yp[1][i]);
-        boundary_ptr->shift_topologic( ym[0][i], ym[1][i], ym[0][i], ym[1][i]);
-        detail::clip_to_boundary( yp[0][i], yp[1][i], boundary_ptr);
-        detail::clip_to_boundary( ym[0][i], ym[1][i], boundary_ptr);
+        boundary_ptr.shift_topologic( yp[0][i], yp[1][i], yp[0][i], yp[1][i]);
+        boundary_ptr.shift_topologic( ym[0][i], ym[1][i], ym[0][i], ym[1][i]);
+        detail::clip_to_boundary( yp[0][i], yp[1][i], &boundary_ptr);
+        detail::clip_to_boundary( ym[0][i], ym[1][i], &boundary_ptr);
     }
     yp_fine=yp, ym_fine=ym;
 }
@@ -333,7 +333,6 @@ template<class ProductGeometry, class IMatrix, class container >
 struct Fieldaligned
 {
 
-    typedef IMatrix InterpolationMatrix; //!< typdef to reveal the interpolation matrix
     ///@brief do not allocate memory; no member call except construct is valid
     Fieldaligned(){}
 
@@ -394,8 +393,8 @@ struct Fieldaligned
     * if Dirichlet boundaries are used the left value is the left function
     value, if Neumann boundaries are used the left value is the left derivative value
     * @param bcz boundary condition
-    * @param left spatially variable left boundary value
-    * @param right spatially variable right boundary value
+    * @param left spatially variable left boundary value (2d size)
+    * @param right spatially variable right boundary value (2d size)
     */
     void set_boundaries( dg::bc bcz, const container& left, const container& right)
     {
@@ -515,15 +514,15 @@ void Fieldaligned<Geometry, IMatrix, container>::construct(const dg::geo::Binary
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     //Resize vector to 2D grid size
     m_perp_size = grid2d_ptr->size();
-    m_limiter = dg::pullback( limit, *grid2d_ptr);
-    m_right = m_left = dg::evaluate( zero, *grid2d_ptr);
-    m_ghostM.resize( m_perp_size); m_ghostP.resize( m_perp_size);
+    dg::blas1::transfer( dg::pullback(limit, *grid2d_ptr), m_limiter);
+    dg::blas1::transfer( dg::evaluate(zero, *grid2d_ptr), m_left);
+    m_ghostM = m_ghostP = m_right = m_left;
     //%%%%%%%%%%%%%%%%%%%%%%%%%%Set starting points and integrate field lines%%%%%%%%%%%%%%
     dg::Timer t;
     std::vector<thrust::host_vector<double> > yp_coarse( 3), ym_coarse(yp_coarse), yp, ym; 
     
     dg::aGeometry2d* g2dField_ptr = grid2d_ptr->clone();//INTEGRATE HIGH ORDER GRID
-    g2dField_ptr->set( 7, g2dField_ptr->Nx(), g2dField_ptr->Ny());
+    //g2dField_ptr->set( 7, g2dField_ptr->Nx(), g2dField_ptr->Ny());
     std::cout << "Start fieldline integration!\n";
     t.tic();
     detail::integrate_all_fieldlines2d( vec, g2dField_ptr, g2dField_ptr, yp_coarse, ym_coarse, deltaPhi, eps);
@@ -531,7 +530,7 @@ void Fieldaligned<Geometry, IMatrix, container>::construct(const dg::geo::Binary
     dg::Grid2d g2dFine((dg::Grid2d(*grid2d_ptr)));//FINE GRID
     g2dFine.multiplyCellNumbers((double)mx, (double)my);
     dg::IHMatrix interpolate = dg::create::interpolation( g2dFine, *g2dField_ptr);  //INTERPOLATE TO FINE GRID
-    dg::geo::detail::interpolate_and_clip( interpolate, &g2dFine, &g2dFine, yp_coarse, ym_coarse, yp, ym);
+    dg::geo::detail::interpolate_and_clip( interpolate, g2dFine, g2dFine, yp_coarse, ym_coarse, yp, ym);
     delete g2dField_ptr;
     t.toc(); 
     std::cout << "Fieldline integration took "<<t.diff()<<"s\n";
diff --git a/inc/geometries/mpi_curvilinear.h b/inc/geometries/mpi_curvilinear.h
index 921602dcc..1200d65bc 100644
--- a/inc/geometries/mpi_curvilinear.h
+++ b/inc/geometries/mpi_curvilinear.h
@@ -191,7 +191,7 @@ struct CurvilinearProductMPIGrid3d : public dg::aProductMPIGeometry3d
 };
 ///@cond
 CurvilinearMPIGrid2d::CurvilinearMPIGrid2d( const CurvilinearProductMPIGrid3d& g):
-    dg::aMPIGeometry2d( g.global().x0(), g.global().x1(), g.global().y0(), g.global().y1(), g.global().n(), g.global().Nx(), g.global().Ny(), g.global().bcx(), g.global().bcy(), get_perp_comm( )),
+    dg::aMPIGeometry2d( g.global().x0(), g.global().x1(), g.global().y0(), g.global().y1(), g.global().n(), g.global().Nx(), g.global().Ny(), g.global().bcx(), g.global().bcy(), g.get_perp_comm() ),
     handle_(g.generator())
 {
     map_=g.map();
diff --git a/inc/geometries/mpi_fieldaligned.h b/inc/geometries/mpi_fieldaligned.h
index eef6dd90e..5543d85dc 100644
--- a/inc/geometries/mpi_fieldaligned.h
+++ b/inc/geometries/mpi_fieldaligned.h
@@ -19,52 +19,55 @@ namespace geo{
 namespace detail{
 
 ///basically a copy across processes
-template<class InputIterator, class OutputIterator>
-void sendForward( InputIterator begin, InputIterator end, OutputIterator result, MPI_Comm comm) //send to next plane
+template<class thrust_vector>
+void sendForward( const thrust_vector& in, thrust_vector& out, MPI_Comm comm) //send to next plane
 {
     int source, dest;
     MPI_Status status;
-    MPI_Cart_shift( comm_, 2, +1, &source, &dest);
+    MPI_Cart_shift( comm, 2, +1, &source, &dest);
 #if THRUST_DEVICE_SYSTEM==THRUST_DEVICE_SYSTEM_CUDA
     cudaDeviceSynchronize();//wait until device functions are finished before sending data
 #endif //THRUST_DEVICE_SYSTEM
-    unsigned size = thrust::distance( begin, end);
-    MPI_Sendrecv(   thrust::raw_pointer_cast(begin), size, MPI_DOUBLE,  //sender
+    unsigned size = in.size();
+    MPI_Sendrecv(   thrust::raw_pointer_cast(in.data()), size, MPI_DOUBLE,  //sender
                     dest, 9,  //destination
-                    thrust::raw_pointer_cast(result), size, MPI_DOUBLE, //receiver
+                    thrust::raw_pointer_cast(out.data()), size, MPI_DOUBLE, //receiver
                     source, 9, //source
                     comm, &status);
 }
 ///basically a copy across processes
-template<class InputIterator, class OutputIterator>
-void sendBackward( InputIterator begin, InputIterator end, OutputIterator result, MPI_Comm comm) //send to next plane
+template<class thrust_vector>
+void sendBackward( const thrust_vector& in, thrust_vector& out, MPI_Comm comm) //send to next plane
 {
     int source, dest;
     MPI_Status status;
-    MPI_Cart_shift( comm_, 2, -1, &source, &dest);
+    MPI_Cart_shift( comm, 2, -1, &source, &dest);
 #if THRUST_DEVICE_SYSTEM==THRUST_DEVICE_SYSTEM_CUDA
     cudaDeviceSynchronize();//wait until device functions are finished before sending data
 #endif //THRUST_DEVICE_SYSTEM
-    unsigned size = thrust::distance( begin, end);
-    MPI_Sendrecv(   thrust::raw_pointer_cast(begin), size, MPI_DOUBLE,  //sender
+    unsigned size = in.size();
+    MPI_Sendrecv(   thrust::raw_pointer_cast(in.data()), size, MPI_DOUBLE,  //sender
                     dest, 3,  //destination
-                    thrust::raw_pointer_cast(result), size, MPI_DOUBLE, //receiver
+                    thrust::raw_pointer_cast(out.data()), size, MPI_DOUBLE, //receiver
                     source, 3, //source
                     comm, &status);
 }
 }//namespace detail
 
-template <class Geometry, class LocalIMatrix, class CommunicatorXY, class LocalContainer>
-struct Fieldaligned< Geometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vector<LocalContainer> > 
+template <class ProductMPIGeometry, class LocalIMatrix, class CommunicatorXY, class LocalContainer>
+struct Fieldaligned< ProductMPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vector<LocalContainer> > 
 {
     Fieldaligned(){}
     template <class Limiter>
-    Fieldaligned(const dg::geo::BinaryVectorLvl0& vec, const Geometry& grid, unsigned multiplyX, unsigned multiplyY, double eps = 1e-5, dg::bc globalbcx = dg::NEU, dg::bc globalbcy = dg::NEU, Limiter limit = FullLimiter(), double deltaPhi = -1)
+    Fieldaligned(const dg::geo::BinaryVectorLvl0& vec, const ProductMPIGeometry& grid, unsigned multiplyX, unsigned multiplyY, bool dependsOnX, bool dependsOnY, double eps = 1e-5, dg::bc globalbcx = dg::NEU, dg::bc globalbcy = dg::NEU, Limiter limit = FullLimiter(), double deltaPhi = -1)
     {
-        construct( vec, grid, multiplyX, multiplyY, eps, limit, globalbcx, globalbcy, limit, deltaPhi);
+        construct( vec, grid, multiplyX, multiplyY, dependsOnX, dependsOnY, eps, globalbcx, globalbcy, limit, deltaPhi);
     }
     template <class Limiter>
-    void construct(const dg::geo::BinaryVectorLvl0& vec, const Geometry& grid, unsigned multiplyX, unsigned multiplyY, double eps = 1e-5, dg::bc globalbcx = dg::NEU, dg::bc globalbcy = dg::NEU, Limiter limit = FullLimiter(), double deltaPhi = -1);
+    void construct(const dg::geo::BinaryVectorLvl0& vec, const ProductMPIGeometry& grid, unsigned multiplyX, unsigned multiplyY, bool dependsOnX, bool dependsOnY, double eps = 1e-5, dg::bc globalbcx = dg::NEU, dg::bc globalbcy = dg::NEU, Limiter limit = FullLimiter(), double deltaPhi = -1);
+
+    bool dependsOnX()const{return m_dependsOnX;}
+    bool dependsOnY()const{return m_dependsOnY;}
 
     void set_boundaries( dg::bc bcz, double left, double right)
     {
@@ -90,7 +93,10 @@ struct Fieldaligned< Geometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vec
     }
 
     template< class BinaryOp>
-    MPI_Vector<LocalContainer> evaluate( BinaryOp f, unsigned plane=0) const;
+    MPI_Vector<LocalContainer> evaluate( BinaryOp binary, unsigned p0=0) const
+    {
+        return evaluate( binary, dg::CONSTANT(1), p0, 0);
+    }
 
     template< class BinaryOp, class UnaryOp>
     MPI_Vector<LocalContainer> evaluate( BinaryOp f, UnaryOp g, unsigned p0, unsigned rounds) const;
@@ -100,145 +106,125 @@ struct Fieldaligned< Geometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vec
     const MPI_Vector<LocalContainer>& hz_inv()const {return m_hz_inv;}
     const MPI_Vector<LocalContainer>& hp_inv()const {return m_hp_inv;}
     const MPI_Vector<LocalContainer>& hm_inv()const {return m_hm_inv;}
-    const Geometry& grid() const{return g_;}
+    const ProductMPIGeometry& grid() const{return m_g.get();}
   private:
-    MPI_Vector<LocalContainer> m_hz_inv, m_hp_inv, m_hm_inv; 
-    LocalContainer m_ghostM, m_ghostP;
-    dg::Handle<Geometry> m_g;
+    void ePlus( enum whichMatrix which, const MPI_Vector<LocalContainer>& in, MPI_Vector<LocalContainer>& out);
+    void eMinus(enum whichMatrix which, const MPI_Vector<LocalContainer>& in, MPI_Vector<LocalContainer>& out);
+    MPIDistMat<LocalIMatrix, CommunicatorXY> m_plus, m_minus, m_plusT, m_minusT; //2d interpolation matrices
+    MPI_Vector<LocalContainer> m_hz_inv, m_hp_inv, m_hm_inv; //3d size
+    MPI_Vector<LocalContainer> m_hp, m_hm;      //2d size
+    MPI_Vector<LocalContainer> m_left, m_right; //2d size
+    MPI_Vector<LocalContainer> m_limiter; //2d size
+    MPI_Vector<LocalContainer> m_ghostM, m_ghostP; //2d size
+    unsigned m_Nz, m_perp_size; 
     dg::bc m_bcz;
-    MPI_Vector<LocalContainer> m_left, m_right;
-    MPI_Vector<LocalContainer> m_limiter;
-    std::vector<LocalContainer> m_temp; 
-    MPIDistMat<LocalIMatrix, CommunicatorXY> m_plus, m_minus, m_plusT, m_minusT;
+    std::vector<MPI_Vector<LocalContainer> > m_f, m_temp;  //split 3d vectors
+    dg::Handle<ProductMPIGeometry> m_g;
+    bool m_dependsOnX, m_dependsOnY;
+    unsigned m_coords2, m_sizeZ; //number of processes in z
 };
 //////////////////////////////////////DEFINITIONS/////////////////////////////////////
 template<class MPIGeometry, class LocalIMatrix, class CommunicatorXY, class LocalContainer>
 template <class Limiter>
 void Fieldaligned<MPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vector<LocalContainer> >::construct(
-    const dg::geo::BinaryVectorLvl0& vec, const MPIGeometry& grid, unsigned mx, unsigned my, double eps, dg::bc globalbcx, dg::bc globalbcy, Limiter limit, double deltaPhi)
+    const dg::geo::BinaryVectorLvl0& vec, const MPIGeometry& grid, unsigned mx, unsigned my, bool bx, bool by, double eps, dg::bc globalbcx, dg::bc globalbcy, Limiter limit, double deltaPhi)
 {
-    temp_(g_.Nz())
-
-    dg::blas1::transfer( dg::evaluate( dg::zero, grid), m_hz_inv), m_hp_inv= m_hz_inv, m_hm_inv= m_hz_inv;
+    m_dependsOnX=bx, m_dependsOnY=by;
     m_Nz=grid.Nz(), m_bcz=grid.bcz(); 
-    m_g=grid;
-    dg::split( m_hz_inv, m_temp);
+    m_g.reset(grid);
+    dg::blas1::transfer( dg::evaluate( dg::zero, grid), m_hz_inv), m_hp_inv= m_hz_inv, m_hm_inv= m_hz_inv;
+    dg::split( m_hz_inv, m_temp, grid);
+    dg::split( m_hz_inv, m_f, grid);
     if( deltaPhi <=0) deltaPhi = grid.hz();
     else assert( grid.Nz() == 1 || grid.hz()==deltaPhi);
+    int dims[3], periods[3], coords[3];
+    MPI_Cart_get( m_g.get().communicator(), 3, dims, periods, coords);
+    m_coords2 = coords[2], m_sizeZ = dims[2];
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-    const aMPIGeometry2d* grid2d_ptr = grid->perp_grid();
+    const aMPIGeometry2d* grid2d_ptr = grid.perp_grid();
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-    unsigned localsize = grid2d_ptr->size();
-    MPI_Vector<dg::HVec> temp = dg::pullback( limit, *grid2d_ptr);
-    dg::blas1::transfer( temp.data(), limiter_);
-    right_ = left_ = dg::evaluate( zero, grid2d_ptr->local());
-    ghostM.resize( localsize); ghostP.resize( localsize);
+    m_perp_size = grid2d_ptr->size();
+    dg::blas1::transfer( dg::pullback(limit, *grid2d_ptr), m_limiter);
+    m_right = m_left = dg::evaluate( zero, *grid2d_ptr);
+    dg::blas1::transfer( dg::evaluate(zero, *grid2d_ptr), m_left);
+    m_ghostM = m_ghostP = m_right = m_left;
     //%%%%%%%%%%%%%%%%%%%%%%%%%%Set starting points and integrate field lines%%%%%%%%%%%%%%
-    std::vector<thrust::host_vector<double> > yp_coarse( 3), ym_coarse(yp_coarse); 
+    std::vector<thrust::host_vector<double> > yp_coarse( 3), ym_coarse(yp_coarse), yp, ym; 
     
     dg::aMPIGeometry2d* g2dField_ptr = grid2d_ptr->clone();//INTEGRATE HIGH ORDER GRID
-    g2dField_ptr->set( 7, g2dField_ptr->global().Nx(), g2dField_ptr->global().Ny());
-    dg::aGeometry2d* global_g2dField_ptr = g2dField_ptr->global_grid();
-    dg::aGrid2d local_g2dField = g2dField_ptr->local();
+    //g2dField_ptr->set( 7, g2dField_ptr->global().Nx(), g2dField_ptr->global().Ny());
+    dg::aGeometry2d* global_g2dField_ptr = g2dField_ptr->global_geometry();
+    dg::Grid2d local_g2dField = g2dField_ptr->local();
     detail::integrate_all_fieldlines2d( vec, global_g2dField_ptr, &local_g2dField, yp_coarse, ym_coarse, deltaPhi, eps);
 
     dg::MPIGrid2d g2dFine((dg::MPIGrid2d(*grid2d_ptr)));//FINE GRID
     g2dFine.multiplyCellNumbers((double)mx, (double)my);
     dg::IHMatrix interpolate = dg::create::interpolation( g2dFine.local(), local_g2dField);  //INTERPOLATE TO FINE GRID
-    dg::geo::detail::interpolate_and_clip( interpolate, g2dFine.local(), global_g2dField_ptr, yp_coarse, ym_coarse, yp, ym);
+    dg::geo::detail::interpolate_and_clip( interpolate, g2dFine.local(), *global_g2dField_ptr, yp_coarse, ym_coarse, yp, ym);
     delete g2dField_ptr;
     delete global_g2dField_ptr;
     //%%%%%%%%%%%%%%%%%%Create interpolation and projection%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-    t.tic();
     dg::IHMatrix plusFine  = dg::create::interpolation( yp[0], yp[1], grid2d_ptr->global(), globalbcx, globalbcy), plus, plusT;
     dg::IHMatrix minusFine = dg::create::interpolation( ym[0], ym[1], grid2d_ptr->global(), globalbcx, globalbcy), minus, minusT;
-    dg::IHMatrix projection = dg::create::projection( grid2d_ptr->local(), g2dFine->local());
-    t.toc();
-    std::cout <<"Creation of interpolation/projection took "<<t.diff()<<"s\n";
-    t.tic();
+    dg::IHMatrix projection = dg::create::projection( grid2d_ptr->local(), g2dFine.local());
     cusp::multiply( projection, plusFine, plus);
     cusp::multiply( projection, minusFine, minus);
-    t.toc();
-    std::cout<< "Multiplication of P*I took: "<<t.diff()<<"s\n";
     //%Transposed matrices work only for csr_matrix due to bad matrix form for ell_matrix!!!
-    dg::convert( plus, m_plus, *grid2d_ptr);
-    dg::convert( minus, m_minus, *grid2d_ptr);
+    dg::MIHMatrix temp = dg::convert( plus, *grid2d_ptr);
+    dg::blas2::transfer( temp, m_plus);
+    temp = dg::convert( minus, *grid2d_ptr);
+    dg::blas2::transfer( temp, m_minus);
     m_plusT = dg::transpose( m_plus);
     m_minusT = dg::transpose( m_minus);
-    dg::blas2::transfer( plus, m_plus);
-    dg::blas2::transfer( plusT, m_plusT);
-    dg::blas2::transfer( minus, m_minus);
-    dg::blas2::transfer( minusT, m_minusT);
     //%%%%%%%%%%%%%%%%%%%%%%%project h and copy into h vectors%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-    thrust::host_vector<double> hp( perp_size_), hm(hp);
-    dg::blas2::symv( projection, yp[2], hp);
-    dg::blas2::symv( projection, ym[2], hm);
-    for( unsigned i=0; i<grid.Nz(); i++)
-    {
-        thrust::copy( hp.begin(), hp.end(), hp_.data().begin() + i*localsize_);
-        thrust::copy( hm.begin(), hm.end(), hm_.data().begin() + i*localsize_);
-    }
-    dg::blas1::scal( hm_, -1.);
-    dg::blas1::axpby(  1., hp_, +1., hm_, hz_);
-    delete grid2d_ptr;
-
-    thrust::host_vector<double> pX, pY;
-    dg::blas1::transfer( cp.global_gather( yp[0]), pX);
-    dg::blas1::transfer( cp.global_gather( yp[1]), pY);
-
-    //construt interpolation matrix
-    plus = dg::create::interpolation( pX, pY, grid2d_ptr.local(), globalbcz);
-    cusp::transpose( plus, plusT);
-
-    dg::blas1::transfer( cm.global_gather( ym[0]), pX);
-    dg::blas1::transfer( cm.global_gather( ym[1]), pY);
-    minus = dg::create::interpolation( pX, pY, grid2d_ptr.local(), globalbcz);
-    cusp::transpose( minus, minusT);
-    for( unsigned i=0; i<g_.Nz(); i++)
-    {
-        temp_[i].resize( localsize);
-    }
+    dg::MHVec hp( dg::evaluate( dg::zero, *grid2d_ptr)), hm(hp), hz(hp);
+    dg::blas2::symv( projection, yp[2], hp.data());
+    dg::blas2::symv( projection, ym[2], hm.data());
+    dg::blas1::scal( hm, -1.);
+    dg::blas1::axpby(  1., hp, +1., hm, hz);
+    dg::blas1::transfer( hp, m_hp);
+    dg::blas1::transfer( hm, m_hm);
+    dg::blas1::transform( hp, hp, dg::INVERT<double>());
+    dg::blas1::transform( hm, hm, dg::INVERT<double>());
+    dg::blas1::transform( hz, hz, dg::INVERT<double>());
+    dg::join( std::vector<dg::MHVec >( m_Nz, hp), m_hp_inv, grid);
+    dg::join( std::vector<dg::MHVec >( m_Nz, hm), m_hm_inv, grid);
+    dg::join( std::vector<dg::MHVec >( m_Nz, hz), m_hz_inv, grid);
     delete grid2d_ptr;
 }
 
-template<class G, class M, class C, class container>
-template< class BinaryOp>
-MPI_Vector<container> Fieldaligned<G,RowDistMat<M,C>,MPI_Vector<container> >::evaluate( BinaryOp binary, unsigned p0) const
-{
-    return evaluate( binary, dg::CONSTANT(1), p0, 0);
-}
-
 template<class G, class M, class C, class container>
 template< class BinaryOp, class UnaryOp>
-MPI_Vector<container> Fieldaligned<G,RowDistMat<M,C>, MPI_Vector<container> >::evaluate( BinaryOp binary, UnaryOp unary, unsigned p0, unsigned rounds) const
+MPI_Vector<container> Fieldaligned<G,MPIDistMat<M,C>, MPI_Vector<container> >::evaluate( BinaryOp binary, UnaryOp unary, unsigned p0, unsigned rounds) const
 {
     //idea: simply apply I+/I- enough times on the init2d vector to get the result in each plane
     //unary function is always such that the p0 plane is at x=0
-    assert( g_.Nz() > 1);
     assert( p0 < m_g.get().global().Nz());
-    const aGeometry2d* g2d_ptr = m_g.get().perp_grid();
-    MPI_Vector<container> init2d = dg::pullback( binary, *g2d_perp); 
+    const aMPIGeometry2d* g2d_ptr = m_g.get().perp_grid();
+    MPI_Vector<container> init2d = dg::pullback( binary, *g2d_ptr); 
+    delete g2d_ptr;
+    unsigned Nz = m_g.get().global().Nz();
 
     MPI_Vector<container> temp(init2d), tempP(init2d), tempM(init2d);
     MPI_Vector<container> vec3d = dg::evaluate( dg::zero, m_g.get());
     std::vector<MPI_Vector<container> >  plus2d, minus2d, result;
-    dg.:split( vec3d, plus2d);
-    dg.:split( vec3d, minus2d);
-    dg.:split( vec3d, result);
+    dg::split( vec3d, plus2d, m_g.get());
+    dg::split( vec3d, minus2d, m_g.get());
+    dg::split( vec3d, result, m_g.get());
     unsigned turns = rounds; 
     if( turns ==0) turns++;
     //first apply Interpolation many times, scale and store results
     for( unsigned r=0; r<turns; r++)
-        for( unsigned i0=0; i0<m_g.get().global().Nz(); i0++)
+        for( unsigned i0=0; i0<Nz; i0++)
         {
             dg::blas1::copy( init2d, tempP);
             dg::blas1::copy( init2d, tempM);
-            unsigned rep = i0 + r*m_g.get().global().Nz(); 
+            unsigned rep = i0 + r*Nz; 
             for(unsigned k=0; k<rep; k++)
             {
-                dg::blas2::symv( plus, tempP, temp);
+                dg::blas2::symv( m_plus, tempP, temp);
                 temp.swap( tempP);
-                dg::blas2::symv( minus, tempM, temp);
+                dg::blas2::symv( m_minus, tempM, temp);
                 temp.swap( tempM);
             }
             dg::blas1::scal( tempP, unary(  (double)rep*m_g.get().hz() ) );
@@ -249,118 +235,115 @@ MPI_Vector<container> Fieldaligned<G,RowDistMat<M,C>, MPI_Vector<container> >::e
     //now we have the plus and the minus filaments
     if( rounds == 0) //there is a limiter
     {
-        for( unsigned i0=0; i0<m_g.get().global().Nz(); i0++)
+        for( unsigned i0=0; i0<Nz; i0++)
         {
-            int idx = (int)(i0+coords[2]*m_g.get().Nz())  - (int)p0;
+            int idx = (int)(i0+m_coords2*m_g.get().Nz())  - (int)p0;
             if(idx>=0)
                 result[i0] = plus2d[idx];
             else
                 result[i0] = minus2d[abs(idx)];
-            thrust::copy( result[i0].data().begin(), result[i0].data().end(), vec3d.data().begin() + i0*g2d_ptr->size());
+            thrust::copy( result[i0].data().begin(), result[i0].data().end(), vec3d.data().begin() + i0*m_perp_size);
         }
     }
     else //sum up plus2d and minus2d
     {
-        for( unsigned i0=0; i0<g_.global().Nz(); i0++)
+        for( unsigned i0=0; i0<Nz; i0++)
         {
-            unsigned revi0 = (m_g.get().global().Nz() - i0)%m_g.get().global().Nz(); //reverted index
+            unsigned revi0 = (Nz - i0)%Nz; //reverted index
             dg::blas1::axpby( 1., plus2d[i0], 0., result[i0]);
             dg::blas1::axpby( 1., minus2d[revi0], 1., result[i0]);
         }
         dg::blas1::axpby( -1., init2d, 1., result[0]);
-        for(unsigned i0=0; i0<g_.Nz(); i0++)
+        for(unsigned i0=0; i0<Nz; i0++)
         {
-            int idx = ((int)i0 + coords[2]*g_.Nz() -(int)p0 + m_g.get().global().Nz())%m_g.get().global().Nz(); //shift index
-            thrust::copy( result[idx].data().begin(), result[idx].data().end(), vec3d.data().begin() + i0*g2d_ptr->size());
+            int idx = ((int)i0 + m_coords2*m_g.get().Nz() -(int)p0 + Nz)%Nz; //shift index
+            thrust::copy( result[idx].data().begin(), result[idx].data().end(), vec3d.data().begin() + i0*m_perp_size);
         }
     }
-    delete g2d_ptr;
     return vec3d;
 }
 
 template<class G, class M, class C, class container>
-void Fieldaligned<G, RowDistMatr<M,C>, MPI_Vector<container> >::operator()(enum whichMatrix which, const MPI_Vector<container>& f, MPI_Vector<container>& fe)
+void Fieldaligned<G, MPIDistMat<M,C>, MPI_Vector<container> >::operator()(enum whichMatrix which, const MPI_Vector<container>& f, MPI_Vector<container>& fe)
 {
     if(which == einsPlus || which == einsMinusT) ePlus( which, f, fe);
     if(which == einsMinus || which == einsPlusT) eMinus( which, f, fe);
 }
 
 template<class G, class M, class C, class container>
-void Fieldaligned<G,RowDistMat<M,C>, MPI_Vector<container> >::ePlus( enum whichMatrix which, const MPI_Vector<container>& f, MPI_Vector<container>& fpe ) 
+void Fieldaligned<G,MPIDistMat<M,C>, MPI_Vector<container> >::ePlus( enum whichMatrix which, const MPI_Vector<container>& f, MPI_Vector<container>& fpe ) 
 {
-    dg::split( f, m_f);
-
+    dg::split( f, m_f, m_g.get());
     //1. compute 2d interpolation in every plane and store in m_temp
     for( unsigned i0=0; i0<m_Nz; i0++)
     {
         unsigned ip = (i0==m_Nz-1) ? 0:i0+1;
-        if(which == einsPlus)           dg::blas2::symv( plus,   m_f[ip], m_temp[i0]);
-        else if(which == einsMinusT)    dg::blas2::symv( minusT, m_f[ip], m_temp[i0]);
+        if(which == einsPlus)           dg::blas2::symv( m_plus,   m_f[ip], m_temp[i0]);
+        else if(which == einsMinusT)    dg::blas2::symv( m_minusT, m_f[ip], m_temp[i0]);
     }
 
     //2. communicate halo in z
     if( m_sizeZ != 1)
     {
         unsigned i0 = m_Nz-1;
-        detail::sendBackward( m_temp[i0].begin(), m_temp[i0].end(), m_buffer.begin());
-        m_temp[i0].swap( m_buffer);
+        detail::sendBackward( m_temp[i0].data(), m_ghostM.data(), m_g.get().communicator());
+        m_temp[i0].swap( m_ghostM);
     }
 
     //3. apply right boundary conditions in last plane
     unsigned i0=m_Nz-1;
-    if( m_bcz != dg::PER && m_g.z1() == m_g.global().z1())
+    if( m_bcz != dg::PER && m_g.get().z1() == m_g.get().global().z1())
     {
-        if( bcz_ == dg::DIR || bcz_ == dg::NEU_DIR)
+        if( m_bcz == dg::DIR || m_bcz == dg::NEU_DIR)
             dg::blas1::axpby( 2, m_right, -1., m_f[i0], m_ghostP);
-        if( bcz_ == dg::NEU || bcz_ == dg::DIR_NEU)
+        if( m_bcz == dg::NEU || m_bcz == dg::DIR_NEU)
         {
-            dg::blas1::pointwiseDivide( m_right, m_hp_inv[i0], m_ghostP);
+            dg::blas1::pointwiseDot( m_right, m_hp, m_ghostP);
             dg::blas1::axpby( 1., m_ghostP, 1., m_f[i0], m_ghostP);
         }
         //interlay ghostcells with periodic cells: L*g + (1-L)*fpe
         dg::blas1::axpby( 1., m_ghostP, -1., m_temp[i0], m_ghostP);
         dg::blas1::pointwiseDot( 1., m_limiter, m_ghostP, 1., m_temp[i0]);
     }
-    dg::join( m_temp, fpe);
+    dg::join( m_temp, fpe, m_g.get());
 }
 
 template<class G, class M, class C, class container>
-void Fieldaligned<G,RowDistMat<M,C>, MPI_Vector<container> >::eMinus( enum whichMatrix which, const MPI_Vector<container>& f, MPI_Vector<container>& fme ) 
+void Fieldaligned<G,MPIDistMat<M,C>, MPI_Vector<container> >::eMinus( enum whichMatrix which, const MPI_Vector<container>& f, MPI_Vector<container>& fme ) 
 {
-    dg::split( f, m_f);
-
+    dg::split( f, m_f, m_g.get());
     //1. compute 2d interpolation in every plane and store in m_temp
     for( unsigned i0=0; i0<m_Nz; i0++)
     {
         unsigned im = (i0==0) ? m_Nz-1:i0-1;
-        if(which == einsPlusT)         dg::blas2::symv( plusT, m_f[im], m_temp[i0]);
-        else if(which == einsMinus)    dg::blas2::symv( minus, m_f[im], m_temp[i0]);
+        if(which == einsPlusT)         dg::blas2::symv( m_plusT, m_f[im], m_temp[i0]);
+        else if(which == einsMinus)    dg::blas2::symv( m_minus, m_f[im], m_temp[i0]);
     }
 
     //2. communicate halo in z
     if( m_sizeZ != 1)
     {
         unsigned i0 = 0;
-        detail::sendForward( m_temp[i0].begin(), m_temp[i0].end(), m_buffer.begin());
-        m_temp[i0].swap( m_buffer);
+        detail::sendForward( m_temp[i0].data(), m_ghostP.data(), m_g.get().communicator());
+        m_temp[i0].swap( m_ghostP);
     }
 
     //3. apply left boundary conditions in first plane
     unsigned i0=0;
-    if( m_bcz != dg::PER)
+    if( m_bcz != dg::PER && m_g.get().z0() == m_g.get().global().z0())
     {
         if( m_bcz == dg::DIR || m_bcz == dg::DIR_NEU)
             dg::blas1::axpby( 2., m_left,  -1., m_f[i0], m_ghostM);
         if( m_bcz == dg::NEU || m_bcz == dg::NEU_DIR)
         {
-            dg::blas1::pointwiseDivide( m_left, m_hm_inv[i0], m_ghostM);
+            dg::blas1::pointwiseDot( m_left, m_hm, m_ghostM);
             dg::blas1::axpby( -1., m_ghostM, 1., m_f[i0], m_ghostM);
         }
         //interlay ghostcells with periodic cells: L*g + (1-L)*fme
         dg::blas1::axpby( 1., m_ghostM, -1., m_temp[i0], m_ghostM);
         dg::blas1::pointwiseDot( 1., m_limiter, m_ghostM, 1., m_temp[i0]);
     }
-    dg::join( m_temp, fme);
+    dg::join( m_temp, fme, m_g.get());
 }
 
 
-- 
GitLab


From bc3004a7a3a830d98abf6d33d28dc29cefa224bc Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Wed, 18 Oct 2017 23:11:32 +0200
Subject: [PATCH 369/453] experiments with code snippets in documentation

---
 inc/dg/Doxyfile                 |  8 ++---
 inc/dg/backend/evaluation.cuh   |  6 +++-
 inc/dg/backend/evaluation_t.cu  |  2 +-
 inc/dg/backend/grid.h           |  3 +-
 inc/dg/backend/mpi_evaluation.h |  8 ++---
 inc/dg/backend/mpi_grid.h       |  2 ++
 inc/dg/backend/mpi_precon.h     | 34 ++++---------------
 inc/dg/backend/weights.cuh      | 60 +++++++++++----------------------
 inc/dg/blas2.h                  |  3 +-
 inc/dg/cg.h                     |  2 ++
 inc/dg/cg2d_t.cu                |  4 +--
 inc/dg/code_snippets.dox        | 56 ++++++++++++++++++++++++++++++
 inc/dg/dg_doc.h                 | 23 ++++---------
 13 files changed, 111 insertions(+), 100 deletions(-)
 create mode 100644 inc/dg/code_snippets.dox

diff --git a/inc/dg/Doxyfile b/inc/dg/Doxyfile
index bbf0519b5..fe89eaf56 100644
--- a/inc/dg/Doxyfile
+++ b/inc/dg/Doxyfile
@@ -291,7 +291,8 @@ OPTIMIZE_OUTPUT_VHDL   = NO
 # Note that for custom extensions you also need to set FILE_PATTERNS otherwise
 # the files are not read by doxygen.
 
-EXTENSION_MAPPING      = cuh=C++
+EXTENSION_MAPPING      = cuh=C++ \
+                         cu=C++
 
 # If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments
 # according to the Markdown format, which allows for more readable
@@ -808,8 +809,7 @@ INPUT_ENCODING         = UTF-8
 # *.m, *.markdown, *.md, *.mm, *.dox, *.py, *.pyw, *.f90, *.f, *.for, *.tcl,
 # *.vhd, *.vhdl, *.ucf, *.qsf, *.as and *.js.
 
-FILE_PATTERNS          = *.h \
-                         *.cuh
+FILE_PATTERNS          = *.h *.cuh *.dox
 
 # The RECURSIVE tag can be used to specify whether or not subdirectories should
 # be searched for input files as well.
@@ -864,7 +864,7 @@ EXCLUDE_SYMBOLS        = hide_*
 # that contain example code fragments that are included (see the \include
 # command).
 
-EXAMPLE_PATH           =
+EXAMPLE_PATH           = cg2d_t.cu
 
 # If the value of the EXAMPLE_PATH tag contains directories, you can use the
 # EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and
diff --git a/inc/dg/backend/evaluation.cuh b/inc/dg/backend/evaluation.cuh
index 680cb8252..ce6688a4b 100644
--- a/inc/dg/backend/evaluation.cuh
+++ b/inc/dg/backend/evaluation.cuh
@@ -27,6 +27,8 @@ namespace dg
  * @param g The grid on which to evaluate f
  *
  * @return  A DG Host Vector with values
+ * @sa <a href="./dg_introduction.pdf" target="_blank">Introduction to dg methods</a>
+ * @copydoc hide_code_evaluate1d
  */
 template< class UnaryOp>
 thrust::host_vector<double> evaluate( UnaryOp f, const Grid1d& g)
@@ -54,6 +56,7 @@ thrust::host_vector<double> evaluate( double (f)(double), const Grid1d& g)
  * @param g The 2d grid on which to evaluate f
  *
  * @return  A dG Host Vector with values
+ * @sa <a href="./dg_introduction.pdf" target="_blank">Introduction to dg methods</a>
  * @copydoc hide_code_evaluate2d
  */
 template< class BinaryOp>
@@ -92,7 +95,8 @@ thrust::host_vector<double> evaluate( double(f)(double, double), const aTopology
  * @param g The 3d grid on which to evaluate f
  *
  * @return  A dG Host Vector with values
- * @note if you don't like to copy f then just pass a (const) reference then the type should adapt
+ * @sa <a href="./dg_introduction.pdf" target="_blank">Introduction to dg methods</a>
+ * @copydoc hide_code_evaluate3d
  */
 template< class TernaryOp>
 thrust::host_vector<double> evaluate( const TernaryOp& f, const aTopology3d& g)
diff --git a/inc/dg/backend/evaluation_t.cu b/inc/dg/backend/evaluation_t.cu
index 0a43f0025..46ae17ca2 100644
--- a/inc/dg/backend/evaluation_t.cu
+++ b/inc/dg/backend/evaluation_t.cu
@@ -48,7 +48,7 @@ int main()
     dg::Grid3d g3d( 0, lx,0, ly,0, lz, n, Nx, Ny, Nz,dg::PER,dg::PER,dg::PER);
 
     //test evaluation functions
-    const HVec h_x = dg::evaluate( function, g1d);
+    const HVec h_x = dg::evaluate( exp, g1d);
     const HVec h_n = dg::evaluate( function, g2d);
     const HVec h_z = dg::evaluate( function, g3d);
     const HVec w1d = dg::create::weights( g1d);
diff --git a/inc/dg/backend/grid.h b/inc/dg/backend/grid.h
index 03059f43d..6c8cbbd51 100644
--- a/inc/dg/backend/grid.h
+++ b/inc/dg/backend/grid.h
@@ -54,7 +54,7 @@ namespace dg{
 /**
 * @brief 1D grid
 * @ingroup grid
-*
+* @copydoc hide_code_evaluate1d
 */
 struct Grid1d
 {
@@ -729,6 +729,7 @@ struct Grid2d : public aTopology2d
 /**
  * @brief The simplest implementation of aTopology3d
  * @ingroup grid
+ * @copydoc hide_code_evaluate3d
  */
 struct Grid3d : public aTopology3d
 {
diff --git a/inc/dg/backend/mpi_evaluation.h b/inc/dg/backend/mpi_evaluation.h
index f15a4bd1c..ed7cc41a4 100644
--- a/inc/dg/backend/mpi_evaluation.h
+++ b/inc/dg/backend/mpi_evaluation.h
@@ -23,8 +23,8 @@ namespace dg
  * @param g The 2d grid on which to evaluate f
  *
  * @return  A MPI Vector with values
- * @note Copies the binary Operator. This function is meant for small function objects, that
-            may be constructed during function call.
+ * @sa <a href="./dg_introduction.pdf" target="_blank">Introduction to dg methods</a>
+ * @copydoc hide_code_mpi_evaluate2d
  */
 template< class BinaryOp>
 MPI_Vector<thrust::host_vector<double> > evaluate( const BinaryOp& f, const aMPITopology2d& g)
@@ -50,8 +50,8 @@ MPI_Vector<thrust::host_vector<double> > evaluate( double(f)(double, double), co
  * @param g The 3d grid on which to evaluate f
  *
  * @return  A MPI Vector with values
- * @note Copies the ternary Operator. This function is meant for small function objects, that
-            may be constructed during function call.
+ * @sa <a href="./dg_introduction.pdf" target="_blank">Introduction to dg methods</a>
+ * @copydoc hide_code_mpi_evaluate3d
  */
 template< class TernaryOp>
 MPI_Vector<thrust::host_vector<double> > evaluate( const TernaryOp& f, const aMPITopology3d& g)
diff --git a/inc/dg/backend/mpi_grid.h b/inc/dg/backend/mpi_grid.h
index 54fcc2404..fed48016b 100644
--- a/inc/dg/backend/mpi_grid.h
+++ b/inc/dg/backend/mpi_grid.h
@@ -716,6 +716,7 @@ void aMPITopology3d::do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, u
 /**
  * @brief The simplest implementation of aMPITopology2d
  * @ingroup grid
+ * @copydoc hide_code_mpi_evaluate2d
  */
 struct MPIGrid2d: public aMPITopology2d
 {
@@ -746,6 +747,7 @@ struct MPIGrid2d: public aMPITopology2d
 /**
  * @brief The simplest implementation of aMPITopology3d
  * @ingroup grid
+ * @copydoc hide_code_mpi_evaluate3d
  */
 struct MPIGrid3d : public aMPITopology3d
 {
diff --git a/inc/dg/backend/mpi_precon.h b/inc/dg/backend/mpi_precon.h
index a9fc44098..c2fde459e 100644
--- a/inc/dg/backend/mpi_precon.h
+++ b/inc/dg/backend/mpi_precon.h
@@ -34,49 +34,27 @@ namespace create
 ///@addtogroup highlevel
 ///@{
 
-/**
-* @brief create Preconditioner containing 2d X-space weight coefficients
-*
-* @param g The grid 
-*
-* @return Preconditioner
-*/
+///@copydoc hide_weights_doc
+///@copydoc hide_code_mpi_evaluate2d
 MPI_Vector<thrust::host_vector<double> > weights( const aMPITopology2d& g)
 {
     thrust::host_vector<double> w = create::weights( g.local());
     return MPI_Vector<thrust::host_vector<double> >( w, g.communicator());
 }
-/**
-* @brief create Preconditioner containing 2d inverse X-space weight coefficients
-*
-* @param g The grid 
-*
-* @return Preconditioner
-*/
+///@copydoc hide_inv_weights_doc
 MPI_Vector<thrust::host_vector<double> > inv_weights( const aMPITopology2d& g)
 {
     thrust::host_vector<double> w = create::inv_weights( g.local());
     return MPI_Vector<thrust::host_vector<double> >( w, g.communicator());
 }
-/**
-* @brief create Preconditioner containing 3d X-space weight coefficients
-*
-* @param g The grid 
-*
-* @return Preconditioner
-*/
+///@copydoc hide_weights_doc
+///@copydoc hide_code_mpi_evaluate3d
 MPI_Vector<thrust::host_vector<double> > weights( const aMPITopology3d& g)
 {
     thrust::host_vector<double> w = create::weights( g.local());
     return MPI_Vector<thrust::host_vector<double> >( w, g.communicator());
 }
-/**
-* @brief create Preconditioner containing 3d inverse X-space weight coefficients
-*
-* @param g The grid 
-*
-* @return Preconditioner
-*/
+///@copydoc hide_inv_weights_doc
 MPI_Vector<thrust::host_vector<double> > inv_weights( const aMPITopology3d& g)
 {
     thrust::host_vector<double> w = create::inv_weights( g.local());
diff --git a/inc/dg/backend/weights.cuh b/inc/dg/backend/weights.cuh
index 0385001db..e84da7a18 100644
--- a/inc/dg/backend/weights.cuh
+++ b/inc/dg/backend/weights.cuh
@@ -38,14 +38,21 @@ thrust::host_vector<double> abscissas( const Grid1d& g)
 ///@addtogroup highlevel
 ///@{
 
-
-/**
-* @brief create host_vector containing 1d X-space weight coefficients
-*
+/*!@class hide_weights_doc
+* @brief create host_vector containing X-space weight coefficients
+* @param g The grid 
+* @return Host Vector
+* @sa <a href="./dg_introduction.pdf" target="_blank">Introduction to dg methods</a>
+*/
+/*!@class hide_inv_weights_doc
+* @brief create host_vector containing inverse X-space weight coefficients
 * @param g The grid 
-*
 * @return Host Vector
+* @sa <a href="./dg_introduction.pdf" target="_blank">Introduction to dg methods</a>
 */
+
+///@copydoc hide_weights_doc
+///@copydoc hide_code_evaluate1d
 thrust::host_vector<double> weights( const Grid1d& g)
 {
     thrust::host_vector<double> v( g.size());
@@ -54,13 +61,7 @@ thrust::host_vector<double> weights( const Grid1d& g)
             v[i*g.n() + j] = g.h()/2.*g.dlt().weights()[j];
     return v;
 }
-/**
-* @brief create host_vector containing 1d X-space inverse weight coefficients
-*
-* @param g The grid 
-*
-* @return Host Vector
-*/
+///@copydoc hide_inv_weights_doc
 thrust::host_vector<double> inv_weights( const Grid1d& g)
 {
     thrust::host_vector<double> v = weights( g);
@@ -79,14 +80,8 @@ int get_j( unsigned n, unsigned Nx, int idx) { return idx%n;}
 }//namespace detail
 ///@endcond
 
-/**
-* @brief create host_vector containing 2d X-space integration weight coefficients
-*
-* @param g The grid 
-*
-* @return Host Vector
-* @copydoc hide_code_evaluate2d
-*/
+///@copydoc hide_weights_doc
+///@copydoc hide_code_evaluate2d
 thrust::host_vector<double> weights( const aTopology2d& g)
 {
     //choose layout
@@ -98,13 +93,7 @@ thrust::host_vector<double> weights( const aTopology2d& g)
                 g.dlt().weights()[detail::get_j(g.n(),g.Nx(), i)];
     return v;
 }
-/**
-* @brief create host_vector containing 2d X-space inverse weight coefficients
-*
-* @param g The grid 
-*
-* @return Host Vector
-*/
+///@copydoc hide_inv_weights_doc
 thrust::host_vector<double> inv_weights( const aTopology2d& g)
 {
     thrust::host_vector<double> v = weights( g);
@@ -113,13 +102,8 @@ thrust::host_vector<double> inv_weights( const aTopology2d& g)
     return v;
 }
 
-/**
-* @brief create host_vector containing 3d X-space weight coefficients for integration
-*
-* @param g The grid 
-*
-* @return Host Vector
-*/
+///@copydoc hide_weights_doc
+///@copydoc hide_code_evaluate3d
 thrust::host_vector<double> weights( const aTopology3d& g)
 {
     thrust::host_vector<double> v( g.size());
@@ -131,13 +115,7 @@ thrust::host_vector<double> weights( const aTopology3d& g)
     return v;
 }
 
-/**
-* @brief create host_vector containing 3d X-space inverse weight coefficients
-*
-* @param g The grid 
-*
-* @return Host Vector
-*/
+///@copydoc hide_inv_weights_doc
 thrust::host_vector<double> inv_weights( const aTopology3d& g)
 {
     thrust::host_vector<double> v = weights( g);
diff --git a/inc/dg/blas2.h b/inc/dg/blas2.h
index 7acd5e181..e1e8b8b49 100644
--- a/inc/dg/blas2.h
+++ b/inc/dg/blas2.h
@@ -71,7 +71,6 @@ inline typename MatrixTraits<DiagonalMatrix>::value_type dot( const container& x
 
 /*! @brief \f$ x^T M x\f$; General dot produt
  *
- * This routine is equivalent to the call blas2::dot( x, m, x):
  * \f[ x^T M x = \sum_{i,j=0}^{N-1} x_i M_{ij} x_j \f]
  * @tparam DiagonalMatrix Right now DiagonalMatrix has to be the same as container, except if the container is a std::vector<container_type>, then the DiagonalMatrix has to be the container_type. 
  * In the latter case the Matrix is applied to all containers in the std::vector and the sum is returned. 
@@ -81,6 +80,8 @@ inline typename MatrixTraits<DiagonalMatrix>::value_type dot( const container& x
  * @return Generalized scalar product
  * @note This routine is always executed synchronously due to the 
     implicit memcpy of the result.
+ * @note This routine is equivalent to the call dg::blas2::dot( x, m, x);
+     which should be prefered because it looks more explicit
  */
 template< class DiagonalMatrix, class container>
 inline typename MatrixTraits<DiagonalMatrix>::value_type dot( const DiagonalMatrix& m, const container& x)
diff --git a/inc/dg/cg.h b/inc/dg/cg.h
index 7fd78e5d5..e8298a312 100644
--- a/inc/dg/cg.h
+++ b/inc/dg/cg.h
@@ -27,6 +27,8 @@ namespace dg{
 
  @note Conjugate gradients might become unstable for positive semidefinite
  matrices arising e.g. in the discretization of the periodic laplacian
+
+ @snippet cg2d_t.cu doxygen
 */
 template< class container>
 class CG
diff --git a/inc/dg/cg2d_t.cu b/inc/dg/cg2d_t.cu
index 211683f69..9632dd4c5 100644
--- a/inc/dg/cg2d_t.cu
+++ b/inc/dg/cg2d_t.cu
@@ -28,16 +28,16 @@ int main()
     std::cout<<"Evaluate initial condition\n";
     dg::HVec x = dg::evaluate( initial, grid);
 
+//! [doxygen]
     std::cout << "Create Laplacian\n";
     dg::Elliptic<dg::CartesianGrid2d, dg::HMatrix, dg::HVec> A( grid);
     dg::CG<dg::HVec > pcg( x, n*n*Nx*Ny);
     std::cout<<"Evaluate right hand side\n";
     dg::HVec b = dg::evaluate ( laplace_fct, grid);
     const dg::HVec solution = dg::evaluate ( fct, grid);
-    //////////////////////////////////////////////////////////////////////
-    //compute S b
     dg::blas2::symv( w2d, b, b);
     std::cout << "Number of pcg iterations "<< pcg( A, x, b, v2d, eps_)<<std::endl;
+//! [doxygen]
     //std::cout << "Number of cg iterations "<< pcg( A, x, b, dg::Identity<double>(), eps)<<std::endl;
     std::cout << "For a precision of "<< eps_<<std::endl;
     //compute error
diff --git a/inc/dg/code_snippets.dox b/inc/dg/code_snippets.dox
new file mode 100644
index 000000000..b702a4418
--- /dev/null
+++ b/inc/dg/code_snippets.dox
@@ -0,0 +1,56 @@
+#error Documentation only
+
+/** @class hide_code_evaluate1d
+ * This code snippet demonstrates how to discretize and integrate a function with dG
+@code{.cpp}
+dg::Grid1d g1d( 0, 2, 3, 20);
+const dg::HVec w1d = dg::create::weights( g1d);
+const dg::HVec h_x = dg::evaluate( exp, g1d);
+double norm = dg::blas2::dot( h_x, w1d, h_x); // norm is now: (exp(4)-exp(0))/2
+@endcode
+*/
+/** @class hide_code_evaluate2d
+ * This code snippet demonstrates how to discretize and integrate a function with dG
+@code{.cpp}
+double function(double x, double y){return exp(x)*exp(y);}
+//...
+dg::Grid2d g2d( 0, 2, 0, 2, 3, 20, 20);
+const dg::HVec w2d = dg::create::weights( g2d);
+const dg::HVec h_x = dg::evaluate( function, g2d);
+double norm = dg::blas2::dot( h_x, w2d, h_x); // norm is now: (exp(4)-exp(0))^2/4
+@endcode
+*/
+/** @class hide_code_evaluate3d
+ * This code snippet demonstrates how to discretize and integrate a function with dG
+@code{.cpp}
+double function(double x, double y, double z){return exp(x)*exp(y)*exp(z);}
+//...
+dg::Grid3d g3d( 0, 2, 0, 2, 0, 2, 3, 20, 20, 20);
+const dg::HVec w3d = dg::create::weights( g3d);
+const dg::HVec h_x = dg::evaluate( function, g3d);
+double norm = dg::blas2::dot(h_x, w3d, h_x); // norm is now: (exp(4)-exp(0))^3/8
+@endcode
+*/
+/** @class hide_code_mpi_evaluate2d
+ * This code snippet demonstrates how to discretize and integrate a function with dG
+@code{.cpp}
+double function(double x, double y){return exp(x)*exp(y);}
+//... use MPI_Cart_create to create 2d Cartesian communicator
+dg::MPIGrid2d g2d( 0, 2, 0, 2, 3, 20, 20, comm2d );
+const dg::MHVec w2d = dg::create::weights( g2d);
+const dg::MHVec h_x = dg::evaluate( function, g2d);
+double norm = dg::blas2::dot( h_x, w2d, h_x); // norm is now: (exp(4)-exp(0))^2/4
+@endcode
+@sa <a href="./dg_introduction.pdf" target="_blank">Introduction to dg methods</a>
+*/
+/** @class hide_code_mpi_evaluate3d
+ * This code snippet demonstrates how to discretize and integrate a function with dG
+@code{.cpp}
+double function(double x, double y, double z){return exp(x)*exp(y)*exp(z);}
+//... use MPI_Cart_create to create 3d Cartesian communicator
+dg::Grid3d g3d( 0, 2, 0, 2, 0, 2, 3, 20, 20, 20, comm3d);
+const dg::MHVec w3d = dg::create::weights( g3d);
+const dg::MHVec h_x = dg::evaluate( function, g3d);
+double norm = dg::blas2::dot(h_x, w3d, h_x); // norm is now: (exp(4)-exp(0))^3/8
+@endcode
+*/
diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index b9ba73085..aa5d4b2df 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -182,31 +182,19 @@
   and specializing MatrixTraits with the SelfMadeMatrixTag as the matrix_category
   */
 
-/** @class hide_code_evaluate2d
- * This code snippet demonstrates how to integrate a function discretized with dG
-@code{.cpp}
-double function(double x, double y){return exp(x)*exp(y);}
-//...
-dg::Grid2d g2d( 0, 2, 0, 2, 3, 20, 20);
-const dg::HVec w2d = dg::create::weights( g2d);
-const dg::HVec h_x = dg::evaluate( function, g2d);
-double norm = dg::blas2::dot( h_x, w2d, h_x); // norm is now: (exp(4)-exp(0))/2
-@endcode
-*/
-
-
 /*!@addtogroup mpi_structures
 @{
-@page mpi_matrix MPI Vectors and the blas1 functions
+@note The mpi backend is activated by including \c mpi.h before any other feltor header file
+@section mpi_vector MPI Vectors and the blas1 functions
 
 In Feltor each mpi process gets an equally sized chunk of a vector.
-The corresponding structure in FELTOR is the dg::MPI_Vector, which is 
+The corresponding structure in FELTOR is the \c dg::MPI_Vector, which is 
 nothing but a wrapper around any container type object and a MPI_Comm. 
 With this the dg::blas1 functions can readily implemented by just redirecting to the
 implementation for the container type. The only functions that need
 communication are the dg::blas1::dot functions (MPI_Allreduce).
 
-@page mpi_vector Row and column distributed matrices
+@section mpi_matrix Row and column distributed matrices
 
 Contrary to a vector
 a matrix can be distributed in two ways, row-wise and column wise. 
@@ -221,6 +209,7 @@ vector it holds.
 When we implement a matrix-vector multiplication the order 
 of communication and computation depends on the distribution 
 of the matrix.
+\subsection row Row distributed
 For the row-distributed matrix each process first has to gather 
 all elements of the input vector it needs to be able to compute the elements of the output. In general this requires MPI communication.
 (read the documentation of dg::aCommunicator for more info of how global scatter/gather operations work).
@@ -236,7 +225,7 @@ and \f$G\f$ is the gather matrix, in which the MPI-communication takes place.
 The dg::RowColDistMat goes one step further and separates the matrix \f$ R\f$ into 
 a part that can be computed entirely on the local process and a part that needs communication.
 
-\section column 
+\subsection column Column distributed
 
 In a column distributed matrix the local matrix-vector multiplication can be executed first because each processor already
 has all vector elements it needs. 
-- 
GitLab


From bf6c85b51543d4ff2ec8da6ca44158950c1d51a2 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <Matthias.Wiesenberger@uibk.ac.at>
Date: Thu, 19 Oct 2017 11:21:05 +0200
Subject: [PATCH 370/453] debugged toefl_hpc and toefl_mpi

---
 src/toefl/toefl_hpc.cu | 4 ++--
 src/toefl/toefl_mpi.cu | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/toefl/toefl_hpc.cu b/src/toefl/toefl_hpc.cu
index 1c493be2e..f3377d847 100644
--- a/src/toefl/toefl_hpc.cu
+++ b/src/toefl/toefl_hpc.cu
@@ -43,8 +43,8 @@ int main( int argc, char* argv[])
     ////////////////////////////////set up computations///////////////////////////
     dg::Grid2d grid( 0, p.lx, 0, p.ly, p.n, p.Nx, p.Ny, p.bc_x, p.bc_y);
     //create RHS 
-    dg::ToeflR< dg::CartesianGrid2d, dg::DMatrix, dg::DVec > test( grid, p); 
-    dg::Diffusion<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> diffusion( grid, p.nu);
+    toefl::Explicit< dg::CartesianGrid2d, dg::DMatrix, dg::DVec > test( grid, p); 
+    toefl::Implicit<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> diffusion( grid, p.nu);
     /////////////////////create initial vector////////////////////////////////////
     dg::Gaussian g( p.posX*p.lx, p.posY*p.ly, p.sigma, p.sigma, p.amp); 
     std::vector<dg::DVec> y0(2, dg::evaluate( g, grid)), y1(y0); // n_e' = gaussian
diff --git a/src/toefl/toefl_mpi.cu b/src/toefl/toefl_mpi.cu
index 34ac09eb6..cc066e67a 100644
--- a/src/toefl/toefl_mpi.cu
+++ b/src/toefl/toefl_mpi.cu
@@ -74,8 +74,8 @@ int main( int argc, char* argv[])
     dg::MPIGrid2d grid( 0, p.lx, 0, p.ly, p.n, p.Nx, p.Ny, p.bc_x, p.bc_y, comm);
     dg::MPIGrid2d grid_out( 0., p.lx, 0.,p.ly, p.n_out, p.Nx_out, p.Ny_out, p.bc_x, p.bc_y, comm);  
     //create RHS 
-    dg::ToeflR< dg::CartesianMPIGrid2d, dg::MDMatrix, dg::MDVec > test( grid, p); 
-    dg::Diffusion<dg::CartesianMPIGrid2d, dg::MDMatrix, dg::MDVec> diffusion( grid, p.nu);
+    toefl::Explicit< dg::CartesianMPIGrid2d, dg::MDMatrix, dg::MDVec > test( grid, p); 
+    toefl::Implicit<dg::CartesianMPIGrid2d, dg::MDMatrix, dg::MDVec> diffusion( grid, p.nu);
     //////////////////create initial vector///////////////////////////////////////
     dg::Gaussian g( p.posX*p.lx, p.posY*p.ly, p.sigma, p.sigma, p.amp); 
     std::vector<dg::MDVec> y0(2, dg::evaluate( g, grid)), y1(y0); // n_e' = gaussian
-- 
GitLab


From f91feeefcfea1e011118ef1df0b0a30c2bd7aa73 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <Matthias.Wiesenberger@uibk.ac.at>
Date: Thu, 19 Oct 2017 11:32:28 +0200
Subject: [PATCH 371/453] added missing doGemv in mpi_matrix_blas

---
 inc/dg/backend/mpi_matrix_blas.h | 7 ++++++-
 src/toefl/toefl_mpi.cu           | 4 ----
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/inc/dg/backend/mpi_matrix_blas.h b/inc/dg/backend/mpi_matrix_blas.h
index 39efe1865..645e51e41 100644
--- a/inc/dg/backend/mpi_matrix_blas.h
+++ b/inc/dg/backend/mpi_matrix_blas.h
@@ -31,7 +31,12 @@ inline void doSymv( typename MatrixTraits<Matrix>::value_type alpha, const Matri
 template< class Matrix, class Vector1, class Vector2>
 inline void doGemv( Matrix& m, Vector1&x, Vector2& y, MPIMatrixTag, MPIVectorTag, MPIVectorTag  )
 {
-    doSymv( m, x, y, MPIMatrixTag(), MPIVectorTag(), MPIVectorTag());
+    m.symv( x, y);
+}
+template< class Matrix, class Vector>
+inline void doGemv( typename MatrixTraits<Matrix>::value_type alpha, const Matrix& m, const Vector& x, typename MatrixTraits<Matrix>::value_type beta, Vector& y, MPIMatrixTag, MPIVectorTag )
+{
+    m.symv( alpha, x, beta, y);
 }
 template< class Matrix, class Vector>
 inline void doSymv( Matrix& m, Vector& x, Vector& y, CuspMatrixTag, MPIVectorTag, MPIVectorTag )
diff --git a/src/toefl/toefl_mpi.cu b/src/toefl/toefl_mpi.cu
index cc066e67a..7678b588a 100644
--- a/src/toefl/toefl_mpi.cu
+++ b/src/toefl/toefl_mpi.cu
@@ -101,10 +101,6 @@ int main( int argc, char* argv[])
     MPI_Info info = MPI_INFO_NULL;
     err = nc_create_par( argv[2],NC_NETCDF4|NC_MPIIO|NC_CLOBBER,comm,info, &ncid);
     err = nc_put_att_text( ncid, NC_GLOBAL, "inputfile", input.size(), input.data());
-    const int version[3] = {FELTOR_MAJOR_VERSION, FELTOR_MINOR_VERSION, FELTOR_SUBMINOR_VERSION}; //write maybe to json file!?
-    err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_major_version", NC_INT, 1, &version[0]);
-    err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_minor_version", NC_INT, 1, &version[1]);
-    err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_subminor_version", NC_INT, 1, &version[2]);
     int dim_ids[3], tvarID;
     err = file::define_dimensions( ncid, dim_ids, &tvarID, grid_out.global());
     //field IDs
-- 
GitLab


From c4a15364cb16754a4b09e6c7fce6b60bdbb6b777 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Thu, 19 Oct 2017 16:00:37 +0200
Subject: [PATCH 372/453] writeup regarding ds_t and ds_mpit and made ds_mpit
 compile

---
 doc/related_pages/newcommands.tex       |  2 +
 doc/related_pages/parallel/parallel.tex | 51 ++++++++++++++
 inc/dg/backend/mpi_matrix.h             |  4 +-
 inc/geometries/ds_mpit.cu               | 92 +++++++------------------
 inc/geometries/ds_t.cu                  | 25 +++----
 5 files changed, 89 insertions(+), 85 deletions(-)

diff --git a/doc/related_pages/newcommands.tex b/doc/related_pages/newcommands.tex
index ce64c7952..5705f445e 100644
--- a/doc/related_pages/newcommands.tex
+++ b/doc/related_pages/newcommands.tex
@@ -1,3 +1,5 @@
+\definecolor{light-gray}{gray}{0.95}
+\newcommand{\code}[1]{\colorbox{light-gray}{\texttt{#1}}}
 \newcommand{\eps}{\varepsilon}
 \renewcommand{\d}{\mathrm{d}}
 \renewcommand{\vec}[1]{{\mathbf{#1}}}
diff --git a/doc/related_pages/parallel/parallel.tex b/doc/related_pages/parallel/parallel.tex
index 460be8663..f7509865f 100644
--- a/doc/related_pages/parallel/parallel.tex
+++ b/doc/related_pages/parallel/parallel.tex
@@ -448,5 +448,56 @@ procedures.
 %..................................................................
 
 
+\section{Numerical test programs}
+The essential test programs for the parallel derivative are located in 
+\code{
+path/to/feltor/inc/geometries}.
+\code{ ds\_t.cu} and \code{ ds\_mpit.cu} test the cylindrical grid for shared and distributed memory 
+systems. \code{ ds\_curvilinear\_t.cu} and \code{ ds\_curvilinear\_mpit.cu} test the implementation
+on a flux-aligned grid.
+The magnetic field in FELTOR is given by
+\begin{align}
+  \vec B = \frac{R_0}{R}( I(\psi_p) \hat e_\varphi + \nabla\psi_p \times\hat e_\varphi)
+\end{align}
+This gives rise to a magnetic field strength and components
+\begin{align}
+  B = \frac{R_0}{R} \sqrt{ I^2 + \left( \nabla\psi_p \right)^2} \\
+  B^R = \frac{R_0}{R}\frac{\partial\psi_p}{\partial Z} \quad
+  B^Z = -\frac{R_0}{R}\frac{\partial\psi_p}{\partial R}\quad 
+  B^\varphi = \frac{R_0I}{R^2}
+  \label{}
+\end{align}
+\subsection{Cylindrical grid and circular flux surfaces}
+A very simple non-trivial choice for the poloidal flux is
+\begin{align}
+  \psi_p = \frac{1}{2} \left( (R-R_0)^2 + Z^2 \right) \equiv \frac{1}{2} r^2
+  \label{eq:circular}
+\end{align}
+for which 
+\begin{align}
+  \vec b^p &= \frac{1}{\sqrt{I^2 + r^2}} \begin{pmatrix} Z \\ -(R-R_0)\end{pmatrix} \\
+  b^\varphi &= \frac{1}{\sqrt{I^2 + r^2}} I/R
+  \label{}
+\end{align}
+for the poloidal and toroidal parts of the magnetic unit vector. 
+We choose $R_0 = 10$ and $I=20$ in order to keep the q-factor for the $r=1$ flux surface at $2$.
+We set up a domain 
+$R\in[R_0-1, R_0+1]$,
+$Z\in[-1,1]$ and 
+$\varphi \in [0,2\pi]$ and choose
+\begin{align}
+  f(R,Z,\varphi) = ((R-R_0)^2 + Z^2)\sin(\varphi)\\
+  \bhat \cdot \nabla f= b^\varphi \cos(\varphi)
+  \label{}
+\end{align}
+This is advantageous since the solution is correct even if we modify the $\bhat$ field
+to be toroidal on the boundary of the domain in order to avoid boundary conditions. 
+Also the function is parabolic, which means the dG polynomials of degree greater than $2$ 
+should be exact. 
+\subsection{Curvilinear grid}
+
+
+
+
 \end{document}
 
diff --git a/inc/dg/backend/mpi_matrix.h b/inc/dg/backend/mpi_matrix.h
index ded961022..5056afcb6 100644
--- a/inc/dg/backend/mpi_matrix.h
+++ b/inc/dg/backend/mpi_matrix.h
@@ -242,7 +242,7 @@ struct MPIDistMat
 
     
     template<class container> 
-    void symv( double alpha, const MPI_Vector<container>& x, double beta, MPI_Vector<container>& y)
+    void symv( double alpha, const MPI_Vector<container>& x, double beta, MPI_Vector<container>& y)const
     {
         if( m_c.size() == 0) //no communication needed
         {
@@ -271,7 +271,7 @@ struct MPIDistMat
         }
     }
     template<class container> 
-    void symv( const MPI_Vector<container>& x, MPI_Vector<container>& y)
+    void symv( const MPI_Vector<container>& x, MPI_Vector<container>& y)const
     {
         if( m_c.get().size() == 0) //no communication needed
         {
diff --git a/inc/geometries/ds_mpit.cu b/inc/geometries/ds_mpit.cu
index b6d0e6f22..5ca941ce5 100644
--- a/inc/geometries/ds_mpit.cu
+++ b/inc/geometries/ds_mpit.cu
@@ -2,50 +2,17 @@
 
 #include <mpi.h>
 
+#include "dg/backend/timer.cuh"
+#include "dg/backend/mpi_evaluation.h"
+#include "dg/backend/mpi_init.h"
+#include "dg/backend/functions.h"
+#include "dg/geometry/geometry.h"
 #include "ds.h"
+#include "toroidal.h"
 
-#include "backend/mpi_evaluation.h"
-#include "backend/mpi_init.h"
-#include "backend/functions.h"
-#include "backend/timer.cuh"
-
-struct Field
-{
-    Field( double R_0, double I_0):R_0(R_0), I_0(I_0){}
-    void operator()( const dg::HVec& y, dg::HVec& yp) const
-    {
-        double gradpsi = ((y[0]-R_0)*(y[0]-R_0) + y[1]*y[1])/I_0/I_0;
-        yp[2] = y[0]*sqrt(1 + gradpsi);
-        yp[0] = y[0]*y[1]/I_0;
-        yp[1] = y[0]/I_0*(R_0-y[0]) ;
-    }
-    double operator()( double x, double y, double z) const
-    {
-        double gradpsi = ((x-R_0)*(x-R_0) + y*y)/I_0/I_0;
-        return  x/sqrt( 1 + gradpsi)/R_0/I_0;
-    }
-    double error( const dg::HVec& x0, const dg::HVec& x1)
-    {
-        return sqrt( (x0[0]-x1[0])*(x0[0]-x1[0]) +(x0[1]-x1[1])*(x0[1]-x1[1])+(x0[2]-x1[2])*(x0[2]-x1[2]));
-    }
-    bool monitor( const dg::HVec& end){ 
-        if ( std::isnan(end[0]) || std::isnan(end[1]) || std::isnan(end[2]) ) 
-        {
-            return false;
-        }
-        //if new integrated point outside domain
-        if ((1e-5 > end[0]  ) || (1e10 < end[0])  ||(-1e10  > end[1]  ) || (1e10 < end[1])||(-1e10 > end[2]  ) || (1e10 < end[2])  )
-        {
-            return false;
-        }
-        return true;
-    }
-    private:
-    double R_0, I_0;
-};
-
-double R_0 = 10;
-double I_0 = 40;
+const double R_0 = 10;
+const double I_0 = 20; //q factor at r=1 is I_0/R_0
+const double a  = 1; //small radius  
 double func(double R, double Z, double phi)
 {
     double r2 = (R-R_0)*(R-R_0)+Z*Z;
@@ -53,7 +20,7 @@ double func(double R, double Z, double phi)
 }
 double deri(double R, double Z, double phi)
 {
-    double r2 = (R-R_0)*(R-R_0)+Z*Z;
+    double r2 = (R-R_0)*(R-R_0)+Z*Z; //(grad psi)^2
     return I_0/R/sqrt(I_0*I_0 + r2)* r2*cos(phi);
 }
 
@@ -67,43 +34,34 @@ int main(int argc, char* argv[])
     dg::mpi_init3d( dg::NEU, dg::NEU, dg::PER, n, Nx, Ny, Nz, comm);
     MPI_Comm_rank( MPI_COMM_WORLD, &rank);
 
-    Field field( R_0, I_0);
-    dg::CylindricalMPIGrid3d<dg::MDVec> g3d( R_0 - 1, R_0+1, -1, 1, 0, 2.*M_PI, n, Nx, Ny, Nz, dg::NEU, dg::NEU, dg::PER, comm);
-    const dg::MDVec w3d = dg::create::volume( g3d);
-    dg::Timer t;
-    t.tic();
-    dg::MDDS::FieldAligned dsFA( field, g3d, 1e-10, dg::DefaultLimiter(), dg::NEU);
-
-    dg::MDDS ds ( dsFA, field, dg::not_normed, dg::centered); 
-    t.toc();
-    if(rank==0)std::cout << "Creation of parallel Derivative took     "<<t.diff()<<"s\n";
+    dg::CylindricalMPIGrid3d g3d( R_0 - 1, R_0+1, -1, 1, 0, 2.*M_PI, n, Nx, Ny, Nz, dg::NEU, dg::NEU, dg::PER, comm);
+    const dg::MDVec vol3d = dg::create::volume( g3d);
+    if(rank==0)std::cout << "Create parallel Derivative!";
+    dg::geo::TokamakMagneticField mag = dg::geo::createCircularField( R_0, I_0);
+    dg::geo::BinaryVectorLvl0 bhat( (dg::geo::BHatR)(mag), (dg::geo::BHatZ)(mag), (dg::geo::BHatP)(mag));
+    dg::geo::Fieldaligned<dg::aProductMPIGeometry3d,dg::MIDMatrix,dg::MDVec>  dsFA( bhat, g3d, 2,2,true,true,1e-10, dg::NEU, dg::NEU, dg::geo::NoLimiter());
+    dg::geo::DS<dg::aProductMPIGeometry3d, dg::MIDMatrix, dg::MDMatrix, dg::MDVec> ds( dsFA, dg::not_normed, dg::centered);
 
     dg::MDVec function = dg::evaluate( func, g3d), derivative(function);
     const dg::MDVec solution = dg::evaluate( deri, g3d);
-    t.tic();
     ds( function, derivative);
-    t.toc();
-    if(rank==0)std::cout << "Application of parallel Derivative took  "<<t.diff()<<"s\n";
     dg::blas1::axpby( 1., solution, -1., derivative);
-    double norm = dg::blas2::dot( w3d, solution);
+    double norm = dg::blas2::dot( vol3d, solution);
     if(rank==0)std::cout << "Norm Solution "<<sqrt( norm)<<"\n";
-    double norm2 = sqrt( dg::blas2::dot( derivative, w3d, derivative)/norm);
+    double norm2 = sqrt( dg::blas2::dot( derivative, vol3d, derivative)/norm);
     if(rank==0)std::cout << "Relative Difference Is "<< norm2<<"\n";    
-    if(rank==0)std::cout << "(Error is from the parallel derivative only if n>2)\n"; //because the function is a parabola
+    if(rank==0)std::cout << "(Since the function is a parabola, the error is from the parallel derivative only if n>2)\n"; 
     dg::Gaussian init0(R_0+0.5, 0, 0.2, 0.2, 1);
     dg::GaussianZ modulate(0, M_PI/3., 1);
-    t.tic();
     function = ds.fieldaligned().evaluate( init0, modulate, Nz/2, 2);
-    t.toc();
-    if(rank==0)std::cout << "Fieldaligned initialization took "<<t.diff()<<"s\n";
     ds( function, derivative);
-    norm = dg::blas2::dot(w3d, derivative);
+    norm = dg::blas2::dot(vol3d, derivative);
     if(rank==0)std::cout << "Norm Centered Derivative "<<sqrt( norm)<<" (compare with that of ds_b)\n";
-    ds.forward( function, derivative);
-    norm = dg::blas2::dot(w3d, derivative);
+    ds.forward(1., function, 0.,  derivative);
+    norm = dg::blas2::dot(vol3d, derivative);
     if(rank==0)std::cout << "Norm Forward  Derivative "<<sqrt( norm)<<" (compare with that of ds_b)\n";
-    ds.backward( function, derivative);
-    norm = dg::blas2::dot(w3d, derivative);
+    ds.backward(1., function, 0., derivative);
+    norm = dg::blas2::dot(vol3d, derivative);
     if(rank==0)std::cout << "Norm Backward Derivative "<<sqrt( norm)<<" (compare with that of ds_b)\n";
     MPI_Finalize();
     return 0;
diff --git a/inc/geometries/ds_t.cu b/inc/geometries/ds_t.cu
index 45574af05..f034fd13c 100644
--- a/inc/geometries/ds_t.cu
+++ b/inc/geometries/ds_t.cu
@@ -8,13 +8,12 @@
 #include "dg/functors.h"
 #include "dg/geometry/geometry.h"
 #include "ds.h"
-#include "solovev.h"
-#include "flux.h"
 #include "toroidal.h"
 
 
 const double R_0 = 10;
-const double I_0 = 2;
+const double I_0 = 20; //q factor at r=1 is I_0/R_0
+const double a  = 1; //small radius  
 double func(double R, double Z, double phi)
 {
     double r2 = (R-R_0)*(R-R_0)+Z*Z;
@@ -22,7 +21,7 @@ double func(double R, double Z, double phi)
 }
 double deri(double R, double Z, double phi)
 {
-    double r2 = (R-R_0)*(R-R_0)+Z*Z;
+    double r2 = (R-R_0)*(R-R_0)+Z*Z; //(grad psi)^2
     return I_0/R/sqrt(I_0*I_0 + r2)* r2*cos(phi);
 }
 
@@ -33,35 +32,29 @@ int main(int argc, char * argv[])
     unsigned n, Nx, Ny, Nz;
     std::cin >> n>> Nx>>Ny>>Nz;
     std::cout << "You typed "<<n<<" "<<Nx<<" "<<Ny<<" "<<Nz<<std::endl;
+    std::cout << "Type mx and my\n";
+    unsigned mx, my;
+    std::cin >> mx>> my;
+    std::cout << "You typed "<<mx<<" "<<my<<std::endl;
     dg::CylindricalGrid3d g3d( R_0 - 1, R_0+1, -1, 1, 0, 2.*M_PI, n, Nx, Ny, Nz, dg::NEU, dg::NEU, dg::PER);
     const dg::DVec vol3d = dg::create::volume( g3d);
-    dg::Timer t;
-    t.tic();
+    if(rank==0)std::cout << "Create parallel Derivative!";
     dg::geo::TokamakMagneticField mag = dg::geo::createCircularField( R_0, I_0);
     dg::geo::BinaryVectorLvl0 bhat( (dg::geo::BHatR)(mag), (dg::geo::BHatZ)(mag), (dg::geo::BHatP)(mag));
     dg::geo::Fieldaligned<dg::aProductGeometry3d,dg::IDMatrix,dg::DVec>  dsFA( bhat, g3d, 2,2,true,true,1e-10, dg::NEU, dg::NEU, dg::geo::NoLimiter());
-
     dg::geo::DS<dg::aProductGeometry3d, dg::IDMatrix, dg::DMatrix, dg::DVec> ds( dsFA, dg::not_normed, dg::centered);
-    t.toc();
-    std::cout << "Creation of parallel Derivative took     "<<t.diff()<<"s\n";
 
     dg::DVec function = dg::evaluate( func, g3d), derivative(function);
     const dg::DVec solution = dg::evaluate( deri, g3d);
-    t.tic();
     ds( function, derivative);
-    t.toc();
-    std::cout << "Application of parallel Derivative took  "<<t.diff()<<"s\n";
     dg::blas1::axpby( 1., solution, -1., derivative);
     double norm = dg::blas2::dot( vol3d, solution);
     std::cout << "Norm Solution "<<sqrt( norm)<<"\n";
     std::cout << "Relative Difference Is "<< sqrt( dg::blas2::dot( derivative, vol3d, derivative)/norm )<<"\n";
-    std::cout << "Error is from the parallel derivative only if n>2\n"; //since the function is a parabola
+    std::cout << "(Since the function is a parabola, the error is from the parallel derivative only if n>2)\n"; 
     dg::Gaussian init0(R_0+0.5, 0, 0.2, 0.2, 1);
     dg::GaussianZ modulate(0., M_PI/3., 1);
-    t.tic();
     function = ds.fieldaligned().evaluate( init0, modulate, Nz/2, 2);
-    t.toc();
-    std::cout << "Fieldaligned initialization took "<<t.diff()<<"s\n";
     ds( function, derivative);
     norm = dg::blas2::dot(vol3d, derivative);
     std::cout << "Norm Centered Derivative "<<sqrt( norm)<<" (compare with that of ds_mpib)\n";
-- 
GitLab


From 731665b1dce0bd6e6aa365d90c0fc2237c1d3fd6 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Thu, 19 Oct 2017 16:28:41 +0200
Subject: [PATCH 373/453] updated ds_t and ds_mpit test programs to test
 forward, backward and centered derivatives

---
 inc/geometries/ds.h       |  6 +++---
 inc/geometries/ds_mpit.cu | 30 ++++++++++++++++--------------
 inc/geometries/ds_t.cu    | 25 ++++++++++++++-----------
 3 files changed, 33 insertions(+), 28 deletions(-)

diff --git a/inc/geometries/ds.h b/inc/geometries/ds.h
index bb1ab5870..b38c53479 100644
--- a/inc/geometries/ds.h
+++ b/inc/geometries/ds.h
@@ -231,8 +231,8 @@ void DS<G,I,M,container>::do_backward( double alpha, const container& f, double
 {
     //direct
     m_fa(einsMinus, f, m_tempM);
-    dg::blas1::axpby( 1., m_tempM, -1., f, m_tempM);
-    dg::blas1::pointwiseDot( alpha, m_tempM, m_fa.hp_inv(), beta, dsf);
+    dg::blas1::axpby( 1., f, -1., m_tempM, m_tempM);
+    dg::blas1::pointwiseDot( alpha, m_tempM, m_fa.hm_inv(), beta, dsf);
 }
 template<class G, class I, class M, class container>
 void DS<G, I,M,container>::do_centered( double alpha, const container& f, double beta, container& dsf)
@@ -261,7 +261,7 @@ void DS<G,I,M,container>::do_backwardAdj( double alpha, const container& f, doub
     dg::blas1::pointwiseDot( m_vol3d, m_temp0, m_temp0);
     dg::blas1::pointwiseDot( m_temp0, m_fa.hm_inv(), m_temp0);
     m_fa(einsMinusT, m_temp0, m_tempM);
-    dg::blas1::axpby( -1., m_tempM, 1., m_temp0, m_temp0);
+    dg::blas1::axpby( -1., m_temp0, 1., m_tempM, m_temp0);
     if(no == dg::normed) 
         dg::blas1::pointwiseDot( alpha, m_inv3d, m_temp0, beta, dsf); 
 }
diff --git a/inc/geometries/ds_mpit.cu b/inc/geometries/ds_mpit.cu
index 5ca941ce5..e647760be 100644
--- a/inc/geometries/ds_mpit.cu
+++ b/inc/geometries/ds_mpit.cu
@@ -36,7 +36,7 @@ int main(int argc, char* argv[])
 
     dg::CylindricalMPIGrid3d g3d( R_0 - 1, R_0+1, -1, 1, 0, 2.*M_PI, n, Nx, Ny, Nz, dg::NEU, dg::NEU, dg::PER, comm);
     const dg::MDVec vol3d = dg::create::volume( g3d);
-    if(rank==0)std::cout << "Create parallel Derivative!";
+    if(rank==0)std::cout << "Create parallel Derivative!\n";
     dg::geo::TokamakMagneticField mag = dg::geo::createCircularField( R_0, I_0);
     dg::geo::BinaryVectorLvl0 bhat( (dg::geo::BHatR)(mag), (dg::geo::BHatZ)(mag), (dg::geo::BHatP)(mag));
     dg::geo::Fieldaligned<dg::aProductMPIGeometry3d,dg::MIDMatrix,dg::MDVec>  dsFA( bhat, g3d, 2,2,true,true,1e-10, dg::NEU, dg::NEU, dg::geo::NoLimiter());
@@ -46,23 +46,25 @@ int main(int argc, char* argv[])
     const dg::MDVec solution = dg::evaluate( deri, g3d);
     ds( function, derivative);
     dg::blas1::axpby( 1., solution, -1., derivative);
-    double norm = dg::blas2::dot( vol3d, solution);
-    if(rank==0)std::cout << "Norm Solution "<<sqrt( norm)<<"\n";
-    double norm2 = sqrt( dg::blas2::dot( derivative, vol3d, derivative)/norm);
-    if(rank==0)std::cout << "Relative Difference Is "<< norm2<<"\n";    
-    if(rank==0)std::cout << "(Since the function is a parabola, the error is from the parallel derivative only if n>2)\n"; 
+    double norm = dg::blas2::dot( derivative, vol3d, derivative);
+    const double sol = dg::blas2::dot( vol3d, solution);
+    if(rank==0)std::cout << "Error centered derivative "<< sqrt( norm/sol )<<"\n";
+    ds.forward( 1., function, 0., derivative);
+    dg::blas1::axpby( 1., solution, -1., derivative);
+    norm = dg::blas2::dot(vol3d, derivative);
+    if(rank==0)std::cout << "Error Forward  Derivative "<<sqrt( norm/sol)<<"\n";
+    ds.backward( 1., function, 0., derivative);
+    dg::blas1::axpby( 1., solution, -1., derivative);
+    norm = dg::blas2::dot(vol3d, derivative);
+    if(rank==0)std::cout << "Error Backward Derivative "<<sqrt( norm/sol)<<"\n";
+    if(rank==0)std::cout << "(Since the function is a parabola, the error is from the parallel derivative only if n>2/ no interpolation error)\n"; 
+    if(rank==0)std::cout << "TEST FIELDALIGNED EVALUATION of a Gaussian\n";
     dg::Gaussian init0(R_0+0.5, 0, 0.2, 0.2, 1);
-    dg::GaussianZ modulate(0, M_PI/3., 1);
+    dg::GaussianZ modulate(0., M_PI/3., 1);
     function = ds.fieldaligned().evaluate( init0, modulate, Nz/2, 2);
     ds( function, derivative);
     norm = dg::blas2::dot(vol3d, derivative);
-    if(rank==0)std::cout << "Norm Centered Derivative "<<sqrt( norm)<<" (compare with that of ds_b)\n";
-    ds.forward(1., function, 0.,  derivative);
-    norm = dg::blas2::dot(vol3d, derivative);
-    if(rank==0)std::cout << "Norm Forward  Derivative "<<sqrt( norm)<<" (compare with that of ds_b)\n";
-    ds.backward(1., function, 0., derivative);
-    norm = dg::blas2::dot(vol3d, derivative);
-    if(rank==0)std::cout << "Norm Backward Derivative "<<sqrt( norm)<<" (compare with that of ds_b)\n";
+    if(rank==0)std::cout << "Norm Centered Derivative "<<sqrt( norm)<<" (compare with that of ds_mpib)\n";
     MPI_Finalize();
     return 0;
 }
diff --git a/inc/geometries/ds_t.cu b/inc/geometries/ds_t.cu
index f034fd13c..c27835ef3 100644
--- a/inc/geometries/ds_t.cu
+++ b/inc/geometries/ds_t.cu
@@ -38,7 +38,7 @@ int main(int argc, char * argv[])
     std::cout << "You typed "<<mx<<" "<<my<<std::endl;
     dg::CylindricalGrid3d g3d( R_0 - 1, R_0+1, -1, 1, 0, 2.*M_PI, n, Nx, Ny, Nz, dg::NEU, dg::NEU, dg::PER);
     const dg::DVec vol3d = dg::create::volume( g3d);
-    if(rank==0)std::cout << "Create parallel Derivative!";
+    std::cout << "Create parallel Derivative!\n";
     dg::geo::TokamakMagneticField mag = dg::geo::createCircularField( R_0, I_0);
     dg::geo::BinaryVectorLvl0 bhat( (dg::geo::BHatR)(mag), (dg::geo::BHatZ)(mag), (dg::geo::BHatP)(mag));
     dg::geo::Fieldaligned<dg::aProductGeometry3d,dg::IDMatrix,dg::DVec>  dsFA( bhat, g3d, 2,2,true,true,1e-10, dg::NEU, dg::NEU, dg::geo::NoLimiter());
@@ -48,22 +48,25 @@ int main(int argc, char * argv[])
     const dg::DVec solution = dg::evaluate( deri, g3d);
     ds( function, derivative);
     dg::blas1::axpby( 1., solution, -1., derivative);
-    double norm = dg::blas2::dot( vol3d, solution);
-    std::cout << "Norm Solution "<<sqrt( norm)<<"\n";
-    std::cout << "Relative Difference Is "<< sqrt( dg::blas2::dot( derivative, vol3d, derivative)/norm )<<"\n";
-    std::cout << "(Since the function is a parabola, the error is from the parallel derivative only if n>2)\n"; 
+    double norm = dg::blas2::dot( derivative, vol3d, derivative);
+    const double sol = dg::blas2::dot( vol3d, solution);
+    std::cout << "Error centered derivative "<< sqrt( norm/sol )<<"\n";
+    ds.forward( 1., function, 0., derivative);
+    dg::blas1::axpby( 1., solution, -1., derivative);
+    norm = dg::blas2::dot(vol3d, derivative);
+    std::cout << "Error Forward  Derivative "<<sqrt( norm/sol)<<"\n";
+    ds.backward( 1., function, 0., derivative);
+    dg::blas1::axpby( 1., solution, -1., derivative);
+    norm = dg::blas2::dot(vol3d, derivative);
+    std::cout << "Error Backward Derivative "<<sqrt( norm/sol)<<"\n";
+    std::cout << "(Since the function is a parabola, the error is from the parallel derivative only if n>2/ no interpolation error)\n"; 
+    std::cout << "TEST FIELDALIGNED EVALUATION of a Gaussian\n";
     dg::Gaussian init0(R_0+0.5, 0, 0.2, 0.2, 1);
     dg::GaussianZ modulate(0., M_PI/3., 1);
     function = ds.fieldaligned().evaluate( init0, modulate, Nz/2, 2);
     ds( function, derivative);
     norm = dg::blas2::dot(vol3d, derivative);
     std::cout << "Norm Centered Derivative "<<sqrt( norm)<<" (compare with that of ds_mpib)\n";
-    ds.forward( 1., function, 0., derivative);
-    norm = dg::blas2::dot(vol3d, derivative);
-    std::cout << "Norm Forward  Derivative "<<sqrt( norm)<<" (compare with that of ds_mpib)\n";
-    ds.backward( 1., function, 0., derivative);
-    norm = dg::blas2::dot(vol3d, derivative);
-    std::cout << "Norm Backward Derivative "<<sqrt( norm)<<" (compare with that of ds_mpib)\n";
 
     return 0;
 }
-- 
GitLab


From 3c1b2370f49da827444c295ae1856af82b4319b3 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Thu, 19 Oct 2017 17:21:55 +0200
Subject: [PATCH 374/453] maded nice \c in doxygen docu

---
 doc/related_pages/parallel/parallel.tex |  9 ++-
 inc/dg/blas2.h                          | 30 ++++----
 inc/dg/dg_doc.h                         | 95 +++++++++++++------------
 3 files changed, 69 insertions(+), 65 deletions(-)

diff --git a/doc/related_pages/parallel/parallel.tex b/doc/related_pages/parallel/parallel.tex
index f7509865f..54ae1a21d 100644
--- a/doc/related_pages/parallel/parallel.tex
+++ b/doc/related_pages/parallel/parallel.tex
@@ -459,14 +459,16 @@ The magnetic field in FELTOR is given by
 \begin{align}
   \vec B = \frac{R_0}{R}( I(\psi_p) \hat e_\varphi + \nabla\psi_p \times\hat e_\varphi)
 \end{align}
-This gives rise to a magnetic field strength and components
+This gives rise to magnetic field strength and components
 \begin{align}
   B = \frac{R_0}{R} \sqrt{ I^2 + \left( \nabla\psi_p \right)^2} \\
   B^R = \frac{R_0}{R}\frac{\partial\psi_p}{\partial Z} \quad
   B^Z = -\frac{R_0}{R}\frac{\partial\psi_p}{\partial R}\quad 
-  B^\varphi = \frac{R_0I}{R^2}
+  B^\varphi = \frac{R_0I}{R^2} \\
+  \nabla \cdot\bhat = -\nabla_\parallel \ln B = -\frac{R_0}{RB^2} [B, \psi_p]  
   \label{}
 \end{align}
+where $[.,.]$ is the Poisson bracket.
 \subsection{Cylindrical grid and circular flux surfaces}
 A very simple non-trivial choice for the poloidal flux is
 \begin{align}
@@ -476,7 +478,8 @@ A very simple non-trivial choice for the poloidal flux is
 for which 
 \begin{align}
   \vec b^p &= \frac{1}{\sqrt{I^2 + r^2}} \begin{pmatrix} Z \\ -(R-R_0)\end{pmatrix} \\
-  b^\varphi &= \frac{1}{\sqrt{I^2 + r^2}} I/R
+  b^\varphi &= \frac{1}{\sqrt{I^2 + r^2}} I/R \\
+  \nabla\cdot \bhat = \frac{Z}{R(I^2 + r^2} 
   \label{}
 \end{align}
 for the poloidal and toroidal parts of the magnetic unit vector. 
diff --git a/inc/dg/blas2.h b/inc/dg/blas2.h
index e1e8b8b49..dc77b3f84 100644
--- a/inc/dg/blas2.h
+++ b/inc/dg/blas2.h
@@ -50,12 +50,12 @@ inline void transfer( const Matrix& x, AnotherMatrix& y)
  * matrix M \f[ x^T M y = \sum_{i,j=0}^{N-1} x_i M_{ij} y_j \f]
  * ( Note that if M is not diagonal it is generally more efficient to 
  * precalculate \f$ My\f$ and then call the dg::blas1::dot() routine!
- * @tparam DiagonalMatrix Right now DiagonalMatrix has to be the same as container, except if the container is a std::vector<container_type>, then the DiagonalMatrix has to be the container_type.
+ * @tparam DiagonalMatrix Right now \c DiagonalMatrix has to be the same as \c container, except if \c container is a \p std::vector<container_type>, then the \c DiagonalMatrix has to be the \c container_type.
  * In the latter case the Matrix is applied to all containers in the std::vector and the sum is returned. 
  * @copydoc hide_container
  * @param x Left container
  * @param m The diagonal Matrix
- * @param y Right container might equal Left container
+ * @param y Right container (may alias \p x)
  * @return Generalized scalar product
  * @note This routine is always executed synchronously due to the 
     implicit memcpy of the result.
@@ -72,7 +72,7 @@ inline typename MatrixTraits<DiagonalMatrix>::value_type dot( const container& x
 /*! @brief \f$ x^T M x\f$; General dot produt
  *
  * \f[ x^T M x = \sum_{i,j=0}^{N-1} x_i M_{ij} x_j \f]
- * @tparam DiagonalMatrix Right now DiagonalMatrix has to be the same as container, except if the container is a std::vector<container_type>, then the DiagonalMatrix has to be the container_type. 
+ * @tparam DiagonalMatrix Right now \c DiagonalMatrix has to be the same as \c container, except if \c container is a \c std::vector<container_type>, then the \c DiagonalMatrix has to be the \c container_type. 
  * In the latter case the Matrix is applied to all containers in the std::vector and the sum is returned. 
  * @copydoc hide_container
  * @param m The diagonal Matrix
@@ -80,7 +80,7 @@ inline typename MatrixTraits<DiagonalMatrix>::value_type dot( const container& x
  * @return Generalized scalar product
  * @note This routine is always executed synchronously due to the 
     implicit memcpy of the result.
- * @note This routine is equivalent to the call dg::blas2::dot( x, m, x);
+ * @note This routine is equivalent to the call \c dg::blas2::dot( x, m, x);
      which should be prefered because it looks more explicit
  */
 template< class DiagonalMatrix, class container>
@@ -99,10 +99,10 @@ inline typename MatrixTraits<DiagonalMatrix>::value_type dot( const DiagonalMatr
  * @copydoc hide_container
  * @param alpha A Scalar
  * @param M The Matrix
- * @param x A container different from y 
+ * @param x A container different from \p y 
  * @param beta A Scalar
- * @param y contains the solution on output (may not alias x)
- * @attention y may never alias x
+ * @param y contains the solution on output (may not alias \p x)
+ * @attention \p y may never alias \p x
  */
 template< class Matrix, class container>
 inline void symv( typename MatrixTraits<Matrix>::value_type alpha, 
@@ -129,8 +129,8 @@ inline void symv( typename MatrixTraits<Matrix>::value_type alpha,
  * @copydoc hide_container
  * @tparam same_or_another_container Currently needs to be the same as container.
  * @param M The Matrix
- * @param x A container different from y 
- * @param y contains the solution on output (may not alias x)
+ * @param x A container different from \p y 
+ * @param y contains the solution on output (may not alias \p x)
  * @attention y may never alias x
  * @note Due to the SelfMadeMatrixTag, M cannot be declared const
  */
@@ -154,9 +154,9 @@ inline void symv( Matrix& M,
  * @copydoc hide_container
  * @tparam same_or_another_container Currently needs to be the same as container.
  * @param M The Matrix
- * @param x A container different from y 
- * @param y contains the solution on output (may not alias x)
- * @attention y may never alias x
+ * @param x A container different from \p y 
+ * @param y contains the solution on output (may not alias \p x)
+ * @attention y may never alias \p x
  */
 template< class Matrix, class container, class same_or_another_container>
 inline void gemv( Matrix& M, 
@@ -177,10 +177,10 @@ inline void gemv( Matrix& M,
  * @copydoc hide_container
  * @param alpha A Scalar
  * @param M The Matrix
- * @param x A container different from y 
+ * @param x A container different from \p y 
  * @param beta A Scalar
- * @param y contains the solution on output (may not alias x)
- * @attention y may never alias x
+ * @param y contains the solution on output (may not alias \p x)
+ * @attention y may never alias \p x
  */
 template< class Matrix, class container>
 inline void gemv( typename MatrixTraits<Matrix>::value_type alpha, 
diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index aa5d4b2df..ec317c0fc 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -63,7 +63,7 @@
  *             The first elements of the resulting vector lie in the cell at (x0,y0) and the last
  *             in (x1, y1).
  *         @defgroup highlevel create weights 
- *              overloads for the dg::create::weights and dg::create::inv_weights functions for all
+ *              overloads for the \c dg::create::weights and \c dg::create::inv_weights functions for all
  *              available topologies
  *         @defgroup creation create derivatives 
  *
@@ -117,89 +117,90 @@
 
  /** @class hide_container
   * @tparam container 
-  * A data container class for which the blas1 functionality is overloaded.
-  * We assume that container is copyable/assignable and has a swap member function. 
+  * A data container class for which the \c blas1 functionality is overloaded.
+  * We assume that \c container is copyable/assignable and has a swap member function. 
   * Currently this is one of 
-  *  - dg::HVec, dg::DVec, dg::MHVec or dg::MDVec  
-  *  - std::vector<dg::HVec>, std::vector<dg::DVec>, std::vector<dg::MHVec> or std::vector<dg::MDVec> . 
+  *  - \c dg::HVec, \c dg::DVec, \c dg::MHVec or \c dg::MDVec  
+  *  - \c std::vector<dg::HVec>, \c std::vector<dg::DVec>, \c std::vector<dg::MHVec> or \c std::vector<dg::MDVec> . 
   *
   */
  /** @class hide_matrix
   * @tparam Matrix 
   * A class for which the blas2 functions are callable in connection with the container class. 
-  * The Matrix type can be one of:
+  * The \c Matrix type can be one of:
   *  - container: A container acts as a  diagonal matrix. 
-  *  - dg::HMatrix and dg::IHMatrix with dg::HVec or std::vector<dg::HVec>
-  *  - dg::DMatrix and dg::IDMatrix with dg::DVec or std::vector<dg::DVec>
-  *  - dg::MHMatrix with dg::MHVec or std::vector<dg::MHVec>
-  *  - dg::MDMatrix with dg::MDVec or std::vector<dg::MDVec>
-  *  - Any type that has the SelfMadeMatrixTag specified in a corresponding 
-  *  MatrixTraits class (e.g. Elliptic). In this case only those blas2 functions 
-  *  that have a corresponding member function in the Matrix class (e.g. symv( const container&, container&); ) can be called.
-  *  If the container is a std::vector, then the Matrix is applied to each of the elements.
+  *  - \c dg::HMatrix and \c dg::IHMatrix with \c dg::HVec or \c std::vector<dg::HVec>
+  *  - \c dg::DMatrix and \c dg::IDMatrix with \c dg::DVec or \c std::vector<dg::DVec>
+  *  - \c dg::MHMatrix with \c dg::MHVec or \c std::vector<dg::MHVec>
+  *  - \c dg::MDMatrix with \c dg::MDVec or \c std::vector<dg::MDVec>
+  *  - Any type that has the \c SelfMadeMatrixTag specified in a corresponding 
+  *  \c MatrixTraits class (e.g. \c dg::Elliptic). In this case only those blas2 functions 
+  *  that have a corresponding member function in the Matrix class (e.g. \c symv( const container&, container&); ) can be called.
+  *  .
+  *  If \c container is a \c std::vector, then the Matrix is applied to each of the elements.
   */
   /** @class hide_geometry
   * @tparam Geometry 
-  A type that is or derives from one of the abstract geometry base classes ( aGeometry2d, aGeometry3d, aMPIGeometry2d, ...). 
+  A type that is or derives from one of the abstract geometry base classes ( \c aGeometry2d, \c aGeometry3d, \c aMPIGeometry2d, ...). 
   */
 
   /** @class hide_container_geometry
   * @tparam container 
-  * A data container class for which the blas1 functionality is overloaded and to which the return type of blas1::evaluate() can be converted. 
-  * We assume that container is copyable/assignable and has a swap member function. 
-  * In connection with Geometry this is one of 
-  *  - dg::HVec, dg::DVec when Geometry is a shared memory geometry
-  *  - dg::MHVec or dg::MDVec when Geometry is one of the MPI geometries
+  * A data container class for which the \c blas1 functionality is overloaded and to which the return type of \c blas1::evaluate() can be converted. 
+  * We assume that \c container is copyable/assignable and has a swap member function. 
+  * In connection with \c Geometry this is one of 
+  *  - \c dg::HVec, \c dg::DVec when \c Geometry is a shared memory geometry
+  *  - \c dg::MHVec or \c dg::MDVec when \c Geometry is one of the MPI geometries
   * @tparam Geometry 
-  A type that is or derives from one of the abstract geometry base classes ( aGeometry2d, aGeometry3d, aMPIGeometry2d, ...). Geometry determines which container type can be used.
+  A type that is or derives from one of the abstract geometry base classes ( \c aGeometry2d, \c aGeometry3d, \c aMPIGeometry2d, ...). \c Geometry determines which \c container type can be used.
   */
 
   /** @class hide_geometry_matrix_container
   * @tparam Geometry 
-  A type that is or derives from one of the abstract geometry base classes ( aGeometry2d, aGeometry3d, aMPIGeometry2d, ...). Geometry determines which Matrix and container types can be used:
+  A type that is or derives from one of the abstract geometry base classes ( \c aGeometry2d, \c aGeometry3d, \c aMPIGeometry2d, ...). \c Geometry determines which \c Matrix and \c container types can be used:
   * @tparam Matrix 
-  * A class for which the blas2 functions are callable in connection with the container class and to which the return type of create::dx() can be converted. 
-  * The Matrix type can be one of:
-  *  - dg::HMatrix with dg::HVec and one of the shared memory geometries
-  *  - dg::DMatrix with dg::DVec and one of the shared memory geometries
-  *  - dg::MHMatrix with dg::MHVec and one of the MPI geometries
-  *  - dg::MDMatrix with dg::MDVec and one of the MPI geometries
+  * A class for which the blas2 functions are callable in connection with the \c container class and to which the return type of \c create::dx() can be converted. 
+  * The \c Matrix type can be one of:
+  *  - \c dg::HMatrix with \c dg::HVec and one of the shared memory geometries
+  *  - \c dg::DMatrix with \c dg::DVec and one of the shared memory geometries
+  *  - \c dg::MHMatrix with \c dg::MHVec and one of the MPI geometries
+  *  - \c dg::MDMatrix with \c dg::MDVec and one of the MPI geometries
   * @tparam container 
-  * A data container class for which the blas1 functionality is overloaded and to which the return type of blas1::evaluate() can be converted. 
-  * We assume that container is copyable/assignable and has a swap member function. 
-  * In connection with Geometry this is one of 
-  *  - dg::HVec, dg::DVec when Geometry is a shared memory geometry
-  *  - dg::MHVec or dg::MDVec when Geometry is one of the MPI geometries
+  * A data container class for which the \c blas1 functionality is overloaded and to which the return type of \c blas1::evaluate() can be converted. 
+  * We assume that \c container is copyable/assignable and has a swap member function. 
+  * In connection with \c Geometry this is one of 
+  *  - \c dg::HVec, \c dg::DVec when \c Geometry is a shared memory geometry
+  *  - \c dg::MHVec or \c dg::MDVec when \c Geometry is one of the MPI geometries
   */
 
  /** @class hide_symmetric_op
  * @tparam SymmetricOp 
- A class for which the blas2::symv(Matrix&, Vector1&, Vector2&) function is callable 
- with the container type as argument. Also, The functions %inv_weights() and %precond() 
+ A class for which the \c blas2::symv(Matrix&, Vector1&, Vector2&) function is callable 
+ with the \c container type as argument. Also, The functions \c %inv_weights() and \c %precond() 
  need to be callable and return inverse weights and the preconditioner for the conjugate 
- gradient method. The Operator is assumed to be linear and symmetric!
- @note you can make your own SymmetricOp by providing the member function void symv(const container&, container&);
-  and specializing MatrixTraits with the SelfMadeMatrixTag as the matrix_category
+ gradient method. The %Operator is assumed to be linear and symmetric!
+ @note you can make your own \c SymmetricOp by providing the member function \c void \c symv(const container&, container&);
+  and specializing \c MatrixTraits with the \c SelfMadeMatrixTag as the matrix_category
   */
 
 /*!@addtogroup mpi_structures
 @{
 @note The mpi backend is activated by including \c mpi.h before any other feltor header file
-@section mpi_vector MPI Vectors and the blas1 functions
+@section mpi_vector MPI Vectors and the \c blas1 functions
 
 In Feltor each mpi process gets an equally sized chunk of a vector.
 The corresponding structure in FELTOR is the \c dg::MPI_Vector, which is 
 nothing but a wrapper around any container type object and a MPI_Comm. 
-With this the dg::blas1 functions can readily implemented by just redirecting to the
+With this the \c dg::blas1 functions can readily implemented by just redirecting to the
 implementation for the container type. The only functions that need
-communication are the dg::blas1::dot functions (MPI_Allreduce).
+communication are the \c dg::blas1::dot functions (MPI_Allreduce).
 
 @section mpi_matrix Row and column distributed matrices
 
 Contrary to a vector
 a matrix can be distributed in two ways, row-wise and column wise. 
-The structure dg::MPIDistMat is a wrapper around a LocalMatrix type object 
-and an instance of dg::aCommunicator
+The structure \c dg::MPIDistMat is a wrapper around a LocalMatrix type object 
+and an instance of \c dg::aCommunicator
 In a row-distributed matrix each process gets the complete 
 rows of the matrix that correspond to the indices in the 
 vector it holds. 
@@ -212,7 +213,7 @@ of the matrix.
 \subsection row Row distributed
 For the row-distributed matrix each process first has to gather 
 all elements of the input vector it needs to be able to compute the elements of the output. In general this requires MPI communication.
-(read the documentation of dg::aCommunicator for more info of how global scatter/gather operations work).
+(read the documentation of \c dg::aCommunicator for more info of how global scatter/gather operations work).
 Formally, the gather operation can be written as a matrix \f$G\f$
 of \f$1'\f$s and \f$0'\f$s.
 After the elements have been gathered into a buffer the local matrix-vector
@@ -222,7 +223,7 @@ M = R\cdot G
 \f]
 where \f$R\f$ is the row-distributed matrix with modified indices 
 and \f$G\f$ is the gather matrix, in which the MPI-communication takes place.
-The dg::RowColDistMat goes one step further and separates the matrix \f$ R\f$ into 
+The \c dg::RowColDistMat goes one step further and separates the matrix \f$ R\f$ into 
 a part that can be computed entirely on the local process and a part that needs communication.
 
 \subsection column Column distributed
@@ -233,7 +234,7 @@ However the resuling elements have to be communicated back to
 the process they belong to. Furthermore, a process has to sum
 all elements it receives from other processes on the same
 index. This is a scatter and reduce operation and
-it can be written as a scatter matrix \f$S\f$ (s.a. dg::aCommunicator). The transpose
+it can be written as a scatter matrix \f$S\f$ (s.a. \c dg::aCommunicator). The transpose
 of the scatter matrix is a gather matrix and vice-versa.
 \f[
 M = S\cdot C
@@ -242,7 +243,7 @@ where \f$S\f$ is the scatter matrix and \f$C\f$ is the column distributed
 matrix with modified indices. 
 
 It turns out that a row-distributed matrix can be transposed
-by transposition of the local matrices and the gather matrix (s.a. dg::transpose).
+by transposition of the local matrices and the gather matrix (s.a. \c dg::transpose).
 The result is then a column distributed matrix.
 The transpose of a column distributed matrix is a row-distributed matrix and vice-versa.
 @}
-- 
GitLab


From 00e127df66d5f1f6688d7cea021bb887b7e71f5f Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 19 Oct 2017 23:52:56 +0200
Subject: [PATCH 375/453] test adjoint in cylindrical grid does not really work
 out

---
 inc/geometries/ds.h    | 19 +++++++++++++------
 inc/geometries/ds_t.cu | 30 ++++++++++++++++++++++++++++--
 2 files changed, 41 insertions(+), 8 deletions(-)

diff --git a/inc/geometries/ds.h b/inc/geometries/ds.h
index b38c53479..14c6417b7 100644
--- a/inc/geometries/ds.h
+++ b/inc/geometries/ds.h
@@ -110,17 +110,17 @@ struct DS
         do_centered( alpha, f, beta, g);
     }
 
-    ///@brief Apply the negative forward adjoint derivative on a 3d vector
+    ///@brief forward adjoint \f$ g = -\alpha \nabla\cdot(\vec v f) + \beta g\f$
     ///@copydetails forward
     void forwardAdj( double alpha, const container& f, double beta, container& g){
         do_forwardAdj( alpha, f, beta, g, dg::normed);
     }
-    ///@brief Apply the negative backward adjoint derivative on a 3d vector
+    ///@brief backward adjoint \f$ g = -\alpha \nabla\cdot(\vec v f) + \beta g\f$
     ///@copydetails forward
     void backwardAdj( double alpha, const container& f, double beta, container& g){
         do_backwardAdj( alpha, f, beta, g, dg::normed);
     }
-    ///@brief Apply the negative centered adjoint derivative on a 3d vector
+    ///@brief centered adjoint \f$ g = -\alpha \nabla\cdot(\vec v f) + \beta g\f$
     ///@copydetails forward
     void centeredAdj(double alpha, const container& f, double beta, container& g){
         do_centeredAdj( alpha, f, beta, g, dg::normed);
@@ -250,20 +250,24 @@ void DS<G,I,M,container>::do_forwardAdj( double alpha, const container& f, doubl
     dg::blas1::pointwiseDot( m_vol3d, f, m_temp0);
     dg::blas1::pointwiseDot( m_temp0, m_fa.hp_inv(), m_temp0);
     m_fa(einsPlusT, m_temp0, m_tempP);
-    dg::blas1::axpby( -1., m_tempP, 1., m_temp0, m_temp0);
+    dg::blas1::axpby( 1., m_tempP, -1., m_temp0, m_temp0);
     if(no == dg::normed) 
         dg::blas1::pointwiseDot( alpha, m_inv3d, m_temp0, beta, dsf); 
+    else
+        dg::blas1::axpby( alpha, m_tempM, beta, dsf);
 }
 template<class G,class I, class M, class container>
 void DS<G,I,M,container>::do_backwardAdj( double alpha, const container& f, double beta, container& dsf, dg::norm no)
 {    
     //adjoint discretisation
-    dg::blas1::pointwiseDot( m_vol3d, m_temp0, m_temp0);
+    dg::blas1::pointwiseDot( m_vol3d, f, m_temp0);
     dg::blas1::pointwiseDot( m_temp0, m_fa.hm_inv(), m_temp0);
     m_fa(einsMinusT, m_temp0, m_tempM);
-    dg::blas1::axpby( -1., m_temp0, 1., m_tempM, m_temp0);
+    dg::blas1::axpby( 1., m_temp0, -1., m_tempM, m_temp0);
     if(no == dg::normed) 
         dg::blas1::pointwiseDot( alpha, m_inv3d, m_temp0, beta, dsf); 
+    else
+        dg::blas1::axpby( alpha, m_tempM, beta, dsf);
 }
 template<class G, class I, class M, class container>
 void DS<G, I,M,container>::do_centeredAdj( double alpha, const container& f, double beta, container& dsf, dg::norm no)
@@ -276,6 +280,9 @@ void DS<G, I,M,container>::do_centeredAdj( double alpha, const container& f, dou
     dg::blas1::axpby( 1., m_tempP, -1., m_tempM);
     if(no == dg::normed) 
         dg::blas1::pointwiseDot( alpha, m_inv3d, m_tempM, beta, dsf); 
+    else
+        dg::blas1::axpby( alpha, m_tempM, beta, dsf);
+
 }
 
 template<class G,class I, class M, class container>
diff --git a/inc/geometries/ds_t.cu b/inc/geometries/ds_t.cu
index c27835ef3..19b666810 100644
--- a/inc/geometries/ds_t.cu
+++ b/inc/geometries/ds_t.cu
@@ -7,6 +7,7 @@
 #include "dg/blas.h"
 #include "dg/functors.h"
 #include "dg/geometry/geometry.h"
+#include "magnetic_field.h"
 #include "ds.h"
 #include "toroidal.h"
 
@@ -14,6 +15,8 @@
 const double R_0 = 10;
 const double I_0 = 20; //q factor at r=1 is I_0/R_0
 const double a  = 1; //small radius  
+dg::geo::TokamakMagneticField mag = dg::geo::createCircularField( R_0, I_0);
+dg::geo::GradLnB gradLnB(mag);
 double func(double R, double Z, double phi)
 {
     double r2 = (R-R_0)*(R-R_0)+Z*Z;
@@ -24,6 +27,10 @@ double deri(double R, double Z, double phi)
     double r2 = (R-R_0)*(R-R_0)+Z*Z; //(grad psi)^2
     return I_0/R/sqrt(I_0*I_0 + r2)* r2*cos(phi);
 }
+double adjoint(double R, double Z, double phi)
+{
+    return -gradLnB(R,Z)*func(R,Z,phi) + deri(R,Z,phi);
+}
 
 int main(int argc, char * argv[])
 {
@@ -39,11 +46,11 @@ int main(int argc, char * argv[])
     dg::CylindricalGrid3d g3d( R_0 - 1, R_0+1, -1, 1, 0, 2.*M_PI, n, Nx, Ny, Nz, dg::NEU, dg::NEU, dg::PER);
     const dg::DVec vol3d = dg::create::volume( g3d);
     std::cout << "Create parallel Derivative!\n";
-    dg::geo::TokamakMagneticField mag = dg::geo::createCircularField( R_0, I_0);
     dg::geo::BinaryVectorLvl0 bhat( (dg::geo::BHatR)(mag), (dg::geo::BHatZ)(mag), (dg::geo::BHatP)(mag));
-    dg::geo::Fieldaligned<dg::aProductGeometry3d,dg::IDMatrix,dg::DVec>  dsFA( bhat, g3d, 2,2,true,true,1e-10, dg::NEU, dg::NEU, dg::geo::NoLimiter());
+    dg::geo::Fieldaligned<dg::aProductGeometry3d,dg::IDMatrix,dg::DVec>  dsFA( bhat, g3d, mx, my, true,true, 1e-6, dg::NEU, dg::NEU, dg::geo::NoLimiter());
     dg::geo::DS<dg::aProductGeometry3d, dg::IDMatrix, dg::DMatrix, dg::DVec> ds( dsFA, dg::not_normed, dg::centered);
 
+    ///##########################################################///
     dg::DVec function = dg::evaluate( func, g3d), derivative(function);
     const dg::DVec solution = dg::evaluate( deri, g3d);
     ds( function, derivative);
@@ -60,6 +67,7 @@ int main(int argc, char * argv[])
     norm = dg::blas2::dot(vol3d, derivative);
     std::cout << "Error Backward Derivative "<<sqrt( norm/sol)<<"\n";
     std::cout << "(Since the function is a parabola, the error is from the parallel derivative only if n>2/ no interpolation error)\n"; 
+    ///##########################################################///
     std::cout << "TEST FIELDALIGNED EVALUATION of a Gaussian\n";
     dg::Gaussian init0(R_0+0.5, 0, 0.2, 0.2, 1);
     dg::GaussianZ modulate(0., M_PI/3., 1);
@@ -67,6 +75,24 @@ int main(int argc, char * argv[])
     ds( function, derivative);
     norm = dg::blas2::dot(vol3d, derivative);
     std::cout << "Norm Centered Derivative "<<sqrt( norm)<<" (compare with that of ds_mpib)\n";
+    ///##########################################################///
+    std::cout << "TEST ADJOINT DERIVATIVE! \n";
+    function = dg::evaluate( func, g3d);
+    const dg::DVec adjoint_solution = dg::evaluate( adjoint, g3d);
+    const double adj = dg::blas2::dot( vol3d, adjoint_solution);
+
+    ds.centeredAdj( -1., function, 0., derivative);
+    dg::blas1::axpby( 1., adjoint_solution, -1., derivative);
+    norm = dg::blas2::dot( vol3d, derivative);
+    std::cout << "Error centered derivative "<< sqrt( norm/adj )<<"\n";
+    ds.forwardAdj( -1., function, 0., derivative);
+    dg::blas1::axpby( 1., adjoint_solution, -1., derivative);
+    norm = dg::blas2::dot(vol3d, derivative);
+    std::cout << "Error Forward  Derivative "<<sqrt( norm/adj)<<"\n";
+    ds.backwardAdj( -1., function, 0., derivative);
+    dg::blas1::axpby( 1., adjoint_solution, -1., derivative);
+    norm = dg::blas2::dot(vol3d, derivative);
+    std::cout << "Error Backward Derivative "<<sqrt( norm/adj)<<"\n";
 
     return 0;
 }
-- 
GitLab


From a663ea46319e7bef50bc6bf1046fdaefc0c38a41 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Fri, 20 Oct 2017 15:59:35 +0200
Subject: [PATCH 376/453] update on mpi fieldaligned

---
 inc/dg/backend/grid.h             |   6 +-
 inc/dg/backend/memory.h           |  16 ++---
 inc/dg/backend/mpi_projection.h   |  10 +--
 inc/dg/backend/transpose.h        |  11 ++--
 inc/geometries/fieldaligned.h     | 104 ++++++++++++------------------
 inc/geometries/mpi_fieldaligned.h |  52 +++++++--------
 6 files changed, 89 insertions(+), 110 deletions(-)

diff --git a/inc/dg/backend/grid.h b/inc/dg/backend/grid.h
index 6c8cbbd51..43636b872 100644
--- a/inc/dg/backend/grid.h
+++ b/inc/dg/backend/grid.h
@@ -178,7 +178,7 @@ struct Grid1d
     const DLT<double>& dlt() const {return dlt_;}
     void display( std::ostream& os = std::cout) const
     {
-        os << "aTopology parameters are: \n"
+        os << "Topology parameters are: \n"
             <<"    n  = "<<n_<<"\n"
             <<"    N = "<<Nx_<<"\n"
             <<"    h = "<<h()<<"\n"
@@ -359,7 +359,7 @@ struct aTopology2d
      */
     void display( std::ostream& os = std::cout) const
     {
-        os << "aTopology parameters are: \n"
+        os << "Topology parameters are: \n"
             <<"    n  = "<<n()<<"\n"
             <<"    Nx = "<<Nx()<<"\n"
             <<"    Ny = "<<Ny()<<"\n"
@@ -590,7 +590,7 @@ struct aTopology3d
      */
     void display( std::ostream& os = std::cout) const
     {
-        os << "aTopology parameters are: \n"
+        os << "Topology parameters are: \n"
             <<"    n  = "<<n()<<"\n"
             <<"    Nx = "<<Nx()<<"\n"
             <<"    Ny = "<<Ny()<<"\n"
diff --git a/inc/dg/backend/memory.h b/inc/dg/backend/memory.h
index d116421b0..347b81ed1 100644
--- a/inc/dg/backend/memory.h
+++ b/inc/dg/backend/memory.h
@@ -4,12 +4,12 @@ namespace dg
 {
 
 //there is probably a better class in boost...
-/*!@brief a manager class that invokes the clone() method on the managed ptr when copied
+/*!@brief a manager class that invokes the \c clone() method on the managed ptr when copied
 *
-*When copied invokes a deep copy using the clone() method.
+*When copied invokes a deep copy using the \c clone() method.
 * This class is most useful when a class needs to hold a polymorphic, cloneable oject as a variable. 
-@tparam cloneable a type that may be uncopyable/unassignable but provides the clone() method with signature
- - cloneable* clone() const;
+@tparam cloneable a type that may be uncopyable/unassignable but provides the \c clone() method with signature
+ -  \c cloneable* \c clone() \c const;
 @ingroup lowlevel
 */
 template<class cloneable>
@@ -29,14 +29,14 @@ struct Handle
     */
     Handle( const cloneable& src): ptr_(src.clone()){}
     /**
-    * @brief deep copy the given handle
+    * @brief deep copy the given handle using the \c clone() method of \c cloneable
     * @param src an oject to copy, clones the contained object if not empty
     */
     Handle( const Handle& src):ptr_(0) {
         if(src.ptr_!=0) ptr_ = src.ptr_->clone(); //deep copy
     }
     /**
-    * @brief deep copy the given handle
+    * @brief deep copy the given handle using the \c clone() method of \c cloneable
     * @param src an oject to copy and swap
     */
     Handle& operator=( Handle src) {
@@ -107,11 +107,11 @@ can be declared const while the data it holds are still writeable.
 template< class T>
 struct Buffer
 {
-    ///new T
+    ///new \c T
     Buffer(){
         ptr = new T;
     }
-    ///new T(t)
+    ///new \c T(t)
     Buffer( const T& t){
         ptr = new T(t);
     }
diff --git a/inc/dg/backend/mpi_projection.h b/inc/dg/backend/mpi_projection.h
index 912761472..315498c06 100644
--- a/inc/dg/backend/mpi_projection.h
+++ b/inc/dg/backend/mpi_projection.h
@@ -75,10 +75,6 @@ dg::MIHMatrix convert( const dg::IHMatrix& global, const ConversionPolicy& polic
     cusp::array1d<int, cusp::host_memory> buffer_idx;
     dg::detail::global2bufferIdx( global.column_indices, buffer_idx, unique_global_idx);
     dg::GeneralComm<dg::iHVec, dg::HVec> comm( unique_global_idx, policy);
-    dg::IHMatrix local( global.num_rows, comm.size(), global.values.size());
-    local.row_offsets=global.row_offsets;
-    local.column_indices=buffer_idx;
-    local.values=global.values;
     if( !comm.isCommunicating() ) 
     {
         cusp::array1d<int, cusp::host_memory> local_idx(global.column_indices), pids(local_idx);
@@ -86,9 +82,13 @@ dg::MIHMatrix convert( const dg::IHMatrix& global, const ConversionPolicy& polic
         for(unsigned i=0; i<local_idx.size(); i++)
             if( !policy.global2localIdx(global.column_indices[i], local_idx[i], pids[i]) ) success = false;
         assert( success);
-        local.column_indices=local_idx;
         comm = dg::GeneralComm< dg::iHVec, dg::HVec>();
+        return dg::MIHMatrix( global, comm, dg::row_dist);
     }
+    dg::IHMatrix local( global.num_rows, comm.size(), global.values.size());
+    local.row_offsets=global.row_offsets;
+    local.column_indices=buffer_idx;
+    local.values=global.values;
     dg::MIHMatrix matrix(   local, comm, dg::row_dist);
     return matrix;
 }
diff --git a/inc/dg/backend/transpose.h b/inc/dg/backend/transpose.h
index 852edf30d..ae990dd9e 100644
--- a/inc/dg/backend/transpose.h
+++ b/inc/dg/backend/transpose.h
@@ -13,16 +13,16 @@ namespace detail
 template <class Matrix>
 Matrix doTranspose( const Matrix& src, CuspMatrixTag)
 {
-    Matrix out(src);
+    Matrix out;
     cusp::transpose( src, out);
     return out;
 }
 #ifdef MPI_VERSION
-template <class Matrix, class Collective>
-MPIDistMat<Matrix, Collective> doTranspose( const MPIDistMat<Matrix, Collective>& src, MPIMatrixTag)
+template <class LocalMatrix, class Collective>
+MPIDistMat<LocalMatrix, Collective> doTranspose( const MPIDistMat<LocalMatrix, Collective>& src, MPIMatrixTag)
 {
-    Matrix tr = doTranspose( src.matrix(), typename MatrixTraits<Matrix>::matrix_category());
-    MPIDistMat<Matrix, Collective> out( tr, src.collective());
+    LocalMatrix tr = doTranspose( src.matrix(), typename MatrixTraits<LocalMatrix>::matrix_category());
+    MPIDistMat<LocalMatrix, Collective> out( tr, src.collective());
     return out;
 }
 #endif// MPI_VERSION
@@ -43,6 +43,7 @@ MPIDistMat<Matrix, Collective> doTranspose( const MPIDistMat<Matrix, Collective>
 template<class Matrix>
 Matrix transpose( const Matrix& src)
 {
+    //%Transposed matrices work only for csr_matrix due to bad matrix form for ell_matrix!!!
     return detail::doTranspose( src, typename MatrixTraits<Matrix>::matrix_category());
 }
 
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index 7a7f646fc..a2f387c6f 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -150,19 +150,19 @@ struct DSField
 
 };
 
-void clip_to_boundary( double& x, double& y, const dg::aTopology2d* grid)
+void clip_to_boundary( double& x, double& y, const dg::aTopology2d& grid)
 {
-    if (!(x > grid->x0())) { x=grid->x0();}
-    if (!(x < grid->x1())) { x=grid->x1();}
-    if (!(y > grid->y0())) { y=grid->y0();}
-    if (!(y < grid->y1())) { y=grid->y1();}
+    if (!(x > grid.x0())) { x=grid.x0();}
+    if (!(x < grid.x1())) { x=grid.x1();}
+    if (!(y > grid.y0())) { y=grid.y0();}
+    if (!(y < grid.y1())) { y=grid.y1();}
 }
-void clip_to_boundary( thrust::host_vector<double>& x, const dg::aTopology2d* grid)
+void clip_to_boundary( thrust::host_vector<double>& x, const dg::aTopology2d& grid)
 {
     clip_to_boundary(x[0], x[1], grid);
 }
 
-void interpolate_and_clip( const dg::IHMatrix& interpolate, const dg::aTopology2d& g2dFine, const dg::aTopology2d& boundary_ptr, //2 different grid on account of the MPI implementation
+void interpolate_and_clip( const dg::IHMatrix& interpolate, const dg::aTopology2d& g2dFine, const dg::aTopology2d& boundary, //2 different grid on account of the MPI implementation
         const std::vector<thrust::host_vector<double> >& yp_coarse,
         const std::vector<thrust::host_vector<double> >& ym_coarse,
         std::vector<thrust::host_vector<double> >& yp_fine,
@@ -177,10 +177,10 @@ void interpolate_and_clip( const dg::IHMatrix& interpolate, const dg::aTopology2
     }
     for( unsigned i=0; i<yp[0].size(); i++)
     {
-        boundary_ptr.shift_topologic( yp[0][i], yp[1][i], yp[0][i], yp[1][i]);
-        boundary_ptr.shift_topologic( ym[0][i], ym[1][i], ym[0][i], ym[1][i]);
-        detail::clip_to_boundary( yp[0][i], yp[1][i], &boundary_ptr);
-        detail::clip_to_boundary( ym[0][i], ym[1][i], &boundary_ptr);
+        boundary.shift_topologic( yp[0][i], yp[1][i], yp[0][i], yp[1][i]);
+        boundary.shift_topologic( ym[0][i], ym[1][i], ym[0][i], ym[1][i]);
+        detail::clip_to_boundary( yp[0][i], yp[1][i], boundary);
+        detail::clip_to_boundary( ym[0][i], ym[1][i], boundary);
     }
     yp_fine=yp, ym_fine=ym;
 }
@@ -266,7 +266,7 @@ void boxintegrator( const Field& field, const Topology& grid,
             phi1 = (dPhiMin+dPhiMax)/2.;
             dg::integrateRK4( field, coords0, coords1, dPhiMin, eps);
         }
-        detail::clip_to_boundary( coords1, &grid);
+        detail::clip_to_boundary( coords1, grid);
         //now assume the rest is purely toroidal
         double deltaS = coords1[2];
         thrust::host_vector<double> temp=coords0;
@@ -276,36 +276,34 @@ void boxintegrator( const Field& field, const Topology& grid,
 }
 
 //used in constructor of Fieldaligned
-void integrate_all_fieldlines2d( const dg::geo::BinaryVectorLvl0& vec, const dg::aGeometry2d* g2dField_ptr, const dg::aTopology2d* evaluate_ptr, std::vector<thrust::host_vector<double> >& yp_result, std::vector<thrust::host_vector<double> >& ym_result , double deltaPhi, double eps)
+void integrate_all_fieldlines2d( const dg::geo::BinaryVectorLvl0& vec, const dg::aGeometry2d& grid_field, const dg::aTopology2d& grid_evaluate, std::vector<thrust::host_vector<double> >& yp_result, std::vector<thrust::host_vector<double> >& ym_result , double deltaPhi, double eps)
 {
-    //g2dField contains the global geometry 
-    //evaluate_ptr contains the points to actually integrate
-    std::vector<thrust::host_vector<double> > y( 3, dg::evaluate( dg::cooX2d, *evaluate_ptr)); //x
-    y[1] = dg::evaluate( dg::cooY2d, *evaluate_ptr); //y
-    y[2] = dg::evaluate( dg::zero, *evaluate_ptr); //s
+    //grid_field contains the global geometry for the field and the boundaries
+    //grid_evaluate contains the points to actually integrate
+    std::vector<thrust::host_vector<double> > y( 3, dg::evaluate( dg::cooX2d, grid_evaluate)); //x
+    y[1] = dg::evaluate( dg::cooY2d, grid_evaluate); //y
+    y[2] = dg::evaluate( dg::zero,   grid_evaluate); //s
     std::vector<thrust::host_vector<double> > yp( 3, y[0]), ym(yp); 
     //construct field on high polynomial grid, then integrate it
-    dg::Timer t;
-    t.tic();
-    dg::geo::detail::DSField field( vec, *g2dField_ptr);
-    t.toc();
-    std::cout << "Generation of interpolate grid took "<<t.diff()<<"s\n";
+    dg::geo::detail::DSField field( vec, grid_field);
     //field in case of cartesian grid
     dg::geo::detail::DSFieldCylindrical cyl_field(vec);
-    for( unsigned i=0; i<g2dField_ptr->size(); i++)
+    for( unsigned i=0; i<grid_evaluate.size(); i++)
     {
         thrust::host_vector<double> coords(3), coordsP(3), coordsM(3);
         coords[0] = y[0][i], coords[1] = y[1][i], coords[2] = y[2][i]; //x,y,s
         double phi1 = deltaPhi;
-        if( dynamic_cast<const dg::CartesianGrid2d*>( g2dField_ptr))
-            boxintegrator( cyl_field, *g2dField_ptr, coords, coordsP, phi1, eps);
+        if( dynamic_cast<const dg::CartesianGrid2d*>( &grid_field))
+        {
+            boxintegrator( cyl_field, grid_field, coords, coordsP, phi1, eps);
+        }
         else 
-            boxintegrator( field, *g2dField_ptr, coords, coordsP, phi1, eps);
+            boxintegrator( field, grid_field, coords, coordsP, phi1, eps);
         phi1 =  - deltaPhi;
-        if( dynamic_cast<const dg::CartesianGrid2d*>( g2dField_ptr))
-            boxintegrator( cyl_field, *g2dField_ptr, coords, coordsM, phi1, eps);
+        if( dynamic_cast<const dg::CartesianGrid2d*>( &grid_field))
+            boxintegrator( cyl_field, grid_field, coords, coordsM, phi1, eps);
         else 
-            boxintegrator( field, *g2dField_ptr, coords, coordsM, phi1, eps);
+            boxintegrator( field, grid_field, coords, coordsM, phi1, eps);
         yp[0][i] = coordsP[0], yp[1][i] = coordsP[1], yp[2][i] = coordsP[2];
         ym[0][i] = coordsM[0], ym[1][i] = coordsM[1], ym[2][i] = coordsM[2];
     }
@@ -510,43 +508,29 @@ void Fieldaligned<Geometry, IMatrix, container>::construct(const dg::geo::Binary
     if( deltaPhi <=0) deltaPhi = grid.hz();
     else assert( grid.Nz() == 1 || grid.hz()==deltaPhi);
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-    const dg::aGeometry2d* grid2d_ptr = grid.perp_grid();
+    dg::Handle<dg::aGeometry2d> grid_coarse( grid.perp_grid()) ;
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-    //Resize vector to 2D grid size
-    m_perp_size = grid2d_ptr->size();
-    dg::blas1::transfer( dg::pullback(limit, *grid2d_ptr), m_limiter);
-    dg::blas1::transfer( dg::evaluate(zero, *grid2d_ptr), m_left);
+    m_perp_size = grid_coarse.get().size();
+    dg::blas1::transfer( dg::pullback(limit, grid_coarse.get()), m_limiter);
+    dg::blas1::transfer( dg::evaluate(zero, grid_coarse.get()), m_left);
     m_ghostM = m_ghostP = m_right = m_left;
     //%%%%%%%%%%%%%%%%%%%%%%%%%%Set starting points and integrate field lines%%%%%%%%%%%%%%
-    dg::Timer t;
     std::vector<thrust::host_vector<double> > yp_coarse( 3), ym_coarse(yp_coarse), yp, ym; 
     
-    dg::aGeometry2d* g2dField_ptr = grid2d_ptr->clone();//INTEGRATE HIGH ORDER GRID
-    //g2dField_ptr->set( 7, g2dField_ptr->Nx(), g2dField_ptr->Ny());
-    std::cout << "Start fieldline integration!\n";
-    t.tic();
-    detail::integrate_all_fieldlines2d( vec, g2dField_ptr, g2dField_ptr, yp_coarse, ym_coarse, deltaPhi, eps);
-
-    dg::Grid2d g2dFine((dg::Grid2d(*grid2d_ptr)));//FINE GRID
-    g2dFine.multiplyCellNumbers((double)mx, (double)my);
-    dg::IHMatrix interpolate = dg::create::interpolation( g2dFine, *g2dField_ptr);  //INTERPOLATE TO FINE GRID
-    dg::geo::detail::interpolate_and_clip( interpolate, g2dFine, g2dFine, yp_coarse, ym_coarse, yp, ym);
-    delete g2dField_ptr;
-    t.toc(); 
-    std::cout << "Fieldline integration took "<<t.diff()<<"s\n";
+    dg::Handle<dg::aGeometry2d> grid_magnetic = grid_coarse;//INTEGRATE HIGH ORDER GRID
+    //grid_magnetic.set( 7, grid_magnetic.Nx(), grid_magnetic.Ny());
+    detail::integrate_all_fieldlines2d( vec, grid_magnetic.get(), grid_coarse.get(), yp_coarse, ym_coarse, deltaPhi, eps);
+
+    dg::Grid2d grid_fine( grid_coarse.get() );//FINE GRID
+    grid_fine.multiplyCellNumbers((double)mx, (double)my);
+    dg::IHMatrix interpolate = dg::create::interpolation( grid_fine, grid_coarse.get());  //INTERPOLATE TO FINE GRID
+    dg::geo::detail::interpolate_and_clip( interpolate, grid_fine, grid_fine, yp_coarse, ym_coarse, yp, ym);
     //%%%%%%%%%%%%%%%%%%Create interpolation and projection%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-    t.tic();
-    dg::IHMatrix plusFine  = dg::create::interpolation( yp[0], yp[1], *grid2d_ptr, globalbcx, globalbcy), plus, plusT;
-    dg::IHMatrix minusFine = dg::create::interpolation( ym[0], ym[1], *grid2d_ptr, globalbcx, globalbcy), minus, minusT;
-    dg::IHMatrix projection = dg::create::projection( *grid2d_ptr, g2dFine);
-    t.toc();
-    std::cout <<"Creation of interpolation/projection took "<<t.diff()<<"s\n";
-    t.tic();
+    dg::IHMatrix plusFine  = dg::create::interpolation( yp[0], yp[1], grid_coarse.get(), globalbcx, globalbcy), plus, plusT;
+    dg::IHMatrix minusFine = dg::create::interpolation( ym[0], ym[1], grid_coarse.get(), globalbcx, globalbcy), minus, minusT;
+    dg::IHMatrix projection = dg::create::projection( grid_coarse.get(), grid_fine);
     cusp::multiply( projection, plusFine, plus);
     cusp::multiply( projection, minusFine, minus);
-    t.toc();
-    std::cout<< "Multiplication of P*I took: "<<t.diff()<<"s\n";
-    //%Transposed matrices work only for csr_matrix due to bad matrix form for ell_matrix!!!
     plusT = dg::transpose( plus);
     minusT = dg::transpose( minus);     
     dg::blas2::transfer( plus, m_plus);
@@ -567,8 +551,6 @@ void Fieldaligned<Geometry, IMatrix, container>::construct(const dg::geo::Binary
     dg::join( std::vector<thrust::host_vector<double> >( m_Nz, hp), m_hp_inv, grid);
     dg::join( std::vector<thrust::host_vector<double> >( m_Nz, hm), m_hm_inv, grid);
     dg::join( std::vector<thrust::host_vector<double> >( m_Nz, hz), m_hz_inv, grid);
-    
-    delete grid2d_ptr;
 }
 
 template<class G, class I, class container>
diff --git a/inc/geometries/mpi_fieldaligned.h b/inc/geometries/mpi_fieldaligned.h
index 5543d85dc..160f57268 100644
--- a/inc/geometries/mpi_fieldaligned.h
+++ b/inc/geometries/mpi_fieldaligned.h
@@ -141,43 +141,40 @@ void Fieldaligned<MPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vec
     MPI_Cart_get( m_g.get().communicator(), 3, dims, periods, coords);
     m_coords2 = coords[2], m_sizeZ = dims[2];
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-    const aMPIGeometry2d* grid2d_ptr = grid.perp_grid();
+    dg::Handle<aMPIGeometry2d> grid_coarse( grid.perp_grid());
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-    m_perp_size = grid2d_ptr->size();
-    dg::blas1::transfer( dg::pullback(limit, *grid2d_ptr), m_limiter);
-    m_right = m_left = dg::evaluate( zero, *grid2d_ptr);
-    dg::blas1::transfer( dg::evaluate(zero, *grid2d_ptr), m_left);
+    m_perp_size = grid_coarse.get().size();
+    dg::blas1::transfer( dg::pullback(limit, grid_coarse.get()), m_limiter);
+    dg::blas1::transfer( dg::evaluate(zero, grid_coarse.get()), m_left);
     m_ghostM = m_ghostP = m_right = m_left;
     //%%%%%%%%%%%%%%%%%%%%%%%%%%Set starting points and integrate field lines%%%%%%%%%%%%%%
     std::vector<thrust::host_vector<double> > yp_coarse( 3), ym_coarse(yp_coarse), yp, ym; 
     
-    dg::aMPIGeometry2d* g2dField_ptr = grid2d_ptr->clone();//INTEGRATE HIGH ORDER GRID
-    //g2dField_ptr->set( 7, g2dField_ptr->global().Nx(), g2dField_ptr->global().Ny());
-    dg::aGeometry2d* global_g2dField_ptr = g2dField_ptr->global_geometry();
-    dg::Grid2d local_g2dField = g2dField_ptr->local();
-    detail::integrate_all_fieldlines2d( vec, global_g2dField_ptr, &local_g2dField, yp_coarse, ym_coarse, deltaPhi, eps);
+    dg::Handle<dg::aMPIGeometry2d> grid_magnetic = grid_coarse;//INTEGRATE HIGH ORDER GRID
+    //grid_magnetic->set( 7, grid_magnetic->global().Nx(), grid_magnetic->global().Ny());
+    dg::Handle<dg::aGeometry2d> global_grid_magnetic = grid_magnetic.get().global_geometry();
+    detail::integrate_all_fieldlines2d( vec, global_grid_magnetic.get(), grid_coarse.get().local(), yp_coarse, ym_coarse, deltaPhi, eps);
 
-    dg::MPIGrid2d g2dFine((dg::MPIGrid2d(*grid2d_ptr)));//FINE GRID
-    g2dFine.multiplyCellNumbers((double)mx, (double)my);
-    dg::IHMatrix interpolate = dg::create::interpolation( g2dFine.local(), local_g2dField);  //INTERPOLATE TO FINE GRID
-    dg::geo::detail::interpolate_and_clip( interpolate, g2dFine.local(), *global_g2dField_ptr, yp_coarse, ym_coarse, yp, ym);
-    delete g2dField_ptr;
-    delete global_g2dField_ptr;
+    dg::MPIGrid2d grid_fine( grid_coarse.get() );//FINE GRID
+    grid_fine.multiplyCellNumbers((double)mx, (double)my);
+    dg::IHMatrix interpolate = dg::create::interpolation( grid_fine.local(), grid_coarse.get().local());  //INTERPOLATE TO FINE GRID
+    dg::geo::detail::interpolate_and_clip( interpolate, grid_fine.local(), grid_fine.global(), yp_coarse, ym_coarse, yp, ym);
     //%%%%%%%%%%%%%%%%%%Create interpolation and projection%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-    dg::IHMatrix plusFine  = dg::create::interpolation( yp[0], yp[1], grid2d_ptr->global(), globalbcx, globalbcy), plus, plusT;
-    dg::IHMatrix minusFine = dg::create::interpolation( ym[0], ym[1], grid2d_ptr->global(), globalbcx, globalbcy), minus, minusT;
-    dg::IHMatrix projection = dg::create::projection( grid2d_ptr->local(), g2dFine.local());
+    dg::IHMatrix plusFine  = dg::create::interpolation( yp[0], yp[1], grid_coarse.get().global(), globalbcx, globalbcy), plus;
+    dg::IHMatrix minusFine = dg::create::interpolation( ym[0], ym[1], grid_coarse.get().global(), globalbcx, globalbcy), minus;
+    dg::IHMatrix projection = dg::create::projection( grid_coarse.get().local(), grid_fine.local());
     cusp::multiply( projection, plusFine, plus);
     cusp::multiply( projection, minusFine, minus);
-    //%Transposed matrices work only for csr_matrix due to bad matrix form for ell_matrix!!!
-    dg::MIHMatrix temp = dg::convert( plus, *grid2d_ptr);
+    dg::MIHMatrix temp = dg::convert( plus, grid_coarse.get()), tempT;
+    tempT  = dg::transpose( temp);
     dg::blas2::transfer( temp, m_plus);
-    temp = dg::convert( minus, *grid2d_ptr);
+    dg::blas2::transfer( tempT, m_plusT);
+    temp = dg::convert( minus, grid_coarse.get());
+    tempT  = dg::transpose( temp);
     dg::blas2::transfer( temp, m_minus);
-    m_plusT = dg::transpose( m_plus);
-    m_minusT = dg::transpose( m_minus);
+    dg::blas2::transfer( tempT, m_minusT);
     //%%%%%%%%%%%%%%%%%%%%%%%project h and copy into h vectors%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-    dg::MHVec hp( dg::evaluate( dg::zero, *grid2d_ptr)), hm(hp), hz(hp);
+    dg::MHVec hp( dg::evaluate( dg::zero, grid_coarse.get())), hm(hp), hz(hp);
     dg::blas2::symv( projection, yp[2], hp.data());
     dg::blas2::symv( projection, ym[2], hm.data());
     dg::blas1::scal( hm, -1.);
@@ -190,7 +187,6 @@ void Fieldaligned<MPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vec
     dg::join( std::vector<dg::MHVec >( m_Nz, hp), m_hp_inv, grid);
     dg::join( std::vector<dg::MHVec >( m_Nz, hm), m_hm_inv, grid);
     dg::join( std::vector<dg::MHVec >( m_Nz, hz), m_hz_inv, grid);
-    delete grid2d_ptr;
 }
 
 template<class G, class M, class C, class container>
@@ -208,9 +204,9 @@ MPI_Vector<container> Fieldaligned<G,MPIDistMat<M,C>, MPI_Vector<container> >::e
     MPI_Vector<container> temp(init2d), tempP(init2d), tempM(init2d);
     MPI_Vector<container> vec3d = dg::evaluate( dg::zero, m_g.get());
     std::vector<MPI_Vector<container> >  plus2d, minus2d, result;
-    dg::split( vec3d, plus2d, m_g.get());
+    dg::split( vec3d, plus2d,  m_g.get());
     dg::split( vec3d, minus2d, m_g.get());
-    dg::split( vec3d, result, m_g.get());
+    dg::split( vec3d, result,  m_g.get());
     unsigned turns = rounds; 
     if( turns ==0) turns++;
     //first apply Interpolation many times, scale and store results
-- 
GitLab


From ad954f4c0efbb6163960a074b2c75f82aadaa939 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sat, 21 Oct 2017 17:27:21 +0200
Subject: [PATCH 377/453] enhanced Documentation with real examples from dg/inc
 test programs

---
 inc/dg/Doxyfile                 |  6 ++-
 inc/dg/Makefile                 |  2 +-
 inc/dg/arakawa.h                |  4 +-
 inc/dg/arakawa_t.cu             | 35 +++++++--------
 inc/dg/blas1.h                  | 71 +++++++++++++++++++++++++++--
 inc/dg/blas1_t.cu               |  3 +-
 inc/dg/blas2.h                  |  6 +--
 inc/dg/cg.h                     | 16 ++++---
 inc/dg/elliptic.h               |  2 +
 inc/dg/elliptic2d_b.cu          | 79 +++++++++++++++------------------
 inc/dg/geometry/base_geometry.h |  1 +
 inc/dg/multigrid.h              | 22 +++++++--
 inc/dg/multistep.h              |  6 ++-
 inc/dg/multistep_t.cu           | 71 ++++++++++++-----------------
 inc/dg/poisson.h                |  3 +-
 inc/dg/poisson_t.cu             | 20 +++++----
 inc/dg/runge_kutta.h            |  3 +-
 inc/dg/runge_kutta2d_t.cu       | 48 +++++++-------------
 18 files changed, 228 insertions(+), 170 deletions(-)

diff --git a/inc/dg/Doxyfile b/inc/dg/Doxyfile
index fe89eaf56..35ee4cd03 100644
--- a/inc/dg/Doxyfile
+++ b/inc/dg/Doxyfile
@@ -864,7 +864,11 @@ EXCLUDE_SYMBOLS        = hide_*
 # that contain example code fragments that are included (see the \include
 # command).
 
-EXAMPLE_PATH           = cg2d_t.cu
+EXAMPLE_PATH           = cg2d_t.cu \
+                         arakawa_t.cu poisson_t.cu \
+                         elliptic2d_b.cu runge_kutta2d_t.cu multistep_t.cu
+
+
 
 # If the value of the EXAMPLE_PATH tag contains directories, you can use the
 # EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and
diff --git a/inc/dg/Makefile b/inc/dg/Makefile
index 216eb230b..d1597176f 100644
--- a/inc/dg/Makefile
+++ b/inc/dg/Makefile
@@ -24,7 +24,7 @@ all: $(CPPFILES:%.cpp=%) $(CUFILES:%.cu=%)
 	$(CC) $(OPT) $(INCLUDE) -DDG_DEBUG $(CFLAGS) $< -o $@ 
 
 %_b: %_b.cu 
-	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) -g 
+	$(CC) $(OPT) $(CFLAGS) -DDG_BENCHMARK $< -o $@ $(INCLUDE) -g 
 
 bathRZ_t: bathRZ_t.cu 
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(GLFLAGS) $(INCLUDE) 
diff --git a/inc/dg/arakawa.h b/inc/dg/arakawa.h
index 573558549..737e38ce8 100644
--- a/inc/dg/arakawa.h
+++ b/inc/dg/arakawa.h
@@ -17,12 +17,14 @@
   */
 namespace dg
 {
-
+//citation missing in documentation
 /**
  * @brief X-space generalized version of Arakawa's scheme
  *
  * Computes \f[ [f,g] := 1/\sqrt{g_{2d}}\left(\partial_x f\partial_y g - \partial_y f\partial_x g\right) \f]
  * where \f$ g_{2d} = g/g_{zz}\f$ is the two-dimensional volume element of the plane in 2x1 product space. 
+ * @snippet arakawa_t.cu function
+ * @snippet arakawa_t.cu doxygen
  * @copydoc hide_geometry_matrix_container
  * @ingroup arakawa
  */
diff --git a/inc/dg/arakawa_t.cu b/inc/dg/arakawa_t.cu
index 52c96d80e..c4cf4a8de 100644
--- a/inc/dg/arakawa_t.cu
+++ b/inc/dg/arakawa_t.cu
@@ -24,23 +24,18 @@ double jacobian( double x, double y)
 }
 */
 
-double left( double x, double y) {
-    return sin(x)*cos(y);
-    //return cos(x)*x*x*cos(y);
-}
-double right( double x, double y) {
-    return sin(y)*cos(x);
-    //return cos(x)*exp(0.1*(x+y));
-} 
+//![function]
 const double lx = 2*M_PI;
 const double ly = 2*M_PI;
 dg::bc bcx = dg::PER; 
 dg::bc bcy = dg::PER;
-//double right2( double x, double y) {return sin(y);}
+double left( double x, double y) { return sin(x)*cos(y); }
+double right( double x, double y) { return sin(y)*cos(x); } 
 double jacobian( double x, double y) 
 {
     return cos(x)*cos(y)*cos(x)*cos(y) - sin(x)*sin(y)*sin(x)*sin(y); 
 }
+//![function]
 double variationRHS( double x, double y) 
 {
     return cos(x)*cos(y)*cos(x)*cos(y) + sin(x)*sin(y)*sin(x)*sin(y); 
@@ -68,20 +63,21 @@ int main()
     unsigned n, Nx, Ny;
     std::cout << "Type n, Nx and Ny! \n";
     std::cin >> n >> Nx >> Ny;
-    dg::Grid2d grid( 0, lx, 0, ly, n, Nx, Ny, bcx, bcy);
-    dg::DVec w2d = dg::create::weights( grid);
     std::cout << "Computing on the Grid " <<n<<" x "<<Nx<<" x "<<Ny <<std::endl;
-    std::cout <<std::fixed<< std::setprecision(2)<<std::endl;
-    dg::DVec lhs = dg::evaluate( left, grid), jac(lhs);
-    dg::DVec rhs = dg::evaluate( right, grid);
-    const dg::DVec sol = dg::evaluate ( jacobian, grid);
-    const dg::DVec variation = dg::evaluate ( variationRHS, grid);
-    dg::DVec eins = dg::evaluate( dg::one, grid);
 
-    dg::ArakawaX<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> arakawa( grid);
+    //![doxygen]
+    const dg::CartesianGrid2d grid( 0, lx, 0, ly, n, Nx, Ny, bcx, bcy);
+    const dg::DVec lhs = dg::evaluate( left, grid);
+    const dg::DVec rhs = dg::evaluate( right, grid);
+    dg::DVec jac(lhs);
+
+    dg::ArakawaX<dg::aGeometry2d, dg::DMatrix, dg::DVec> arakawa( grid);
     arakawa( lhs, rhs, jac);
+    //![doxygen]
 
-    std::cout << std::scientific;
+    dg::DVec w2d = dg::create::weights( grid);
+    dg::DVec eins = dg::evaluate( dg::one, grid);
+    const dg::DVec sol = dg::evaluate ( jacobian, grid);
     std::cout << "Mean     Jacobian is "<<dg::blas2::dot( eins, w2d, jac)<<"\n";
     std::cout << "Mean rhs*Jacobian is "<<dg::blas2::dot( rhs,  w2d, jac)<<"\n";
     std::cout << "Mean lhs*Jacobian is "<<dg::blas2::dot( lhs,  w2d, jac)<<"\n";
@@ -95,6 +91,7 @@ int main()
     //n = 5 -> p = 5    |
     // quantities are all conserved to 1e-15 for periodic bc
     // for dirichlet bc these are not better conserved than normal jacobian
+    const dg::DVec variation = dg::evaluate ( variationRHS, grid);
     arakawa.variation( rhs, jac);
     dg::blas1::axpby( 1., variation, -1., jac);
     std::cout << "Variation distance to solution "<<sqrt( dg::blas2::dot( w2d, jac))<<std::endl; //don't forget sqrt when comuting errors
diff --git a/inc/dg/blas1.h b/inc/dg/blas1.h
index 83f27ba10..ae318aa34 100644
--- a/inc/dg/blas1.h
+++ b/inc/dg/blas1.h
@@ -38,6 +38,12 @@ namespace blas1
  * @param x source
  * @param y sink
  * @note y gets resized properly
+
+ * @code
+ dg::HVec host = dg::evaluate( dg::one, grid);
+ dg::DVec device;
+ dg::blas1::transfer( host, device); //device now equals host
+ * @endcode
  */
 template<class container, class other_container>
 inline void transfer( const container& x, other_container& y)
@@ -66,8 +72,12 @@ inline void copy( const Assignable& x, Assignable& y){y=x;}
  * @param y Right container may alias x
  * @return Scalar product as defined above
  * @note This routine is always executed synchronously due to the 
-        implicit memcpy of the result. With mpi the result is broadcasted to all
-        processes
+        implicit memcpy of the result. With mpi the result is broadcasted to all processes
+
+@code
+    dg::DVec two( 100,2), three(100,3);
+    double temp = dg::blas1::dot( two, three); //temp = 30 (5*(2*3))
+@endcode
  */
 template< class container>
 inline typename VectorTraits<container>::value_type dot( const container& x, const container& y)
@@ -84,6 +94,11 @@ inline typename VectorTraits<container>::value_type dot( const container& x, con
  * @param x container x may alias y 
  * @param beta Scalar
  * @param y container y contains solution on output
+
+@code
+    dg::DVec two( 100,2), three(100,3);
+    dg::blas1::axpby( 2, two, 3., three); //three[i] = 13 (2*2+3*3)
+@endcode
  */
 template< class container>
 inline void axpby( typename VectorTraits<container>::value_type alpha, const container& x, typename VectorTraits<container>::value_type beta, container& y)
@@ -103,6 +118,11 @@ inline void axpby( typename VectorTraits<container>::value_type alpha, const con
  * @param beta Scalar
  * @param y container y may alias z
  * @param z container z contains solution on output
+
+@code
+    dg::DVec two( 100,2), three(100,3), result(100);
+    dg::blas1::axpby( 2, two, 3., three, result); //result[i] = 13 (2*2+3*3)
+@endcode
  */
 template< class container>
 inline void axpby( typename VectorTraits<container>::value_type alpha, const container& x, typename VectorTraits<container>::value_type beta, const container& y, container& z)
@@ -123,6 +143,12 @@ inline void axpby( typename VectorTraits<container>::value_type alpha, const con
  * @param y container y may alias result
  * @param gamma Scalar
  * @param z container contains solution on output
+
+@code
+    dg::DVec two(100,2), five(100,5), result(100, 12);
+    dg::blas1::axpbypgz( 2.5, two, 2., five, -3.,result); 
+    //result[i] = -21 (2.5*2+2*5-3*12) 
+@endcode
  */
 template< class container>
 inline void axpbypgz( typename VectorTraits<container>::value_type alpha, const container& x, typename VectorTraits<container>::value_type beta, const container& y, typename VectorTraits<container>::value_type gamma, container& z)
@@ -140,6 +166,12 @@ inline void axpbypgz( typename VectorTraits<container>::value_type alpha, const
  * @param x container x may alias y
  * @param y container y contains result, may alias x
  * @param op unary Operator to use on every element
+
+@code
+    dg::DVec two( 100,2), result(100);
+    dg::blas1::transform( two, result, dg::EXP()); 
+    //result[i] = 7.389056... (e^2)
+@endcode
  */
 template< class container, class UnaryOp>
 inline void transform( const container& x, container& y, UnaryOp op)
@@ -154,6 +186,11 @@ inline void transform( const container& x, container& y, UnaryOp op)
  * @copydoc hide_container
  * @param alpha Scalar  
  * @param x container x 
+
+@code
+    dg::DVec two( 100,2);
+    dg::blas1::scal( two,  0.5 )); //result[i] = 1. 
+@endcode
  */
 template< class container>
 inline void scal( container& x, typename VectorTraits<container>::value_type alpha)
@@ -168,6 +205,11 @@ inline void scal( container& x, typename VectorTraits<container>::value_type alp
  * @copydoc hide_container
  * @param alpha Scalar  
  * @param x container x 
+
+@code
+    dg::DVec two( 100,2);
+    dg::blas1::plus( two,  2. )); //result[i] = 4. 
+@endcode
  */
 template< class container>
 inline void plus( container& x, typename VectorTraits<container>::value_type alpha)
@@ -185,6 +227,11 @@ inline void plus( container& x, typename VectorTraits<container>::value_type alp
 * @param x1 container x1  
 * @param x2 container x2 may alias x1
 * @param y  container y contains result on output ( may alias x1 or x2)
+
+@code
+    dg::DVec two( 100,2), three( 100,3), result(100);
+    dg::blas1::pointwiseDot( two,  three, result ); //result[i] = 6. 
+@endcode
 */
 template< class container>
 inline void pointwiseDot( const container& x1, const container& x2, container& y)
@@ -205,6 +252,12 @@ inline void pointwiseDot( const container& x1, const container& x2, container& y
 * @param x2 container x2 may alias x1
 * @param beta scalar
 * @param y  container y contains result on output ( may alias x1 or x2)
+
+@code
+    dg::DVec two( 100,2), three( 100,3), result(100,6);
+    dg::blas1::pointwiseDot(2., two,  three, -4., result ); 
+    //result[i] = -12. (2*2*3-4*6)
+@endcode
 */
 template< class container>
 inline void pointwiseDot( typename VectorTraits<container>::value_type alpha, const container& x1, const container& x2, typename VectorTraits<container>::value_type beta, container& y)
@@ -222,6 +275,12 @@ inline void pointwiseDot( typename VectorTraits<container>::value_type alpha, co
 * @param x1 container x1  
 * @param x2 container x2 may alias x1
 * @param y  container y contains result on output ( may alias x1 and/or x2)
+
+@code
+    dg::DVec two( 100,2), three( 100,3), result(100);
+    dg::blas1::pointwiseDivide( two,  three, result ); 
+    //result[i] = -0.666... (2/3)
+@endcode
 */
 template< class container>
 inline void pointwiseDivide( const container& x1, const container& x2, container& y)
@@ -246,7 +305,13 @@ inline void pointwiseDivide( const container& x1, const container& x2, container
 * @param y2 container y2 
 * @param gamma scalar
 * @param z  container z contains result on output 
-* @note aliases are allowed
+* @note all aliases are allowed 
+
+@code
+    dg::DVec two(100,2), three(100,3), four(100,5), five(100,5), result(100,6);
+    dg::blas1::pointwiseDot(2., two,  three, -4., four, five, 2., result ); 
+    //result[i] = -56.
+@endcode
 */
 template<class container>
 void pointwiseDot(  typename VectorTraits<container>::value_type alpha, const container& x1, const container& y1, 
diff --git a/inc/dg/blas1_t.cu b/inc/dg/blas1_t.cu
index 465461e42..871556347 100644
--- a/inc/dg/blas1_t.cu
+++ b/inc/dg/blas1_t.cu
@@ -13,8 +13,7 @@ typedef thrust::device_vector<double>  Vector;
 //typedef cusp::array1d<double, cusp::device_memory>  Vector;
 int main()
 {
-    Vector v1( 5, 2), v2( 5, 3);
-    Vector v3(5);
+    Vector v1( 5, 2), v2( 5, 3), v3(5);
     double temp = dg::blas1::dot(v1,v2);
     std::cout << "5*(2*3) = "<<temp << " (30)\n"; 
     dg::blas1::axpby( 2., v1, 3., v2, v3);
diff --git a/inc/dg/blas2.h b/inc/dg/blas2.h
index dc77b3f84..af625f07c 100644
--- a/inc/dg/blas2.h
+++ b/inc/dg/blas2.h
@@ -127,12 +127,12 @@ inline void symv( typename MatrixTraits<Matrix>::value_type alpha,
  * where \f$ M\f$ is a matrix. 
  * @copydoc hide_matrix
  * @copydoc hide_container
- * @tparam same_or_another_container Currently needs to be the same as container.
+ * @tparam same_or_another_container Currently needs to be the same as \c container.
  * @param M The Matrix
  * @param x A container different from \p y 
  * @param y contains the solution on output (may not alias \p x)
  * @attention y may never alias x
- * @note Due to the SelfMadeMatrixTag, M cannot be declared const
+ * @note Due to the \c SelfMadeMatrixTag, M cannot be declared const
  */
 template< class Matrix, class container, class same_or_another_container>
 inline void symv( Matrix& M, 
@@ -152,7 +152,7 @@ inline void symv( Matrix& M,
  * Does exactly the same as symv. 
  * @copydoc hide_matrix
  * @copydoc hide_container
- * @tparam same_or_another_container Currently needs to be the same as container.
+ * @tparam same_or_another_container Currently needs to be the same as \c container.
  * @param M The Matrix
  * @param x A container different from \p y 
  * @param y contains the solution on output (may not alias \p x)
diff --git a/inc/dg/cg.h b/inc/dg/cg.h
index e8298a312..d4cf1e683 100644
--- a/inc/dg/cg.h
+++ b/inc/dg/cg.h
@@ -22,13 +22,13 @@ namespace dg{
 * @brief Functor class for the preconditioned conjugate gradient method to solve
 * \f[ Ax=b\f]
 *
- @ingroup invert
- @copydoc hide_container
-
- @note Conjugate gradients might become unstable for positive semidefinite
- matrices arising e.g. in the discretization of the periodic laplacian
-
- @snippet cg2d_t.cu doxygen
+* @ingroup invert
+* @copydoc hide_container
+*
+* @note Conjugate gradients might become unstable for positive semidefinite
+* matrices arising e.g. in the discretization of the periodic laplacian
+*
+* @snippet cg2d_t.cu doxygen
 */
 template< class container>
 class CG
@@ -355,6 +355,7 @@ struct Extrapolation
  * by appropriate weights \f$W\f$ (s. comment below). 
  * It uses solutions from the last two calls to 
  * extrapolate a solution for the current call.
+ * @snippet elliptic2d_b.cu invert
  * @copydoc hide_container
  * @note A note on weights, inverse weights and preconditioning. 
  * A normalized DG-discretized derivative or operator is normally not symmetric. 
@@ -365,6 +366,7 @@ struct Extrapolation
  * Independent from this, a preconditioner should be used to solve the
  * symmetric matrix equation. The inverse of \f$W\f$ is 
  * a good general purpose preconditioner. 
+ * @sa Extrapolation
  */
 template<class container>
 struct Invert
diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index 4a7b5f343..9cf6ab64a 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -49,6 +49,8 @@ namespace dg
  * @note The inverse of \f$ \chi\f$ makes a good general purpose preconditioner
  * @note the jump term \f$ \alpha J\f$  adds artificial numerical diffusion as discussed above
  * @attention Pay attention to the negative sign 
+ * 
+ * @snippet elliptic2d_b.cu multigrid
  */
 template <class Geometry, class Matrix, class container>
 class Elliptic
diff --git a/inc/dg/elliptic2d_b.cu b/inc/dg/elliptic2d_b.cu
index 3510e9eb1..a2bebfe60 100644
--- a/inc/dg/elliptic2d_b.cu
+++ b/inc/dg/elliptic2d_b.cu
@@ -36,19 +36,14 @@ double der(double x, double y)  { return cos( x)*sin(y);}
 
 int main()
 {
-    dg::Timer t;
     unsigned n, Nx, Ny; 
     double eps;
     double jfactor;
-    unsigned scheme;
 
 	n = 3;
 	Nx = Ny = 64;
 	eps = 1e-6;
 	jfactor = 1;
-	
-    std::cout << "scheme type? \n";
-    std::cin >> scheme;
 
 	/*std::cout << "Type n, Nx and Ny and epsilon and jfactor (1)! \n";
     std::cin >> n >> Nx >> Ny; //more N means less iterations for same error
@@ -72,35 +67,33 @@ int main()
 
     //std::cout << "Create Polarisation object and set chi!\n";
     {
-		t.tic();
-
-		unsigned stages = 3;
-
-		dg::MultigridCG2d<dg::aGeometry2d, dg::DMatrix, dg::DVec > multigrid( grid, stages, scheme);
-		
-		std::vector<dg::DVec> chi_ = multigrid.project( chi);
-		std::vector<dg::Elliptic<dg::aGeometry2d, dg::DMatrix, dg::DVec> > multi_pol( stages);
-		
-		for(unsigned u=0; u<stages; u++)
-		{
-			multi_pol[u].construct( multigrid.grids()[u].get(), dg::not_normed, dg::centered, jfactor); 
-			multi_pol[u].set_chi( chi_[u]);
-		}
-
-		t.toc();
+    //! [multigrid]
+    dg::Timer t;
+    t.tic();
+
+    const unsigned stages = 3;
+
+    dg::MultigridCG2d<dg::aGeometry2d, dg::DMatrix, dg::DVec > multigrid( grid, stages);
+    
+    const std::vector<dg::DVec> multi_chi = multigrid.project( chi);
+    std::vector<dg::Elliptic<dg::aGeometry2d, dg::DMatrix, dg::DVec> > multi_pol( stages);
     
-		//std::cout << "Creation of polarisation object took: "<<t.diff()<<"s\n";
-		//std::cout << eps<<" \n";
-    	
-		t.tic();
-
-		std::vector<unsigned> number = multigrid.solve(multi_pol, x, b, eps);
-		
-		//for( unsigned u=0; u<number.size(); u++)
-		//	std::cout << " # iterations stage "<< number.size()-1-u << " " << number[number.size()-1-u] << " \n";
-
-		t.toc();
-		std::cout << "Took "<< t.diff() <<"s\n";
+    for(unsigned u=0; u<stages; u++)
+    {
+        multi_pol[u].construct( multigrid.grids()[u].get(), dg::not_normed, dg::centered, jfactor); 
+        multi_pol[u].set_chi( multi_chi[u]);
+    }
+
+    t.toc();
+
+    std::cout << "Creation of multigrid took: "<<t.diff()<<"s\n";
+    t.tic();
+    std::vector<unsigned> number = multigrid.direct_solve(multi_pol, x, b, eps);
+    t.toc();
+    std::cout << "Solution took "<< t.diff() <<"s\n";
+    for( unsigned u=0; u<number.size(); u++)
+    	std::cout << " # iterations stage "<< number.size()-1-u << " " << number[number.size()-1-u] << " \n";
+    //! [multigrid]
     }
 
     //compute error
@@ -112,16 +105,18 @@ int main()
     double err = dg::blas2::dot( w2d, error);
     //std::cout << "L2 Norm2 of Error is                       " << err << std::endl;
     const double norm = dg::blas2::dot( w2d, solution);
-    std::cout << " "<<sqrt( err/norm);
+    std::cout << " "<<sqrt( err/norm) << "\n";
     {
-		dg::Elliptic<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> pol_forward( grid, dg::not_normed, dg::centered, jfactor);
-		pol_forward.set_chi( chi);
-		x = temp;
-		dg::Invert<dg::DVec > invert_fw( x, n*n*Nx*Ny, eps);
-		std::cout << " "<< invert_fw( pol_forward, x, b, w2d, v2d, chi_inv);
-		dg::blas1::axpby( 1.,x,-1., solution, error);
-		err = dg::blas2::dot( w2d, error);
-		std::cout << " "<<sqrt( err/norm);
+    x = temp;
+    //![invert]
+    dg::Elliptic<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> pol_forward( grid, dg::not_normed, dg::centered, jfactor);
+    pol_forward.set_chi( chi);
+    dg::Invert<dg::DVec > invert_fw( x, n*n*Nx*Ny, eps);
+    std::cout << " "<< invert_fw( pol_forward, x, b, w2d, v2d, chi_inv);
+    dg::blas1::axpby( 1.,x,-1., solution, error);
+    err = dg::blas2::dot( w2d, error);
+    std::cout << " "<<sqrt( err/norm) << "\n";
+    //![invert]
     }
 
     {
diff --git a/inc/dg/geometry/base_geometry.h b/inc/dg/geometry/base_geometry.h
index ed6818940..d7190da1e 100644
--- a/inc/dg/geometry/base_geometry.h
+++ b/inc/dg/geometry/base_geometry.h
@@ -205,6 +205,7 @@ struct aProductGeometry3d : public aGeometry3d
 
 /**
  * @brief two-dimensional Grid with Cartesian metric
+ * @snippet arakawa_t.cu doxygen
  */
 struct CartesianGrid2d: public dg::aGeometry2d
 {
diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index 9df4c3ac0..1bc73a00d 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -6,7 +6,9 @@
 #include "backend/memory.h"
 #include "blas.h"
 #include "cg.h"
+#ifdef DG_BENCHMARK
 #include "backend/timer.cuh"
+#endif //DG_BENCHMARK
 
 namespace dg
 {
@@ -15,8 +17,11 @@ namespace dg
 * @brief Class for the solution of symmetric matrix equation discretizeable on multiple grids
 *
 * We use conjugate gradien (CG) at each stage and refine the grids in the first two dimensions (2d / x and y) 
+* @snippet elliptic2d_b.cu multigrid
 * @copydoc hide_geometry_matrix_container
 * @ingroup multigrid
+* @sa Extrapolation  to generate an initial guess
+* 
 */
 template< class Geometry, class Matrix, class container> 
 struct MultigridCG2d
@@ -94,7 +99,7 @@ struct MultigridCG2d
 
             //
             // debug print
-            std::cout << "pass: " << i << ", stage: " << u << ", max iter: " << m_schemeLayout[i].m_niter << ", iter: " << number[i] << std::endl;
+            //std::cout << "pass: " << i << ", stage: " << u << ", max iter: " << m_schemeLayout[i].m_niter << ", iter: " << number[i] << std::endl;
 
             if (m_schemeLayout[i].m_step > 0)
             {
@@ -106,7 +111,7 @@ struct MultigridCG2d
                 // transfer residual to the rhs of the coarser grid
                 dg::blas2::symv(interT_[u], m_r[u], b_[w]);
                 //dg::blas2::symv(project_[u], x_[u], x_[w]);
-                std::cout << "zeroed " << w << ", ";
+                //std::cout << "zeroed " << w << ", ";
                 dg::blas1::scal(x_[w], 0.0);
             }
             else if (m_schemeLayout[i].m_step < 0)
@@ -138,6 +143,7 @@ struct MultigridCG2d
     * @param b The right hand side (will be multiplied by weights)
     * @param eps the accuracy: iteration stops if \f$ ||b - Ax|| < \epsilon( ||b|| + 1) \f$ 
     * @return the number of iterations in each of the stages
+     * @note If the Macro DG_BENCHMARK is defined this function will write timings to std::cout
     */
     template<class SymmetricOp>
     std::vector<unsigned> direct_solve( std::vector<SymmetricOp>& op, container&  x, const container& b, double eps)
@@ -150,28 +156,38 @@ struct MultigridCG2d
         for( unsigned u=0; u<stages_-1; u++)
             dg::blas2::gemv( interT_[u], m_r[u], m_r[u+1]);
         std::vector<unsigned> number(stages_);
+#ifdef DG_BENCHMARK
         Timer t;
+#endif //DG_BENCHMARK
         
         dg::blas1::scal( x_[stages_-1], 0.0);
         //now solve residual equations
 		for( unsigned u=stages_-1; u>0; u--)
         {
+#ifdef DG_BENCHMARK
             t.tic();
+#endif //DG_BENCHMARK
             cg_[u].set_max(grids_[u].get().size());
             number[u] = cg_[u]( op[u], x_[u], m_r[u], op[u].precond(), op[u].inv_weights(), eps/2, 1.);
             dg::blas2::symv( inter_[u-1], x_[u], x_[u-1]);
+#ifdef DG_BENCHMARK
             t.toc();
             std::cout << "stage: " << u << ", iter: " << number[u] << ", took "<<t.diff()<<"s\n";
+#endif //DG_BENCHMARK
 
         }
+#ifdef DG_BENCHMARK
         t.tic();
+#endif //DG_BENCHMARK
 
         //update initial guess
         dg::blas1::axpby( 1., x_[0], 1., x);
         cg_[0].set_max(grids_[0].get().size());
         number[0] = cg_[0]( op[0], x, b_[0], op[0].precond(), op[0].inv_weights(), eps);
+#ifdef DG_BENCHMARK
         t.toc();
         std::cout << "stage: " << 0 << ", iter: " << number[0] << ", took "<<t.diff()<<"s\n";
+#endif //DG_BENCHMARK
         
         return number;
     }
@@ -250,7 +266,7 @@ private:
         m_schemeLayout.back().m_niter = x_[0].size();
         m_schemeLayout.back().m_step = 0;
 
-        PrintScheme();
+        //PrintScheme();
 
         // checks:
         // (0) the last entry should be the stage before the original grid
diff --git a/inc/dg/multistep.h b/inc/dg/multistep.h
index ad5e28c12..fedf16a9d 100644
--- a/inc/dg/multistep.h
+++ b/inc/dg/multistep.h
@@ -209,7 +209,9 @@ struct MatrixTraits< detail::Implicit<M, V> >
 * Uses blas1::axpby routines to integrate one step
 * and only one right-hand-side evaluation per step. 
 * Uses a conjugate gradient method for the implicit operator  
-@note To our experience the implicit treatment of diffusive or hyperdiffusive 
+@snippet multistep_t.cu function
+@snippet multistep_t.cu doxygen
+@note In our experience the implicit treatment of diffusive or hyperdiffusive 
 terms can significantly reduce the required number of time steps. This
 far outweighs the increased computational cost of the additional matrix inversions.
 * @ingroup time
@@ -354,6 +356,8 @@ with rational coefficients
 We solve the implicit substeps by a conjugate gradient method, which works as long 
 as the implicit part remains symmetric and linear. 
 
+@snippet multistep_t.cu function
+@snippet multistep_t.cu doxygen
 @note To our experience the implicit treatment of diffusive or hyperdiffusive 
 terms can significantly reduce the required number of time steps. This
 far outweighs the increased computational cost of the additional matrix inversions.
diff --git a/inc/dg/multistep_t.cu b/inc/dg/multistep_t.cu
index e14575309..28ba2bd25 100644
--- a/inc/dg/multistep_t.cu
+++ b/inc/dg/multistep_t.cu
@@ -3,59 +3,48 @@
 #include "multistep.h"
 #include "elliptic.h"
 
-template < class Matrix, class container = thrust::device_vector<double> >
-struct RHS
-{
-    typedef container Vector;
-    RHS( const dg::Grid2d& g, double D): D_(D), laplaceM(g, dg::normed)
-    { }
-    void operator()( const std::vector<container>& y, std::vector<container>& yp)
-    {
-        dg::blas1::axpby( 0., y, 0., yp);
-    }
-  private:
-    double D_;
-    dg::Elliptic<dg::CartesianGrid2d, Matrix, container> laplaceM;
-};
+//![function]
+template<class Vector>
+void zero(const std::vector<Vector>& y, std::vector<Vector>& yp){ dg::blas1::scal(yp,0.);}
 
 template< class Matrix, class container>
 struct Diffusion
 {
-    Diffusion( const dg::Grid2d& g, double nu): nu_(nu),
-        w2d( dg::create::weights(g)), 
-        v2d( dg::create::inv_weights(g)),
-        LaplacianM( g, dg::normed) 
+    Diffusion( const dg::Grid2d& g, double nu): m_nu(nu),
+        m_w2d( dg::create::weights(g)), 
+        m_v2d( dg::create::inv_weights(g)),
+        m_LaplacianM( g, dg::normed) 
         { }
 
     void operator()( const std::vector<container>& x, std::vector<container>& y)
     {
         for(unsigned i=0; i<x.size(); i++)
         {
-            dg::blas2::gemv( LaplacianM, x[i], y[i]);
+            dg::blas2::gemv( m_LaplacianM, x[i], y[i]);
         }
-        dg::blas1::axpby( 0.,y, -nu_, y);
+        dg::blas1::scal( y, -m_nu);
     }
-    const container& inv_weights(){return v2d;}
-    const container& weights(){return w2d;}
-    const container& precond(){return v2d;}
+    const container& inv_weights(){return m_v2d;}
+    const container& weights(){return m_w2d;}
+    const container& precond(){return m_v2d;}
   private:
-    double nu_;
-    const container w2d, v2d;
-    dg::Elliptic<dg::CartesianGrid2d, Matrix, container> LaplacianM;
+    double m_nu;
+    const container m_w2d, m_v2d;
+    dg::Elliptic<dg::CartesianGrid2d, Matrix, container> m_LaplacianM;
 };
+double sine( double x, double y) {return sin(x)*sin(y);}
+const double T = 1.0;
+const double nu = 0.01;
+double sol( double x, double y) {return exp( -2.*nu*T)*sine(x, y);}
+//![function]
 
 
 const unsigned n = 3;
 const double lx = 2.*M_PI;
 const double ly = 2.*M_PI;
 
-const unsigned k = 3;
-const double nu = 0.01;
-const double T = 1.0;
 //const unsigned NT = (unsigned)(nu*T*n*n*N*N/0.01/lx/lx);
 
-double sine( double x, double y) {return sin(x)*sin(y);}
-double sol( double x, double y) {return exp( -2.*nu*T)*sine(x, y);}
 
 
 int main()
@@ -67,28 +56,24 @@ int main()
     NT = (unsigned)(T/dt);
 
     std::cout << "Test Karniadakis scheme on diffusion equation\n";
-    std::cout << "RK order K:               "<< k <<std::endl;
     std::cout << "Number of gridpoints:     "<<Nx*Ny<<std::endl;
     std::cout << "# of timesteps:           "<<NT<<std::endl;
 
+    //![doxygen]
     dg::Grid2d grid( 0, lx, 0, ly, n, Nx, Ny, dg::PER, dg::PER);
-    dg::DVec w2d = dg::create::weights( grid);
-
     std::vector<dg::DVec> y0(2, dg::evaluate( sine, grid)), y1(y0);
-
-    RHS<dg::DMatrix, dg::DVec> rhs( grid, nu);
     Diffusion<dg::DMatrix, dg::DVec> diffusion( grid, nu);
-    dg::Karniadakis< std::vector<dg::DVec> > tvb( y0, y0[0].size(), eps);
-    tvb.init( rhs, diffusion, y0, dt);
+    dg::Karniadakis< std::vector<dg::DVec> > karniadakis( y0, y0[0].size(), eps);
+    karniadakis.init( zero<dg::DVec>, diffusion, y0, dt);
     dg::SIRK< std::vector<dg::DVec> > sirk( y0, y0[0].size(), eps);
-
-    //thrust::swap(y0, y1);
     for( unsigned i=0; i<NT; i++)
     {
-        tvb( rhs, diffusion, y0);
-        //sirk( rhs, diffusion, y0, y1, dt);
-        y0.swap(y1);
+        karniadakis( zero<dg::DVec>, diffusion, y0);
+        //sirk( explicit, diffusion, y0, y1, dt);
+        //y0.swap(y1);
     }
+    //![doxygen]
+    dg::DVec w2d = dg::create::weights( grid);
     double norm_y0 = dg::blas2::dot( w2d, y0[0]);
     std::cout << "Normalized y0 after "<< NT <<" steps is "<< norm_y0 << std::endl;
     dg::DVec solution = dg::evaluate( sol, grid), error( solution);
diff --git a/inc/dg/poisson.h b/inc/dg/poisson.h
index 9c0132a4d..9a0cee4a2 100644
--- a/inc/dg/poisson.h
+++ b/inc/dg/poisson.h
@@ -21,7 +21,8 @@ namespace dg
 /**
  * @brief Poisson bracket scheme
  *
- * Equal to the Arakawa class except for the possitility to use mixed boundary conditions
+ * Has the possitility to use mixed boundary conditions
+ * @snippet poisson_t.cu doxygen
  * @ingroup arakawa
  * @copydoc hide_geometry_matrix_container
  */
diff --git a/inc/dg/poisson_t.cu b/inc/dg/poisson_t.cu
index 1b0136dbd..868f12d8d 100644
--- a/inc/dg/poisson_t.cu
+++ b/inc/dg/poisson_t.cu
@@ -47,19 +47,21 @@ int main()
     unsigned n, Nx, Ny;
     std::cout << "Type n, Nx and Ny! \n";
     std::cin >> n >> Nx >> Ny;
-    dg::Grid2d grid( 0, lx, 0, ly, n, Nx, Ny);
-    dg::DVec w2d = dg::create::weights( grid);
     std::cout << "Computing on the Grid " <<n<<" x "<<Nx<<" x "<<Ny <<std::endl;
-    std::cout <<std::fixed<< std::setprecision(2)<<std::endl;
-    dg::DVec lhs = dg::evaluate( left, grid), jac(lhs);
-    dg::DVec rhs = dg::evaluate( right, grid);
-    const dg::DVec sol = dg::evaluate ( jacobian, grid);
-    const dg::DVec variation = dg::evaluate ( variationRHS, grid);
-    dg::DVec eins = dg::evaluate( dg::one, grid);
+    //![doxygen]
+    const dg::CartesianGrid2d grid( 0, lx, 0, ly, n, Nx, Ny);
+    const dg::DVec lhs = dg::evaluate( left, grid);
+    const dg::DVec rhs = dg::evaluate( right, grid);
+    dg::DVec jac(lhs);
 
-    dg::Poisson<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> poisson( grid, bcxlhs, bcylhs,bcxrhs, bcyrhs );
+    dg::Poisson<dg::aGeometry2d, dg::DMatrix, dg::DVec> poisson( grid, bcxlhs, bcylhs,bcxrhs, bcyrhs );
     poisson( lhs, rhs, jac);
+    //![doxygen]
 
+    const dg::DVec w2d = dg::create::weights( grid);
+    const dg::DVec eins = dg::evaluate( dg::one, grid);
+    const dg::DVec sol = dg::evaluate ( jacobian, grid);
+    const dg::DVec variation = dg::evaluate ( variationRHS, grid);
     std::cout << std::scientific;
     std::cout << "Mean     Jacobian is "<<dg::blas2::dot( eins, w2d, jac)<<"\n";
     std::cout << "Mean rhs*Jacobian is "<<dg::blas2::dot( rhs,  w2d, jac)<<"\n";
diff --git a/inc/dg/runge_kutta.h b/inc/dg/runge_kutta.h
index 1bdc1f095..04c00bfcc 100644
--- a/inc/dg/runge_kutta.h
+++ b/inc/dg/runge_kutta.h
@@ -401,7 +401,8 @@ void RK<k, container>::operator()( Functor& f, const container& u0, container& u
     k_j = f\left( u^n + \Delta t \sum_{l=1}^j a_{jl} k_l\right)
  \end{align}
 \f]
-*
+@snippet runge_kutta2d_t.cu function
+@snippet runge_kutta2d_t.cu doxygen
 * @ingroup time 
 *
 * Uses only dg::blas1::axpby() routines to integrate one step.
diff --git a/inc/dg/runge_kutta2d_t.cu b/inc/dg/runge_kutta2d_t.cu
index 9face0fe0..d2746e216 100644
--- a/inc/dg/runge_kutta2d_t.cu
+++ b/inc/dg/runge_kutta2d_t.cu
@@ -15,62 +15,44 @@ const unsigned Ny = 20;
 const double lx = 2.*M_PI;
 const double ly = 2.*M_PI;
 
-const unsigned s = 17;
-const double T = 1.;
 //const unsigned NT = 2*T/0.01/lx*(double)Nx; //respect courant condition
-unsigned NT = 200;
-
 
+//![function]
+const unsigned NT = 20;
+const unsigned s = 17;
+const double T = 1.;
+template<class Vector>
+void function(const std::vector<Vector>& y, std::vector<Vector>& yp){ yp = y;}
 double initial( double x, double y) { return sin(x)*sin(y); }
+//![function]
+
 double function( double x, double y){ return sin(y); }
 //double result( double x, double y)  { return initial( x-cos(y)*T, y); }
 double arak   ( double x, double y) { return -cos(y)*sin(y)*cos(x); }
 double result( double x, double y)  { return initial(x,y)*exp(T);}
 
-template< class Vector_Type>
-struct RHS
-{
-    typedef std::vector<Vector_Type> Vector;
-    RHS(const dg::Grid2d& grid): arakawa( grid), phi( evaluate( function, grid)),
-                                     temp(phi)
-    { }
-    void operator()(const Vector& y, Vector& yp)
-    {
-        yp = y;
-        //for( unsigned i=0; i<y.size(); i++)
-        //{
-        //    dg::blas1::axpby( 1., y[i], 0, temp);
-        //    arakawa( phi, temp, yp[i]);
-        //}
-    }
-  private:
-    dg::ArakawaX<dg::CartesianGrid2d, dg::DMatrix, Vector_Type> arakawa;
-    Vector_Type phi, temp;
-};
 
 int main()
 {
-    std::cout << "Type NT!\n";
-    std::cin >> NT;
-    const double dt = T/(double)NT;
-    dg::Grid2d grid( 0, lx, 0, ly, n, Nx, Ny, dg::PER, dg::PER);
-    dg::DVec w2d = dg::create::weights( grid);
     //Also test std::vector<DVec> functionality
     std::cout << "# of 2d cells                     " << Nx*Ny <<std::endl;
     std::cout << "# of Legendre nodes per dimension "<< n <<std::endl;
     std::cout << "# of timesteps                    "<< NT <<std::endl;
+    //![doxygen]
+    dg::Grid2d grid( 0, lx, 0, ly, n, Nx, Ny, dg::PER, dg::PER);
     dg::DVec init = dg::evaluate ( initial, grid );
-    const dg::DVec solution = dg::evaluate ( result, grid);
     std::vector<dg::DVec> y0( 2, init), y1(y0);
-
-    RHS<dg::DVec> rhs( grid);
+    const double dt = T/(double)NT;
 
     dg::RK_classic<s, std::vector<dg::DVec> >  rk( y0);
     for( unsigned i=0; i<NT; i++)
     {
-        rk( rhs, y0, y1, dt);
+        rk( function<dg::DVec>, y0, y1, dt);
         y0.swap( y1);
     }
+    //![doxygen]
+    dg::DVec w2d = dg::create::weights( grid);
+    const dg::DVec solution = dg::evaluate ( result, grid);
 
     dg::blas1::axpby( 1., solution, -1., y0[0]);
     std::cout << std::scientific;
-- 
GitLab


From a45ba81d9faf323160c2f8ae44f7a215ccd9a05e Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sat, 21 Oct 2017 17:38:23 +0200
Subject: [PATCH 378/453] updated multistep_t.cu

---
 inc/dg/multistep_t.cu | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/inc/dg/multistep_t.cu b/inc/dg/multistep_t.cu
index 28ba2bd25..a84bbe899 100644
--- a/inc/dg/multistep_t.cu
+++ b/inc/dg/multistep_t.cu
@@ -69,8 +69,7 @@ int main()
     for( unsigned i=0; i<NT; i++)
     {
         karniadakis( zero<dg::DVec>, diffusion, y0);
-        //sirk( explicit, diffusion, y0, y1, dt);
-        //y0.swap(y1);
+        //sirk( explicit, diffusion, y0, y1, dt); y0.swap(y1);
     }
     //![doxygen]
     dg::DVec w2d = dg::create::weights( grid);
@@ -81,7 +80,7 @@ int main()
     dg::blas1::axpby( -1., y0[0], 1., error);
     std::cout << "Normalized solution is "<<  norm_sol<< std::endl;
     double norm_error = dg::blas2::dot( w2d, error);
-    std::cout << "Relative error is      "<< sqrt( norm_error/norm_sol)<<" (0.0020084 Karniadakis) (0.000148647 SIRK)\n";
+    std::cout << "Relative error is      "<< sqrt( norm_error/norm_sol)<<" (0.000149144 Karniadakis) (0.000148647 SIRK)\n";
     //n = 1 -> p = 1 (Sprung in laplace macht n=1 eine Ordng schlechter) 
     //n = 2 -> p = 2
     //n = 3 -> p = 3
-- 
GitLab


From c43575b8d704b165878e6c3f78fceb2b10d9539c Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sat, 21 Oct 2017 18:53:38 +0200
Subject: [PATCH 379/453] maded code example for blas2::symv

---
 inc/dg/Doxyfile              |   3 +-
 inc/dg/backend/derivatives.h |   1 +
 inc/dg/backend/exceptions.h  |  18 +++---
 inc/dg/blas2.h               |   3 +
 inc/dg/code_snippets.dox     |  15 +++++
 inc/dg/elliptic.h            |   4 +-
 inc/dg/elliptic2d_b.cu       |   3 +-
 inc/dg/helmholtz.h           |  22 +++----
 inc/dg/helmholtz_t.cu        |  23 --------
 inc/dg/helmholtzg2_b.cu      | 109 ++++++++++-------------------------
 inc/dg/helmholtzg2_t.cu      |  28 +--------
 inc/dg/multistep.h           |  13 +++--
 12 files changed, 88 insertions(+), 154 deletions(-)

diff --git a/inc/dg/Doxyfile b/inc/dg/Doxyfile
index 35ee4cd03..f9f42adcd 100644
--- a/inc/dg/Doxyfile
+++ b/inc/dg/Doxyfile
@@ -866,7 +866,8 @@ EXCLUDE_SYMBOLS        = hide_*
 
 EXAMPLE_PATH           = cg2d_t.cu \
                          arakawa_t.cu poisson_t.cu \
-                         elliptic2d_b.cu runge_kutta2d_t.cu multistep_t.cu
+                         elliptic2d_b.cu runge_kutta2d_t.cu multistep_t.cu \
+                         helmholtzg2_b.cu
 
 
 
diff --git a/inc/dg/backend/derivatives.h b/inc/dg/backend/derivatives.h
index 377ae0e3b..733f49563 100644
--- a/inc/dg/backend/derivatives.h
+++ b/inc/dg/backend/derivatives.h
@@ -45,6 +45,7 @@ EllSparseBlockMat<double> dx( const aTopology2d& g, bc bcx, direction dir = cent
  * @param dir The direction of the first derivative
  *
  * @return A host matrix
+ * @copydoc hide_code_blas2_symv
  */
 EllSparseBlockMat<double> dx( const aTopology2d& g, direction dir = centered) { return dx( g, g.bcx(), dir);}
 
diff --git a/inc/dg/backend/exceptions.h b/inc/dg/backend/exceptions.h
index b835f4f4a..1d0fc09ff 100644
--- a/inc/dg/backend/exceptions.h
+++ b/inc/dg/backend/exceptions.h
@@ -21,6 +21,10 @@ namespace dg
 
 ///@brief small class holding a stringstream 
 ///@ingroup misc
+///\code
+///try{ throw Error(Message(_ping_)<<"This is error number "<<number);}
+///catch( Error& m) {std::cerr << m.what();}
+///\endcode
 class Message 
 {
   private:
@@ -32,9 +36,9 @@ class Message
     Message(){}
     /*!@brief Initiate message with the file and line it comes from
 
-     * @param file The file in which the exception is thrown (contained in the predefined Macro __FILE__)
-     * @param line The line in which the exception is thrown (contained in the predefined Macro __LINE__)
-     * \note The Macro _ping_ combines __FILE__, __LINE__ in one. 
+     * @param file The file in which the exception is thrown (contained in the predefined Macro \c \__FILE__)
+     * @param line The line in which the exception is thrown (contained in the predefined Macro \c \__LINE__)
+     * \note The Macro \c \_ping_ expands to \c \__FILE__, \c \__LINE__ 
      */
     Message(const char* file, const int line){
         sstream_ << "\n    Message from file **"<<file<<"** in line **" <<line<<"**:\n    ";
@@ -46,7 +50,7 @@ class Message
     Message( std::string m){ sstream_<<m;}
     ~Message(){}
     ///@brief add values to the message stream
-    /// @note you can't use std::endl or std::flush in here
+    /// @note you can't use \c std::endl or \c std::flush in here
     template<class T>
     Message & operator << (const T& value)
     {
@@ -68,7 +72,7 @@ class Message
 /*! @brief class intended for the use in throw statements
  *
  * The objects of this class store a message (that describes the error when thrown)
- * that can then be displayed in a catch block
+ * that can then be displayed in a \c catch block
  * \code
  * try{ throw Error(Message(_ping_)<<"This is error number "<<number);}
  * catch( Error& m) {std::cerr << m.what();}
@@ -108,13 +112,13 @@ struct Fail : public std::exception
     /**
      * @brief Construct from error limit
      *
-     * @param eps error not reached
+     * @param eps accuracy not reached
      */
     Fail( double eps): eps( eps) {}
     /**
      * @brief Return error limit
      *
-     * @return eps
+     * @return eps 
      */
     double epsilon() const { return eps;}
     /**
diff --git a/inc/dg/blas2.h b/inc/dg/blas2.h
index af625f07c..df5bf8395 100644
--- a/inc/dg/blas2.h
+++ b/inc/dg/blas2.h
@@ -37,6 +37,7 @@ namespace blas2{
  * @param x source
  * @param y sink 
  * @note y gets resized properly
+ * @copydoc hide_code_blas2_symv
  */
 template<class Matrix, class AnotherMatrix>
 inline void transfer( const Matrix& x, AnotherMatrix& y)
@@ -103,6 +104,7 @@ inline typename MatrixTraits<DiagonalMatrix>::value_type dot( const DiagonalMatr
  * @param beta A Scalar
  * @param y contains the solution on output (may not alias \p x)
  * @attention \p y may never alias \p x
+ * @copydoc hide_code_blas2_symv
  */
 template< class Matrix, class container>
 inline void symv( typename MatrixTraits<Matrix>::value_type alpha, 
@@ -133,6 +135,7 @@ inline void symv( typename MatrixTraits<Matrix>::value_type alpha,
  * @param y contains the solution on output (may not alias \p x)
  * @attention y may never alias x
  * @note Due to the \c SelfMadeMatrixTag, M cannot be declared const
+ * @copydoc hide_code_blas2_symv
  */
 template< class Matrix, class container, class same_or_another_container>
 inline void symv( Matrix& M, 
diff --git a/inc/dg/code_snippets.dox b/inc/dg/code_snippets.dox
index b702a4418..96cecc0fd 100644
--- a/inc/dg/code_snippets.dox
+++ b/inc/dg/code_snippets.dox
@@ -54,3 +54,18 @@ const dg::MHVec h_x = dg::evaluate( function, g3d);
 double norm = dg::blas2::dot(h_x, w3d, h_x); // norm is now: (exp(4)-exp(0))^3/8
 @endcode
 */
+/** @class hide_code_blas2_symv
+ * This code snippet demonstrates how to derive a function with dG on a device
+@code{.cpp}
+double function(double x, double y){return exp(x)*exp(y);}
+dg::Grid2d g2d( 0, 2, 0, 2, 3, 20, 20);
+dg::DMatrix dx;
+dg::blas2::transfer( dg::create::dx(g2d), dx);
+const dg::DVec  x, y;
+dg::blas1::transfer( dg::evaluate( function, g2d), x);
+y=x;
+dg::blas2::symv(dx, x,y); 
+//or 
+dg::blas2::symv(1., dx, x, 0., y); 
+@endcode
+*/
diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index 9cf6ab64a..52408a1b5 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -40,7 +40,8 @@ namespace dg
  In a time dependent problem the value of \f$\alpha\f$ determines the 
  numerical diffusion, i.e. for too low values numerical oscillations may appear. 
  Also note that a forward discretization has more diffusion than a centered discretization.
-
+ The following code snippet demonstrates the use of \c Elliptic in a multigrid algorithm:
+ * @snippet elliptic2d_b.cu multigrid
  * @copydoc hide_geometry_matrix_container
  * This class has the SelfMadeMatrixTag so it can be used in blas2::symv functions 
  * and thus in a conjugate gradient solver. 
@@ -50,7 +51,6 @@ namespace dg
  * @note the jump term \f$ \alpha J\f$  adds artificial numerical diffusion as discussed above
  * @attention Pay attention to the negative sign 
  * 
- * @snippet elliptic2d_b.cu multigrid
  */
 template <class Geometry, class Matrix, class container>
 class Elliptic
diff --git a/inc/dg/elliptic2d_b.cu b/inc/dg/elliptic2d_b.cu
index a2bebfe60..61832f6fa 100644
--- a/inc/dg/elliptic2d_b.cu
+++ b/inc/dg/elliptic2d_b.cu
@@ -74,10 +74,9 @@ int main()
     const unsigned stages = 3;
 
     dg::MultigridCG2d<dg::aGeometry2d, dg::DMatrix, dg::DVec > multigrid( grid, stages);
-    
     const std::vector<dg::DVec> multi_chi = multigrid.project( chi);
+
     std::vector<dg::Elliptic<dg::aGeometry2d, dg::DMatrix, dg::DVec> > multi_pol( stages);
-    
     for(unsigned u=0; u<stages; u++)
     {
         multi_pol[u].construct( multigrid.grids()[u].get(), dg::not_normed, dg::centered, jfactor); 
diff --git a/inc/dg/helmholtz.h b/inc/dg/helmholtz.h
index 7fa0243a8..29939680d 100644
--- a/inc/dg/helmholtz.h
+++ b/inc/dg/helmholtz.h
@@ -18,9 +18,10 @@ namespace dg{
  *
  * Unnormed discretization of \f[ (\chi+\alpha\Delta) \f]
  * where \f$ \chi\f$ is a function and \f$\alpha\f$ a scalar.
- * Can be used by the Invert class
+ * Can be used by the Invert class. The following example shows how the class can be used to act as a \c Helmholtz2 operator:
+ @snippet helmholtzg2_b.cu doxygen
  * @copydoc hide_geometry_matrix_container
- * @attention The Laplacian in this formula is positive as opposed to the negative sign in the Elliptic operator
+ * @attention The Laplacian in this formula is positive as opposed to the negative sign in the \c Elliptic operator
  */
 template< class Geometry, class Matrix, class container> 
 struct Helmholtz
@@ -28,20 +29,20 @@ struct Helmholtz
     ///@brief empty object ( no memory allocation)
     Helmholtz() {}
     /**
-     * @brief Construct Helmholtz operator
+     * @brief Construct \c Helmholtz operator
      *
      * @param g The grid to use (boundary conditions are taken from there)
      * @param alpha Scalar in the above formula
      * @param dir Direction of the Laplace operator
      * @param jfactor The jfactor used in the Laplace operator (probably 1 is always the best factor but one never knows...)
-     * @note The default value of \f$\chi\f$ is one. Helmholtz is never normed
+     * @note The default value of \f$\chi\f$ is one. \c Helmholtz is never normed
      */
     Helmholtz( const Geometry& g, double alpha = 1., direction dir = dg::forward, double jfactor=1.)
     { 
         construct( g, alpha, dir, jfactor);
     }
     /**
-     * @brief Construct Helmholtz operator
+     * @brief Construct \c Helmholtz operator
      *
      * @param g The grid to use
      * @param bcx boundary condition in x
@@ -144,15 +145,16 @@ struct Helmholtz
  * where \f$ \chi\f$ is a function and \f$\alpha\f$ a scalar.
  * Can be used by the Invert class
  * @copydoc hide_geometry_matrix_container
- * @attention The Laplacian in this formula is positive as opposed to the negative sign in the Elliptic operator
- * @attention It is MUCH better to solve the normal Helmholtz operator twice,
- * consecutively, than solving the Helmholtz2 operator once. 
+ * @attention The Laplacian in this formula is positive as opposed to the negative sign in the \c Elliptic operator
+ * @attention It is MUCH better to solve the normal \c Helmholtz operator twice,
+ * consecutively, than solving the \c Helmholtz2 operator once. The following code snippet shows how to do it:
+ @snippet helmholtzg2_b.cu doxygen
  */
 template< class Geometry, class Matrix, class container> 
 struct Helmholtz2
 {
     /**
-     * @brief Construct Helmholtz operator
+     * @brief Construct \c Helmholtz2 operator
      *
      * @param g The grid to use
      * @param alpha Scalar in the above formula
@@ -167,7 +169,7 @@ struct Helmholtz2
     { 
     }
     /**
-     * @brief Construct Helmholtz operator
+     * @brief Construct \c Helmholtz2 operator
      *
      * @param g The grid to use
      * @param bcx boundary condition in x
diff --git a/inc/dg/helmholtz_t.cu b/inc/dg/helmholtz_t.cu
index bd9b4da12..c896ef75a 100644
--- a/inc/dg/helmholtz_t.cu
+++ b/inc/dg/helmholtz_t.cu
@@ -8,29 +8,6 @@
 
 #include "cg.h"
 
-//template< class container>
-//struct Diffusion
-//{
-//    Diffusion( const dg::Grid2d& g, double nu):
-//        nu_(nu),
-//        w2d( dg::create::weights( g)), v2d( dg::create::inv_weights(g)) { 
-//        dg::Matrix Laplacian_ = dg::create::laplacianM( g, dg::normed); 
-//        cusp::blas::scal( Laplacian_.values, -nu);
-//        Laplacian = Laplacian_;
-//        }
-//    void operator()( const container& x, container& y)
-//    {
-//        //dg::blas1::axpby( 0., x, 0., y);
-//        dg::blas2::gemv( Laplacian, x, y);
-//    }
-//    const container& weights(){return w2d;}
-//    const container& precond(){return v2d;}
-//  private:
-//    double nu_;
-//    const container w2d, v2d;
-//    dg::DMatrix Laplacian;
-//};
-
 const double eps = 1e-4;
 const double alpha = -0.5; 
 double lhs( double x, double y){ return sin(x)*sin(y);}
diff --git a/inc/dg/helmholtzg2_b.cu b/inc/dg/helmholtzg2_b.cu
index 75a1cf898..c66e3d592 100644
--- a/inc/dg/helmholtzg2_b.cu
+++ b/inc/dg/helmholtzg2_b.cu
@@ -12,126 +12,79 @@
 const double eps = 1e-4;
 const double tau=1.0; 
 const double alpha = -0.5*tau;
-double lhs2( double x,double y){ return sin(x);}
-// double lhs1( double x,double y){ return sin(x)*sin(y);}
-double lhs1( double x,double y){ return sin(x);}
-double rhs2( double x,double y){
+double lhs( double x,double y){ return sin(x);}
+double rhs( double x,double y){
     return  (-2.-2.*x+2.*cos(x)+2*x*cos(x)+sin(x)+2*x*sin(x)+x*x*sin(x)
     -2*alpha*sin(x)-2*x*alpha*sin(x)+alpha*alpha*sin(x))/(1.0+x)/alpha;
     
 }
-// double rhs1( double x,double y){ return  (1.-2.*(-0.5*tau))*sin(x)*sin(y);}
-double rhs1( double x,double y){ return  (1.-alpha)*sin(x);}
-// double dx2rhs2( double x,double y){ return (1.0 - 2.*alpha + alpha*alpha)*sin(x);} //// chi=1
-double dx2rhs2( double x,double y){ return (1.+x)*sin(x)-2*alpha*sin(x)+alpha*alpha*(2*cos(x)/(1.+x)/(1.+x)-2*sin(x)/(1.+x)/(1.+x)/(1.+x)+sin(x)/(1.+x));} // chi=x
+// double dxrhs( double x,double y){ return (1.0 - 2.*alpha + alpha*alpha)*sin(x);} //// chi=1
+double dxrhs( double x,double y){ return (1.+x)*sin(x)-2*alpha*sin(x)+alpha*alpha*(2*cos(x)/(1.+x)/(1.+x)-2*sin(x)/(1.+x)/(1.+x)/(1.+x)+sin(x)/(1.+x));} // chi=x
 
 
 int main()
 {
     
-    unsigned n, Nx, Ny, Nz; 
-    std::cout << "Type n, Nx Ny and Nz\n";
-    std::cin >> n>> Nx >> Ny >> Nz;
+    unsigned n, Nx, Ny; 
+    std::cout << "Type n, Nx and Ny\n";
+    std::cin >> n>> Nx >> Ny;
     dg::Grid2d grid2d( 0, 2.*M_PI, 0, 2.*M_PI, n, Nx, Ny,dg::DIR,dg::PER);
     const dg::DVec w2d = dg::create::weights( grid2d);
     const dg::DVec v2d = dg::create::inv_weights( grid2d);
     const dg::DVec one = dg::evaluate( dg::one, grid2d);
-     dg::DVec rho = dg::evaluate( rhs2, grid2d);
-    const dg::DVec rho1 = dg::evaluate( rhs1, grid2d);
-    dg::DVec rholap = dg::evaluate( dx2rhs2, grid2d);
-    const dg::DVec sol = dg::evaluate( lhs2, grid2d);
-    const dg::DVec sol1 = dg::evaluate( lhs1, grid2d);
-    dg::DVec x(rho.size(), 0.), rho_(rho);
-
+    const dg::DVec sol = dg::evaluate( lhs, grid2d);
+    dg::DVec x(sol.size(), 0.);
     const dg::DVec chi = dg::evaluate( dg::LinearX(1.0,1.0), grid2d);
     
-    dg::Helmholtz< dg::CartesianGrid2d, dg::DMatrix, dg::DVec > gamma1inv(grid2d,grid2d.bcx(),grid2d.bcy(), alpha ,dg::centered, 1);
-    dg::Helmholtz2< dg::CartesianGrid2d, dg::DMatrix, dg::DVec > gamma2barinv(grid2d,grid2d.bcx(),grid2d.bcy(), alpha,dg::centered, 1);
-    dg::Elliptic< dg::CartesianGrid2d, dg::DMatrix, dg::DVec > gamma2tilde(grid2d,grid2d.bcx(), grid2d.bcy(), dg::normed, dg::centered);
+    dg::Helmholtz2< dg::CartesianGrid2d, dg::DMatrix, dg::DVec > gamma2barinv(grid2d, alpha,dg::centered);
+    dg::Elliptic< dg::CartesianGrid2d, dg::DMatrix, dg::DVec > gamma2tilde(grid2d, dg::normed, dg::centered);
     gamma2barinv.set_chi(chi); 
-//     gamma2barinv.set_chi(one); 
 
-    dg::DVec x_(rho.size(), 0.);
-    dg::Invert<dg::DVec> invertg2( x_, grid2d.size(), eps);
-    dg::Invert<dg::DVec> invertg1( x_, grid2d.size(), eps);
+    dg::Invert<dg::DVec> invertg2( x, grid2d.size(), eps);
+    dg::Invert<dg::DVec> invertg1( x, grid2d.size(), eps);
     
+    dg::DVec rho = dg::evaluate( rhs, grid2d);
+    dg::DVec rholap = dg::evaluate( dxrhs, grid2d);
     dg::blas2::gemv(gamma2tilde,rho,rholap); //lambda = - nabla_perp^2 phi
     dg::blas1::scal(rholap,alpha); // lambda = 0.5*tau_i*nabla_perp^2 phi
     
-//test gamma2    
+    //test gamma2    
     dg::Timer t;
     t.tic();
-    unsigned number = invertg2( gamma2barinv, x_, rholap);
+    unsigned number = invertg2( gamma2barinv, x, rholap);
             if(  number == invertg2.get_max())
             throw dg::Fail( eps);
     t.toc();
 
     //Evaluation
-    dg::blas1::axpby( 1., sol, -1., x_);
+    dg::blas1::axpby( 1., sol, -1., x);
 
     std::cout << "number of iterations:  "<<number<<std::endl;
-    std::cout << "abs error " << sqrt( dg::blas2::dot( w2d, x_))<<std::endl;
-    std::cout << "rel error " << sqrt( dg::blas2::dot( w2d, x_)/ dg::blas2::dot( w2d, sol))<<std::endl;
+    std::cout << "abs error " << sqrt( dg::blas2::dot( w2d, x))<<std::endl;
+    std::cout << "rel error " << sqrt( dg::blas2::dot( w2d, x)/ dg::blas2::dot( w2d, sol))<<std::endl;
     std::cout << "took  " << t.diff()<<"s"<<std::endl;
-    t.tic();
-    dg::blas1::scal(x_,0.); //x_=0
-
-    number = invertg1( gamma1inv, x_, rho1);
-            if(  number == invertg1.get_max())
-            throw dg::Fail( eps);
-    t.toc();
-
-//test gamma 1
-    //Evaluation
-    dg::blas1::axpby( 1., sol1, -1., x_);
-
-    std::cout << "number of iterations:  "<<number<<std::endl;
-    std::cout << "abs error " << sqrt( dg::blas2::dot( w2d, x_))<<std::endl;
-    std::cout << "rel error " << sqrt( dg::blas2::dot( w2d, x_)/ dg::blas2::dot( w2d, sol1))<<std::endl;
-        std::cout << "took  " << t.diff()<<"s"<<std::endl;
-
-
-    
-/*    
-    std::cout << "Test 3d cylincdrical norm:\n";
-    dg::Grid3d g3d( R_0, R_0+lx, 0, ly, 0,lz, n, Nx, Ny,Nz, bcx, dg::PER, dg::PER, dg::cylindrical);
-    dg::DVec fct_ = dg::evaluate(fct, g3d );
-    dg::DVec laplace_fct_ = dg::evaluate( laplace_fct, g3d);
-    dg::DVec helmholtz_fct_ = dg::evaluate( helmholtz_fct, g3d);
-    dg::DVec temp_(fct_);
-    dg::Elliptic< dg::DMatrix, dg::DVec, dg::DVec > laplaceM( g3d, dg::normed);
-    dg::Helmholtz< dg::DMatrix, dg::DVec, dg::DVec > helmholtz( g3d, alpha);
-    dg::blas2::symv( laplaceM, fct_, temp_);
-    dg::blas1::axpby( 1., laplace_fct_, -1., temp_);
-    std::cout << "error Laplace " << sqrt( dg::blas2::dot( laplaceM.weights(), temp_))<<" (Note the supraconvergence!)"<<std::endl;
-    dg::blas2::symv( helmholtz, fct_, temp_);
-    dg::blas2::symv( helmholtz.precond(), temp_, temp_);
-    dg::blas1::axpby( 1., helmholtz_fct_, -1, temp_);
-    std::cout << "error " << sqrt( dg::blas2::dot( helmholtz.weights(), temp_))<<" (Note the supraconvergence!)"<<std::endl;*/
 
+    dg::DVec phi(x.size(), 0.);
+    dg::blas1::scal(x,0.); //x=0
+    //![doxygen]
     std::cout << "Alternative test with two Helmholtz operators\n";
+    dg::Helmholtz< dg::CartesianGrid2d, dg::DMatrix, dg::DVec > gamma1inv(grid2d, alpha ,dg::centered);
     gamma1inv.set_chi( chi);
-    dg::DVec phi(x_.size(), 0.);
-    dg::blas1::scal(x_,0.); //x_=0
-    dg::Invert<dg::DVec> invertO( x_, grid2d.size(), eps/100);
-    dg::Invert<dg::DVec> invertOO( x_, grid2d.size(), eps/100);
+    dg::Invert<dg::DVec> invertO(  x, grid2d.size(), eps/100);
+    dg::Invert<dg::DVec> invertOO( x, grid2d.size(), eps/100);
     t.tic();
     unsigned number1 = invertO( gamma1inv, phi, rholap);
     dg::blas1::pointwiseDot( phi, chi, phi);
-    unsigned number2 = invertOO( gamma1inv, x_, phi);
+    unsigned number2 = invertOO( gamma1inv, x, phi);
     t.toc();
     //Evaluation
-    dg::blas1::axpby( 1., sol, -1., x_);
+    dg::blas1::axpby( 1., sol, -1., x);
+    //![doxygen]
 
     std::cout << "number of iterations:  "<<number1<<" and "<<number2<<std::endl;
-    std::cout << "abs error " << sqrt( dg::blas2::dot( w2d, x_))<<std::endl;
-    std::cout << "rel error " << sqrt( dg::blas2::dot( w2d, x_)/ dg::blas2::dot( w2d, sol))<<std::endl;
+    std::cout << "abs error " << sqrt( dg::blas2::dot( w2d, x))<<std::endl;
+    std::cout << "rel error " << sqrt( dg::blas2::dot( w2d, x)/ dg::blas2::dot( w2d, sol))<<std::endl;
     std::cout << "took  " << t.diff()<<"s"<<std::endl;
 
-    //Tests GAMMA2
-
     return 0;
 }
-
-
-
diff --git a/inc/dg/helmholtzg2_t.cu b/inc/dg/helmholtzg2_t.cu
index 8b15ff1e7..ec0f7b123 100644
--- a/inc/dg/helmholtzg2_t.cu
+++ b/inc/dg/helmholtzg2_t.cu
@@ -56,12 +56,11 @@ int main()
     dg::blas2::gemv(lapperp,rho,rholap); //lambda = - nabla_perp^2 phi
     dg::blas1::scal(rholap,alpha); // lambda = 0.5*tau_i*nabla_perp^2 phi
     
-//test gamma2    
+    //test gamma2    
     unsigned number = invert( gamma2inv, x_, rholap);
             if(  number == invert.get_max())
             throw dg::Fail( eps);
 
-    //Evaluation
     dg::blas1::axpby( 1., sol, -1., x_);
 
     std::cout << "number of iterations:  "<<number<<std::endl;
@@ -71,36 +70,13 @@ int main()
     number = invert( gamma1inv, x_, rho1);
             if(  number == invert.get_max())
             throw dg::Fail( eps);
-//test gamma 1
-    //Evaluation
+    //test gamma 1
     dg::blas1::axpby( 1., sol1, -1., x_);
 
     std::cout << "number of iterations:  "<<number<<std::endl;
     std::cout << "abs error " << sqrt( dg::blas2::dot( w2d, x_))<<std::endl;
     std::cout << "rel error " << sqrt( dg::blas2::dot( w2d, x_)/ dg::blas2::dot( w2d, sol1))<<std::endl;
 
-    
-/*    
-    std::cout << "Test 3d cylincdrical norm:\n";
-    dg::Grid3d g3d( R_0, R_0+lx, 0, ly, 0,lz, n, Nx, Ny,Nz, bcx, dg::PER, dg::PER, dg::cylindrical);
-    dg::DVec fct_ = dg::evaluate(fct, g3d );
-    dg::DVec laplace_fct_ = dg::evaluate( laplace_fct, g3d);
-    dg::DVec helmholtz_fct_ = dg::evaluate( helmholtz_fct, g3d);
-    dg::DVec temp_(fct_);
-    dg::Elliptic< dg::DMatrix, dg::DVec > laplaceM( g3d, dg::normed);
-    dg::Helmholtz< dg::DMatrix, dg::DVec > helmholtz( g3d, alpha);
-    dg::blas2::symv( laplaceM, fct_, temp_);
-    dg::blas1::axpby( 1., laplace_fct_, -1., temp_);
-    std::cout << "error Laplace " << sqrt( dg::blas2::dot( laplaceM.weights(), temp_))<<" (Note the supraconvergence!)"<<std::endl;
-    dg::blas2::symv( helmholtz, fct_, temp_);
-    dg::blas2::symv( helmholtz.precond(), temp_, temp_);
-    dg::blas1::axpby( 1., helmholtz_fct_, -1, temp_);
-    std::cout << "error " << sqrt( dg::blas2::dot( helmholtz.weights(), temp_))<<" (Note the supraconvergence!)"<<std::endl;*/
-
-
-
-    //Tests GAMMA2
-
     return 0;
 }
 
diff --git a/inc/dg/multistep.h b/inc/dg/multistep.h
index fedf16a9d..88653fa86 100644
--- a/inc/dg/multistep.h
+++ b/inc/dg/multistep.h
@@ -206,10 +206,11 @@ struct MatrixTraits< detail::Implicit<M, V> >
     \gamma_0 = \frac{11}{6} 
 \f]
 *
-* Uses blas1::axpby routines to integrate one step
-* and only one right-hand-side evaluation per step. 
-* Uses a conjugate gradient method for the implicit operator  
+* Uses only one right-hand-side evaluation per step. 
+* Uses a conjugate gradient method for the implicit operator. 
+The following code example demonstrates how to integrate the 2d diffusion equation with the dg library:
 @snippet multistep_t.cu function
+In the main function:
 @snippet multistep_t.cu doxygen
 @note In our experience the implicit treatment of diffusive or hyperdiffusive 
 terms can significantly reduce the required number of time steps. This
@@ -226,7 +227,7 @@ struct Karniadakis
     *
     * @param copyable container of size which is used in integration. 
     * @param max_iter parameter for cg
-    * @param eps  parameter for cg
+    * @param eps  accuracy parameter for cg
     * A container object must be copy-constructible from copyable.
     */
     Karniadakis( const container& copyable, unsigned max_iter, double eps): u_(3, container(copyable)), f_(3, container(copyable)), pcg( copyable, max_iter), eps_(eps){
@@ -356,7 +357,9 @@ with rational coefficients
 We solve the implicit substeps by a conjugate gradient method, which works as long 
 as the implicit part remains symmetric and linear. 
 
+The following code example demonstrates how to integrate the 2d diffusion equation with the dg library:
 @snippet multistep_t.cu function
+In the main function:
 @snippet multistep_t.cu doxygen
 @note To our experience the implicit treatment of diffusive or hyperdiffusive 
 terms can significantly reduce the required number of time steps. This
@@ -372,7 +375,7 @@ struct SIRK
      *
      * @param copyable container of right size
      * @param max_iter maximum iterations for conjugate gradient
-     * @param eps error for conjugate gradient
+     * @param eps accuracy for conjugate gradient
      */
     SIRK(const container& copyable, unsigned max_iter, double eps): k_(3, copyable), f_(copyable), g_(copyable), rhs( f_), pcg( copyable, max_iter), eps_(eps)
     {
-- 
GitLab


From 9c7d6f23c357c4fe06f0ea61396c266374eefc07 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sat, 21 Oct 2017 19:18:41 +0200
Subject: [PATCH 380/453] documented SelfMadeMatrixTag

---
 inc/dg/backend/matrix_categories.h | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/inc/dg/backend/matrix_categories.h b/inc/dg/backend/matrix_categories.h
index 8adb6a8b1..e15177480 100644
--- a/inc/dg/backend/matrix_categories.h
+++ b/inc/dg/backend/matrix_categories.h
@@ -8,9 +8,8 @@ struct AnyMatrixTag{};
 //normal matrices
 struct CuspMatrixTag: public AnyMatrixTag {};
 
-struct SelfMadeMatrixTag {}; //A selfmade matrix can with any Vector
-
-struct dx_matrixTag: public AnyMatrixTag {}; // Small banded block matrix for derivatives
+/// indicates that the \c Matrix type has the \c void \c symv( const Vector&, Vector& ) member function.
+struct SelfMadeMatrixTag {}; 
 
 struct MPIMatrixTag: public AnyMatrixTag {};
 
-- 
GitLab


From c62bae1fcfca9d3d8e4414841c42d73340f2ca3f Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sun, 22 Oct 2017 16:53:02 +0200
Subject: [PATCH 381/453] update documentation with memops numbers, removed
 some commented out code

---
 doc/related_pages/parallel/parallel.tex       |   2 +-
 inc/dg/arakawa.h                              |   2 +-
 inc/dg/cg.h                                   | 109 ++++++++----------
 inc/dg/elliptic.h                             |   1 +
 inc/dg/geometry/base_geometry.h               |   1 +
 inc/dg/geometry/multiply.h                    |   8 +-
 ...ds_curvilinear_mpit.cu => ds_curv_mpit.cu} |   0
 .../{ds_curvilinear_t.cu => ds_curv_t.cu}     |   0
 inc/geometries/ribeiro_t.cu                   |  34 ------
 inc/geometries/simple_orthogonal_t.cu         |  28 -----
 src/feltor/feltor.cuh                         |  76 ++----------
 11 files changed, 65 insertions(+), 196 deletions(-)
 rename inc/geometries/{ds_curvilinear_mpit.cu => ds_curv_mpit.cu} (100%)
 rename inc/geometries/{ds_curvilinear_t.cu => ds_curv_t.cu} (100%)

diff --git a/doc/related_pages/parallel/parallel.tex b/doc/related_pages/parallel/parallel.tex
index 54ae1a21d..18ea42c68 100644
--- a/doc/related_pages/parallel/parallel.tex
+++ b/doc/related_pages/parallel/parallel.tex
@@ -453,7 +453,7 @@ The essential test programs for the parallel derivative are located in
 \code{
 path/to/feltor/inc/geometries}.
 \code{ ds\_t.cu} and \code{ ds\_mpit.cu} test the cylindrical grid for shared and distributed memory 
-systems. \code{ ds\_curvilinear\_t.cu} and \code{ ds\_curvilinear\_mpit.cu} test the implementation
+systems. \code{ ds\_curv\_t.cu} and \code{ ds\_curv\_mpit.cu} test the implementation
 on a flux-aligned grid.
 The magnetic field in FELTOR is given by
 \begin{align}
diff --git a/inc/dg/arakawa.h b/inc/dg/arakawa.h
index 737e38ce8..6f1c3ea48 100644
--- a/inc/dg/arakawa.h
+++ b/inc/dg/arakawa.h
@@ -45,7 +45,7 @@ struct ArakawaX
     ArakawaX( const Geometry& g, bc bcx, bc bcy);
 
     /**
-     * @brief Compute poisson's bracket
+     * @brief Compute poisson's bracket (25+2 memops)
      *
      * Computes \f[ [f,g] := 1/\sqrt{g_{2d}}\left(\partial_x f\partial_y g - \partial_y f\partial_x g\right) \f]
      * where \f$ g_{2d} = g/g_{zz}\f$ is the two-dimensional volume element of the plane in 2x1 product space. 
diff --git a/inc/dg/cg.h b/inc/dg/cg.h
index d4cf1e683..dbca65b4b 100644
--- a/inc/dg/cg.h
+++ b/inc/dg/cg.h
@@ -37,31 +37,20 @@ class CG
     typedef typename VectorTraits<container>::value_type value_type;//!< value type of the container class
     ///@brief Allocate nothing, 
     CG(){}
-      /**
-       * @brief Reserve memory for the pcg method
-       *
-       * @param copyable A container must be copy-constructible from this
-       * @param max_iter Maximum number of iterations to be used
-       */
-    CG( const container& copyable, unsigned max_iter):r(copyable), p(r), ap(r), max_iter(max_iter){}
-    /**
-     * @brief Set the maximum number of iterations 
-     *
-     * @param new_max New maximum number
-     */
+    ///@copydoc construct()
+    CG( const container& copyable, unsigned max_iterations):r(copyable), p(r), ap(r), max_iter(max_iterations){}
+    ///@brief Set the maximum number of iterations 
+    ///@param new_max New maximum number
     void set_max( unsigned new_max) {max_iter = new_max;}
-    /**
-     * @brief Get the current maximum number of iterations
-     *
-     * @return the current maximum
-     */
+    ///@brief Get the current maximum number of iterations
+    ///@return the current maximum
     unsigned get_max() const {return max_iter;}
 
     /**
-     * @brief Set internal storage and maximum number of iterations
+     * @brief Allocate memory for the pcg method
      *
-     * @param copyable
-     * @param max_iterations
+     * @param copyable A container must be copy-constructible from this
+     * @param max_iter Maximum number of iterations to be used
      */
     void construct( const container& copyable, unsigned max_iterations) { 
         ap = p = r = copyable;
@@ -84,12 +73,13 @@ class CG
      * @attention This version uses the Preconditioner to compute the norm for the error condition (this safes one scalar product)
      *
      * @return Number of iterations used to achieve desired precision
+     * @note Requires (11 + N) memops/iteration, where N is the number of memops for \c A, and \c P is assumed vector
      */
     template< class Matrix, class Preconditioner >
     unsigned operator()( Matrix& A, container& x, const container& b, Preconditioner& P , value_type eps = 1e-12, value_type nrmb_correction = 1);
     //version of CG where Preconditioner is not trivial
     /**
-     * @brief Solve the system A*x = b using a preconditioned conjugate gradient method
+     * @brief Solve \f$ Ax = b\f$ using a preconditioned conjugate gradient method
      *
      * The iteration stops if \f$ ||Ax||_S < \epsilon( ||b||_S + C) \f$ where \f$C\f$ is 
      * a correction factor to the absolute error and \f$ S \f$ defines a square norm
@@ -105,6 +95,7 @@ class CG
      * @param nrmb_correction Correction factor C for norm of b
      *
      * @return Number of iterations used to achieve desired precision
+     * @note Requires (15 + N) memops/iteration, where N is the number of memops for \c A, and \c P and \c S are assumed vectors
      */
     template< class Matrix, class Preconditioner, class SquareNorm >
     unsigned operator()( Matrix& A, container& x, const container& b, Preconditioner& P, SquareNorm& S, value_type eps = 1e-12, value_type nrmb_correction = 1);
@@ -263,42 +254,41 @@ can be used to get initial guesses based on past solutions
 template<class container>
 struct Extrapolation
 {
-    /*! @brief Set extrapolation type without initializing values
-     * @param type number of vectors to use for extrapolation ( 0<=type<=3)
-     * @attention the update function must be used at least type times before the extrapolate function can be called
+    /*! @brief Set extrapolation number without initializing values
+     * @param number number of vectors to use for extrapolation ( 0<=number<=3)
+     * @attention the update function must be used at least \c number times before the extrapolate function can be called
      */
-    Extrapolation( unsigned type = 2){ set_type(type); }
-    /*! @brief Set extrapolation type and initialize values
-     * @param type number of vectors to use for extrapolation ( 0<=type<=3)
+    Extrapolation( unsigned number = 2){ set_number(number); }
+    /*! @brief Set extrapolation number and initialize values
+     * @param number number of vectors to use for extrapolation ( 0<=number<=3)
      * @param init the vectors are initialized with this value
      */
-    Extrapolation( unsigned type, const container& init) { 
-        set_type(type, init); 
+    Extrapolation( unsigned number, const container& init) { 
+        set_number(number, init); 
     }
     ///@copydoc Extrapolation(unsigned)
-    void set_type( unsigned type)
+    void set_number( unsigned number)
     {
-        m_type = type;
-        m_x.resize( type);
-        assert( m_type <= 3 );
+        m_number = number;
+        m_x.resize( number);
+        assert( m_number <= 3 );
     }
     ///@copydoc Extrapolation(unsigned,const container&)
-    void set_type( unsigned type, const container& init)
+    void set_number( unsigned number, const container& init)
     {
-        m_x.assign( type, init);
-        m_type = type;
-        assert( m_type <= 3 );
+        m_x.assign( number, init);
+        m_number = number;
+        assert( m_number <= 3 );
     }
-    ///read the current extrapolation type
-    unsigned get_type( ) const{return m_type;}
+    ///read the current extrapolation number
+    unsigned get_number( ) const{return m_number;}
 
     /**
-    * @brief Extrapolate values 
-    *
+    * @brief Extrapolate values (\c number +1 memops)
     * @param new_x (write only) contains extrapolated value on output ( may alias the tail)
     */
     void extrapolate( container& new_x) const{
-        switch(m_type)
+        switch(m_number)
         {
             case(0): 
                      break;
@@ -316,29 +306,28 @@ struct Extrapolation
     
     /**
     * @brief move the all values one step back and copy the given vector as current head
-    *
     * @param new_head the new head ( may alias the tail)
     */
     void update( const container& new_head){
-        if( m_type == 0) return;
+        if( m_number == 0) return;
         //push out last value
-        for (unsigned u=m_type-1; u>0; u--)
+        for (unsigned u=m_number-1; u>0; u--)
             m_x[u].swap( m_x[u-1]);
         blas1::copy( new_head, m_x[0]);
     }
 
     /**
      * @brief return the current head 
-     * @return current head (undefined if m_type==0)
+     * @return current head (undefined if number==0)
      */
     const container& head()const{return m_x[0];}
     ///write access to tail value ( the one that will be deleted in the next update
-    container& tail(){return m_x[m_type-1];}
+    container& tail(){return m_x[m_number-1];}
     ///read access to tail value ( the one that will be deleted in the next update
-    const container& tail()const{return m_x[m_type-1];}
+    const container& tail()const{return m_x[m_number-1];}
 
     private:
-    unsigned m_type;
+    unsigned m_number;
     std::vector<container> m_x;
 };
 
@@ -376,23 +365,14 @@ struct Invert
     ///@brief Allocate nothing
     Invert() { multiplyWeights_ = true; nrmb_correction_ = 1.; }
 
-    /**
-     * @brief Constructor
-     *
-     * @param copyable Needed to construct the two previous solutions
-     * @param max_iter maximum iteration in conjugate gradient
-     * @param eps relative error in conjugate gradient
-     * @param extrapolationType number of last values to use for extrapolation of the current guess
-     * @param multiplyWeights if true the rhs shall be multiplied by the weights before cg is applied
-     * @param nrmb_correction Correction factor for norm of b (cf. CG)
-     */
+    ///@copydoc construct()
     Invert(const container& copyable, unsigned max_iter, value_type eps, int extrapolationType = 2, bool multiplyWeights = true, value_type nrmb_correction = 1)
     {
         construct( copyable, max_iter, eps, extrapolationType, multiplyWeights, nrmb_correction);
     }
 
     /**
-     * @brief to be called after default constructor
+     * @brief Allocate memory
      *
      * @param copyable Needed to construct the two previous solutions
      * @param max_iter maximum iteration in conjugate gradient
@@ -403,7 +383,7 @@ struct Invert
      */
     void construct( const container& copyable, unsigned max_iter, value_type eps, int extrapolationType = 2, bool multiplyWeights = true, value_type nrmb_correction = 1.) 
     {
-        m_ex.set_type( extrapolationType);
+        m_ex.set_number( extrapolationType);
         set_size( copyable, max_iter);
         set_accuracy( eps, nrmb_correction);
         multiplyWeights_=multiplyWeights;
@@ -418,7 +398,7 @@ struct Invert
      */
     void set_size( const container& assignable, unsigned max_iterations) {
         cg.construct(assignable, max_iterations);
-        m_ex.set_type( m_ex.get_type(), assignable);
+        m_ex.set_number( m_ex.get_number(), assignable);
     }
 
     /**
@@ -438,7 +418,7 @@ struct Invert
      * @param extrapolationType number of last values to use for next extrapolation of initial guess
      */
     void set_extrapolationType( int extrapolationType) {
-        m_ex.set_type( extrapolationType);
+        m_ex.set_number( extrapolationType);
     }
     /**
      * @brief Set the maximum number of iterations 
@@ -489,7 +469,8 @@ struct Invert
      * @param weights The weights that normalize the symmetric operator
      * @param inv_weights The inverse of the weights that normalize the symmetric operator
      * @param p The preconditioner  
-     * @note If the Macro DG_BENCHMARK is defined this function will write timings to std::cout
+     * @note (15+N)memops per iteration where N is the memops contained in \c op.
+     *   If the Macro DG_BENCHMARK is defined this function will write timings to std::cout
      *
      * @return number of iterations used 
      */
diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index 52408a1b5..e29cf745b 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -174,6 +174,7 @@ class Elliptic
      *
      * @param x left-hand-side
      * @param y result
+     * @note Requires 24 memops if geometry is curvilinear; 20 memops if geometry is orthogonal or chi is set; 16 memops if geometry is Cartesian and chi is not set;
      */
     void symv( const container& x, container& y) 
     {
diff --git a/inc/dg/geometry/base_geometry.h b/inc/dg/geometry/base_geometry.h
index d7190da1e..8a8a5aa60 100644
--- a/inc/dg/geometry/base_geometry.h
+++ b/inc/dg/geometry/base_geometry.h
@@ -205,6 +205,7 @@ struct aProductGeometry3d : public aGeometry3d
 
 /**
  * @brief two-dimensional Grid with Cartesian metric
+
  * @snippet arakawa_t.cu doxygen
  */
 struct CartesianGrid2d: public dg::aGeometry2d
diff --git a/inc/dg/geometry/multiply.h b/inc/dg/geometry/multiply.h
index d31caf562..8dd5fc3e0 100644
--- a/inc/dg/geometry/multiply.h
+++ b/inc/dg/geometry/multiply.h
@@ -123,7 +123,8 @@ void pointwiseDivide( const container& in, const SparseElement<container>& mu, c
 ///@cond
 namespace detail
 {
-//i0 must be the diagonal index, out0 may alias in0 but not in1
+//multiply_add given containers with given tensor indices
+//i0 must be the diagonal index, out0 may alias in0 but not in1 
 template<class container>
 void multiply2d_helper( const SparseTensor<container>& t, const container& in0, const container& in1, container& out0, int i0[2], int i1[2])
 {
@@ -151,6 +152,11 @@ void multiply2d_helper( const SparseTensor<container>& t, const container& in0,
  * @param out0 (output) first component  (restricted)
  * @param out1 (output) second component (may alias in1)
  * @attention aliasing only allowed between out1 and in1
+ * @note Currently requires:
+         - 10 memops if all values in t are set; 
+         - 6  memops if t is diagonal; 
+         - 4  memops if t is empty
+         - (-1 memop if alias is used)
  */
 template<class container>
 void multiply2d( const SparseTensor<container>& t, const container& in0, const container& in1, container& out0, container& out1)
diff --git a/inc/geometries/ds_curvilinear_mpit.cu b/inc/geometries/ds_curv_mpit.cu
similarity index 100%
rename from inc/geometries/ds_curvilinear_mpit.cu
rename to inc/geometries/ds_curv_mpit.cu
diff --git a/inc/geometries/ds_curvilinear_t.cu b/inc/geometries/ds_curv_t.cu
similarity index 100%
rename from inc/geometries/ds_curvilinear_t.cu
rename to inc/geometries/ds_curv_t.cu
diff --git a/inc/geometries/ribeiro_t.cu b/inc/geometries/ribeiro_t.cu
index add55358f..ba61fafdb 100644
--- a/inc/geometries/ribeiro_t.cu
+++ b/inc/geometries/ribeiro_t.cu
@@ -174,39 +174,5 @@ int main( int argc, char* argv[])
     std::cout << "relative difference in volume is "<<fabs(volumeRZP - volume)/volume<<std::endl;
     std::cout << "Note that the error might also come from the volume in RZP!\n"; //since integration of jacobian is fairly good probably
 
-    /////////////////////////TEST 3d grid//////////////////////////////////////
-    //std::cout << "Start DS test!"<<std::endl;
-    //const dg::HVec vol3d = dg::create::volume( g3d);
-    //t.tic();
-    //DFA fieldaligned( dg::ribeiro::Field( gp, g3d.x(), g3d.f_x()), g3d, gp.rk4eps, dg::NoLimiter()); 
-
-    //dg::DS<DFA, dg::DMatrix, dg::HVec> ds( fieldaligned, dg::ribeiro::Field(gp, g3d.x(), g3d.f_x()), dg::normed, dg::centered);
-
-    //t.toc();
-    //std::cout << "Construction took "<<t.diff()<<"s\n";
-    //dg::HVec B = dg::pullback( dg::geo::InvB(gp), g3d), divB(B);
-    //dg::HVec lnB = dg::pullback( dg::geo::LnB(gp), g3d), gradB(B);
-    //dg::HVec gradLnB = dg::pullback( dg::geo::GradLnB(gp), g3d);
-    //dg::blas1::pointwiseDivide( ones3d, B, B);
-    //dg::HVec function = dg::pullback( dg::geo::FuncNeu(gp), g3d), derivative(function);
-    //ds( function, derivative);
-
-    //ds.centeredT( B, divB);
-    //double norm =  sqrt( dg::blas2::dot(divB, vol3d, divB));
-    //std::cout << "Divergence of B is "<<norm<<"\n";
-
-    //ds.centered( lnB, gradB);
-    //std::cout << "num. norm of gradLnB is "<<sqrt( dg::blas2::dot( gradB,vol3d, gradB))<<"\n";
-    //norm = sqrt( dg::blas2::dot( gradLnB, vol3d, gradLnB) );
-    //std::cout << "ana. norm of gradLnB is "<<norm<<"\n";
-    //dg::blas1::axpby( 1., gradB, -1., gradLnB, gradLnB);
-    //X = divB;
-    //err = nc_put_var_double( ncid, divBID, periodify(X, g2d_periodic).data());
-    //double norm2 = sqrt(dg::blas2::dot(gradLnB, vol3d,gradLnB));
-    //std::cout << "rel. error of lnB is    "<<norm2/norm<<"\n";
-    //err = nc_close( ncid);
-
-
-
     return 0;
 }
diff --git a/inc/geometries/simple_orthogonal_t.cu b/inc/geometries/simple_orthogonal_t.cu
index 013298512..9600988b7 100644
--- a/inc/geometries/simple_orthogonal_t.cu
+++ b/inc/geometries/simple_orthogonal_t.cu
@@ -183,37 +183,9 @@ int main( int argc, char* argv[])
     std::cout << "relative difference in volume is "<<fabs(volumeRZP - volume)/volume<<std::endl;
     std::cout << "Note that the error might also come from the volume in RZP!\n"; //since integration of jacobian is fairly good probably
 
-//     /////////////////////////TEST 3d grid//////////////////////////////////////
-//     std::cout << "Start DS test!"<<std::endl;
-//     const dg::HVec vol3d = dg::create::volume( g3d);
-//     t.tic();
-//     DFA fieldaligned( OrthogonalField( gp, g2d, g2d.f2_xy()), g3d, gp.rk4eps, dg::NoLimiter()); 
-// 
-//     dg::DS<DFA, dg::DMatrix, dg::HVec> ds( fieldaligned, OrthogonalField(gp, g2d, g2d.f2_xy()), dg::normed, dg::centered);
-//     t.toc();
-//     std::cout << "Construction took "<<t.diff()<<"s\n";
-//     dg::HVec B = dg::pullback( dg::geo::InvB(gp), g3d), divB(B);
-//     dg::HVec lnB = dg::pullback( dg::geo::LnB(gp), g3d), gradB(B);
-//     dg::HVec gradLnB = dg::pullback( dg::geo::GradLnB(gp), g3d);
-//     dg::blas1::pointwiseDivide( ones3d, B, B);
-//     dg::HVec function = dg::pullback( solovev::FuncNeu(gp), g3d), derivative(function);
-//     ds( function, derivative);
-// 
-//     ds.centeredT( B, divB);
-//     double norm =  sqrt( dg::blas2::dot(divB, vol3d, divB));
-//     std::cout << "Divergence of B is "<<norm<<"\n";
-// 
-//     ds.centered( lnB, gradB);
-//     std::cout << "num. norm of gradLnB is "<<sqrt( dg::blas2::dot( gradB,vol3d, gradB))<<"\n";
-//     norm = sqrt( dg::blas2::dot( gradLnB, vol3d, gradLnB) );
-//     std::cout << "ana. norm of gradLnB is "<<norm<<"\n";
-//     dg::blas1::axpby( 1., gradB, -1., gradLnB, gradLnB);
-     //X = g2d.lapx();
     dg::geo::TokamakMagneticField c=dg::geo::createSolovevField(gp);
      X = dg::pullback(dg::geo::FuncDirNeu(c, psi_0, psi_1, 550, -150, 30., 1), g2d);
      err = nc_put_var_double( ncid, divBID, periodify(X, g2d_periodic).data());
-//     double norm2 = sqrt(dg::blas2::dot(gradLnB, vol3d,gradLnB));
-//     std::cout << "rel. error of lnB is    "<<norm2/norm<<"\n";
     err = nc_close( ncid);
 
 
diff --git a/src/feltor/feltor.cuh b/src/feltor/feltor.cuh
index ec9b32367..28fcde5db 100644
--- a/src/feltor/feltor.cuh
+++ b/src/feltor/feltor.cuh
@@ -31,14 +31,6 @@ template<class Geometry, class IMatrix, class Matrix, class container>
 struct Implicit
 {
 
-    /**
-     * @brief Construct from parameters
-     *
-     * @tparam Grid3d three-dimensional grid class 
-     * @param g The grid
-     * @param p the physics parameters
-     * @param gp the geometry parameters
-     */
     Implicit( const Geometry& g, feltor::Parameters p, dg::geo::solovev::GeomParameters gp, DS& dsN, DS& dsDIR):
         p(p),
         gp(gp),
@@ -52,13 +44,7 @@ struct Implicit
         dg::blas1::transfer( dg::pullback( dg::geo::GaussianDamping(Psip(gp), gp.psipmaxcut, gp.alpha), g), dampgauss_);
     }
 
-    /**
-     * @brief Return implicit terms
-     *
-     * @param x input vector (x[0] := N_e -1, x[1] := N_i-1, x[2] := U_e, x[3] = U_i)
-     * @param y output vector
-     */
-    void operator()( std::vector<container>& x, std::vector<container>& y)
+    void operator()( const std::vector<container>& x, std::vector<container>& y)
     {
         /* x[0] := N_e - 1
            x[1] := N_i - 1
@@ -96,31 +82,11 @@ struct Implicit
         }
     }
 
-    /**
-     * @brief Return the laplacian with dirichlet BC
-     *
-     * @return 
-     */
     dg::Elliptic<Geometry, Matrix, container>& laplacianM() {return LaplacianM_perpDIR;}
 
-    /**
-     * @brief Model function for Inversion
-     *
-     * @return weights for the inversion function in
-     */
     const container& weights(){return LaplacianM_perpDIR.weights();}
-    /**
-     * @brief Model function for Inversion
-     *
-     * @return preconditioner for the inversion function in
-     */
+    const container& inv_weights(){return LaplacianM_perpDIR.inv_weights();}
     const container& precond(){return LaplacianM_perpDIR.precond();}
-    /**
-     * @brief Damping used in the diffusion equations
-     *
-     * @return Vector containing damping 
-     */
-    //const container& damping(){return dampprof_;}
   private:
     const feltor::Parameters p;
     const dg::geo::solovev::GeomParameters gp;
@@ -133,22 +99,9 @@ struct Implicit
 template< class Geometry, class IMatrix, class Matrix, class container >
 struct Explicit
 {
-    /**
-     * @brief Construct from parameters
-     *
-     * @tparam Grid3d three-dimensional grid class 
-     * @param g The grid
-     * @param p the physics parameters
-     * @param gp the geometry parameters
-     */
     Explicit( const Geometry& g, const dg::geo::TokamakMagneticField& mag, feltor::Parameters p, dg::geo::solovev::GeomParameters gp);
 
 
-    /**
-     * @brief Return a ds class for evaluation purposes
-     *
-     * @return 
-     */
     dg::DS<Geometry, IMatrix, Matrix, container>& ds(){return dsN_;}
     dg::DS<Geometry, IMatrix, Matrix, container>& dsDIR(){return dsDIR_;}
 
@@ -168,13 +121,8 @@ struct Explicit
      */
     void initializene( const container& y, container& target);
 
-    /**
-     * @brief Compute explicit rhs of Explicit equations
-     *
-     * @param y y[0] := N_e - 1, y[1] := N_i - 1, y[2] := U_e, y[3] := U_i
-     * @param yp Result
-     */
-    void operator()( std::vector<container>& y, std::vector<container>& yp);
+    ///@param y y[0] := N_e - 1, y[1] := N_i - 1, y[2] := U_e, y[3] := U_i
+    void operator()( const std::vector<container>& y, std::vector<container>& yp);
 
     /**
      * @brief \f[ M := \int_V (n_e-1) dV \f]
@@ -183,12 +131,6 @@ struct Explicit
      * @note call energies() before use
      */
     double mass( ) {return mass_;}
-    /**
-     * @brief Do not use! Not implemented yet!
-     *
-     * @return 0
-     */
-    double mass_diffusion( ) {return diff_;}
     /**
      * @brief 
      \f[
@@ -237,9 +179,9 @@ struct Explicit
     void vecdotnablaN(const container& x, const container& y, container& z, container& target);
     void vecdotnablaDIR(const container& x, const container& y, container& z, container& target);
     //extrapolates and solves for phi[1], then adds square velocity ( omega)
-    container& compute_psi( container& potential);
+    container& compute_psi( const container& potential);
     container& polarisation( const std::vector<container>& y); //solves polarisation equation
-    double add_parallel_dynamics( std::vector<container>& y, std::vector<container>& yp);
+    double add_parallel_dynamics( const std::vector<container>& y, std::vector<container>& yp);
 
     container chi, omega, lambda; //!!Attention: chi and omega are helper variables and may be changed at any time and by any method!!
 
@@ -361,7 +303,7 @@ container& Explicit<Geometry, DS, Matrix, container>::polarisation( const std::v
 }
 
 template< class Geometry, class DS, class Matrix, class container>
-container& Explicit<Geometry, DS, Matrix,container>::compute_psi( container& potential)
+container& Explicit<Geometry, DS, Matrix,container>::compute_psi( const container& potential)
 {
     invert_invgammaPhi(invgammaDIR,chi,potential);                    //chi  Gamma phi
     poissonN.variationRHS(potential, omega);
@@ -379,7 +321,7 @@ void Explicit<Geometry, DS, Matrix, container>::initializene( const container& s
 
 
 template<class G, class DS, class M, class V>
-double Explicit<G, DS, M, V>::add_parallel_dynamics( std::vector<V>& y, std::vector<V>& yp)
+double Explicit<G, DS, M, V>::add_parallel_dynamics( const std::vector<V>& y, std::vector<V>& yp)
 {
     double z[2]     = {-1.0,1.0};
     double Dpar[4]  = {0.0, 0.0,0.0,0.0};
@@ -498,7 +440,7 @@ double Explicit<G, DS, M, V>::add_parallel_dynamics( std::vector<V>& y, std::vec
 
 
 template<class Geometry, class DS, class Matrix, class container>
-void Explicit<Geometry, DS, Matrix, container>::operator()( std::vector<container>& y, std::vector<container>& yp)
+void Explicit<Geometry, DS, Matrix, container>::operator()( const std::vector<container>& y, std::vector<container>& yp)
 {
     /* y[0] := N_e - 1
        y[1] := N_i - 1
-- 
GitLab


From 98a917fc8bfa0890f6a46691d01ecff175a3fee1 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 23 Oct 2017 15:08:48 +0200
Subject: [PATCH 382/453] debugged evaluate in ds_mpit further and added code
 examples in geometries

---
 inc/geometries/Doxyfile           |  5 ++--
 inc/geometries/curvilinear.h      |  3 ++
 inc/geometries/ds.h               |  1 +
 inc/geometries/ds_mpit.cu         |  2 +-
 inc/geometries/ds_t.cu            | 46 ++++++++++---------------------
 inc/geometries/fieldaligned.h     | 31 +++++++++++----------
 inc/geometries/fluxfunctions.h    |  4 +++
 inc/geometries/hector.h           |  1 +
 inc/geometries/hector_t.cu        |  2 ++
 inc/geometries/magnetic_field.h   |  1 +
 inc/geometries/mpi_fieldaligned.h | 21 ++++++--------
 src/toefl/parameters.h            |  2 +-
 src/toefl/toeflR.cuh              |  1 -
 13 files changed, 56 insertions(+), 64 deletions(-)

diff --git a/inc/geometries/Doxyfile b/inc/geometries/Doxyfile
index 803732e04..f15c0afd6 100644
--- a/inc/geometries/Doxyfile
+++ b/inc/geometries/Doxyfile
@@ -291,7 +291,8 @@ OPTIMIZE_OUTPUT_VHDL   = NO
 # Note that for custom extensions you also need to set FILE_PATTERNS otherwise
 # the files are not read by doxygen.
 
-EXTENSION_MAPPING      =
+EXTENSION_MAPPING      = cuh=C++ \
+                         cu=C++
 
 # If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments
 # according to the Markdown format, which allows for more readable
@@ -855,7 +856,7 @@ EXCLUDE_SYMBOLS        = hide_*
 # that contain example code fragments that are included (see the \include
 # command).
 
-EXAMPLE_PATH           =
+EXAMPLE_PATH           = ds_t.cu hector_t.cu
 
 # If the value of the EXAMPLE_PATH tag contains directories, you can use the
 # EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and
diff --git a/inc/geometries/curvilinear.h b/inc/geometries/curvilinear.h
index a5c260780..f71dac0a2 100644
--- a/inc/geometries/curvilinear.h
+++ b/inc/geometries/curvilinear.h
@@ -74,6 +74,8 @@ void square( const dg::SparseTensor<thrust::host_vector<double> >& jac, const th
 
 /**
  * @brief A two-dimensional grid based on curvilinear coordinates
+ *
+ * @snippet hector_t.cu doxygen
  */
 struct CurvilinearGrid2d : public dg::aGeometry2d
 {
@@ -117,6 +119,7 @@ struct CurvilinearGrid2d : public dg::aGeometry2d
  * @brief A 2x1 curvilinear product space grid
 
  * The base coordinate system is the cylindrical coordinate system R,Z,phi
+ * @snippet hector_t.cu doxygen
  */
 struct CurvilinearProductGrid3d : public dg::aProductGeometry3d
 {
diff --git a/inc/geometries/ds.h b/inc/geometries/ds.h
index 14c6417b7..68f7261af 100644
--- a/inc/geometries/ds.h
+++ b/inc/geometries/ds.h
@@ -27,6 +27,7 @@ namespace geo{
 \f$\nabla_\parallel^\dagger\f$ and 
 \f$\Delta_\parallel=\nabla_\parallel^\dagger\cdot\nabla_\parallel\f$
 in arbitrary coordinates
+@snippet ds_t.cu doxygen
 * @ingroup fieldaligned
 * @tparam ProductGeometry must be either dg::aProductGeometry3d or dg::aProductMPIGeometry3d or any derivative 
 * @tparam IMatrix The type of the interpolation matrix 
diff --git a/inc/geometries/ds_mpit.cu b/inc/geometries/ds_mpit.cu
index e647760be..08bfdd68b 100644
--- a/inc/geometries/ds_mpit.cu
+++ b/inc/geometries/ds_mpit.cu
@@ -64,7 +64,7 @@ int main(int argc, char* argv[])
     function = ds.fieldaligned().evaluate( init0, modulate, Nz/2, 2);
     ds( function, derivative);
     norm = dg::blas2::dot(vol3d, derivative);
-    if(rank==0)std::cout << "Norm Centered Derivative "<<sqrt( norm)<<" (compare with that of ds_mpib)\n";
+    if(rank==0)std::cout << "Norm Centered Derivative "<<sqrt( norm)<<" (compare with that of ds_t)\n";
     MPI_Finalize();
     return 0;
 }
diff --git a/inc/geometries/ds_t.cu b/inc/geometries/ds_t.cu
index 19b666810..1e44a83f5 100644
--- a/inc/geometries/ds_t.cu
+++ b/inc/geometries/ds_t.cu
@@ -15,8 +15,6 @@
 const double R_0 = 10;
 const double I_0 = 20; //q factor at r=1 is I_0/R_0
 const double a  = 1; //small radius  
-dg::geo::TokamakMagneticField mag = dg::geo::createCircularField( R_0, I_0);
-dg::geo::GradLnB gradLnB(mag);
 double func(double R, double Z, double phi)
 {
     double r2 = (R-R_0)*(R-R_0)+Z*Z;
@@ -27,10 +25,6 @@ double deri(double R, double Z, double phi)
     double r2 = (R-R_0)*(R-R_0)+Z*Z; //(grad psi)^2
     return I_0/R/sqrt(I_0*I_0 + r2)* r2*cos(phi);
 }
-double adjoint(double R, double Z, double phi)
-{
-    return -gradLnB(R,Z)*func(R,Z,phi) + deri(R,Z,phi);
-}
 
 int main(int argc, char * argv[])
 {
@@ -43,18 +37,24 @@ int main(int argc, char * argv[])
     unsigned mx, my;
     std::cin >> mx>> my;
     std::cout << "You typed "<<mx<<" "<<my<<std::endl;
-    dg::CylindricalGrid3d g3d( R_0 - 1, R_0+1, -1, 1, 0, 2.*M_PI, n, Nx, Ny, Nz, dg::NEU, dg::NEU, dg::PER);
-    const dg::DVec vol3d = dg::create::volume( g3d);
     std::cout << "Create parallel Derivative!\n";
-    dg::geo::BinaryVectorLvl0 bhat( (dg::geo::BHatR)(mag), (dg::geo::BHatZ)(mag), (dg::geo::BHatP)(mag));
-    dg::geo::Fieldaligned<dg::aProductGeometry3d,dg::IDMatrix,dg::DVec>  dsFA( bhat, g3d, mx, my, true,true, 1e-6, dg::NEU, dg::NEU, dg::geo::NoLimiter());
-    dg::geo::DS<dg::aProductGeometry3d, dg::IDMatrix, dg::DMatrix, dg::DVec> ds( dsFA, dg::not_normed, dg::centered);
 
+    //![doxygen]
+    const dg::CylindricalGrid3d g3d( R_0 - 1, R_0+1, -1, 1, 0, 2.*M_PI, n, Nx, Ny, Nz, dg::NEU, dg::NEU, dg::PER);
+    //create magnetic field
+    const dg::geo::TokamakMagneticField mag = dg::geo::createCircularField( R_0, I_0);
+    const dg::geo::BinaryVectorLvl0 bhat( (dg::geo::BHatR)(mag), (dg::geo::BHatZ)(mag), (dg::geo::BHatP)(mag));
+    //create Fieldaligned object and construct DS from it
+    const dg::geo::Fieldaligned<dg::aProductGeometry3d,dg::IDMatrix,dg::DVec>  dsFA( bhat, g3d, mx, my, true,true, 1e-6, dg::NEU, dg::NEU, dg::geo::NoLimiter());
+    dg::geo::DS<dg::aProductGeometry3d, dg::IDMatrix, dg::DMatrix, dg::DVec> ds( dsFA, dg::not_normed, dg::centered);
     ///##########################################################///
+    //apply to function 
     dg::DVec function = dg::evaluate( func, g3d), derivative(function);
-    const dg::DVec solution = dg::evaluate( deri, g3d);
     ds( function, derivative);
+    //![doxygen]
+    const dg::DVec solution = dg::evaluate( deri, g3d);
     dg::blas1::axpby( 1., solution, -1., derivative);
+    const dg::DVec vol3d = dg::create::volume( g3d);
     double norm = dg::blas2::dot( derivative, vol3d, derivative);
     const double sol = dg::blas2::dot( vol3d, solution);
     std::cout << "Error centered derivative "<< sqrt( norm/sol )<<"\n";
@@ -71,28 +71,10 @@ int main(int argc, char * argv[])
     std::cout << "TEST FIELDALIGNED EVALUATION of a Gaussian\n";
     dg::Gaussian init0(R_0+0.5, 0, 0.2, 0.2, 1);
     dg::GaussianZ modulate(0., M_PI/3., 1);
-    function = ds.fieldaligned().evaluate( init0, modulate, Nz/2, 2);
+    function = dsFA.evaluate( init0, modulate, Nz/2, 2);
     ds( function, derivative);
     norm = dg::blas2::dot(vol3d, derivative);
-    std::cout << "Norm Centered Derivative "<<sqrt( norm)<<" (compare with that of ds_mpib)\n";
-    ///##########################################################///
-    std::cout << "TEST ADJOINT DERIVATIVE! \n";
-    function = dg::evaluate( func, g3d);
-    const dg::DVec adjoint_solution = dg::evaluate( adjoint, g3d);
-    const double adj = dg::blas2::dot( vol3d, adjoint_solution);
-
-    ds.centeredAdj( -1., function, 0., derivative);
-    dg::blas1::axpby( 1., adjoint_solution, -1., derivative);
-    norm = dg::blas2::dot( vol3d, derivative);
-    std::cout << "Error centered derivative "<< sqrt( norm/adj )<<"\n";
-    ds.forwardAdj( -1., function, 0., derivative);
-    dg::blas1::axpby( 1., adjoint_solution, -1., derivative);
-    norm = dg::blas2::dot(vol3d, derivative);
-    std::cout << "Error Forward  Derivative "<<sqrt( norm/adj)<<"\n";
-    ds.backwardAdj( -1., function, 0., derivative);
-    dg::blas1::axpby( 1., adjoint_solution, -1., derivative);
-    norm = dg::blas2::dot(vol3d, derivative);
-    std::cout << "Error Backward Derivative "<<sqrt( norm/adj)<<"\n";
+    std::cout << "Norm Centered Derivative "<<sqrt( norm)<<" (compare with that of ds_mpit)\n";
 
     return 0;
 }
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index a2f387c6f..b8c8e8b45 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -320,6 +320,7 @@ void integrate_all_fieldlines2d( const dg::geo::BinaryVectorLvl0& vec, const dg:
 * @brief Create and manage interpolation matrices from fieldline integration
 *
 * @ingroup fieldaligned
+* @snippet ds_t.cu doxygen
 * @tparam ProductGeometry must be either aProductGeometry3d or aProductMPIGeometry3d or any derivative 
 * @tparam IMatrix The type of the interpolation matrix 
     - dg::IHMatrix, or dg::IDMatrix, dg::MIHMatrix, or dg::MIDMatrix
@@ -426,7 +427,7 @@ struct Fieldaligned
      * get the values in the 3rd dimension. Uses the grid given in the constructor.
      * @tparam BinaryOp Binary Functor
      * @param binary Functor to evaluate
-     * @param p0 The number of the plane to start
+     * @param p0 The index of the plane to start
      *
      * @return Returns an instance of container
      */
@@ -439,16 +440,19 @@ struct Fieldaligned
     /**
      * @brief Evaluate a 2d functor and transform to all planes along the fieldlines
      *
-     * Evaluates the given functor on a 2d plane and then follows fieldlines to
-     * get the values in the 3rd dimension. Uses the grid given in the constructor.
-     * The second functor is used to scale the values along the fieldlines.
-     * The fieldlines are assumed to be periodic.
+     * The algorithm does the equivalent of the following: 
+     *  - Evaluate the given \c BinaryOp on a 2d plane 
+     *  - Apply the plus and minus transformation each \f$ r N_z\f$ times where \f$ N_z\f$ is the number of planes in the global 3d grid and \f$ r\f$ is the number of rounds.  
+     *  - Scale the transformations with \f$ u ( \pm (iN_z + j)h_z) \f$, where \c u is the given \c UnarayOp, \c i is the round index and \c j is the plane index. 
+     *  - Sum all transformations with the same plane index \c j , where the minus transformations get the inverted index \f$ N_z - j\f$.
+     *  - Shift the index by \f$ p_0\f$
+     *  .
      * @tparam BinaryOp Binary Functor
      * @tparam UnaryOp Unary Functor
      * @param binary Functor to evaluate in x-y
      * @param unary Functor to evaluate in z
-     * @param p0 The number of the plane to start
-     * @param rounds The number of rounds to follow a fieldline
+     * @param p0 The index of the plane to start
+     * @param rounds The number of rounds \c r to follow a fieldline; can be zero, then the fieldlines are only followed within the current box ( no periodicity) 
      * @note g is evaluated such that p0 corresponds to z=0, p0+1 corresponds to z=hz, p0-1 to z=-hz, ...
      *
      * @return Returns an instance of container
@@ -560,16 +564,13 @@ container Fieldaligned<G, I,container>::evaluate( const BinaryOp& binary, const
     //idea: simply apply I+/I- enough times on the init2d vector to get the result in each plane
     //unary function is always such that the p0 plane is at x=0
     assert( p0 < m_g.get().Nz());
-    const dg::aGeometry2d* g2d_ptr = m_g.get().perp_grid();
-    container init2d = dg::pullback( binary, *g2d_ptr); 
-    delete g2d_ptr;
+    const dg::Handle<aGeometry2d> g2d = m_g.get().perp_grid();
+    container init2d = dg::pullback( binary, g2d.get()); 
+    container zero2d = dg::evaluate( dg::zero, g2d.get()); 
 
     container temp(init2d), tempP(init2d), tempM(init2d);
     container vec3d = dg::evaluate( dg::zero, m_g.get());
-    std::vector<container>  plus2d, minus2d, result; 
-    dg::split( vec3d, plus2d, m_g.get());
-    dg::split( vec3d, minus2d, m_g.get());
-    dg::split( vec3d, result, m_g.get());
+    std::vector<container>  plus2d(m_Nz, zero2d), minus2d(plus2d), result(plus2d); 
     unsigned turns = rounds; 
     if( turns ==0) turns++;
     //first apply Interpolation many times, scale and store results
@@ -578,7 +579,7 @@ container Fieldaligned<G, I,container>::evaluate( const BinaryOp& binary, const
         {
             dg::blas1::copy( init2d, tempP);
             dg::blas1::copy( init2d, tempM);
-            unsigned rep = i0 + r*m_Nz;
+            unsigned rep = r*m_Nz + i0;
             for(unsigned k=0; k<rep; k++)
             {
                 dg::blas2::symv( m_plus, tempP, temp);
diff --git a/inc/geometries/fluxfunctions.h b/inc/geometries/fluxfunctions.h
index 9b9d8dd3b..47b6e6d1d 100644
--- a/inc/geometries/fluxfunctions.h
+++ b/inc/geometries/fluxfunctions.h
@@ -109,6 +109,8 @@ aBinaryFunctor* make_aBinaryFunctor(const BinaryFunctor& f){return new BinaryFun
 
 /**
 * @brief This struct bundles a function and its first derivatives
+*
+* @snippet hector_t.cu doxygen 
 */
 struct BinaryFunctorsLvl1 
 {
@@ -196,6 +198,7 @@ struct BinaryFunctorsLvl2
 };
 
 /// A symmetric 2d tensor field and its divergence
+///@snippet hector_t.cu doxygen 
 struct BinarySymmTensorLvl1
 {
     BinarySymmTensorLvl1( ){}
@@ -251,6 +254,7 @@ struct BinarySymmTensorLvl1
 };
 
 /// A vector field with three components that depend only on the first two coordinates
+///@snippet ds_t.cu doxygen
 struct BinaryVectorLvl0
 {
     BinaryVectorLvl0(){}
diff --git a/inc/geometries/hector.h b/inc/geometries/hector.h
index 4c86d1ed5..ba5de9b29 100644
--- a/inc/geometries/hector.h
+++ b/inc/geometries/hector.h
@@ -200,6 +200,7 @@ void transform(
 /**
  * @brief The High PrEcision Conformal grid generaTOR 
  *
+ * @snippet hector_t.cu doxygen
  * @ingroup generators_geo
  * @tparam IMatrix The interpolation matrix type
  * @copydoc hide_matrix
diff --git a/inc/geometries/hector_t.cu b/inc/geometries/hector_t.cu
index 552e4474f..dc1dea9bc 100644
--- a/inc/geometries/hector_t.cu
+++ b/inc/geometries/hector_t.cu
@@ -73,6 +73,7 @@ int main( int argc, char* argv[])
     //solovev::detail::Fpsi fpsi( gp, -10);
     std::cout << "Constructing conformal grid ... \n";
     t.tic();
+    //![doxygen]
     Hector<dg::IDMatrix, dg::DMatrix, dg::DVec>* hector;
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     int construction = 0;
@@ -93,6 +94,7 @@ int main( int argc, char* argv[])
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     dg::geo::CurvilinearProductGrid3d g3d(*hector, n, Nx, Ny,Nz, dg::DIR);
     dg::geo::CurvilinearGrid2d g2d = g3d.perp_grid();
+    //![doxygen]
 
     dg::Grid2d g2d_periodic(g2d.x0(), g2d.x1(), g2d.y0(), g2d.y1(), g2d.n(), g2d.Nx(), g2d.Ny()+1); 
     t.toc();
diff --git a/inc/geometries/magnetic_field.h b/inc/geometries/magnetic_field.h
index 055c741c8..a8b46a470 100644
--- a/inc/geometries/magnetic_field.h
+++ b/inc/geometries/magnetic_field.h
@@ -23,6 +23,7 @@ namespace geo
  \f]
  where \f$ R_0\f$ is a normalization constant, \f$ I\f$ the poloidal current 
  and \f$ \psi_p\f$ the poloidal flux function.
+ @snippet ds_t.cu doxygen
  @note We implicitly also assume the toroidal field line approximation, i.e. all curvature
  and other perpendicular terms created with this field will assume that the perpendicular 
  direction lies within the
diff --git a/inc/geometries/mpi_fieldaligned.h b/inc/geometries/mpi_fieldaligned.h
index 160f57268..7f1df1f5b 100644
--- a/inc/geometries/mpi_fieldaligned.h
+++ b/inc/geometries/mpi_fieldaligned.h
@@ -196,17 +196,14 @@ MPI_Vector<container> Fieldaligned<G,MPIDistMat<M,C>, MPI_Vector<container> >::e
     //idea: simply apply I+/I- enough times on the init2d vector to get the result in each plane
     //unary function is always such that the p0 plane is at x=0
     assert( p0 < m_g.get().global().Nz());
-    const aMPIGeometry2d* g2d_ptr = m_g.get().perp_grid();
-    MPI_Vector<container> init2d = dg::pullback( binary, *g2d_ptr); 
-    delete g2d_ptr;
+    const dg::Handle<aMPIGeometry2d> g2d = m_g.get().perp_grid();
+    MPI_Vector<container> init2d = dg::pullback( binary, g2d.get()); 
+    MPI_Vector<container> zero2d = dg::evaluate( dg::zero, g2d.get()); 
     unsigned Nz = m_g.get().global().Nz();
 
     MPI_Vector<container> temp(init2d), tempP(init2d), tempM(init2d);
     MPI_Vector<container> vec3d = dg::evaluate( dg::zero, m_g.get());
-    std::vector<MPI_Vector<container> >  plus2d, minus2d, result;
-    dg::split( vec3d, plus2d,  m_g.get());
-    dg::split( vec3d, minus2d, m_g.get());
-    dg::split( vec3d, result,  m_g.get());
+    std::vector<MPI_Vector<container> >  plus2d(Nz, zero2d), minus2d(plus2d), result(plus2d);
     unsigned turns = rounds; 
     if( turns ==0) turns++;
     //first apply Interpolation many times, scale and store results
@@ -215,7 +212,7 @@ MPI_Vector<container> Fieldaligned<G,MPIDistMat<M,C>, MPI_Vector<container> >::e
         {
             dg::blas1::copy( init2d, tempP);
             dg::blas1::copy( init2d, tempM);
-            unsigned rep = i0 + r*Nz; 
+            unsigned rep = r*Nz + i0; 
             for(unsigned k=0; k<rep; k++)
             {
                 dg::blas2::symv( m_plus, tempP, temp);
@@ -231,9 +228,9 @@ MPI_Vector<container> Fieldaligned<G,MPIDistMat<M,C>, MPI_Vector<container> >::e
     //now we have the plus and the minus filaments
     if( rounds == 0) //there is a limiter
     {
-        for( unsigned i0=0; i0<Nz; i0++)
+        for( unsigned i0=0; i0<m_Nz; i0++)
         {
-            int idx = (int)(i0+m_coords2*m_g.get().Nz())  - (int)p0;
+            int idx = (int)(i0+m_coords2*m_Nz)  - (int)p0;
             if(idx>=0)
                 result[i0] = plus2d[idx];
             else
@@ -250,9 +247,9 @@ MPI_Vector<container> Fieldaligned<G,MPIDistMat<M,C>, MPI_Vector<container> >::e
             dg::blas1::axpby( 1., minus2d[revi0], 1., result[i0]);
         }
         dg::blas1::axpby( -1., init2d, 1., result[0]);
-        for(unsigned i0=0; i0<Nz; i0++)
+        for(unsigned i0=0; i0<m_Nz; i0++)
         {
-            int idx = ((int)i0 + m_coords2*m_g.get().Nz() -(int)p0 + Nz)%Nz; //shift index
+            int idx = ((int)i0 + m_coords2*m_Nz -(int)p0 + Nz)%Nz; //shift index
             thrust::copy( result[idx].data().begin(), result[idx].data().end(), vec3d.data().begin() + i0*m_perp_size);
         }
     }
diff --git a/src/toefl/parameters.h b/src/toefl/parameters.h
index 6bb4abbab..78e4b4844 100644
--- a/src/toefl/parameters.h
+++ b/src/toefl/parameters.h
@@ -1,6 +1,6 @@
 #pragma once
 #include <string>
-#include "dg/enums.h"
+#include "dg/algorithm.h"
 #include "json/json.h"
 
 /**
diff --git a/src/toefl/toeflR.cuh b/src/toefl/toeflR.cuh
index a64c46d14..b580f55e4 100644
--- a/src/toefl/toeflR.cuh
+++ b/src/toefl/toeflR.cuh
@@ -2,7 +2,6 @@
 #include <exception>
 
 #include "dg/algorithm.h"
-#include "dg/backend/typedefs.cuh"
 #include "parameters.h"
 
 #ifdef DG_BENCHMARK
-- 
GitLab


From 792cef7d128a52b915c3618512e9a783b6a22346 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 23 Oct 2017 15:25:04 +0200
Subject: [PATCH 383/453] more doku

---
 inc/geometries/hector_t.cu     |  2 +-
 inc/geometries/ribeiro_mpit.cu | 31 -------------------------------
 2 files changed, 1 insertion(+), 32 deletions(-)

diff --git a/inc/geometries/hector_t.cu b/inc/geometries/hector_t.cu
index dc1dea9bc..14a8ff1fe 100644
--- a/inc/geometries/hector_t.cu
+++ b/inc/geometries/hector_t.cu
@@ -72,11 +72,11 @@ int main( int argc, char* argv[])
     dg::Timer t;
     //solovev::detail::Fpsi fpsi( gp, -10);
     std::cout << "Constructing conformal grid ... \n";
+    int construction = 0;
     t.tic();
     //![doxygen]
     Hector<dg::IDMatrix, dg::DMatrix, dg::DVec>* hector;
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-    int construction = 0;
     if( construction == 0)
     {
         hector = new Hector<dg::IDMatrix, dg::DMatrix, dg::DVec>( psip, psi_0, psi_1, gp.R_0, 0., nGrid, NxGrid, NyGrid, epsHector, true);
diff --git a/inc/geometries/ribeiro_mpit.cu b/inc/geometries/ribeiro_mpit.cu
index 9cf2738c4..82ab2749a 100644
--- a/inc/geometries/ribeiro_mpit.cu
+++ b/inc/geometries/ribeiro_mpit.cu
@@ -172,37 +172,6 @@ int main( int argc, char* argv[])
     if(rank==0)std::cout << "relative difference in volume is "<<fabs(volumeRZP - volume)/volume<<std::endl;
     if(rank==0)std::cout << "Note that the error might also come from the volume in RZP!\n"; //since integration of jacobian is fairly good probably
 
-    /////////////////////////TEST 3d grid//////////////////////////////////////
-    //if(rank==0)std::cout << "Start DS test!"<<std::endl;
-    //const dg::MHVec vol3d = dg::create::volume( g3d);
-    //t.tic();
-    ////DFA fieldaligned( CurvilinearField( gp, g3d.x(), g3d.f_x()), g3d, gp.rk4eps, dg::NoLimiter()); 
-    //DFA fieldaligned( OrthogonalField( gp, g2d.global(), g2d.f2_xy()), g3d, gp.rk4eps, dg::NoLimiter()); 
-
-    ////dg::DS<DFA, dg::MHMatrix, dg::MHVec> ds( fieldaligned, CurvilinearField(gp, g3d.x(), g3d.f_x()), dg::normed, dg::centered);
-    //dg::DS<DFA, dg::MHMatrix, dg::MHVec> ds( fieldaligned, OrthogonalField(gp, g2d.global(), g2d.f2_xy()), dg::normed, dg::centered);
-    //t.toc();
-    //if(rank==0)std::cout << "Construction took "<<t.diff()<<"s\n";
-    //dg::MHVec B = dg::pullback( dg::geo::InvB(gp), g3d), divB(B);
-    //dg::MHVec lnB = dg::pullback( dg::geo::LnB(gp), g3d), gradB(B); //dg::MHVec gradLnB = dg::pullback( dg::geo::GradLnB(gp), g3d);
-    //dg::blas1::pointwiseDivide( ones3d, B, B);
-    //dg::MHVec function = dg::pullback( dg::geo::FuncNeu(gp), g3d), derivative(function);
-    //ds( function, derivative);
-
-    //ds.centeredT( B, divB);
-    //double norm =  sqrt( dg::blas2::dot(divB, vol3d, divB));
-    //if(rank==0)std::cout << "Divergence of B is "<<norm<<"\n";
-
-    //ds.centered( lnB, gradB);
-    //norm = sqrt(dg::blas2::dot(gradB,vol3d,gradB) );
-    //if(rank==0)std::cout << "num. norm of gradLnB is "<<norm<<"\n";
-    //norm = sqrt( dg::blas2::dot( gradLnB, vol3d, gradLnB) );
-    //if(rank==0)std::cout << "ana. norm of gradLnB is "<<norm<<"\n";
-    //dg::blas1::axpby( 1., gradB, -1., gradLnB, gradLnB);
-    //X = divB.data();
-    //err = nc_put_vara_double( ncid, divBID, start,count, X.data());
-    //double norm2 = sqrt(dg::blas2::dot(gradLnB, vol3d,gradLnB));
-    //if(rank==0)std::cout << "rel. error of lnB is    "<<norm2/norm<<"\n";
     err = nc_close( ncid);
     MPI_Finalize();
 
-- 
GitLab


From aae572722fbddf8e1634050e57bd87f1fa3436af Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 23 Oct 2017 17:10:24 +0200
Subject: [PATCH 384/453] make toefl_hpc and toefl_mpi do exactly the same

---
 inc/geometries/hector_t.cu |  12 +++--
 src/toefl/toefl_hpc.cu     |  66 ++++++++++++------------
 src/toefl/toefl_mpi.cu     | 103 ++++++++++++++-----------------------
 3 files changed, 80 insertions(+), 101 deletions(-)

diff --git a/inc/geometries/hector_t.cu b/inc/geometries/hector_t.cu
index 14a8ff1fe..06b16f730 100644
--- a/inc/geometries/hector_t.cu
+++ b/inc/geometries/hector_t.cu
@@ -79,17 +79,21 @@ int main( int argc, char* argv[])
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     if( construction == 0)
     {
-        hector = new Hector<dg::IDMatrix, dg::DMatrix, dg::DVec>( psip, psi_0, psi_1, gp.R_0, 0., nGrid, NxGrid, NyGrid, epsHector, true);
+        hector = new Hector<dg::IDMatrix, dg::DMatrix, dg::DVec>( 
+                psip, psi_0, psi_1, gp.R_0, 0., nGrid, NxGrid, NyGrid, epsHector, true);
     }
     else if( construction == 1)
     {
         dg::geo::BinaryFunctorsLvl1 nc = dg::geo::make_NablaPsiInvCollective( psip);
-        hector = new Hector<dg::IDMatrix, dg::DMatrix, dg::DVec>( psip, nc, psi_0, psi_1, gp.R_0, 0., nGrid, NxGrid, NyGrid, epsHector, true);
+        hector = new Hector<dg::IDMatrix, dg::DMatrix, dg::DVec>( 
+                psip, nc, psi_0, psi_1, gp.R_0, 0., nGrid, NxGrid, NyGrid, epsHector, true);
     }
     else
     {
-        dg::geo::BinarySymmTensorLvl1 lc = dg::geo::make_LiseikinCollective( psip, 0.1, 0.001);
-        hector = new Hector<dg::IDMatrix, dg::DMatrix, dg::DVec>( psip,lc, psi_0, psi_1, gp.R_0, 0., nGrid, NxGrid, NyGrid, epsHector, true);
+        dg::geo::BinarySymmTensorLvl1 lc = dg::geo::make_LiseikinCollective( 
+                psip, 0.1, 0.001);
+        hector = new Hector<dg::IDMatrix, dg::DMatrix, dg::DVec>( 
+                psip,lc, psi_0, psi_1, gp.R_0, 0., nGrid, NxGrid, NyGrid, epsHector, true);
     }
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     dg::geo::CurvilinearProductGrid3d g3d(*hector, n, Nx, Ny,Nz, dg::DIR);
diff --git a/src/toefl/toefl_hpc.cu b/src/toefl/toefl_hpc.cu
index f3377d847..130577eec 100644
--- a/src/toefl/toefl_hpc.cu
+++ b/src/toefl/toefl_hpc.cu
@@ -13,13 +13,6 @@
 #include "dg/backend/timer.cuh"
 
 
-/*
-   - reads parameters from input.txt or any other given file, 
-   - integrates the ToeflR - functor and 
-   - writes outputs to a given outputfile using hdf5. 
-        density fields are the real densities in XSPACE ( not logarithmic values)
-*/
-
 int main( int argc, char* argv[])
 {
     //Parameter initialisation
@@ -42,9 +35,10 @@ int main( int argc, char* argv[])
 
     ////////////////////////////////set up computations///////////////////////////
     dg::Grid2d grid( 0, p.lx, 0, p.ly, p.n, p.Nx, p.Ny, p.bc_x, p.bc_y);
+    dg::Grid2d grid_out( 0, p.lx, 0, p.ly, p.n_out, p.Nx_out, p.Ny_out, p.bc_x, p.bc_y);
     //create RHS 
     toefl::Explicit< dg::CartesianGrid2d, dg::DMatrix, dg::DVec > test( grid, p); 
-    toefl::Implicit<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> diffusion( grid, p.nu);
+    toefl::Implicit< dg::CartesianGrid2d, dg::DMatrix, dg::DVec > diffusion( grid, p.nu);
     /////////////////////create initial vector////////////////////////////////////
     dg::Gaussian g( p.posX*p.lx, p.posY*p.ly, p.sigma, p.sigma, p.amp); 
     std::vector<dg::DVec> y0(2, dg::evaluate( g, grid)), y1(y0); // n_e' = gaussian
@@ -56,21 +50,18 @@ int main( int argc, char* argv[])
     if( p.equations == "gravity_local" || p.equations == "gravity_global" || p.equations == "drift_global"){
         y0[1] = dg::evaluate( dg::zero, grid);
     }
-
     //////////////////initialisation of timestepper and first step///////////////////
-    std::cout << "init timestepper...\n";
     double time = 0;
-    //dg::AB< k, std::vector<dg::DVec> > ab( y0);
-    dg::Karniadakis< std::vector<dg::DVec> > ab( y0, y0[0].size(), 1e-9);
+    dg::Karniadakis< std::vector<dg::DVec> > ab( y0, y0[0].size(), p.eps_time);
     ab.init( test, diffusion, y0, p.dt);
-    y0.swap( y1); //y1 now contains value at zero time
+    y1 = y0;
     /////////////////////////////set up netcdf/////////////////////////////////////
     file::NC_Error_Handle err;
     int ncid;
     err = nc_create( argv[2],NC_NETCDF4|NC_CLOBBER, &ncid);
     err = nc_put_att_text( ncid, NC_GLOBAL, "inputfile", input.size(), input.data());
     int dim_ids[3], tvarID;
-    err = file::define_dimensions( ncid, dim_ids, &tvarID, grid);
+    err = file::define_dimensions( ncid, dim_ids, &tvarID, grid_out);
     //field IDs
     std::string names[4] = {"electrons", "ions", "potential", "vorticity"}; 
     int dataIDs[4]; 
@@ -86,20 +77,24 @@ int main( int argc, char* argv[])
     err = nc_def_var( ncid, "dissipation", NC_DOUBLE, 1, &EtimeID, &dissID);
     err = nc_def_var( ncid, "dEdt",        NC_DOUBLE, 1, &EtimeID, &dEdtID);
     err = nc_enddef(ncid);
+    dg::DVec transfer( dg::evaluate( dg::zero, grid));
+    ///////////////////////////////////first output/////////////////////////
     size_t start[3] = {0, 0, 0};
-    size_t count[3] = {1, grid.n()*grid.Ny(), grid.n()*grid.Nx()};
+    size_t count[3] = {1, grid_out.n()*grid_out.Ny(), grid_out.n()*grid_out.Nx()};
     size_t Estart[] = {0};
     size_t Ecount[] = {1};
-    ///////////////////////////////////first output/////////////////////////
-    //output all three fields
-    std::vector<dg::DVec> transferD(4);
-    std::vector<dg::HVec> output(4);
-    transferD[0] = y1[0], transferD[1] = y1[1], transferD[2] = test.potential()[0], transferD[3] = test.potential()[0]; //electrons
-    start[0] = 0;
+    std::vector<dg::DVec> transferD(4, dg::evaluate(dg::zero, grid_out));
+    dg::HVec transferH(dg::evaluate(dg::zero, grid_out));
+    dg::IDMatrix interpolate = dg::create::interpolation( grid_out, grid); 
+    dg::blas2::symv( interpolate, y1[0], transferD[0]);
+    dg::blas2::symv( interpolate, y1[1], transferD[1]);
+    dg::blas2::symv( interpolate, test.potential()[0], transferD[2]);
+    dg::blas2::symv( diffusion.laplacianM(), test.potential()[0], transfer);
+    dg::blas2::symv( interpolate, transfer, transferD[3]);
     for( int k=0;k<4; k++)
     {
-        dg::blas1::transfer( transferD[k], output[k]);
-        err = nc_put_vara_double( ncid, dataIDs[k], start, count, output[k].data() );
+        dg::blas1::transfer( transferD[k], transferH);
+        err = nc_put_vara_double( ncid, dataIDs[k], start, count, transferH.data() );
     }
     err = nc_put_vara_double( ncid, tvarID, start, count, &time);
     err = nc_close(ncid);
@@ -122,8 +117,7 @@ int main( int argc, char* argv[])
 #endif//DG_BENCHMARK
         for( unsigned j=0; j<p.itstp; j++)
         {
-            ab( test, diffusion, y0);
-            y0.swap( y1); //attention on -O3 ?
+            ab( test, diffusion, y1);
             //store accuracy details
             {
                 std::cout << "(m_tot-m_0)/m_0: "<< (test.mass()-mass0)/mass_blob0<<"\t";
@@ -131,7 +125,7 @@ int main( int argc, char* argv[])
                 E1 = test.energy();
                 diff = (E1 - E0)/p.dt;
                 double diss = test.energy_diffusion( );
-                std::cout << "diff diss: "<< diff<<" "<<diss<<"\t";
+                std::cout << "diff: "<< diff<<" diss: "<<diss<<"\t";
                 std::cout << "Accuracy: "<< 2.*(diff-diss)/(diff+diss)<<"\n";
             }
             time+=p.dt;
@@ -147,15 +141,19 @@ int main( int argc, char* argv[])
                 err = nc_close(ncid);
             }
         }
-        //output all three fields and vorticity
-        transferD[0] = y1[0], transferD[1] = y1[1], transferD[2] = test.potential()[0]; //electrons
-        dg::blas2::symv( diffusion.laplacianM(), transferD[2], transferD[3]);
-        for( int k=0;k<4; k++)
-            dg::blas1::transfer( transferD[k], output[k]);
-        err = nc_open(argv[2], NC_WRITE, &ncid);
+        //////////////////////////write fields////////////////////////
         start[0] = i;
-        for( int k=0; k<4; k++)
-            err = nc_put_vara_double( ncid, dataIDs[k], start, count, output[k].data() );
+        dg::blas2::symv( interpolate, y1[0], transferD[0]);
+        dg::blas2::symv( interpolate, y1[1], transferD[1]);
+        dg::blas2::symv( interpolate, test.potential()[0], transferD[2]);
+        dg::blas2::symv( diffusion.laplacianM(), test.potential()[0], transfer);
+        dg::blas2::symv( interpolate, transfer, transferD[3]);
+        err = nc_open(argv[2], NC_WRITE, &ncid);
+        for( int k=0;k<4; k++)
+        {
+            dg::blas1::transfer( transferD[k], transferH);
+            err = nc_put_vara_double( ncid, dataIDs[k], start, count, transferH.data() );
+        }
         err = nc_put_vara_double( ncid, tvarID, start, count, &time);
         err = nc_close(ncid);
 
diff --git a/src/toefl/toefl_mpi.cu b/src/toefl/toefl_mpi.cu
index 7678b588a..c7a1a7452 100644
--- a/src/toefl/toefl_mpi.cu
+++ b/src/toefl/toefl_mpi.cu
@@ -15,13 +15,6 @@
 #include "dg/backend/timer.cuh"
 
 
-/*
-   - reads parameters from input.txt or any other given file, 
-   - integrates the ToeflR - functor and 
-   - writes outputs to a given outputfile using hdf5. 
-        density fields are the real densities in XSPACE ( not logarithmic values)
-*/
-
 int main( int argc, char* argv[])
 {
     ////////////////////////////////setup MPI///////////////////////////////
@@ -75,7 +68,7 @@ int main( int argc, char* argv[])
     dg::MPIGrid2d grid_out( 0., p.lx, 0.,p.ly, p.n_out, p.Nx_out, p.Ny_out, p.bc_x, p.bc_y, comm);  
     //create RHS 
     toefl::Explicit< dg::CartesianMPIGrid2d, dg::MDMatrix, dg::MDVec > test( grid, p); 
-    toefl::Implicit<dg::CartesianMPIGrid2d, dg::MDMatrix, dg::MDVec> diffusion( grid, p.nu);
+    toefl::Implicit< dg::CartesianMPIGrid2d, dg::MDMatrix, dg::MDVec > diffusion( grid, p.nu);
     //////////////////create initial vector///////////////////////////////////////
     dg::Gaussian g( p.posX*p.lx, p.posY*p.ly, p.sigma, p.sigma, p.amp); 
     std::vector<dg::MDVec> y0(2, dg::evaluate( g, grid)), y1(y0); // n_e' = gaussian
@@ -87,18 +80,14 @@ int main( int argc, char* argv[])
     if( p.equations == "gravity_local" || p.equations == "gravity_global" || p.equations == "drift_global" ){
         y0[1] = dg::evaluate( dg::zero, grid);
     }
-    //////////////////////////////////////////////////////////////////////
-
     //////////////////initialisation of timestepper and first step///////////////////
     double time = 0;
-    //dg::AB< k, std::vector<dg::MDVec> > ab( y0);
-    dg::Karniadakis< std::vector<dg::MDVec> > ab( y0, y0[0].size(), 1e-9);
+    dg::Karniadakis< std::vector<dg::MDVec> > ab( y0, y0[0].size(), p.eps_time);
     ab.init( test, diffusion, y0, p.dt);
-    y0.swap( y1); //y1 now contains value at zero time
+    y1 = y0;
     /////////////////////////////set up netcdf/////////////////////////////////////
     file::NC_Error_Handle err;
-    int ncid;
-    MPI_Info info = MPI_INFO_NULL;
+    int ncid; MPI_Info info = MPI_INFO_NULL;
     err = nc_create_par( argv[2],NC_NETCDF4|NC_MPIIO|NC_CLOBBER,comm,info, &ncid);
     err = nc_put_att_text( ncid, NC_GLOBAL, "inputfile", input.size(), input.data());
     int dim_ids[3], tvarID;
@@ -117,48 +106,41 @@ int main( int argc, char* argv[])
     err = nc_def_var( ncid, "mass",        NC_DOUBLE, 1, &EtimeID, &massID);
     err = nc_def_var( ncid, "dissipation", NC_DOUBLE, 1, &EtimeID, &dissID);
     err = nc_def_var( ncid, "dEdt",        NC_DOUBLE, 1, &EtimeID, &dEdtID);
-    for(unsigned i=0; i<4; i++)
-        err = nc_var_par_access( ncid, dataIDs[i], NC_COLLECTIVE);
-    err = nc_var_par_access( ncid, tvarID, NC_COLLECTIVE);
-    err = nc_var_par_access( ncid, EtimevarID, NC_COLLECTIVE);
-    err = nc_var_par_access( ncid, energyID, NC_COLLECTIVE);
-    err = nc_var_par_access( ncid, massID, NC_COLLECTIVE);
-    err = nc_var_par_access( ncid, dissID, NC_COLLECTIVE);
-    err = nc_var_par_access( ncid, dEdtID, NC_COLLECTIVE);
-    err = nc_enddef(ncid);
-
+      //mpi specific part
+      for(unsigned i=0; i<4; i++)
+          err = nc_var_par_access( ncid, dataIDs[i], NC_COLLECTIVE);
+      err = nc_var_par_access( ncid, tvarID, NC_COLLECTIVE);
+      err = nc_var_par_access( ncid, EtimevarID, NC_COLLECTIVE);
+      err = nc_var_par_access( ncid, energyID, NC_COLLECTIVE);
+      err = nc_var_par_access( ncid, massID, NC_COLLECTIVE);
+      err = nc_var_par_access( ncid, dissID, NC_COLLECTIVE);
+      err = nc_var_par_access( ncid, dEdtID, NC_COLLECTIVE);
+      err = nc_enddef(ncid);
+      int dims[2],  coords[2];
+      MPI_Cart_get( comm, 2, dims, periods, coords);
+      dg::MDVec transfer( dg::evaluate( dg::zero, grid));
     ///////////////////////////////////first output/////////////////////////
-    int dims[2],  coords[2];
-    MPI_Cart_get( comm, 2, dims, periods, coords);
-    size_t count[3] = {1, grid_out.n()*grid_out.Ny(), grid_out.n()*grid_out.Nx()};
     size_t start[3] = {0, coords[1]*count[1], coords[0]*count[2]};
+    size_t count[3] = {1, grid_out.n()*grid_out.Ny(), grid_out.n()*grid_out.Nx()};
     size_t Estart[] = {0};
     size_t Ecount[] = {1};
-    dg::MDVec transfer( dg::evaluate(dg::zero, grid));
-    dg::DVec transferD( dg::evaluate(dg::zero, grid_out.local()));
-    dg::HVec transferH( dg::evaluate(dg::zero, grid_out.local()));
-    dg::IDMatrix interpolate = dg::create::interpolation( grid_out.local(), grid.local()); //create local interpolation matrix
-    for( unsigned i=0; i<2; i++)
+    std::vector<dg::DVec> transferD(4, dg::evaluate(dg::zero, grid_out.local()));
+    dg::HVec transferH(dg::evaluate(dg::zero, grid_out.local()));
+    dg::IDMatrix interpolate = dg::create::interpolation( grid_out.local(), grid.local()); 
+    dg::blas2::symv( interpolate, y1[0].data(), transferD[0]);
+    dg::blas2::symv( interpolate, y1[1].data(), transferD[1]);
+    dg::blas2::symv( interpolate, test.potential()[0].data(), transferD[2]);
+    dg::blas2::symv( diffusion.laplacianM(), test.potential()[0], transfer);
+    dg::blas2::symv( interpolate, transfer.data(), transferD[3]);
+    for( unsigned k=0; k<4; k++)
     {
-        dg::blas2::gemv( interpolate, y0[i].data(), transferD);
-        dg::blas1::transfer( transferD, transferH);
+        dg::blas1::transfer( transferD[k], transferH);
         err = nc_put_vara_double( ncid, dataIDs[i], start, count, transferH.data() );
     }
-    //pot
-    transfer = test.potential()[0];
-    dg::blas2::gemv( interpolate, transfer.data(), transferD);
-    dg::blas1::transfer( transferD, transferH);
-    err = nc_put_vara_double( ncid, dataIDs[2], start, count, transferH.data() );
-    //Vor
-    transfer = test.potential()[0];
-    dg::blas2::gemv( diffusion.laplacianM(), transfer, y1[1]);        
-    dg::blas2::gemv( interpolate,y1[1].data(), transferD);
-    dg::blas1::transfer( transferD, transferH);
-    err = nc_put_vara_double( ncid, dataIDs[3], start, count, transferH.data() );
     err = nc_put_vara_double( ncid, tvarID, start, count, &time);
     //err = nc_close(ncid);
     ///////////////////////////////////////Timeloop/////////////////////////////////
-    const double mass0 = test.mass(), mass_blob0 = mass0 - grid.lx()*grid.ly();
+    const double mass0 = test.mass(), mass_blob0 = mass0 - grid.global().lx()*grid.global().ly();
     double E0 = test.energy(), energy0 = E0, E1 = 0, diff = 0;
     dg::Timer t;
     t.tic();
@@ -176,8 +158,7 @@ int main( int argc, char* argv[])
 #endif//DG_BENCHMARK
         for( unsigned j=0; j<p.itstp; j++)
         {
-            ab( test, diffusion, y0);
-            y0.swap( y1); //attention on -O3 ?
+            ab( test, diffusion, y1);
             //store accuracy details
             {
                 if(rank==0)std::cout << "(m_tot-m_0)/m_0: "<< (test.mass()-mass0)/mass_blob0<<"\t";
@@ -185,7 +166,7 @@ int main( int argc, char* argv[])
                 E1 = test.energy();
                 diff = (E1 - E0)/p.dt;
                 double diss = test.energy_diffusion( );
-                if(rank==0)std::cout << "(E_tot-E_0)/E_0: "<< (E1-energy0)/energy0<<"\t";
+                if(rank==0)std::cout << "diff: "<< diff<<" diss: "<<diss<<"\t";
                 if(rank==0)std::cout << "Accuracy: "<< 2.*(diff-diss)/(diff+diss)<<"\n";
             }
             time+=p.dt;
@@ -203,22 +184,18 @@ int main( int argc, char* argv[])
         }
         //////////////////////////write fields////////////////////////
         start[0] = i;
-        for( unsigned j=0; j<2; j++)
+        dg::blas2::symv( interpolate, y1[0].data(), transferD[0]);
+        dg::blas2::symv( interpolate, y1[1].data(), transferD[1]);
+        dg::blas2::symv( interpolate, test.potential()[0].data(), transferD[2]);
+        dg::blas2::symv( diffusion.laplacianM(), transferD[2], transferD[3]);
+        //err = nc_open(argv[2], NC_WRITE, &ncid);
+        for( int k=0;k<4; k++)
         {
-            dg::blas2::gemv( interpolate, y0[j].data(), transferD);
-            dg::blas1::transfer( transferD, transferH);
-            err = nc_put_vara_double( ncid, dataIDs[j], start, count, transferH.data());
+            dg::blas1::transfer( transferD[k], transferH);
+            err = nc_put_vara_double( ncid, dataIDs[k], start, count, transferH.data() );
         }
-        transfer = test.potential()[0];
-        dg::blas2::gemv( interpolate, transfer.data(), transferD);
-        dg::blas1::transfer( transferD, transferH);
-        err = nc_put_vara_double( ncid, dataIDs[2], start, count, transferH.data() );
-        transfer = test.potential()[0];
-        dg::blas2::gemv( diffusion.laplacianM(), transfer, y1[1]);        //correct?    
-        dg::blas2::gemv( interpolate,y1[1].data(), transferD);
-        dg::blas1::transfer( transferD, transferH);
-        err = nc_put_vara_double( ncid, dataIDs[3], start, count, transferH.data() );
         err = nc_put_vara_double( ncid, tvarID, start, count, &time);
+        //err = nc_close(ncid);
 
 #ifdef DG_BENCHMARK
         ti.toc();
-- 
GitLab


From fa030372586a4f82bc4c83366fcf81bcb4e11f9b Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 23 Oct 2017 17:16:02 +0200
Subject: [PATCH 385/453] little bugs in toefl_mpi.cu

---
 src/toefl/toefl_hpc.cu | 4 ++--
 src/toefl/toefl_mpi.cu | 9 +++++----
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/src/toefl/toefl_hpc.cu b/src/toefl/toefl_hpc.cu
index 130577eec..2ab51a600 100644
--- a/src/toefl/toefl_hpc.cu
+++ b/src/toefl/toefl_hpc.cu
@@ -79,10 +79,10 @@ int main( int argc, char* argv[])
     err = nc_enddef(ncid);
     dg::DVec transfer( dg::evaluate( dg::zero, grid));
     ///////////////////////////////////first output/////////////////////////
-    size_t start[3] = {0, 0, 0};
     size_t count[3] = {1, grid_out.n()*grid_out.Ny(), grid_out.n()*grid_out.Nx()};
-    size_t Estart[] = {0};
+    size_t start[3] = {0, 0, 0};
     size_t Ecount[] = {1};
+    size_t Estart[] = {0};
     std::vector<dg::DVec> transferD(4, dg::evaluate(dg::zero, grid_out));
     dg::HVec transferH(dg::evaluate(dg::zero, grid_out));
     dg::IDMatrix interpolate = dg::create::interpolation( grid_out, grid); 
diff --git a/src/toefl/toefl_mpi.cu b/src/toefl/toefl_mpi.cu
index c7a1a7452..17ce61da6 100644
--- a/src/toefl/toefl_mpi.cu
+++ b/src/toefl/toefl_mpi.cu
@@ -120,10 +120,10 @@ int main( int argc, char* argv[])
       MPI_Cart_get( comm, 2, dims, periods, coords);
       dg::MDVec transfer( dg::evaluate( dg::zero, grid));
     ///////////////////////////////////first output/////////////////////////
-    size_t start[3] = {0, coords[1]*count[1], coords[0]*count[2]};
     size_t count[3] = {1, grid_out.n()*grid_out.Ny(), grid_out.n()*grid_out.Nx()};
-    size_t Estart[] = {0};
+    size_t start[3] = {0, coords[1]*count[1], coords[0]*count[2]};
     size_t Ecount[] = {1};
+    size_t Estart[] = {0};
     std::vector<dg::DVec> transferD(4, dg::evaluate(dg::zero, grid_out.local()));
     dg::HVec transferH(dg::evaluate(dg::zero, grid_out.local()));
     dg::IDMatrix interpolate = dg::create::interpolation( grid_out.local(), grid.local()); 
@@ -135,7 +135,7 @@ int main( int argc, char* argv[])
     for( unsigned k=0; k<4; k++)
     {
         dg::blas1::transfer( transferD[k], transferH);
-        err = nc_put_vara_double( ncid, dataIDs[i], start, count, transferH.data() );
+        err = nc_put_vara_double( ncid, dataIDs[k], start, count, transferH.data() );
     }
     err = nc_put_vara_double( ncid, tvarID, start, count, &time);
     //err = nc_close(ncid);
@@ -187,7 +187,8 @@ int main( int argc, char* argv[])
         dg::blas2::symv( interpolate, y1[0].data(), transferD[0]);
         dg::blas2::symv( interpolate, y1[1].data(), transferD[1]);
         dg::blas2::symv( interpolate, test.potential()[0].data(), transferD[2]);
-        dg::blas2::symv( diffusion.laplacianM(), transferD[2], transferD[3]);
+        dg::blas2::symv( diffusion.laplacianM(), test.potential()[0], transfer);
+        dg::blas2::symv( interpolate, transfer.data(), transferD[3]);
         //err = nc_open(argv[2], NC_WRITE, &ncid);
         for( int k=0;k<4; k++)
         {
-- 
GitLab


From de0855619af65cec4f0f446debc430e38c6a1968 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <Matthias.Wiesenberger@uibk.ac.at>
Date: Tue, 24 Oct 2017 13:28:52 +0200
Subject: [PATCH 386/453] debugged ribeiro_mpit and introduced c++14 as the
 default standard

---
 config/default.mk                         |  2 +-
 config/marconi.mk                         |  2 +-
 inc/dg/geometry/mpi_base.h                |  1 +
 inc/geometries/conformal_elliptic_b.cu    |  4 +--
 inc/geometries/ds_geom_t.cu               |  4 +--
 inc/geometries/flux_t.cu                  |  2 +-
 inc/geometries/geometry_advection_b.cu    |  2 +-
 inc/geometries/geometry_advection_mpib.cu |  2 +-
 inc/geometries/geometry_diag.cu           |  2 +-
 inc/geometries/geometry_elliptic_b.cu     |  2 +-
 inc/geometries/geometry_elliptic_mpib.cu  |  2 +-
 inc/geometries/guenther_ds_b.cu           |  2 +-
 inc/geometries/hector_t.cu                |  2 +-
 inc/geometries/init.h                     | 16 +++++-----
 inc/geometries/ribeiroX_t.cu              | 16 ++++------
 inc/geometries/ribeiro_mpit.cu            | 37 +++++++++++------------
 inc/geometries/ribeiro_t.cu               |  3 +-
 inc/geometries/separatrix_orthogonal.h    |  2 +-
 inc/geometries/separatrix_orthogonal_t.cu |  2 +-
 inc/geometries/simple_orthogonal_t.cu     |  2 +-
 inc/geometries/solovev.h                  |  6 ++--
 src/toefl/Makefile                        |  2 +-
 22 files changed, 54 insertions(+), 61 deletions(-)

diff --git a/config/default.mk b/config/default.mk
index ae19833f8..b15f57d5b 100644
--- a/config/default.mk
+++ b/config/default.mk
@@ -4,7 +4,7 @@ INCLUDED=1
 #default machine values
 INCLUDE = -I$(HOME)/include#  # cusp and thrust and the draw libraries
 GLFLAGS =$$(pkg-config --static --libs glfw3) #glfw3 installation
-CC=g++ #C++ compiler
+CC=g++ -std=c++14 #C++ compiler
 MPICC=mpic++  #mpi compiler
 OPT=-O3 # optimization flag
 NVCCARCH=-arch sm_20 #nvcc gpu compute capability
diff --git a/config/marconi.mk b/config/marconi.mk
index 5f7a50d8e..8c2ff4cbe 100644
--- a/config/marconi.mk
+++ b/config/marconi.mk
@@ -3,7 +3,7 @@ ifeq ($(strip $(HPC_SYSTEM)),marconi)
 INCLUDE += -I$(HOME)/include # cusp, thrust
 INCLUDE += -I$(NETCDF_INC) -I$(HDF5_INC)
 GLFLAGS  = -lm 
-CC=icc
+CC=icc -std=c++14
 MPICC=mpiicc
 OPT=-O3 -xHost  # overwritten for mic in devices.mk
 #MPICFLAGS+= -DMPICH_IGNORE_CXX_SEEK
diff --git a/inc/dg/geometry/mpi_base.h b/inc/dg/geometry/mpi_base.h
index 2f3580316..bb7c7a5b6 100644
--- a/inc/dg/geometry/mpi_base.h
+++ b/inc/dg/geometry/mpi_base.h
@@ -1,6 +1,7 @@
 #pragma once
 
 #include "dg/backend/mpi_grid.h"
+#include "base_geometry.h"
 #include "tensor.h"
 
 namespace dg
diff --git a/inc/geometries/conformal_elliptic_b.cu b/inc/geometries/conformal_elliptic_b.cu
index 0e867e26d..9e6a1d0c6 100644
--- a/inc/geometries/conformal_elliptic_b.cu
+++ b/inc/geometries/conformal_elliptic_b.cu
@@ -15,8 +15,6 @@
 #include "testfunctors.h"
 #include "dg/backend/timer.cuh"
 
-using namespace dg::geo::solovev;
-
 const unsigned nIter=6;
 template<class Geometry>
 void compute_error_elliptic( const dg::geo::TokamakMagneticField& c, const Geometry& g2d, double psi_0, double psi_1, double eps)
@@ -98,7 +96,7 @@ int main(int argc, char**argv)
         reader.parse(is,js,false);
     }
     //write parameters from file into variables
-    GeomParameters gp(js);
+    dg::geo::solovev::Parameters gp(js);
     gp.display( std::cout);
     dg::geo::TokamakMagneticField c = dg::geo::createSolovevField( gp); 
     const double eps = 1e-10;
diff --git a/inc/geometries/ds_geom_t.cu b/inc/geometries/ds_geom_t.cu
index 32b33ac91..b4152d29f 100644
--- a/inc/geometries/ds_geom_t.cu
+++ b/inc/geometries/ds_geom_t.cu
@@ -55,7 +55,7 @@ struct Parameters
 
 struct InvNormR
 {
-    InvNormR( dg::geo::solovev::GeomParameters gp): R_0(gp.R_0){}
+    InvNormR( dg::geo::solovev::Parameters gp): R_0(gp.R_0){}
     double operator()( double R, double Z, double phi)const
     {
         return R_0/R;
@@ -86,7 +86,7 @@ int main( int argc, char* argv[])
         reader.parse( isG, geom_js, false);
     }
     const Parameters p(input_js);
-    const dg::geo::solovev::GeomParameters gp(geom_js);
+    const dg::geo::solovev::Parameters gp(geom_js);
     p.display( std::cout);
     gp.display( std::cout);
 
diff --git a/inc/geometries/flux_t.cu b/inc/geometries/flux_t.cu
index 9f4b3e8e2..cb5debbc2 100644
--- a/inc/geometries/flux_t.cu
+++ b/inc/geometries/flux_t.cu
@@ -61,7 +61,7 @@ int main( int argc, char* argv[])
         reader.parse(is,js,false);
     }
     //write parameters from file into variables
-    dg::geo::solovev::GeomParameters gp(js);
+    dg::geo::solovev::Parameters gp(js);
     dg::geo::TokamakMagneticField c = dg::geo::createSolovevField( gp);
     std::cout << "Psi min "<<c.psip()(gp.R_0, 0)<<"\n";
     std::cout << "Type psi_0 (-20) and psi_1 (-4)\n";
diff --git a/inc/geometries/geometry_advection_b.cu b/inc/geometries/geometry_advection_b.cu
index 50a61e354..e21c3a70f 100644
--- a/inc/geometries/geometry_advection_b.cu
+++ b/inc/geometries/geometry_advection_b.cu
@@ -120,7 +120,7 @@ int main(int argc, char** argv)
         std::ifstream is(argv[1]);
         reader.parse(is,js,false);
     }
-    dg::geo::solovev::GeomParameters gp(js);
+    dg::geo::solovev::Parameters gp(js);
     dg::geo::TokamakMagneticField c = dg::geo::createSolovevField( gp);
 
     std::cout << "Psi min "<<c.psip()(gp.R_0, 0)<<"\n";
diff --git a/inc/geometries/geometry_advection_mpib.cu b/inc/geometries/geometry_advection_mpib.cu
index 60c345a55..9eef02050 100644
--- a/inc/geometries/geometry_advection_mpib.cu
+++ b/inc/geometries/geometry_advection_mpib.cu
@@ -124,7 +124,7 @@ int main(int argc, char** argv)
         std::ifstream is(argv[1]);
         reader.parse(is,js,false);
     }
-    dg::geo::solovev::GeomParameters gp(js);
+    dg::geo::solovev::Parameters gp(js);
     dg::geo::TokamakMagneticField c = dg::geo::createSolovevField( gp);
     if(rank==0)std::cout << "Psi min "<<c.psip()(gp.R_0, 0)<<"\n";
     if(rank==0)std::cout << "Type psi_0 and psi_1\n";
diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index de4918a35..05ad6eafb 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -108,7 +108,7 @@ int main( int argc, char* argv[])
         reader.parse( geom, geom_js, false);
     }
     const Parameters p(input_js);
-    const dg::geo::solovev::GeomParameters gp(geom_js);
+    const dg::geo::solovev::Parameters gp(geom_js);
     p.display( std::cout);
     gp.display( std::cout);
     std::string input = input_js.toStyledString();
diff --git a/inc/geometries/geometry_elliptic_b.cu b/inc/geometries/geometry_elliptic_b.cu
index f9215a46e..e8f47af12 100644
--- a/inc/geometries/geometry_elliptic_b.cu
+++ b/inc/geometries/geometry_elliptic_b.cu
@@ -36,7 +36,7 @@ int main(int argc, char**argv)
         reader.parse(is,js,false);
     }
     //write parameters from file into variables
-    dg::geo::solovev::GeomParameters gp(js);
+    dg::geo::solovev::Parameters gp(js);
     dg::geo::TokamakMagneticField c = dg::geo::createSolovevField(gp);
     gp.display( std::cout);
     dg::Timer t;
diff --git a/inc/geometries/geometry_elliptic_mpib.cu b/inc/geometries/geometry_elliptic_mpib.cu
index 5b0e16146..45cd6797e 100644
--- a/inc/geometries/geometry_elliptic_mpib.cu
+++ b/inc/geometries/geometry_elliptic_mpib.cu
@@ -38,7 +38,7 @@ int main(int argc, char**argv)
         std::ifstream is(argv[1]);
         reader.parse(is,js,false);
     }
-    dg::geo::solovev::GeomParameters gp(js);
+    dg::geo::solovev::Parameters gp(js);
     dg::geo::TokamakMagneticField c = dg::geo::solovev::createMagField(gp);
     if(rank==0)std::cout << "Psi min "<<c.psip()(gp.R_0, 0)<<"\n";
     if(rank==0)std::cout << "Type psi_0 and psi_1\n";
diff --git a/inc/geometries/guenther_ds_b.cu b/inc/geometries/guenther_ds_b.cu
index eb28a3378..a34cf3de1 100644
--- a/inc/geometries/guenther_ds_b.cu
+++ b/inc/geometries/guenther_ds_b.cu
@@ -27,7 +27,7 @@ int main( )
     Json::Value js;
     std::ifstream is("guenther_params.js");
     reader.parse(is,js,false);
-    GeomParameters gp(js);
+    Parameters gp(js);
     gp.display( std::cout);
 
     //////////////////////////////////////////////////////////////////////////
diff --git a/inc/geometries/hector_t.cu b/inc/geometries/hector_t.cu
index 06b16f730..766a67d90 100644
--- a/inc/geometries/hector_t.cu
+++ b/inc/geometries/hector_t.cu
@@ -62,7 +62,7 @@ int main( int argc, char* argv[])
         reader.parse(is,js,false);
     }
     //write parameters from file into variables
-    dg::geo::solovev::GeomParameters gp(js);
+    dg::geo::solovev::Parameters gp(js);
     dg::geo::BinaryFunctorsLvl2 psip = dg::geo::solovev::createPsip( gp); 
     std::cout << "Psi min "<<psip.f()(gp.R_0, 0)<<"\n";
     std::cout << "Type psi_0 and psi_1\n";
diff --git a/inc/geometries/init.h b/inc/geometries/init.h
index fee6fabcf..97abd1013 100644
--- a/inc/geometries/init.h
+++ b/inc/geometries/init.h
@@ -168,7 +168,7 @@ struct GaussianProfDamping : public aCloneableBinaryFunctor<GaussianProfDamping>
  */ 
 struct GaussianProfXDamping : public aCloneableBinaryFunctor<GaussianProfXDamping>
 {
-    GaussianProfXDamping( const aBinaryFunctor& psi, dg::geo::solovev::GeomParameters gp):
+    GaussianProfXDamping( const aBinaryFunctor& psi, dg::geo::solovev::Parameters gp):
         gp_(gp),
         psip_(psi) { }
     private:
@@ -178,7 +178,7 @@ struct GaussianProfXDamping : public aCloneableBinaryFunctor<GaussianProfXDampin
         if( psip_.get()(R,Z) < (gp_.psipmax-4.*gp_.alpha)) return 1.;
         return exp( -( psip_.get()(R,Z)-(gp_.psipmax-4.*gp_.alpha))*( psip_.get()(R,Z)-(gp_.psipmax-4.*gp_.alpha))/2./gp_.alpha/gp_.alpha);
     }
-    dg::geo::solovev::GeomParameters gp_;
+    dg::geo::solovev::Parameters gp_;
     Handle<aBinaryFunctor> psip_;
 };
 
@@ -202,7 +202,7 @@ struct TanhSource : public aCloneableBinaryFunctor<TanhSource>
 
 // struct Gradient : public aCloneableBinaryFunctor<Gradient>
 // {
-//     Gradient(  eule::Parameters p, GeomParameters gp):
+//     Gradient(  eule::Parameters p, Parameters gp):
 //         p_(p),
 //         gp_(gp),
 //         psip_(gp) {
@@ -215,7 +215,7 @@ struct TanhSource : public aCloneableBinaryFunctor<TanhSource>
 //         return p_.bgprofamp;
 //     }
 //     eule::Parameters p_;
-//     GeomParameters gp_;
+//     Parameters gp_;
 //     Handle<aBinaryFunctor> psip_;
 // };
 
@@ -229,7 +229,7 @@ struct TanhSource : public aCloneableBinaryFunctor<TanhSource>
  */ 
 struct Nprofile : public aCloneableBinaryFunctor<Nprofile>
 {
-     Nprofile( double bgprofamp, double peakamp, dg::geo::solovev::GeomParameters gp, const aBinaryFunctor& psi):
+     Nprofile( double bgprofamp, double peakamp, dg::geo::solovev::Parameters gp, const aBinaryFunctor& psi):
          bgamp(bgprofamp), namp( peakamp),
          gp_(gp),
          psip_(psi) { }
@@ -241,7 +241,7 @@ struct Nprofile : public aCloneableBinaryFunctor<Nprofile>
         return bgamp;
     }
     double bgamp, namp;
-    dg::geo::solovev::GeomParameters gp_;
+    dg::geo::solovev::Parameters gp_;
     Handle<aBinaryFunctor> psip_;
 };
 
@@ -255,7 +255,7 @@ struct Nprofile : public aCloneableBinaryFunctor<Nprofile>
  */ 
 struct ZonalFlow : public aCloneableBinaryFunctor<ZonalFlow>
 {
-    ZonalFlow(  double amplitude, double k_psi, dg::geo::solovev::GeomParameters gp, const aBinaryFunctor& psi):
+    ZonalFlow(  double amplitude, double k_psi, dg::geo::solovev::Parameters gp, const aBinaryFunctor& psi):
         amp_(amplitude), k_(k_psi),
         gp_(gp),
         psip_(psi) { }
@@ -268,7 +268,7 @@ struct ZonalFlow : public aCloneableBinaryFunctor<ZonalFlow>
 
     }
     double amp_, k_;
-    dg::geo::solovev::GeomParameters gp_;
+    dg::geo::solovev::Parameters gp_;
     Handle<aBinaryFunctor> psip_;
 };
 
diff --git a/inc/geometries/ribeiroX_t.cu b/inc/geometries/ribeiroX_t.cu
index 4b50edcd9..88544e1b3 100644
--- a/inc/geometries/ribeiroX_t.cu
+++ b/inc/geometries/ribeiroX_t.cu
@@ -18,8 +18,6 @@
 #include "dg/ds.h"
 #include "init.h"
 
-using namespace dg::geo;
-
 //typedef dg::FieldAligned< solovev::ConformalXGrid3d<dg::DVec> , dg::IDMatrix, dg::DVec> DFA;
 double sine( double x) {return sin(x);}
 double cosine( double x) {return cos(x);}
@@ -89,7 +87,7 @@ int main( int argc, char* argv[])
         std::ifstream is(argv[1]);
         reader.parse(is,js,false);
     }
-    solovev::GeomParameters gp(js);
+    dg::geo::solovev::Parameters gp(js);
     dg::Timer t;
     std::cout << "Type psi_0 \n";
     double psi_0 = -16;
@@ -100,18 +98,16 @@ int main( int argc, char* argv[])
     gp.display( std::cout);
     std::cout << "Constructing orthogonal grid ... \n";
     t.tic();
-    solovev::Psip psip( gp); 
-    std::cout << "Psi min "<<psip(gp.R_0, 0)<<"\n";
-    solovev::PsipR psipR(gp); solovev::PsipZ psipZ(gp);
-    solovev::PsipRR psipRR(gp); solovev::PsipRZ psipRZ(gp); solovev::PsipZZ psipZZ(gp);
+    dg::geo::BinaryFunctorsLvl2 psip = dg::geo::solovev::createPsip(gp);
+    std::cout << "Psi min "<<psip.f()(gp.R_0, 0)<<"\n";
     double R_X = gp.R_0-1.1*gp.triangularity*gp.a;
     double Z_X = -1.1*gp.elongation*gp.a;
     dg::geo::findXpoint( psipR, psipZ, psipRR, psipRZ, psipZZ, R_X, Z_X);
 
     double R0 = gp.R_0, Z0 = 0;
-    dg::geo::RibeiroX<solovev::Psip,solovev::PsipR,solovev::PsipZ,solovev::PsipRR, solovev::PsipRZ, solovev::PsipZZ> generator(psip, psipR, psipZ, psipRR, psipRZ, psipZZ, psi_0, fx_0, R_X,Z_X, R0, Z0);
-    dg::geo::CurvilinearGridX3d<dg::DVec> g3d(generator, psi_0, fx_0, fy_0, n, Nx, Ny,Nz, dg::DIR, dg::NEU);
-    dg::geo::CurvilinearGridX2d<dg::DVec> g2d = g3d.perp_grid();
+    dg::geo::RibeiroX generator(psip, psi_0, fx_0, R_X,Z_X, R0, Z0);
+    dg::geo::CurvilinearGridX3d g3d(generator, psi_0, fx_0, fy_0, n, Nx, Ny,Nz, dg::DIR, dg::NEU);
+    dg::geo::CurvilinearGridX2d g2d = g3d.perp_grid();
     t.toc();
     dg::GridX3d g3d_periodic(g3d.x0(), g3d.x1(), g3d.y0(), g3d.y1(), g3d.z0(), g3d.z1(), g3d.fx(), g3d.fy(), g3d.n(), g3d.Nx(), g3d.Ny(), 2); 
     std::cout << "Construction took "<<t.diff()<<"s"<<std::endl;
diff --git a/inc/geometries/ribeiro_mpit.cu b/inc/geometries/ribeiro_mpit.cu
index 82ab2749a..f47fc04eb 100644
--- a/inc/geometries/ribeiro_mpit.cu
+++ b/inc/geometries/ribeiro_mpit.cu
@@ -23,7 +23,6 @@
 #include <netcdf_par.h>
 #include "file/nc_utilities.h"
 
-using namespace dg::geo::solovev;
 double sineX( double x, double y) {return sin(x)*sin(y);}
 double cosineX( double x, double y) {return cos(x)*sin(y);}
 double sineY( double x, double y) {return sin(x)*sin(y);}
@@ -50,8 +49,8 @@ int main( int argc, char* argv[])
         std::ifstream is(argv[1]);
         reader.parse(is,js,false);
     }
-    GeomParameters gp(js);
-    dg::geo::BinaryFunctorsLvl2 psip = createPsip( gp);
+    dg::geo::solovev::Parameters gp(js);
+    dg::geo::BinaryFunctorsLvl2 psip = dg::geo::solovev::createPsip( gp);
     if(rank==0)std::cout << "Psi min "<<psip.f()(gp.R_0, 0)<<"\n";
     if(rank==0)std::cout << "Type psi_0 and psi_1\n";
     double psi_0, psi_1;
@@ -65,15 +64,15 @@ int main( int argc, char* argv[])
     t.tic();
     dg::geo::Ribeiro ribeiro( psip, psi_0, psi_1, gp.R_0, 0., 1);
     dg::geo::CurvilinearProductMPIGrid3d g3d(ribeiro, n, Nx, Ny,Nz, dg::DIR,dg::PER, dg::PER,comm);
-    dg::geo::CurvilinearMPIGrid2d g2d = g3d.perp_grid();
+    dg::Handle<dg::aMPIGeometry2d> g2d = g3d.perp_grid();
     t.toc();
     if(rank==0)std::cout << "Construction took "<<t.diff()<<"s"<<std::endl;
     int ncid;
     file::NC_Error_Handle err;
     MPI_Info info = MPI_INFO_NULL;
-    err = nc_create_par( "test_mpi.nc", NC_NETCDF4|NC_MPIIO|NC_CLOBBER, g2d.communicator(), info, &ncid); //MPI ON
+    err = nc_create_par( "test_mpi.nc", NC_NETCDF4|NC_MPIIO|NC_CLOBBER, g2d.get().communicator(), info, &ncid); //MPI ON
     int dim3d[2];
-    err = file::define_dimensions(  ncid, dim3d, g2d.global());
+    err = file::define_dimensions(  ncid, dim3d, g2d.get().global());
     int coordsID[2], onesID, defID,confID, volID, divBID;
     err = nc_def_var( ncid, "x_XYP", NC_DOUBLE, 2, dim3d, &coordsID[0]);
     err = nc_def_var( ncid, "y_XYP", NC_DOUBLE, 2, dim3d, &coordsID[1]);
@@ -85,8 +84,8 @@ int main( int argc, char* argv[])
     err = nc_def_var( ncid, "divB", NC_DOUBLE, 2, dim3d, &divBID);
 
     int dims[2], periods[2],  coords[2];
-    MPI_Cart_get( g2d.communicator(), 2, dims, periods, coords);
-    size_t count[2] = {g2d.n()*g2d.Ny(), g2d.n()*g2d.Nx()};
+    MPI_Cart_get( g2d.get().communicator(), 2, dims, periods, coords);
+    size_t count[2] = {g2d.get().n()*g2d.get().Ny(), g2d.get().n()*g2d.get().Nx()};
     size_t start[2] = {coords[1]*count[0], coords[0]*count[1]};
     err = nc_var_par_access( ncid, coordsID[0], NC_COLLECTIVE);
     err = nc_var_par_access( ncid, coordsID[1], NC_COLLECTIVE);
@@ -94,30 +93,30 @@ int main( int argc, char* argv[])
     err = nc_var_par_access( ncid, defID, NC_COLLECTIVE);
     err = nc_var_par_access( ncid, divBID, NC_COLLECTIVE);
 
-    dg::MHVec psi_p = dg::pullback( psip.f(), g2d);
+    dg::MHVec psi_p = dg::pullback( psip.f(), g2d.get());
     //g.display();
     err = nc_put_vara_double( ncid, onesID, start, count, psi_p.data().data());
-    dg::HVec X( g2d.size()), Y(X); //P = dg::pullback( dg::coo3, g);
-    for( unsigned i=0; i<g2d.size(); i++)
+    dg::HVec X( g2d.get().size()), Y(X); //P = dg::pullback( dg::coo3, g);
+    for( unsigned i=0; i<g2d.get().size(); i++)
     {
-        X[i] = g2d.map()[0].data()[i];
-        Y[i] = g2d.map()[0].data()[i];
+        X[i] = g2d.get().map()[0].data()[i];
+        Y[i] = g2d.get().map()[0].data()[i];
     }
 
-    dg::MHVec temp0( dg::evaluate(dg::zero, g2d)), temp1(temp0);
-    dg::MHVec w2d = dg::create::weights( g2d);
+    dg::MHVec temp0( dg::evaluate(dg::zero, g2d.get())), temp1(temp0);
+    dg::MHVec w2d = dg::create::weights( g2d.get());
 
     err = nc_put_vara_double( ncid, coordsID[0], start,count, X.data());
     err = nc_put_vara_double( ncid, coordsID[1], start,count, Y.data());
 
-    dg::SparseTensor<dg::MHVec> metric = g2d.metric();
+    dg::SparseTensor<dg::MHVec> metric = g2d.get().metric();
     dg::MHVec g_xx = metric.value(0,0), g_xy = metric.value(0,1), g_yy=metric.value(1,1);
     dg::SparseElement<dg::MHVec> vol_ = dg::tensor::volume(metric);
     dg::MHVec vol = vol_.value();
     //err = nc_put_vara_double( ncid, coordsID[2], g.z().data());
-    //dg::blas1::pointwiseDivide( g2d.g_xy(), g2d.g_xx(), temp0);
+    //dg::blas1::pointwiseDivide( g2d.get().g_xy(), g2d.get().g_xx(), temp0);
     dg::blas1::pointwiseDivide( g_yy, g_xx, temp0);
-    const dg::MHVec ones = dg::evaluate( dg::one, g2d);
+    const dg::MHVec ones = dg::evaluate( dg::one, g2d.get());
     dg::blas1::axpby( 1., ones, -1., temp0, temp0);
     dg::blas1::transfer( temp0.data(), X);
     err = nc_put_vara_double( ncid, defID, start,count, X.data());
@@ -162,7 +161,7 @@ int main( int argc, char* argv[])
     else               gp.psipmax = psi_0, gp.psipmin = psi_1;
     dg::geo::Iris iris(psip.f(), gp.psipmin, gp.psipmax);
     //dg::CylindricalGrid3d<dg::HVec> g3d( gp.R_0 -2.*gp.a, gp.R_0 + 2*gp.a, -2*gp.a, 2*gp.a, 0, 2*M_PI, 3, 2200, 2200, 1, dg::PER, dg::PER, dg::PER);
-    dg::CartesianMPIGrid2d g2dC( gp.R_0 -2.*gp.a, gp.R_0 + 2.*gp.a, -2.*gp.a, 2.*gp.a, 1, 2e3, 2e3, dg::DIR, dg::PER, g2d.communicator());
+    dg::CartesianMPIGrid2d g2dC( gp.R_0 -2.*gp.a, gp.R_0 + 2.*gp.a, -2.*gp.a, 2.*gp.a, 1, 2e3, 2e3, dg::DIR, dg::PER, g2d.get().communicator());
     dg::MHVec vec  = dg::evaluate( iris, g2dC);
     dg::MHVec R  = dg::evaluate( dg::cooX2d, g2dC);
     dg::MHVec g2d_weights = dg::create::volume( g2dC);
diff --git a/inc/geometries/ribeiro_t.cu b/inc/geometries/ribeiro_t.cu
index ba61fafdb..524012d2d 100644
--- a/inc/geometries/ribeiro_t.cu
+++ b/inc/geometries/ribeiro_t.cu
@@ -18,7 +18,6 @@
 
 #include "file/nc_utilities.h"
 
-using namespace dg::geo::solovev;
 thrust::host_vector<double> periodify( const thrust::host_vector<double>& in, const dg::Grid2d& g)
 {
     thrust::host_vector<double> out(g.size());
@@ -60,7 +59,7 @@ int main( int argc, char* argv[])
         reader.parse(is,js,false);
     }
     //write parameters from file into variables
-    dg::geo::solovev::GeomParameters gp(js);
+    dg::geo::solovev::Parameters gp(js);
     dg::geo::BinaryFunctorsLvl2 psip = dg::geo::solovev::createPsip( gp);
     std::cout << "Psi min "<<psip.f()(gp.R_0, 0)<<"\n";
     std::cout << "Type psi_0 and psi_1\n";
diff --git a/inc/geometries/separatrix_orthogonal.h b/inc/geometries/separatrix_orthogonal.h
index 0ad647b3a..0bb150ee8 100644
--- a/inc/geometries/separatrix_orthogonal.h
+++ b/inc/geometries/separatrix_orthogonal.h
@@ -341,7 +341,7 @@ struct SeparatrixOrthogonal : public aGeneratorX2d
 // */ 
 //struct XField
 //{
-//    XField( dg::geo::solovev::GeomParameters gp,const dg::GridX2d& gXY, const thrust::host_vector<double>& g):
+//    XField( dg::geo::solovev::Parameters gp,const dg::GridX2d& gXY, const thrust::host_vector<double>& g):
 //        gp_(gp),
 //        psipR_(gp), psipZ_(gp),
 //        ipol_(gp), invB_(gp), gXY_(gXY), g_(dg::create::forward_transform(g, gXY)) 
diff --git a/inc/geometries/separatrix_orthogonal_t.cu b/inc/geometries/separatrix_orthogonal_t.cu
index 30b3702d5..78fd1a35e 100644
--- a/inc/geometries/separatrix_orthogonal_t.cu
+++ b/inc/geometries/separatrix_orthogonal_t.cu
@@ -99,7 +99,7 @@ int main( int argc, char* argv[])
         std::ifstream is(argv[1]);
         reader.parse(is,js,false);
     }
-    dg::geo::taylor::GeomParameters gp(js);
+    dg::geo::taylor::Parameters gp(js);
     dg::Timer t;
     std::cout << "Type psi_0 \n";
     double psi_0 = -16;
diff --git a/inc/geometries/simple_orthogonal_t.cu b/inc/geometries/simple_orthogonal_t.cu
index 9600988b7..a5c194b65 100644
--- a/inc/geometries/simple_orthogonal_t.cu
+++ b/inc/geometries/simple_orthogonal_t.cu
@@ -61,7 +61,7 @@ int main( int argc, char* argv[])
         std::ifstream is(argv[1]);
         reader.parse(is,js,false);
     }
-    dg::geo::solovev::GeomParameters gp(js);
+    dg::geo::solovev::Parameters gp(js);
     dg::geo::BinaryFunctorsLvl2 psip=dg::geo::solovev::createPsip(gp);
     std::cout << "Psi min "<<psip.f()(gp.R_0, 0)<<"\n";
     std::cout << "Type psi_0 and psi_1\n";
diff --git a/inc/geometries/solovev.h b/inc/geometries/solovev.h
index c25d6d454..4e9046d95 100644
--- a/inc/geometries/solovev.h
+++ b/inc/geometries/solovev.h
@@ -352,18 +352,18 @@ struct IpolZ: public aCloneableBinaryFunctor<IpolZ>
     PsipZ psipZ_;
 };
 
-BinaryFunctorsLvl2 createPsip( Parameters gp)
+dg::geo::BinaryFunctorsLvl2 createPsip( Parameters gp)
 {
     BinaryFunctorsLvl2 psip( new Psip(gp), new PsipR(gp), new PsipZ(gp),new PsipRR(gp), new PsipRZ(gp), new PsipZZ(gp));
     return psip;
 }
-BinaryFunctorsLvl1 createIpol( Parameters gp)
+dg::geo::BinaryFunctorsLvl1 createIpol( Parameters gp)
 {
     BinaryFunctorsLvl1 ipol( new Ipol(gp), new IpolR(gp), new IpolZ(gp));
     return ipol;
 }
 
-TokamakMagneticField createMagField( Parameters gp)
+dg::geo::TokamakMagneticField createMagField( Parameters gp)
 {
     return TokamakMagneticField( gp.R_0, createPsip(gp), createIpol(gp));
 }
diff --git a/src/toefl/Makefile b/src/toefl/Makefile
index d22bdbba9..febec8d4d 100644
--- a/src/toefl/Makefile
+++ b/src/toefl/Makefile
@@ -17,7 +17,7 @@ toefl_hpc: toefl_hpc.cu toeflR.cuh
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -DDG_BENCHMARK  -g
 
 toefl_mpi: toefl_mpi.cu toeflR.cuh 
-	$(MPICC) $(OPT) $(MPICFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -DDG_BENCHMARK
+	$(MPICC) $(OPT) $(MPICFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) #-DDG_BENCHMARK
 
 doc: 
 	mkdir -p doc; \
-- 
GitLab


From 9e7277ff0a7d8612dbc646ef9de70e24df8ac5f1 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Tue, 24 Oct 2017 14:23:31 +0200
Subject: [PATCH 387/453] generate more code examples for documentation

---
 inc/dg/elliptic.h               |  2 ++
 inc/geometries/Doxyfile         |  2 +-
 inc/geometries/curvilinear.h    |  2 +-
 inc/geometries/flux.h           |  4 +++-
 inc/geometries/flux_t.cu        | 16 +++++++++++-----
 inc/geometries/fluxfunctions.h  |  2 ++
 inc/geometries/generator.h      |  3 ++-
 inc/geometries/geometries_doc.h |  4 ++--
 inc/geometries/hector_t.cu      | 30 +++++++++++++++---------------
 9 files changed, 39 insertions(+), 26 deletions(-)

diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index e29cf745b..7df62ffaf 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -603,6 +603,8 @@ struct TensorElliptic
         set( chixx_, chixy_, chiyy_);
     }
 
+    ///@copydoc Elliptic::weights()
+    const container& weights()const {return weights_;}
     ///@copydoc Elliptic::inv_weights()
     const container& inv_weights()const {return inv_weights_;}
     ///@copydoc GeneralElliptic::precond()
diff --git a/inc/geometries/Doxyfile b/inc/geometries/Doxyfile
index f15c0afd6..272887edc 100644
--- a/inc/geometries/Doxyfile
+++ b/inc/geometries/Doxyfile
@@ -856,7 +856,7 @@ EXCLUDE_SYMBOLS        = hide_*
 # that contain example code fragments that are included (see the \include
 # command).
 
-EXAMPLE_PATH           = ds_t.cu hector_t.cu
+EXAMPLE_PATH           = ds_t.cu hector_t.cu flux_t.cu
 
 # If the value of the EXAMPLE_PATH tag contains directories, you can use the
 # EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and
diff --git a/inc/geometries/curvilinear.h b/inc/geometries/curvilinear.h
index f71dac0a2..d63efba49 100644
--- a/inc/geometries/curvilinear.h
+++ b/inc/geometries/curvilinear.h
@@ -75,7 +75,7 @@ void square( const dg::SparseTensor<thrust::host_vector<double> >& jac, const th
 /**
  * @brief A two-dimensional grid based on curvilinear coordinates
  *
- * @snippet hector_t.cu doxygen
+ * @snippet flux_t.cu doxygen
  */
 struct CurvilinearGrid2d : public dg::aGeometry2d
 {
diff --git a/inc/geometries/flux.h b/inc/geometries/flux.h
index 2f3ba3d6a..e6b8b9c6d 100644
--- a/inc/geometries/flux.h
+++ b/inc/geometries/flux.h
@@ -123,7 +123,9 @@ struct Fpsi
 
 /**
  * @brief A symmetry flux generator
+ *
  * @ingroup generators_geo
+ * @snippet flux_t.cu doxygen
  */
 struct FluxGenerator : public aGenerator2d
 {
@@ -139,7 +141,7 @@ struct FluxGenerator : public aGenerator2d
      * @param mode This parameter indicates the adaption type used to create the grid: 0 is no adaption, 1 is an equalarc adaption
      * @note If mode == 1 then this class does the same as the RibeiroFluxGenerator
      */
-    FluxGenerator( const BinaryFunctorsLvl2& psi, const BinaryFunctorsLvl1 ipol, double psi_0, double psi_1, double x0, double y0, int mode=0):
+    FluxGenerator( const BinaryFunctorsLvl2& psi, const BinaryFunctorsLvl1& ipol, double psi_0, double psi_1, double x0, double y0, int mode=0):
         psi_(psi), ipol_(ipol), mode_(mode)
     {
         psi0_ = psi_0, psi1_ = psi_1;
diff --git a/inc/geometries/flux_t.cu b/inc/geometries/flux_t.cu
index cb5debbc2..00f55072e 100644
--- a/inc/geometries/flux_t.cu
+++ b/inc/geometries/flux_t.cu
@@ -38,9 +38,9 @@ thrust::host_vector<double> periodify( const thrust::host_vector<double>& in, co
     return out;
 }
 
-double sineX( double x, double y) {return sin(x)*sin(y);}
+double sineX(   double x, double y) {return sin(x)*sin(y);}
 double cosineX( double x, double y) {return cos(x)*sin(y);}
-double sineY( double x, double y) {return sin(x)*sin(y);}
+double sineY(   double x, double y) {return sin(x)*sin(y);}
 double cosineY( double x, double y) {return sin(x)*cos(y);}
 
 int main( int argc, char* argv[])
@@ -62,8 +62,8 @@ int main( int argc, char* argv[])
     }
     //write parameters from file into variables
     dg::geo::solovev::Parameters gp(js);
-    dg::geo::TokamakMagneticField c = dg::geo::createSolovevField( gp);
-    std::cout << "Psi min "<<c.psip()(gp.R_0, 0)<<"\n";
+    {dg::geo::TokamakMagneticField c = dg::geo::createSolovevField( gp);
+    std::cout << "Psi min "<<c.psip()(gp.R_0, 0)<<"\n";}
     std::cout << "Type psi_0 (-20) and psi_1 (-4)\n";
     double psi_0, psi_1;
     std::cin >> psi_0>> psi_1;
@@ -72,9 +72,15 @@ int main( int argc, char* argv[])
     //solovev::detail::Fpsi fpsi( gp, -10);
     std::cout << "Constructing flux grid ... \n";
     t.tic();
+    //![doxygen]
+    //create the magnetic field
+    dg::geo::TokamakMagneticField c = dg::geo::createSolovevField( gp);
+    //create a grid generator
     dg::geo::FluxGenerator flux( c.get_psip(), c.get_ipol(), psi_0, psi_1, gp.R_0, 0., 1);
-    dg::geo::CurvilinearProductGrid3d g3d(flux, n, Nx, Ny,Nz, dg::DIR);
+    //create a grid 
     dg::geo::CurvilinearGrid2d g2d(flux, n, Nx,Ny, dg::NEU);
+    //![doxygen]
+    dg::geo::CurvilinearProductGrid3d g3d(flux, n, Nx, Ny,Nz, dg::DIR);
     dg::Grid2d g2d_periodic(g2d.x0(), g2d.x1(), g2d.y0(), g2d.y1(), g2d.n(), g2d.Nx(), g2d.Ny()+1); 
     t.toc();
     std::cout << "Construction took "<<t.diff()<<"s"<<std::endl;
diff --git a/inc/geometries/fluxfunctions.h b/inc/geometries/fluxfunctions.h
index 47b6e6d1d..360212e82 100644
--- a/inc/geometries/fluxfunctions.h
+++ b/inc/geometries/fluxfunctions.h
@@ -155,6 +155,8 @@ struct BinaryFunctorsLvl1
 };
 /**
 * @brief This struct bundles a function and its first and second derivatives
+*
+* @snippet hector_t.cu doxygen
 */
 struct BinaryFunctorsLvl2 
 {
diff --git a/inc/geometries/generator.h b/inc/geometries/generator.h
index f42eb20fc..b4c4e8fc4 100644
--- a/inc/geometries/generator.h
+++ b/inc/geometries/generator.h
@@ -10,7 +10,8 @@ namespace geo
 
 A generator is there to construct coordinate transformations from physical coordinates
 \f$ x,y\f$ to the computational domain \f$\zeta, \eta\f$, which
-is a product space. 
+is a product space. It can be used to construct curvilinear grids like the following code snippet demonstrates:
+@snippet hector_t.cu doxygen 
 @note the origin of the computational space is assumed to be (0,0)
  @ingroup generators_geo
 */
diff --git a/inc/geometries/geometries_doc.h b/inc/geometries/geometries_doc.h
index 69ce6d7ab..d889c3c02 100644
--- a/inc/geometries/geometries_doc.h
+++ b/inc/geometries/geometries_doc.h
@@ -1,11 +1,11 @@
 #error Documentation only
 /*! 
  * 
- * @defgroup grids 1. New geometric grids
- * @defgroup generators_geo 2. Grid generators
+ * @defgroup generators_geo 1. Grid generators
  *
       All the grids introduced by this extension can be constructed with 
       generator classes. 
+ * @defgroup grids 2. New geometric grids
  * @defgroup fluxfunctions 3. New functors based on the magnetic field geometry
 
  All functors in this section model two or three-dimensional functions, i.e. they all overload the operator() like aBinaryOperator
diff --git a/inc/geometries/hector_t.cu b/inc/geometries/hector_t.cu
index 766a67d90..cf200a390 100644
--- a/inc/geometries/hector_t.cu
+++ b/inc/geometries/hector_t.cu
@@ -4,6 +4,7 @@
 #include <fstream>
 #include <sstream>
 #include <cmath>
+#include <memory>
 
 #include "dg/backend/xspacelib.cuh"
 #include "dg/functors.h"
@@ -18,8 +19,6 @@
 
 #include "file/nc_utilities.h"
 
-using namespace dg::geo;
-
 thrust::host_vector<double> periodify( const thrust::host_vector<double>& in, const dg::Grid2d& g)
 {
     thrust::host_vector<double> out(g.size());
@@ -63,8 +62,8 @@ int main( int argc, char* argv[])
     }
     //write parameters from file into variables
     dg::geo::solovev::Parameters gp(js);
-    dg::geo::BinaryFunctorsLvl2 psip = dg::geo::solovev::createPsip( gp); 
-    std::cout << "Psi min "<<psip.f()(gp.R_0, 0)<<"\n";
+    {const dg::geo::BinaryFunctorsLvl2 psip = dg::geo::solovev::createPsip( gp); 
+    std::cout << "Psi min "<<psip.f()(gp.R_0, 0)<<"\n";}
     std::cout << "Type psi_0 and psi_1\n";
     double psi_0, psi_1;
     std::cin >> psi_0>> psi_1;
@@ -75,30 +74,32 @@ int main( int argc, char* argv[])
     int construction = 0;
     t.tic();
     //![doxygen]
-    Hector<dg::IDMatrix, dg::DMatrix, dg::DVec>* hector;
+    std::unique_ptr< dg::geo::aGenerator2d > hector;
+    const dg::geo::BinaryFunctorsLvl2 psip = dg::geo::solovev::createPsip( gp); 
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     if( construction == 0)
     {
-        hector = new Hector<dg::IDMatrix, dg::DMatrix, dg::DVec>( 
-                psip, psi_0, psi_1, gp.R_0, 0., nGrid, NxGrid, NyGrid, epsHector, true);
+        hector.reset( new dg::geo::Hector<dg::IDMatrix, dg::DMatrix, dg::DVec>( 
+                psip, psi_0, psi_1, gp.R_0, 0., nGrid, NxGrid, NyGrid, epsHector, true));
     }
     else if( construction == 1)
     {
         dg::geo::BinaryFunctorsLvl1 nc = dg::geo::make_NablaPsiInvCollective( psip);
-        hector = new Hector<dg::IDMatrix, dg::DMatrix, dg::DVec>( 
-                psip, nc, psi_0, psi_1, gp.R_0, 0., nGrid, NxGrid, NyGrid, epsHector, true);
+        hector.reset( new dg::geo::Hector<dg::IDMatrix, dg::DMatrix, dg::DVec>( 
+                psip, nc, psi_0, psi_1, gp.R_0, 0., nGrid, NxGrid, NyGrid, epsHector, true));
     }
     else
     {
         dg::geo::BinarySymmTensorLvl1 lc = dg::geo::make_LiseikinCollective( 
                 psip, 0.1, 0.001);
-        hector = new Hector<dg::IDMatrix, dg::DMatrix, dg::DVec>( 
-                psip,lc, psi_0, psi_1, gp.R_0, 0., nGrid, NxGrid, NyGrid, epsHector, true);
+        hector.reset( new dg::geo::Hector<dg::IDMatrix, dg::DMatrix, dg::DVec>( 
+                psip,lc, psi_0, psi_1, gp.R_0, 0., nGrid, NxGrid, NyGrid, epsHector, true));
     }
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     dg::geo::CurvilinearProductGrid3d g3d(*hector, n, Nx, Ny,Nz, dg::DIR);
-    dg::geo::CurvilinearGrid2d g2d = g3d.perp_grid();
+    std::unique_ptr<const dg::aGeometry2d> g2d_ptr( g3d.perp_grid());
     //![doxygen]
+    const dg::aGeometry2d& g2d = *g2d_ptr;
 
     dg::Grid2d g2d_periodic(g2d.x0(), g2d.x1(), g2d.y0(), g2d.y1(), g2d.n(), g2d.Nx(), g2d.Ny()+1); 
     t.toc();
@@ -167,13 +168,12 @@ int main( int argc, char* argv[])
     dg::HVec ones2d = dg::evaluate( dg::one, g2d);
     double volumeUV = dg::blas1::dot( vol.value(), ones2d);
 
-    volume = dg::create::volume( hector->internal_grid());
-    ones2d = dg::evaluate( dg::one, hector->internal_grid());
+    volume = dg::create::volume( dynamic_cast<dg::geo::Hector<dg::IDMatrix, dg::DMatrix, dg::DVec>*>( hector.get())->internal_grid());
+    ones2d = dg::evaluate( dg::one, dynamic_cast<dg::geo::Hector<dg::IDMatrix, dg::DMatrix, dg::DVec>*>( hector.get())->internal_grid());
     double volumeZE = dg::blas1::dot( vol.value(), ones2d);
     std::cout << "volumeUV is "<< volumeUV<<std::endl;
     std::cout << "volumeZE is "<< volumeZE<<std::endl;
     std::cout << "relative difference in volume is "<<fabs(volumeUV - volumeZE)/volumeZE<<std::endl;
     err = nc_close( ncid);
-    delete hector;
     return 0;
 }
-- 
GitLab


From a941b1864732c368f3575022ccfab92b9e6259c7 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Tue, 24 Oct 2017 17:27:03 +0200
Subject: [PATCH 388/453] corrected gradien -> gradient

---
 inc/dg/multigrid.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index 1bc73a00d..f9295918c 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -16,7 +16,7 @@ namespace dg
 /**
 * @brief Class for the solution of symmetric matrix equation discretizeable on multiple grids
 *
-* We use conjugate gradien (CG) at each stage and refine the grids in the first two dimensions (2d / x and y) 
+* We use conjugate gradient (CG) at each stage and refine the grids in the first two dimensions (2d / x and y) 
 * @snippet elliptic2d_b.cu multigrid
 * @copydoc hide_geometry_matrix_container
 * @ingroup multigrid
-- 
GitLab


From 13652157ac5d5672e5354cfdc7842da1d5651231 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Wed, 25 Oct 2017 13:36:21 +0200
Subject: [PATCH 389/453] update documentation of interpolation routines

---
 inc/dg/Doxyfile                   |  4 ++-
 inc/dg/backend/interpolation.cuh  | 31 +++++++++++++++-------
 inc/dg/backend/interpolation_t.cu | 43 ++++++++++++++++++++-----------
 inc/dg/backend/projection.cuh     | 19 ++++++++++----
 inc/dg/cg.h                       |  2 +-
 5 files changed, 67 insertions(+), 32 deletions(-)

diff --git a/inc/dg/Doxyfile b/inc/dg/Doxyfile
index f9f42adcd..38d44516e 100644
--- a/inc/dg/Doxyfile
+++ b/inc/dg/Doxyfile
@@ -867,7 +867,9 @@ EXCLUDE_SYMBOLS        = hide_*
 EXAMPLE_PATH           = cg2d_t.cu \
                          arakawa_t.cu poisson_t.cu \
                          elliptic2d_b.cu runge_kutta2d_t.cu multistep_t.cu \
-                         helmholtzg2_b.cu
+                         helmholtzg2_b.cu \
+                         backend/interpolation_t.cu
+
 
 
 
diff --git a/inc/dg/backend/interpolation.cuh b/inc/dg/backend/interpolation.cuh
index 7ce9646dc..60911d635 100644
--- a/inc/dg/backend/interpolation.cuh
+++ b/inc/dg/backend/interpolation.cuh
@@ -70,7 +70,10 @@ std::vector<double> coefficients( double xn, unsigned n)
 /**
  * @brief Create interpolation matrix
  *
- * The matrix, when applied to a vector, interpolates its values to the given coordinates
+ * The created matrix has \c g.size() columns and \c x.size() rows. It uses 
+ * polynomial interpolation given by the dG polynomials, i.e. the interpolation has order \c g.n() .
+ * When applied to a vector the result contains the interpolated values at the given interpolation points. 
+ * @sa <a href="./dg_introduction.pdf" target="_blank">Introduction to dg methods</a>
  * @param x X-coordinates of interpolation points
  * @param g The Grid on which to operate
  *
@@ -115,16 +118,20 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const thrust::ho
 /**
  * @brief Create interpolation matrix
  *
- * The matrix, when applied to a vector, interpolates its values to the given coordinates
+ * The created matrix has \c g.size() columns and \c x.size() rows. It uses 
+ * polynomial interpolation given by the dG polynomials, i.e. the interpolation has order \c g.n() .
+ * When applied to a vector the result contains the interpolated values at the given interpolation points. 
+ * @snippet backend/interpolation_t.cu doxygen
+ * @sa <a href="./dg_introduction.pdf" target="_blank">Introduction to dg methods</a>
  * @param x X-coordinates of interpolation points
- * @param y Y-coordinates of interpolation points
+ * @param y Y-coordinates of interpolation points ( has to have equal size as x)
  * @param g The Grid on which to operate
  * @param bcx determines what to do when a point lies exactly on the boundary in x:  DIR generates zeroes in the interpolation matrix, 
  NEU and PER interpolate the inner side polynomial. (DIR_NEU and NEU_DIR apply NEU / DIR to the respective left or right boundary )
  * @param bcy determines what to do when a point lies exactly on the boundary in y. Behaviour correponds to bcx.
- * @attention all points (x,y) must lie within or on the boundaries of g.
  *
  * @return interpolation matrix
+ * @attention all points (x,y) must lie within or on the boundaries of g.
  */
 cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const thrust::host_vector<double>& x, const thrust::host_vector<double>& y, const aTopology2d& g , dg::bc bcx = dg::NEU, dg::bc bcy = dg::NEU)
 {
@@ -255,7 +262,11 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const thrust::ho
 /**
  * @brief Create interpolation matrix
  *
- * The matrix, when applied to a vector, interpolates its values to the given coordinates. In z-direction only a nearest neighbor interpolation is used
+ * The created matrix has \c g.size() columns and \c x.size() rows. It uses 
+ * polynomial interpolation given by the dG polynomials, i.e. the interpolation has order \c g.n() .
+ * When applied to a vector the result contains the interpolated values at the given interpolation points. 
+ * @snippet backend/interpolation_t.cu doxygen3d
+ * @sa <a href="./dg_introduction.pdf" target="_blank">Introduction to dg methods</a>
  * @param x X-coordinates of interpolation points
  * @param y Y-coordinates of interpolation points
  * @param z Z-coordinates of interpolation points
@@ -400,13 +411,13 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolation( const thrust::ho
 /**
  * @brief Create interpolation between two grids
  *
- * This matrix can be applied to vectors defined on the old grid to obtain
- * its values on the new grid.
+ * This matrix interpolates vectors on the old grid \c g_old to the %Gaussian nodes of the new grid \c g_new. The interpolation is of the order \c g_old.n()
+ * @sa <a href="./dg_introduction.pdf" target="_blank">Introduction to dg methods</a>
  * 
- * @param g_new The new points 
+ * @param g_new The new grid
  * @param g_old The old grid
  *
- * @return Interpolation matrix
+ * @return Interpolation matrix with \c g_old.size() columns and \c g_new.size() rows
  * @note The boundaries of the old grid must lie within the boundaries of the new grid
  * @note also check the transformation matrix, which is the more general solution
  */
@@ -488,7 +499,7 @@ thrust::host_vector<double> forward_transform( const thrust::host_vector<double>
  *
  * @ingroup interpolation
  * @return interpolated point
- * @note g.contains(x,y) must return true
+ * @note \c g.contains(x,y) must return true
  */
 double interpolate( double x, double y,  const thrust::host_vector<double>& v, const aTopology2d& g )
 {
diff --git a/inc/dg/backend/interpolation_t.cu b/inc/dg/backend/interpolation_t.cu
index b0b6c5053..dc8537e83 100644
--- a/inc/dg/backend/interpolation_t.cu
+++ b/inc/dg/backend/interpolation_t.cu
@@ -20,16 +20,16 @@ int main()
 {
 
     {
-    dg::Grid2d g( -10, 10, -5, 5, n, Nx, Ny);
     std::cout << "First test grid set functions: \n";
+    dg::Grid2d g( -10, 10, -5, 5, n, Nx, Ny);
     g.display( std::cout);
     g.set(2,2,3);
     g.display( std::cout);
     g.set(n, Nx, Ny);
     g.display( std::cout);
-    Matrix A = dg::create::backscatter( g);
-    //A.sort_by_row_and_column();
 
+    //![doxygen]
+    //create equidistant values
     thrust::host_vector<double> x( g.size()), y(x);
     for( unsigned i=0; i<g.Ny()*g.n(); i++)
         for( unsigned j=0; j<g.Nx()*g.n(); j++)
@@ -39,13 +39,24 @@ int main()
             y[i*g.Nx()*g.n() + j] = 
                     g.y0() + (i+0.5)*g.hy()/(double)(g.n());
         }
+    //typedef cusp::coo_matrix<int, double, cusp::host_memory> Matrix;
     Matrix B = dg::create::interpolation( x, y, g);
-    thrust::host_vector<double> vec = dg::evaluate( function, g), inter1(vec), inter2(vec);
+
+    const thrust::host_vector<double> vec = dg::evaluate( function, g); 
+    thrust::host_vector<double> inter(vec);
+    dg::blas2::symv( B, vec, inter);
+    //inter now contains the values of vec interpolated at equidistant points
+    //![doxygen]
+    Matrix A = dg::create::backscatter( g);
+    thrust::host_vector<double> inter1(vec);
     dg::blas2::symv( A, vec, inter1);
-    dg::blas2::symv( B, vec, inter2);
-    dg::blas1::axpby( 1., inter1, -1., inter2, vec);
-    double error = dg::blas1::dot( vec, vec);
+    dg::blas1::axpby( 1., inter1, -1., inter, inter1);
+    double error = dg::blas1::dot( inter1, inter1);
     std::cout << "Error is "<<error<<" (should be small)!\n";
+    if( error > 1e-14) 
+        std::cout<< "2D TEST FAILED!\n";
+    else
+        std::cout << "2D TEST PASSED!\n";
     //cusp::print(A);
     //cusp::print(B);
     //ATTENTION: backscatter might delete zeroes in matrices
@@ -62,10 +73,6 @@ int main()
     //    std::cerr << "Number of entries not equal!\n";
     //    passed = false;
     //}
-    if( error > 1e-14) 
-        std::cout<< "2D TEST FAILED!\n";
-    else
-        std::cout << "2D TEST PASSED!\n";
 
 
     bool passed = true;
@@ -99,6 +106,8 @@ int main()
     Matrix A = dg::create::backscatter( g);
     //A.sort_by_row_and_column();
 
+    //![doxygen3d]
+    //create equidistant values
     std::vector<double> x( g.size()), y(x), z(x);
     for( unsigned k=0; k<g.Nz(); k++)
         for( unsigned i=0; i<g.Ny()*g.n(); i++)
@@ -111,12 +120,16 @@ int main()
                 z[(k*g.Ny()*g.n() + i)*g.Nx()*g.n() + j] = 
                         g.z0() + (k+0.5)*g.hz();
             }
+    //typedef cusp::coo_matrix<int, double, cusp::host_memory> Matrix;
     Matrix B = dg::create::interpolation( x, y, z, g);
-    thrust::host_vector<double> vec = dg::evaluate( function, g), inter1(vec), inter2(vec);
+    const thrust::host_vector<double> vec = dg::evaluate( function, g); 
+    thrust::host_vector<double> inter(vec);
+    dg::blas2::symv( B, vec, inter);
+    //![doxygen3d]
+    thrust::host_vector<double> inter1(vec);
     dg::blas2::symv( A, vec, inter1);
-    dg::blas2::symv( B, vec, inter2);
-    dg::blas1::axpby( 1., inter1, -1., inter2, vec);
-    double error = dg::blas1::dot( vec, vec);
+    dg::blas1::axpby( 1., inter1, -1., inter, inter1);
+    double error = dg::blas1::dot( inter1, inter1);
     std::cout << "Error is "<<error<<" (should be small)!\n";
     if( error > 1e-14) 
         std::cout<< "3D TEST FAILED!\n";
diff --git a/inc/dg/backend/projection.cuh b/inc/dg/backend/projection.cuh
index a6293e2c2..28a63d890 100644
--- a/inc/dg/backend/projection.cuh
+++ b/inc/dg/backend/projection.cuh
@@ -54,6 +54,12 @@ namespace create{
 /**
  * @brief Create the transpose of the interpolation matrix from new to old
  *
+ * Does the equivalent of the following
+ * @code
+   Matrix A = dg::create::interpolation( g_old, g_new);
+   return A.transpose();
+   @endcode
+ * @sa <a href="./dg_introduction.pdf" target="_blank">Introduction to dg methods</a>
  * @param g_new The new grid 
  * @param g_old The old grid
  *
@@ -84,21 +90,23 @@ cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const aTopology
 /**
  * @brief Create a projection between two grids
  *
- * The projection matrix is the adjoint of the interpolation matrix
  * This matrix can be applied to vectors defined on the old (fine) grid to obtain
- * its values on the new (coarse) grid. 
+ * its values projected on the new (coarse) grid. (Projection means that the
+ * projection integrals over the base polynomials are computed).
  * If the fine grid is a multiple of the coarse grid, the integral value
  of the projected vector will be conserved and the difference in the L2 norm 
  between old and new vector small. 
+ * The projection matrix is the adjoint of the interpolation matrix
+ * @sa <a href="./dg_introduction.pdf" target="_blank">Introduction to dg methods</a>
  * 
  * @param g_new The new (coarse) grid 
  * @param g_old The old (fine) grid
  *
  * @return Projection matrix
  * @note The boundaries of the old grid must lie within the boundaries of the new grid
- * @note also check the transformation matrix, which is the more general solution
- @attention Projection only works if the number of cells in the
- fine grid are multiple of the number of cells in the coarse grid
+ * @note also check \c dg::create::transformation, which is the more general solution
+ * @attention Projection only works if the number of cells in the
+ * fine grid is a multiple of the number of cells in the coarse grid
  */
 cusp::coo_matrix< int, double, cusp::host_memory> projection( const Grid1d& g_new, const Grid1d& g_old)
 {
@@ -193,6 +201,7 @@ cusp::coo_matrix< int, double, cusp::host_memory> projection( const aTopology3d&
  \f] 
  where \f$ Q\f$ is the interpolation matrix and \f$ P \f$ the projection. If either new or
  old grid is already the lcm grid this function reduces to the interpolation/projection function. 
+ * @sa <a href="./dg_introduction.pdf" target="_blank">Introduction to dg methods</a>
  * 
  * @param g_new The new grid 
  * @param g_old The old grid
diff --git a/inc/dg/cg.h b/inc/dg/cg.h
index dbca65b4b..c8056adae 100644
--- a/inc/dg/cg.h
+++ b/inc/dg/cg.h
@@ -50,7 +50,7 @@ class CG
      * @brief Allocate memory for the pcg method
      *
      * @param copyable A container must be copy-constructible from this
-     * @param max_iter Maximum number of iterations to be used
+     * @param max_iterations Maximum number of iterations to be used
      */
     void construct( const container& copyable, unsigned max_iterations) { 
         ap = p = r = copyable;
-- 
GitLab


From 2e01f89a6c46ed88373f397ab23a21a53430108c Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Wed, 25 Oct 2017 17:05:35 +0200
Subject: [PATCH 390/453] write a max_iter function in multigrid.h to indicate
 failure of convergence

---
 inc/dg/elliptic.h      |  3 +--
 inc/dg/elliptic2d_b.cu |  4 ++++
 inc/dg/multigrid.h     | 53 ++++++++++++++++++++++++------------------
 src/toefl/toeflR.cuh   | 16 ++++---------
 4 files changed, 40 insertions(+), 36 deletions(-)

diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index 7df62ffaf..3d0cb20a7 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -117,7 +117,6 @@ class Elliptic
      * @brief Change Chi 
      *
      * @param chi The new chi (all elements must be >0)
-     * @note There is no get_chi because chi is multiplied with volume elements
      */
     void set_chi( const container& chi)
     {
@@ -151,7 +150,7 @@ class Elliptic
     /**
      * @brief Return the default preconditioner to use in conjugate gradient
      *
-     * Currently returns the inverse of the weights without volume elment multiplied by the inverse of \f$ \chi\f$. 
+     * Currently returns the inverse weights without volume elment divided by the current \f$ \chi\f$. 
      * This is especially good when \f$ \chi\f$ exhibits large amplitudes or variations
      * @return the inverse of \f$\chi\f$.       
      */
diff --git a/inc/dg/elliptic2d_b.cu b/inc/dg/elliptic2d_b.cu
index 61832f6fa..703f98d84 100644
--- a/inc/dg/elliptic2d_b.cu
+++ b/inc/dg/elliptic2d_b.cu
@@ -73,7 +73,9 @@ int main()
 
     const unsigned stages = 3;
 
+    const dg::CartesianGrid2d grid( 0, lx, 0, ly, n, Nx, Ny, bcx, bcy);
     dg::MultigridCG2d<dg::aGeometry2d, dg::DMatrix, dg::DVec > multigrid( grid, stages);
+    const dg::DVec chi =  dg::evaluate( pol, grid);
     const std::vector<dg::DVec> multi_chi = multigrid.project( chi);
 
     std::vector<dg::Elliptic<dg::aGeometry2d, dg::DMatrix, dg::DVec> > multi_pol( stages);
@@ -86,6 +88,8 @@ int main()
     t.toc();
 
     std::cout << "Creation of multigrid took: "<<t.diff()<<"s\n";
+    const dg::DVec b =    dg::evaluate( rhs,     grid);
+    dg::DVec x       =    dg::evaluate( initial, grid);
     t.tic();
     std::vector<unsigned> number = multigrid.direct_solve(multi_pol, x, b, eps);
     t.toc();
diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index f9295918c..7d520643e 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -14,25 +14,26 @@ namespace dg
 {
 
 /**
-* @brief Class for the solution of symmetric matrix equation discretizeable on multiple grids
+* @brief Class for the solution of a symmetric matrix equation discretizeable on multiple grids
 *
-* We use conjugate gradient (CG) at each stage and refine the grids in the first two dimensions (2d / x and y) 
 * @snippet elliptic2d_b.cu multigrid
+* We use conjugate gradient (CG) at each stage and refine the grids in the first two dimensions (2d / x and y) 
+* @note The preconditioner for the CG solver is taken from the \c precond() method in the \c SymmetricOp class
 * @copydoc hide_geometry_matrix_container
 * @ingroup multigrid
-* @sa Extrapolation  to generate an initial guess
+* @sa \c Extrapolation  to generate an initial guess
 * 
 */
 template< class Geometry, class Matrix, class container> 
 struct MultigridCG2d
 {
     /**
-    * @brief Construct the grids and the interpolation/projection operators
-    *
-    * @param grid the original grid (Nx() and Ny() must be evenly divisable by pow(2, stages-1)
-    * @param stages number of grids in total (The second grid contains half the points of the original grids,  
-    *   The third grid contains half of the second grid ...). Must be > 1
-    * @param scheme_type scheme type in the solve function
+     * @brief Construct the grids and the interpolation/projection operators
+     *
+     * @param grid the original grid (Nx() and Ny() must be evenly divisable by pow(2, stages-1)
+     * @param stages number of grids in total (The second grid contains half the points of the original grids,  
+     *   The third grid contains half of the second grid ...). Must be > 1
+     * @param scheme_type scheme type in the solve function (irrelevant for \c direct_solve method)
     */
     MultigridCG2d( const Geometry& grid, const unsigned stages, const int scheme_type = 0 )
     {
@@ -131,19 +132,20 @@ struct MultigridCG2d
 	}
 
     /**
-    * @brief Nested iterations
-    *
-    * - Compute residual with given initial guess. 
-    * - Project residual down to the coarsest grid. 
-    * - Solve equation on the coarse grid 
-    * - interpolate solution up to next finer grid and repeat until the original grid is reached. 
-    * @copydoc hide_symmetric_op
-    * @param op Index 0 is the matrix on the original grid, 1 on the half grid, 2 on the quarter grid, ...
-    * @param x (read/write) contains initial guess on input and the solution on output
-    * @param b The right hand side (will be multiplied by weights)
-    * @param eps the accuracy: iteration stops if \f$ ||b - Ax|| < \epsilon( ||b|| + 1) \f$ 
-    * @return the number of iterations in each of the stages
-     * @note If the Macro DG_BENCHMARK is defined this function will write timings to std::cout
+     * @brief Nested iterations
+     *
+     * - Compute residual with given initial guess. 
+     * - Project residual down to the coarsest grid. 
+     * - Solve equation on the coarse grid 
+     * - interpolate solution up to next finer grid and repeat until the original grid is reached. 
+     * @note The preconditioner for the CG solver is taken from the \c precond() method in the \c SymmetricOp class
+     * @copydoc hide_symmetric_op
+     * @param op Index 0 is the \c SymmetricOp on the original grid, 1 on the half grid, 2 on the quarter grid, ...
+     * @param x (read/write) contains initial guess on input and the solution on output
+     * @param b The right hand side (will be multiplied by \c weights)
+     * @param eps the accuracy: iteration stops if \f$ ||b - Ax|| < \epsilon( ||b|| + 1) \f$ 
+     * @return the number of iterations in each of the stages
+     * @note If the Macro \c DG_BENCHMARK is defined this function will write timings to \c std::cout
     */
     template<class SymmetricOp>
     std::vector<unsigned> direct_solve( std::vector<SymmetricOp>& op, container&  x, const container& b, double eps)
@@ -221,8 +223,14 @@ struct MultigridCG2d
     ///@return number of stages 
     unsigned stages()const{return stages_;}
 
+    ///observe the grids at all stages
     const std::vector<dg::Handle< Geometry > > grids()const { return grids_; }
 
+    
+    ///After a call to a solution method returns the maximum number of iterations allowed at stage  0
+    ///(if the solution method returns this number, failure is indicated)
+    unsigned max_iter() const{return cg_[0].get_max();} //could this be a problem in MPI?
+
 private:
 
 	void set_scheme(const int scheme_type)
@@ -282,7 +290,6 @@ private:
         assert(u == 0);
 	}
 
-    ///print scheme information to std::cout
     void PrintScheme(void)
     {
         std::cout << "Scheme: " << std::endl;
diff --git a/src/toefl/toeflR.cuh b/src/toefl/toeflR.cuh
index b580f55e4..bfc12433f 100644
--- a/src/toefl/toeflR.cuh
+++ b/src/toefl/toeflR.cuh
@@ -133,12 +133,11 @@ struct Explicit
     std::vector<dg::Helmholtz<Geometry,  Matrix, container> > multi_gamma1;
     dg::ArakawaX< Geometry, Matrix, container> arakawa; 
 
-    dg::Invert<container> invert_invgamma, invert_pol;
     dg::MultigridCG2d<Geometry, Matrix, container> multigrid;
     dg::Extrapolation<container> old_phi, old_psi, old_gammaN;
     std::vector<container> multi_chi;
 
-    const container w2d, v2d, one;
+    const container w2d, one;
     const double eps_pol, eps_gamma; 
     const double kappa, friction, nu, tau;
     const std::string equations;
@@ -158,11 +157,9 @@ Explicit< Geometry, M, container>::Explicit( const Geometry& grid, const Paramet
     pol(     grid, dg::not_normed, dg::centered, p.jfactor), 
     laplaceM( grid, dg::normed, dg::centered),
     arakawa( grid), 
-    invert_invgamma( omega, p.Nx*p.Ny*p.n*p.n, p.eps_gamma),
-    invert_pol(      omega, p.Nx*p.Ny*p.n*p.n, p.eps_pol),
     multigrid( grid, 3),
     old_phi( 2, chi), old_psi( 2, chi), old_gammaN( 2, chi), 
-    w2d( dg::create::volume(grid)), v2d( dg::create::inv_volume(grid)), one( dg::evaluate(dg::one, grid)),
+    w2d( dg::create::volume(grid)), one( dg::evaluate(dg::one, grid)),
     eps_pol(p.eps_pol), eps_gamma( p.eps_gamma), kappa(p.kappa), friction(p.friction), nu(p.nu), tau( p.tau), equations( p.equations), boussinesq(p.boussinesq)
 { 
     multi_chi= multigrid.project( chi);
@@ -185,8 +182,7 @@ const container& Explicit<G, M, container>::compute_psi( const container& potent
         old_psi.extrapolate( phi[1]);
         std::vector<unsigned> number = multigrid.direct_solve( multi_gamma1, phi[1], potential, eps_gamma);
         old_psi.update( phi[1]);
-        //unsigned number = invert_invgamma( gamma1, phi[1], potential);
-        if(  number[0] == invert_invgamma.get_max())
+        if(  number[0] == multigrid.max_iter())
             throw dg::Fail( eps_gamma);
     }
     //compute (nabla phi)^2
@@ -262,8 +258,7 @@ const container& Explicit<G, M, container>::polarisation( const std::vector<cont
         old_gammaN.extrapolate( gamma_n);
         std::vector<unsigned> number = multigrid.direct_solve( multi_gamma1, gamma_n, y[1], eps_gamma);
         old_gammaN.update( gamma_n);
-        //unsigned number = invert_invgamma( gamma1, gamma_n, y[1]);
-        if(  number[0] == invert_invgamma.get_max())
+        if(  number[0] == multigrid.max_iter())
             throw dg::Fail( eps_gamma);
         dg::blas1::axpby( -1., y[0], 1., gamma_n, omega); //omega = a_i\Gamma n_i - n_e
     }
@@ -274,11 +269,10 @@ const container& Explicit<G, M, container>::polarisation( const std::vector<cont
             dg::blas1::pointwiseDivide( omega, chi, omega);
     //invert 
 
-    //unsigned number = invert_pol( pol, phi[0], omega, v2d, chi);
     old_phi.extrapolate( phi[0]);
     std::vector<unsigned> number = multigrid.direct_solve( multi_pol, phi[0], omega, eps_pol);
     old_phi.update( phi[0]);
-    if(  number[0] == invert_pol.get_max())
+    if(  number[0] == multigrid.max_iter())
         throw dg::Fail( eps_pol);
     return phi[0];
 }
-- 
GitLab


From dd50de7ca3c94f68e4db4a8df76a88c80fbdd4b7 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Wed, 25 Oct 2017 22:37:17 +0200
Subject: [PATCH 391/453] made mpi grid represent the global grid

---
 inc/dg/backend/mpi_grid.h       | 345 ++++++++++++++------------------
 inc/dg/backend/mpi_projection.h |   4 +-
 inc/dg/dg_doc.h                 |   2 +-
 3 files changed, 149 insertions(+), 202 deletions(-)

diff --git a/inc/dg/backend/mpi_grid.h b/inc/dg/backend/mpi_grid.h
index fed48016b..c103589e9 100644
--- a/inc/dg/backend/mpi_grid.h
+++ b/inc/dg/backend/mpi_grid.h
@@ -26,12 +26,12 @@ namespace dg
 /**
  * @brief 2D MPI abstract grid class 
  *
- * Represents the local grid coordinates and the process topology. 
+ * Represents the global grid coordinates and the process topology. 
  * It just divides the given (global) box into nonoverlapping (local) subboxes that are attributed to each process
  * @note a single cell is never divided across processes.
- * @note although it is abstract objects are not meant to be hold on the heap via a base class pointer ( we protected the destructor)
+ * @note although it is abstract objects, are not meant to be hold on the heap via a base class pointer ( we protected the destructor)
  * @attention
- * The boundaries in the constructors are global boundaries, the boundaries returned by the access functions are local boundaries, this is because the grid represents the information given to one process
+ * The access functions \c n() \c Nx() ,... all return the global parameters. If you want to have the local ones call the \c local() function.
  * @ingroup basictopology
  */
 struct aMPITopology2d
@@ -39,147 +39,106 @@ struct aMPITopology2d
     typedef MPITag memory_category;
     typedef TwoDimensionalTag dimensionality;
 
-
     /**
-     * @brief Return local x0
+     * @brief Return global x0
      *
-     * @return local left boundary
+     * @return global left boundary
      */
-    double x0() const {
-        int dims[2], periods[2], coords[2];
-        MPI_Cart_get( comm, 2, dims, periods, coords);
-        return g.x0() + g.lx()/(double)dims[0]*(double)coords[0]; 
-    }
-
+    double x0() const { return g.x0();}
     /**
-     * @brief Return local x1
+     * @brief Return global x1
      *
-     * @return local right boundary
+     * @return global right boundary
      */
-    double x1() const {
-        int dims[2], periods[2], coords[2];
-        MPI_Cart_get( comm, 2, dims, periods, coords);
-        return g.x0() + g.lx()/(double)dims[0]*(double)(coords[0]+1); 
-    }
-
+    double x1() const { return g.x1(); }
     /**
-     * @brief Return local y0
+     * @brief Return global y0
      *
-     * @return local left boundary
+     * @return global left boundary
      */
-    double y0() const {
-        int dims[2], periods[2], coords[2];
-        MPI_Cart_get( comm, 2, dims, periods, coords);
-        return g.y0() + g.ly()/(double)dims[1]*(double)coords[1]; 
-    }
-
+    double y0() const { return g.y0();}
     /**
-     * @brief Return local y1
+     * @brief Return global y1
      *
-     * @return local right boundary
+     * @return global right boundary
      */
-    double y1() const {
-        int dims[2], periods[2], coords[2];
-        MPI_Cart_get( comm, 2, dims, periods, coords);
-        return g.y0() + g.ly()/(double)dims[1]*(double)(coords[1]+1); 
-    }
-
+    double y1() const { return g.y1();}
     /**
-     * @brief Return local lx
+     * @brief Return global lx
      *
-     * @return local length
+     * @return global length
      */
-    double lx() const {return x1()-x0();}
-
+    double lx() const {return g.lx();}
     /**
-     * @brief Return local ly
+     * @brief Return global ly
      *
-     * @return local length
+     * @return global length
      */
-    double ly() const {return y1()-y0();}
-
+    double ly() const {return g.ly();}
     /**
-     * @brief Return local hx
+     * @brief Return global hx
      *
-     * @return local grid constant
+     * @return global grid constant
      */
     double hx() const {return g.hx();}
-
     /**
-     * @brief Return local hy
+     * @brief Return global hy
      *
-     * @return local grid constant
+     * @return global grid constant
      */
     double hy() const {return g.hy();}
-
     /**
      * @brief Return n
      *
      * @return number of polynomial coefficients
      */
     unsigned n() const {return g.n();}
-
     /**
-     * @brief Return the local number of cells 
+     * @brief Return the global number of cells 
      *
      * @return number of cells
      */
-    unsigned Nx() const {
-        int dims[2], periods[2], coords[2];
-        MPI_Cart_get( comm, 2, dims, periods, coords);
-        return g.Nx()/dims[0];
-    }
-
+    unsigned Nx() const { return g.Nx();}
     /**
-     * @brief Return the local number of cells 
+     * @brief Return the global number of cells 
      *
      * Not the one given in the constructor
      * @return number of cells
      */
-    unsigned Ny() const {
-        int dims[2], periods[2], coords[2];
-        MPI_Cart_get( comm, 2, dims, periods, coords);
-        return g.Ny()/dims[1];
-    }
-
+    unsigned Ny() const { return g.Ny();}
     /**
      * @brief global x boundary
      *
      * @return boundary condition
      */
     bc bcx() const {return g.bcx();}
-
     /**
      * @brief global y boundary
      *
      * @return boundary condition
      */
     bc bcy() const {return g.bcy();}
-
     /**
      * @brief Return mpi cartesian communicator that is used in this grid
      *
      * @return Communicator
      */
     MPI_Comm communicator() const{return comm;}
-
     /**
      * @brief The Discrete Legendre Transformation 
      *
      * @return DLT corresponding to n given in the constructor
      */
     const DLT<double>& dlt() const{return g.dlt();}
-
     /**
-     * @brief The total local number of points
+     * @brief The total global number of points
      *
-     * @note for the total global number of points call grid.global().size()
-     * @return n*n*Nx*Ny
+     * @note for the total local number of points call grid.local().size()
+     * @return n()*n()*Nx()*Ny()
      */
-    unsigned size() const { return n()*n()*Nx()*Ny();}
-
+    unsigned size() const { return g.size();}
     /**
-     * @brief Display 
+     * @brief Display global and local grid
      *
      * @param os output stream
      */
@@ -203,7 +162,6 @@ struct aMPITopology2d
      * @return pid of a process, or -1 if non of the grids matches
      */
     int pidOf( double x, double y) const;
-
     /**
     * @brief Multiply the number of cells with a given factor
     *
@@ -216,15 +174,12 @@ struct aMPITopology2d
     }
     /**
     * @copydoc Grid2d::set(unsigned,unsigned,unsigned)
-    * @attention these are global parameters, i.e. set( g.n(), 2*g.Nx(), 2*g.Ny()) is NOT(!) what you want
-    *           use the multiplyCellNumbers function instead, or set( g.n(), 2*g.global().Nx(), 2*g.global().Ny())
     */
     void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny) {
         check_division( new_Nx, new_Ny, g.bcx(), g.bcy());
         if( new_n == g.n() && new_Nx == g.Nx() && new_Ny == g.Ny()) return;
         do_set( new_n,new_Nx,new_Ny);
     }
-
     /**
     * @brief Map a local index plus the PID to a global vector index
     *
@@ -235,14 +190,15 @@ struct aMPITopology2d
     */
     bool local2globalIdx( int localIdx, int PID, int& globalIdx)const
     {
+        Grid2d l = local();
         if( localIdx < 0 || localIdx >= (int)size()) return -1;
         int coords[2];
         if( MPI_Cart_coords( comm, PID, 2, coords) != MPI_SUCCESS)
             return false;
-        int lIdx0 = localIdx %(n()*Nx());
-        int lIdx1 = localIdx /(n()*Nx());
-        int gIdx0 = coords[0]*n()*Nx()+lIdx0;
-        int gIdx1 = coords[1]*n()*Ny()+lIdx1;
+        int lIdx0 = localIdx %(l.n()*l.Nx());
+        int lIdx1 = localIdx /(l.n()*l.Nx());
+        int gIdx0 = coords[0]*l.n()*l.Nx()+lIdx0;
+        int gIdx1 = coords[1]*l.n()*l.Ny()+lIdx1;
         globalIdx = gIdx1*g.n()*g.Nx() + gIdx0;
         return true;
     }
@@ -256,15 +212,16 @@ struct aMPITopology2d
     */
     bool global2localIdx( int globalIdx, int& localIdx, int& PID)const
     {
+        Grid2d l = local();
         if( globalIdx < 0 || globalIdx >= (int)g.size()) return -1;
         int coords[2];
         int gIdx0 = globalIdx%(g.n()*g.Nx());
         int gIdx1 = globalIdx/(g.n()*g.Nx());
-        coords[0] = gIdx0/(n()*Nx());
-        coords[1] = gIdx1/(n()*Ny());
-        int lIdx0 = gIdx0%(n()*Nx());
-        int lIdx1 = gIdx1%(n()*Ny());
-        localIdx = lIdx1*n()*Nx() + lIdx0;
+        coords[0] = gIdx0/(l.n()*l.Nx());
+        coords[1] = gIdx1/(l.n()*l.Ny());
+        int lIdx0 = gIdx0%(l.n()*l.Nx());
+        int lIdx1 = gIdx1%(l.n()*l.Ny());
+        localIdx = lIdx1*l.n()*l.Nx() + lIdx0;
         if( MPI_Cart_rank( comm, coords, &PID) == MPI_SUCCESS ) 
             return true;
         else
@@ -273,22 +230,35 @@ struct aMPITopology2d
             return false;
         }
     }
-
     /**
      * @brief Return a non-MPI grid local for the calling process
      *
-     * The local grid returns the same values for x0(), x1(), ..., Nx(), Ny(), ... as the grid
-     * class itself 
+     * The local grid contains the boundaries and cell numbers the calling process sees and is in charge of. 
      * @return Grid object
      * @note the boundary conditions in the local grid are not well defined since there might not actually be any boundaries
      */
-    Grid2d local() const {return Grid2d(x0(), x1(), y0(), y1(), n(), Nx(), Ny(), bcx(), bcy());}
-
+    Grid2d local() const {
+        int dims[2], periods[2], coords[2];
+        MPI_Cart_get( comm, 2, dims, periods, coords);
+        double x0 = g.x0() + g.lx()/(double)dims[0]*(double)coords[0]; 
+        double x1 = g.x0() + g.lx()/(double)dims[0]*(double)(coords[0]+1); 
+        if( coords[0] == dims[0]-1) 
+            x1 = g.x1();
+        double y0 = g.y0() + g.ly()/(double)dims[1]*(double)coords[1]; 
+        double y1 = g.y0() + g.ly()/(double)dims[1]*(double)(coords[1]+1); 
+        if( coords[1] == dims[1]-1) 
+            y1 = g.y1();
+        unsigned Nx = g.Nx()/dims[0];
+        unsigned Ny = g.Ny()/dims[1];
+        return Grid2d(x0, x1, y0, y1, g.n(), Nx, Ny, g.bcx(), g.bcy());
+    }
     /**
      * @brief Return the global non-MPI grid 
      *
      * The global grid contains the global boundaries and cell numbers. 
      * This is the grid that we would have to use in a non-MPI implementation.
+     * The global grid returns the same values for x0(), x1(), ..., Nx(), Ny(), ... as the grid
+     * class itself 
      * @return non-MPI Grid object
      */
     Grid2d global() const {return g;}
@@ -337,20 +307,13 @@ struct aMPITopology2d
     }
     Grid2d g; //global grid
     MPI_Comm comm; //just an integer...
-
 };
 
 
 /**
  * @brief 3D MPI Grid class 
  *
- * Represents the local grid coordinates and the process topology. 
- * It just divides the given box into nonoverlapping subboxes that are attributed to each process
- * @attention
- * The boundaries in the constructors are global boundaries, the boundaries returned by the access functions are local boundaries, this is because the grid represents the information given to one process
- *
- * @note Note that a single cell is never divided across processes.
- * @note although it is abstract objects are not meant to be hold on the heap via a base class pointer ( we protected the destructor)
+ * @copydetails aMPITopology2d
  * @ingroup basictopology
  */
 struct aMPITopology3d
@@ -358,104 +321,76 @@ struct aMPITopology3d
     typedef MPITag memory_category;
     typedef ThreeDimensionalTag dimensionality;
 
-
     /**
-     * @brief Return local x0
+     * @brief Return global x0
      *
-     * @return local left boundary
+     * @return global left boundary
      */
-    double x0() const {
-        int dims[3], periods[3], coords[3];
-        MPI_Cart_get( comm, 3, dims, periods, coords);
-        return g.x0() + g.lx()/(double)dims[0]*(double)coords[0]; 
-    }
+    double x0() const { return g.x0();}
     /**
-     * @brief Return local x1
+     * @brief Return global x1
      *
-     * @return local right boundary
+     * @return global right boundary
      */
-    double x1() const {
-        int dims[3], periods[3], coords[3];
-        MPI_Cart_get( comm, 3, dims, periods, coords);
-        if( coords[0] == dims[0]-1) return g.x1();
-        return g.x0() + g.lx()/(double)dims[0]*(double)(coords[0]+1); 
-    }
+    double x1() const { return g.x1();}
     /**
-     * @brief Return local y0
+     * @brief Return global y0
      *
-     * @return local left boundary
+     * @return global left boundary
      */
-    double y0() const {
-        int dims[3], periods[3], coords[3];
-        MPI_Cart_get( comm, 3, dims, periods, coords);
-        return g.y0() + g.ly()/(double)dims[1]*(double)coords[1]; 
-    }
+    double y0() const { return g.y0();}
     /**
-     * @brief Return local y1
+     * @brief Return global y1
      *
-     * @return local right boundary
+     * @return global right boundary
      */
-    double y1() const {
-        int dims[3], periods[3], coords[3];
-        MPI_Cart_get( comm, 3, dims, periods, coords);
-        if( coords[1] == dims[1]-1) return g.y1();
-        return g.y0() + g.ly()/(double)dims[1]*(double)(coords[1]+1); 
-    }
+    double y1() const { return g.y1();}
     /**
-     * @brief Return local z0
+     * @brief Return global z0
      *
-     * @return local left boundary
+     * @return global left boundary
      */
-    double z0() const {
-        int dims[3], periods[3], coords[3];
-        MPI_Cart_get( comm, 3, dims, periods, coords);
-        return g.z0() + g.lz()/(double)dims[2]*(double)coords[2]; 
-    }
+    double z0() const { return g.z0();}
     /**
-     * @brief Return local z1
+     * @brief Return global z1
      *
-     * @return local right boundary
+     * @return global right boundary
      */
-    double z1() const {
-        int dims[3], periods[3], coords[3];
-        MPI_Cart_get( comm, 3, dims, periods, coords);
-        if( coords[2] == dims[2]-1) return g.z1();
-        return g.z0() + g.lz()/(double)dims[2]*(double)(coords[2]+1); 
-    }
+    double z1() const { return g.z1();}
     /**
-     * @brief Return local lx
+     * @brief Return global lx
      *
-     * @return local length
+     * @return global length
      */
-    double lx() const {return x1()-x0();}
+    double lx() const {return g.lx();}
     /**
-     * @brief Return local ly
+     * @brief Return global ly
      *
-     * @return local length
+     * @return global length
      */
-    double ly() const {return y1()-y0();}
+    double ly() const {return g.ly();}
     /**
-     * @brief Return local lz
+     * @brief Return global lz
      *
-     * @return local length
+     * @return global length
      */
-    double lz() const {return z1()-z0();}
+    double lz() const {return g.lz();}
     /**
-     * @brief Return local hx
+     * @brief Return global hx
      *
-     * @return local grid constant
+     * @return global grid constant
      */
     double hx() const {return g.hx();}
     /**
-     * @brief Return local hy
+     * @brief Return global hy
      *
-     * @return local grid constant
+     * @return global grid constant
      */
     double hy() const {return g.hy();}
     /**
-     * @brief Return local hz
+     * @brief Return global hz
      *
-     * @return local grid constant
+     * @return global grid constant
      */
     double hz() const {return g.hz();}
     /**
@@ -465,38 +400,26 @@ struct aMPITopology3d
      */
     unsigned n() const {return g.n();}
     /**
-     * @brief Return the local number of cells 
+     * @brief Return the global number of cells 
      *
      * Not the one given in the constructor
      * @return number of cells
      */
-    unsigned Nx() const {
-        int dims[3], periods[3], coords[3];
-        MPI_Cart_get( comm, 3, dims, periods, coords);
-        return g.Nx()/dims[0];
-    }
+    unsigned Nx() const { return g.Nx();}
     /**
-     * @brief Return the local number of cells 
+     * @brief Return the global number of cells 
      *
      * Not the one given in the constructor
      * @return number of cells
      */
-    unsigned Ny() const {
-        int dims[3], periods[3], coords[3];
-        MPI_Cart_get( comm, 3, dims, periods, coords);
-        return g.Ny()/dims[1];
-    }
+    unsigned Ny() const { return g.Ny();}
     /**
-     * @brief Return the local number of cells 
+     * @brief Return the global number of cells 
      *
      * Not the one given in the constructor
      * @return number of cells
      */
-    unsigned Nz() const {
-        int dims[3], periods[3], coords[3];
-        MPI_Cart_get( comm, 3, dims, periods, coords);
-        return g.Nz()/dims[2];
-    }
+    unsigned Nz() const { return g.Nz();}
     /**
      * @brief global x boundary
      *
@@ -538,13 +461,13 @@ struct aMPITopology3d
      */
     const DLT<double>& dlt() const{return g.dlt();}
     /**
-     * @brief The total local number of points
+     * @brief The total global number of points
      *
-     * @return n*n*Nx*Ny*Nz
+     * @return n()*n()*Nx()*Ny()*Nz()
      */
-    unsigned size() const { return n()*n()*Nx()*Ny()*Nz();}
+    unsigned size() const { return g.size();}
     /**
-     * @brief Display 
+     * @brief Display global and local grid paramters 
      *
      * @param os output stream
      */
@@ -574,8 +497,6 @@ struct aMPITopology3d
     }
     /**
      * @copydoc Grid3d::set(unsigned,unsigned,unsigned,unsigned)
-     * @attention these are global parameters, i.e. set( g.n(), 2*g.Nx(), 2*g.Ny(), 2*g.Nz()) is NOT(!) what you want
-     *           use the multiplyCellNumbers function instead, or set( g.n(), 2*g.global().Nx(), 2*g.global().Ny(), 2*g.global().Nz())
      */
     void set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz) {
         check_division( new_Nx,new_Ny,new_Nz,g.bcx(),g.bcy(),g.bcz());
@@ -585,41 +506,65 @@ struct aMPITopology3d
     ///@copydoc aMPITopology2d::local2globalIdx(int,int,int&)const
     bool local2globalIdx( int localIdx, int PID, int& globalIdx)const
     {
+        Grid3d l = local();
         if( localIdx < 0 || localIdx >= (int)size()) return false;
         int coords[3];
         if( MPI_Cart_coords( comm, PID, 3, coords) != MPI_SUCCESS)
             return false;
-        int lIdx0 = localIdx %(n()*Nx());
-        int lIdx1 = (localIdx /(n()*Nx())) % (n()*Ny());
-        int lIdx2 = localIdx / (n()*n()*Nx()*Ny());
-        int gIdx0 = coords[0]*n()*Nx()+lIdx0;
-        int gIdx1 = coords[1]*n()*Ny()+lIdx1;
-        int gIdx2 = coords[2]*Nz()  + lIdx2;
+        int lIdx0 = localIdx %(l.n()*l.Nx());
+        int lIdx1 = (localIdx /(l.n()*l.Nx())) % (l.n()*l.Ny());
+        int lIdx2 = localIdx / (l.n()*l.n()*l.Nx()*l.Ny());
+        int gIdx0 = coords[0]*l.n()*l.Nx()+lIdx0;
+        int gIdx1 = coords[1]*l.n()*l.Ny()+lIdx1;
+        int gIdx2 = coords[2]*l.Nz()  + lIdx2;
         globalIdx = (gIdx2*g.n()*g.Ny() + gIdx1)*g.n()*g.Nx() + gIdx0;
         return true;
     }
     ///@copydoc aMPITopology2d::global2localIdx(int,int&,int&)const
     bool global2localIdx( int globalIdx, int& localIdx, int& PID)const
     {
+        Grid3d l = local();
         if( globalIdx < 0 || globalIdx >= (int)g.size()) return false;
         int coords[3];
         int gIdx0 = globalIdx%(g.n()*g.Nx());
         int gIdx1 = (globalIdx/(g.n()*g.Nx())) % (g.n()*g.Ny());
         int gIdx2 = globalIdx/(g.n()*g.n()*g.Nx()*g.Ny());
-        coords[0] = gIdx0/(n()*Nx());
-        coords[1] = gIdx1/(n()*Ny());
-        coords[2] = gIdx2/Nz();
-        int lIdx0 = gIdx0%(n()*Nx());
-        int lIdx1 = gIdx1%(n()*Ny());
-        int lIdx2 = gIdx2%Nz();
-        localIdx = (lIdx2*n()*Ny() + lIdx1)*n()*Nx() + lIdx0;
+        coords[0] = gIdx0/(l.n()*l.Nx());
+        coords[1] = gIdx1/(l.n()*l.Ny());
+        coords[2] = gIdx2/l.Nz();
+        int lIdx0 = gIdx0%(l.n()*l.Nx());
+        int lIdx1 = gIdx1%(l.n()*l.Ny());
+        int lIdx2 = gIdx2%l.Nz();
+        localIdx = (lIdx2*l.n()*l.Ny() + lIdx1)*l.n()*l.Nx() + lIdx0;
         if( MPI_Cart_rank( comm, coords, &PID) == MPI_SUCCESS ) 
             return true;
         else
             return false;
     }
     ///@copydoc aMPITopology2d::local()const
-    Grid3d local() const {return Grid3d(x0(), x1(), y0(), y1(), z0(), z1(), n(), Nx(), Ny(), Nz(), bcx(), bcy(), bcz());}
+    Grid3d local() const {
+        int dims[3], periods[3], coords[3];
+        MPI_Cart_get( comm, 3, dims, periods, coords);
+        double x0 = g.x0() + g.lx()/(double)dims[0]*(double)coords[0]; 
+        double x1 = g.x0() + g.lx()/(double)dims[0]*(double)(coords[0]+1); 
+        if( coords[0] == dims[0]-1) 
+            x1 = g.x1();
+
+        double y0 = g.y0() + g.ly()/(double)dims[1]*(double)coords[1]; 
+        double y1 = g.y0() + g.ly()/(double)dims[1]*(double)(coords[1]+1); 
+        if( coords[1] == dims[1]-1) 
+            y1 = g.y1();
+
+        double z0 = g.z0() + g.lz()/(double)dims[2]*(double)coords[2]; 
+        double z1 = g.z0() + g.lz()/(double)dims[2]*(double)(coords[2]+1); 
+        if( coords[2] == dims[2]-1) 
+            z1 = g.z1();
+        Nx = g.Nx()/dims[0];
+        Ny = g.Ny()/dims[1];
+        Nz = g.Nz()/dims[2];
+        
+        return Grid3d(x0, x1, y0, y1, z0, z1, g.n(), Nx, Ny, Nz, g.bcx(), g.bcy(), g.bcz());
+    }
      ///@copydoc aMPITopology2d::global()const
     Grid3d global() const {return g;}
     protected:
diff --git a/inc/dg/backend/mpi_projection.h b/inc/dg/backend/mpi_projection.h
index 315498c06..d4b15b499 100644
--- a/inc/dg/backend/mpi_projection.h
+++ b/inc/dg/backend/mpi_projection.h
@@ -19,7 +19,9 @@ namespace dg
 //interpolation matrices
 typedef MPIDistMat< dg::IHMatrix, GeneralComm< dg::iHVec, dg::HVec > > MIHMatrix; //!< MPI distributed CSR host Matrix
 typedef MPIDistMat< dg::IDMatrix, GeneralComm< dg::iDVec, dg::DVec > > MIDMatrix; //!< MPI distributed CSR device Matrix
+///@}
 
+///@cond
 namespace detail{
 //given global indices -> make a sorted unique indices vector -> make a gather map into the unique vector
 //buffer_idx -> (gather map/ new column indices) same size as global_idx ( can alias global_idx, index into unique_global_idx
@@ -49,7 +51,7 @@ void global2bufferIdx( const cusp::array1d<int, cusp::host_memory>& global_idx,
     thrust::scatter( gather_map.begin(), gather_map.end(), index.begin(), buffer_idx.begin());
 }
 }//namespace detail
-///@}
+///@endcond
 
 /**
  * @brief Convert a matrix with local row and global column indices to a row distributed MPI matrix
diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index ec317c0fc..7f36fd60a 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -186,7 +186,7 @@
 /*!@addtogroup mpi_structures
 @{
 @note The mpi backend is activated by including \c mpi.h before any other feltor header file
-@section mpi_vector MPI Vectors and the \c blas1 functions
+@section mpi_vector MPI Vectors and the blas1 functions
 
 In Feltor each mpi process gets an equally sized chunk of a vector.
 The corresponding structure in FELTOR is the \c dg::MPI_Vector, which is 
-- 
GitLab


From d3ee183074e6df31c5a80732ef1c122a89def902 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 26 Oct 2017 00:08:04 +0200
Subject: [PATCH 392/453] made documentation for average.h

---
 inc/dg/Doxyfile                   |  3 ++-
 inc/dg/backend/average.cuh        | 15 +++++----------
 inc/dg/backend/average.h          | 21 ++++++++++++---------
 inc/dg/backend/average_mpit.cu    | 12 +++++++-----
 inc/dg/backend/average_t.cu       | 15 +++++++++------
 inc/dg/backend/mpi_collective.h   | 20 ++++++++++----------
 inc/dg/backend/mpi_communicator.h |  2 +-
 inc/dg/backend/mpi_derivatives.h  | 20 ++++++++++----------
 inc/dg/backend/mpi_evaluation.h   | 26 +++++++++++++-------------
 inc/dg/backend/mpi_grid.h         |  6 +++---
 inc/dg/backend/split_and_join.h   | 25 ++++++++++++++-----------
 inc/dg/dg_doc.h                   |  2 +-
 12 files changed, 87 insertions(+), 80 deletions(-)

diff --git a/inc/dg/Doxyfile b/inc/dg/Doxyfile
index 38d44516e..09576dbd7 100644
--- a/inc/dg/Doxyfile
+++ b/inc/dg/Doxyfile
@@ -868,7 +868,8 @@ EXAMPLE_PATH           = cg2d_t.cu \
                          arakawa_t.cu poisson_t.cu \
                          elliptic2d_b.cu runge_kutta2d_t.cu multistep_t.cu \
                          helmholtzg2_b.cu \
-                         backend/interpolation_t.cu
+                         backend/interpolation_t.cu \
+                         backend/average_t.cu backend/average_mpit.cu
 
 
 
diff --git a/inc/dg/backend/average.cuh b/inc/dg/backend/average.cuh
index 76e1c6204..59f2d68d7 100644
--- a/inc/dg/backend/average.cuh
+++ b/inc/dg/backend/average.cuh
@@ -9,20 +9,15 @@
   @brief contains classes for poloidal and toroidal average computations.
   */
 namespace dg{
-//struct printf_functor
-//{
-//__host__ __device__
-//void operator()(double x)
-//{
-//    printf("%f\n",x);
-//}
-//};
+
 /**
  * @brief Class for y average computations
  *
+ * @snippet backend/average_t.cu doxygen
  * @ingroup utilities
- * @tparam container Vector class to be used
- * @tparam IndexContainer Class for scatter maps
+ * @tparam container Currently this is one of 
+ *  - \c dg::HVec, \c dg::DVec, \c dg::MHVec or \c dg::MDVec  
+ * @tparam IndexContainer Type of index vectors; May equal \c container
  */
 template< class container, class IndexContainer>
 struct PoloidalAverage
diff --git a/inc/dg/backend/average.h b/inc/dg/backend/average.h
index bc17c4261..0e620c6f4 100644
--- a/inc/dg/backend/average.h
+++ b/inc/dg/backend/average.h
@@ -13,9 +13,11 @@ namespace dg{
 /**
  * @brief MPI specialized class for y average computations
  *
+ * @snippet backend/average_mpit.cu doxygen
  * @ingroup utilities
- * @tparam container Vector class to be used
- * @tparam IndexContainer Class for scatter maps
+ * @tparam container Currently this is one of 
+ *  - \c dg::HVec, \c dg::DVec, \c dg::MHVec or \c dg::MDVec  
+ * @tparam IndexContainer Type of index vectors; May equal \c container
  */
 template< class container, class IndexContainer>
 struct PoloidalAverage<MPI_Vector<container>, MPI_Vector<IndexContainer> >
@@ -25,18 +27,19 @@ struct PoloidalAverage<MPI_Vector<container>, MPI_Vector<IndexContainer> >
      * @param g 2d MPITopology
      */
     PoloidalAverage( const aMPITopology2d& g): 
-        helper1d_( g.n()*g.Nx()), hhelper1d_(g.n()*g.Nx()),
-        recv_(hhelper1d_),dummy( g.n()*g.Nx()), 
-        helper_( g.size()), ly_(g.global().ly())
+        helper1d_( g.n()*g.local().Nx()), hhelper1d_(helper1d_),
+        recv_(helper1d_),dummy( helper1d_), 
+        helper_( g.local().size()), ly_( g.global().ly())
     {
         int remain[] = {false, true};
         MPI_Cart_sub( g.communicator(), remain, &comm1d_);
 
-        invertxy = create::scatterMapInvertxy( g.n(), g.Nx(), g.Ny());
-        lines = create::contiguousLineNumbers( g.n()*g.Nx(), g.n()*g.Ny());
-        Grid2d gTr( g.y0(), g.y1(), g.x0(), g.x1(), g.n(), g.Ny(), g.Nx());
+        Grid2d l = g.local();
+        invertxy = create::scatterMapInvertxy( l.n(), l.Nx(), l.Ny());
+        lines = create::contiguousLineNumbers( l.n()*l.Nx(), l.n()*l.Ny());
+        Grid2d gTr( l.y0(), l.y1(), l.x0(), l.x1(), l.n(), l.Ny(), l.Nx());
         w2d = dg::create::weights( gTr);
-        Grid1d g1x( 0, g.lx(), g.n(), g.Nx());
+        Grid1d g1x( l.x0(), l.x1(), l.n(), l.Nx());
         v1d = dg::create::inv_weights( g1x);
     }
     /**
diff --git a/inc/dg/backend/average_mpit.cu b/inc/dg/backend/average_mpit.cu
index 5b3d1ccb4..41463fa82 100644
--- a/inc/dg/backend/average_mpit.cu
+++ b/inc/dg/backend/average_mpit.cu
@@ -29,20 +29,22 @@ int main(int argc, char* argv[])
     mpi_init2d( bcx, bcy, n, Nx, Ny, comm);
     MPI_Comm_rank( MPI_COMM_WORLD, &rank);
 
+    //![doxygen]
     dg::MPIGrid2d g( 0, lx, 0, ly, n, Nx, Ny, bcx, bcy, comm);
-    
 
     if(rank==0)std::cout << "constructing polavg" << std::endl;
     dg::PoloidalAverage<dg::MDVec, dg::MDVec > pol(g);
     if(rank==0)std::cout << "constructing polavg end" << std::endl;
-    dg::MDVec vector = dg::evaluate( function ,g), average_y( vector);
-    const dg::MDVec solution = dg::evaluate( pol_average, g);
+    const dg::MDVec vector = dg::evaluate( function ,g); 
+    dg::MDVec average_y( vector);
     if(rank==0)std::cout << "Averaging ... \n";
     pol( vector, average_y);
-    dg::blas1::axpby( 1., solution, -1., average_y, vector);
+    //![doxygen]
+    const dg::MDVec solution = dg::evaluate( pol_average, g);
+    dg::blas1::axpby( 1., solution, -1., average_y);
 
     dg::MDVec w2d = dg::create::weights(g);
-    double norm = dg::blas2::dot(vector, w2d, vector);
+    double norm = dg::blas2::dot(average_y, w2d, average_y);
     if(rank==0)std::cout << "Distance to solution is: "<<        sqrt(norm)<<std::endl;
 
     MPI_Finalize();
diff --git a/inc/dg/backend/average_t.cu b/inc/dg/backend/average_t.cu
index 12a29cf2e..bc6c7b2f3 100644
--- a/inc/dg/backend/average_t.cu
+++ b/inc/dg/backend/average_t.cu
@@ -14,17 +14,20 @@ int main()
     unsigned n, Nx, Ny;
     std::cout << "Type n, Nx and Ny!\n";
     std::cin >> n >> Nx >> Ny;
+    //![doxygen]
     const dg::Grid2d g( 0, lx, 0, ly, n, Nx, Ny);
-    dg::HVec w2d = dg::create::weights( g);
 
-    dg::PoloidalAverage<dg::HVec, thrust::host_vector<int> > pol(g);
+    dg::PoloidalAverage<dg::HVec, dg::iHVec > pol(g);
 
-    dg::HVec vector = dg::evaluate( function ,g), average_y( vector);
-    const dg::HVec solution = dg::evaluate( pol_average, g);
+    const dg::HVec vector = dg::evaluate( function ,g); 
+    dg::HVec average_y( vector);
     std::cout << "Averaging ... \n";
     pol( vector, average_y);
-    dg::blas1::axpby( 1., solution, -1., average_y, vector);
-    std::cout << "Distance to solution is: "<<sqrt(dg::blas2::dot( vector, w2d, vector))<<std::endl;
+    //![doxygen]
+    const dg::HVec w2d = dg::create::weights( g);
+    const dg::HVec solution = dg::evaluate( pol_average, g);
+    dg::blas1::axpby( 1., solution, -1., average_y);
+    std::cout << "Distance to solution is: "<<sqrt(dg::blas2::dot( average_y, w2d, average_y))<<std::endl;
 
 
 
diff --git a/inc/dg/backend/mpi_collective.h b/inc/dg/backend/mpi_collective.h
index 6e4978607..4547d0df8 100644
--- a/inc/dg/backend/mpi_collective.h
+++ b/inc/dg/backend/mpi_collective.h
@@ -253,8 +253,8 @@ struct BijectiveComm : public aCommunicator<Vector>
  * This Communicator performs surjective global gather and
  scatter operations, which means that the gather/scatter map
  is surjective, i.e. all elements in a source vector get gathered. 
- Compared to BijectiveComm in the global_gather function there is an additional 
- gather and in the global_scatter_reduce function a reduction 
+ Compared to \c BijectiveComm in the \c global_gather function there is an additional 
+ gather and in the \c global_scatter_reduce function a reduction 
  needs to be performed.
  * @tparam Index an integer thrust Vector
  * @tparam Vector a thrust Vector
@@ -360,7 +360,7 @@ struct SurjectiveComm : public aCommunicator<Vector>
  * @brief Struct that performs general collective scatter and gather operations across processes on distributed vectors using mpi
  *
  * This Communicator can perform general global gather and
- scatter operations. Compared to SurjectiveComm the global_scatter_reduce function needs
+ scatter operations. Compared to \c SurjectiveComm the \c global_scatter_reduce function needs
  to perform an additional scatter as some elements of the source vector might be left empty
  * @tparam Index an integer thrust Vector 
  * @tparam Vector a thrust Vector 
@@ -373,10 +373,10 @@ struct GeneralComm : public aCommunicator<Vector>
     /**
     * @brief Construct from local indices and PIDs gather map
     *
-    * The indices in the gather map are written with respect to the buffer vector (unlike in BijectiveComm, where it is given wrt the source vector).
+    * The indices in the gather map are written with respect to the buffer vector (unlike in \c BijectiveComm, where it is given wrt the source vector).
     * Each location in the source vector is uniquely specified by a local vector index and the process rank. 
-    * @param localGatherMap Each element localGatherMap[i] represents a local vector index from where to gather the value. There are "local buffer size" elements.
-    * @param pidGatherMap Each element pidGatherMap[i] represents the pid/rank from where to gather the corresponding local index localGatherMap[i].  Same size as localGatherMap.
+    * @param localGatherMap Each element \c localGatherMap[i] represents a local vector index from where to gather the value. There are "local buffer size" elements.
+    * @param pidGatherMap Each element \c pidGatherMap[i] represents the pid/rank from where to gather the corresponding local index \c localGatherMap[i].  Same size as localGatherMap.
      *   The pid/rank needs to be element of the given communicator.
     * @param comm The MPI communicator participating in the scatter/gather operations
     */
@@ -393,13 +393,13 @@ struct GeneralComm : public aCommunicator<Vector>
     /**
      * @brief Construct from global indices gather map
      *
-     * Uses the global2localIdx() member of MPITopology to generate localGatherMap and pidGatherMap 
+     * Uses the \c global2localIdx() member of MPITopology to generate localGatherMap and pidGatherMap 
      * @tparam ConversionPolicy has to have the members: 
-     *  - global2localIdx(unsigned,unsigned,unsigned) const;
+     *  - \c global2localIdx(unsigned,unsigned,unsigned) \c const;
      * where the first parameter is the global index and the 
      * other two are the pair (local idx, rank). 
-     *  - MPI_Comm %communicator() const;  returns the communicator to use in the gather/scatter
-     * @param globalGatherMap Each element globalGatherMap[i] represents a global vector index from where to take the value. There are "local buffer size == size()" elements.
+     *  - \c MPI_Comm \c %communicator() \c const;  returns the communicator to use in the gather/scatter
+     * @param globalGatherMap Each element \c globalGatherMap[i] represents a global vector index from where to take the value. There are "local buffer size == size()" elements.
      * @param p the conversion object
      * @sa basictopology the MPI %grids defined in Level 3 can all be used as a ConversionPolicy
      */
diff --git a/inc/dg/backend/mpi_communicator.h b/inc/dg/backend/mpi_communicator.h
index 939ddac86..a24ef54f2 100644
--- a/inc/dg/backend/mpi_communicator.h
+++ b/inc/dg/backend/mpi_communicator.h
@@ -6,7 +6,7 @@ namespace dg
 {
 ///@cond
 template<class value_type>
-MPI_Datatype getMPIDataType(){ assert( false && "Type not supported!\n" ); return; }
+MPI_Datatype getMPIDataType(){ assert( false && "Type not supported!\n" ); return MPI_DOUBLE; }
 template<>
 MPI_Datatype getMPIDataType<double>(){ return MPI_DOUBLE;}
 template<>
diff --git a/inc/dg/backend/mpi_derivatives.h b/inc/dg/backend/mpi_derivatives.h
index 763ae619c..f974caa14 100644
--- a/inc/dg/backend/mpi_derivatives.h
+++ b/inc/dg/backend/mpi_derivatives.h
@@ -122,7 +122,7 @@ EllSparseBlockMat<double> distribute_rows( const EllSparseBlockMat<double>& src,
 RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dx( const aMPITopology2d& g, bc bcx, direction dir = centered)
 {
     EllSparseBlockMat<double> matrix = dg::create::dx( g.global(), bcx, dir);
-    unsigned vector_dimensions[] = {(unsigned)(g.n()*g.Nx()), (unsigned)(g.n()*g.Ny()), 1}; //x, y, z
+    unsigned vector_dimensions[] = {(unsigned)(g.n()*g.local().Nx()), (unsigned)(g.n()*g.local().Ny()), 1}; //x, y, z
     MPI_Comm comm = g.communicator();
     int ndims;
     MPI_Cartdim_get( comm, &ndims);
@@ -151,7 +151,7 @@ RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dx( c
 RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dy( const aMPITopology2d& g, bc bcy, direction dir = centered)
 {
     EllSparseBlockMat<double> matrix = dg::create::dy( g.global(), bcy, dir);
-    unsigned vector_dimensions[] = {(unsigned)(g.n()*g.Nx()), (unsigned)(g.n()*g.Ny()), 1}; //x, y, z
+    unsigned vector_dimensions[] = {(unsigned)(g.n()*g.local().Nx()), (unsigned)(g.n()*g.local().Ny()), 1}; //x, y, z
     MPI_Comm comm = g.communicator();
     int ndims;
     MPI_Cartdim_get( comm, &ndims);
@@ -178,7 +178,7 @@ RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dy( c
 RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpX( const aMPITopology2d& g, bc bcx)
 {
     EllSparseBlockMat<double> matrix = dg::create::jumpX( g.global(), bcx);
-    unsigned vector_dimensions[] = {(unsigned)(g.n()*g.Nx()), (unsigned)(g.n()*g.Ny()), 1}; //x, y, z
+    unsigned vector_dimensions[] = {(unsigned)(g.n()*g.local().Nx()), (unsigned)(g.n()*g.local().Ny()), 1}; //x, y, z
     MPI_Comm comm = g.communicator();
     int ndims;
     MPI_Cartdim_get( comm, &ndims);
@@ -204,7 +204,7 @@ RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpX
 RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpY( const aMPITopology2d& g, bc bcy)
 {
     EllSparseBlockMat<double> matrix = dg::create::jumpY( g.global(), bcy);
-    unsigned vector_dimensions[] = {(unsigned)(g.n()*g.Nx()), (unsigned)(g.n()*g.Ny()), 1}; //x, y, z
+    unsigned vector_dimensions[] = {(unsigned)(g.n()*g.local().Nx()), (unsigned)(g.n()*g.local().Ny()), 1}; //x, y, z
     MPI_Comm comm = g.communicator();
     int ndims;
     MPI_Cartdim_get( comm, &ndims);
@@ -232,7 +232,7 @@ RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpY
 RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dx( const aMPITopology3d& g, bc bcx, direction dir = centered)
 {
     EllSparseBlockMat<double> matrix = dg::create::dx( g.global(), bcx, dir);
-    unsigned vector_dimensions[] = {(unsigned)(g.n()*g.Nx()), (unsigned)(g.n()*g.Ny()), (unsigned)(g.Nz())}; //x, y, z
+    unsigned vector_dimensions[] = {(unsigned)(g.n()*g.local().Nx()), (unsigned)(g.n()*g.local().Ny()), (unsigned)(g.local().Nz())}; //x, y, z
     MPI_Comm comm = g.communicator();
     int ndims;
     MPI_Cartdim_get( comm, &ndims);
@@ -259,7 +259,7 @@ RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dx( c
 RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dy( const aMPITopology3d& g, bc bcy, direction dir = centered)
 {
     EllSparseBlockMat<double> matrix = dg::create::dy( g.global(), bcy, dir);
-    unsigned vector_dimensions[] = {(unsigned)(g.n()*g.Nx()), (unsigned)(g.n()*g.Ny()), (unsigned)(g.Nz())}; //x, y, z
+    unsigned vector_dimensions[] = {(unsigned)(g.n()*g.local().Nx()), (unsigned)(g.n()*g.local().Ny()), (unsigned)(g.local().Nz())}; //x, y, z
     MPI_Comm comm = g.communicator();
     int ndims;
     MPI_Cartdim_get( comm, &ndims);
@@ -286,7 +286,7 @@ RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dy( c
 RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dz( const aMPITopology3d& g, bc bcz, direction dir = centered)
 {
     EllSparseBlockMat<double> matrix = dg::create::dz( g.global(), bcz, dir);
-    unsigned vector_dimensions[] = {(unsigned)(g.n()*g.Nx()), (unsigned)(g.n()*g.Ny()), (unsigned)(g.Nz())}; //x, y, z
+    unsigned vector_dimensions[] = {(unsigned)(g.n()*g.local().Nx()), (unsigned)(g.n()*g.local().Ny()), (unsigned)(g.local().Nz())}; //x, y, z
     MPI_Comm comm = g.communicator();
     int ndims;
     MPI_Cartdim_get( comm, &ndims);
@@ -313,7 +313,7 @@ RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> dz( c
 RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpX( const aMPITopology3d& g, bc bcx)
 {
     EllSparseBlockMat<double> matrix = dg::create::jumpX( g.global(), bcx);
-    unsigned vector_dimensions[] = {(unsigned)(g.n()*g.Nx()), (unsigned)(g.n()*g.Ny()), (unsigned)(g.Nz())}; //x, y, z
+    unsigned vector_dimensions[] = {(unsigned)(g.n()*g.local().Nx()), (unsigned)(g.n()*g.local().Ny()), (unsigned)(g.local().Nz())}; //x, y, z
     MPI_Comm comm = g.communicator();
     int ndims;
     MPI_Cartdim_get( comm, &ndims);
@@ -340,7 +340,7 @@ RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpX
 RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpY( const aMPITopology3d& g, bc bcy)
 {
     EllSparseBlockMat<double> matrix = dg::create::jumpY( g.global(), bcy);
-    unsigned vector_dimensions[] = {(unsigned)(g.n()*g.Nx()), (unsigned)(g.n()*g.Ny()), (unsigned)(g.Nz())}; //x, y, z
+    unsigned vector_dimensions[] = {(unsigned)(g.n()*g.local().Nx()), (unsigned)(g.n()*g.local().Ny()), (unsigned)(g.local().Nz())}; //x, y, z
     MPI_Comm comm = g.communicator();
     int ndims;
     MPI_Cartdim_get( comm, &ndims);
@@ -366,7 +366,7 @@ RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpY
 RowColDistMat< EllSparseBlockMat<double>, CooSparseBlockMat<double>, NNCH> jumpZ( const aMPITopology3d& g, bc bcz)
 {
     EllSparseBlockMat<double> matrix = dg::create::jumpZ( g.global(), bcz);
-    unsigned vector_dimensions[] = {(unsigned)(g.n()*g.Nx()), (unsigned)(g.n()*g.Ny()), (unsigned)(g.Nz())}; //x, y, z
+    unsigned vector_dimensions[] = {(unsigned)(g.n()*g.local().Nx()), (unsigned)(g.n()*g.local().Ny()), (unsigned)(g.local().Nz())}; //x, y, z
     MPI_Comm comm = g.communicator();
     int ndims;
     MPI_Cartdim_get( comm, &ndims);
diff --git a/inc/dg/backend/mpi_evaluation.h b/inc/dg/backend/mpi_evaluation.h
index ed7cc41a4..01f0ebe8d 100644
--- a/inc/dg/backend/mpi_evaluation.h
+++ b/inc/dg/backend/mpi_evaluation.h
@@ -31,7 +31,6 @@ MPI_Vector<thrust::host_vector<double> > evaluate( const BinaryOp& f, const aMPI
 {
     thrust::host_vector<double> w = evaluate( f, g.local());
     MPI_Vector<thrust::host_vector<double> > v( w, g.communicator());
-    //v.data() = evaluate(f,g.local());
     return v;
 };
 ///@cond
@@ -58,7 +57,6 @@ MPI_Vector<thrust::host_vector<double> > evaluate( const TernaryOp& f, const aMP
 {
     thrust::host_vector<double> w = evaluate( f, g.local());
     MPI_Vector<thrust::host_vector<double> > v( w, g.communicator());
-    //v.data() = evaluate(f, g.local());
     return v;
 };
 ///@cond
@@ -81,17 +79,18 @@ MPI_Vector<thrust::host_vector<double> > evaluate( double(f)(double, double, dou
 MPI_Vector<thrust::host_vector<double> > global2local( const thrust::host_vector<double>& global, const aMPITopology3d& g)
 {
     assert( global.size() == g.global().size());
-    thrust::host_vector<double> temp(g.size());
+    Grid3d l = g.local();
+    thrust::host_vector<double> temp(l.size());
     int dims[3], periods[3], coords[3];
     MPI_Cart_get( g.communicator(), 3, dims, periods, coords);
-    for( unsigned s=0; s<g.Nz(); s++)
+    for( unsigned s=0; s<l.Nz(); s++)
         //for( unsigned py=0; py<dims[1]; py++)
-            for( unsigned i=0; i<g.n()*g.Ny(); i++)
+            for( unsigned i=0; i<l.n()*l.Ny(); i++)
                 //for( unsigned px=0; px<dims[0]; px++)
-                    for( unsigned j=0; j<g.n()*g.Nx(); j++)
+                    for( unsigned j=0; j<l.n()*l.Nx(); j++)
                     {
-                        unsigned idx1 = (s*g.n()*g.Ny()+i)*g.n()*g.Nx() + j;
-                        unsigned idx2 = (((s*dims[1]+coords[1])*g.n()*g.Ny()+i)*dims[0] + coords[0])*g.n()*g.Nx() + j;
+                        unsigned idx1 = (s*l.n()*l.Ny()+i)*l.n()*l.Nx() + j;
+                        unsigned idx2 = (((s*dims[1]+coords[1])*l.n()*l.Ny()+i)*dims[0] + coords[0])*l.n()*l.Nx() + j;
                         temp[idx1] = global[idx2];
                     }
     return MPI_Vector<thrust::host_vector<double> >(temp, g.communicator()); 
@@ -103,16 +102,17 @@ MPI_Vector<thrust::host_vector<double> > global2local( const thrust::host_vector
 MPI_Vector<thrust::host_vector<double> > global2local( const thrust::host_vector<double>& global, const aMPITopology2d& g)
 {
     assert( global.size() == g.global().size());
-    thrust::host_vector<double> temp(g.size());
+    Grid2d l = g.local();
+    thrust::host_vector<double> temp(l.size());
     int dims[2], periods[2], coords[2];
     MPI_Cart_get( g.communicator(), 2, dims, periods, coords);
     //for( unsigned py=0; py<dims[1]; py++)
-        for( unsigned i=0; i<g.n()*g.Ny(); i++)
+        for( unsigned i=0; i<l.n()*l.Ny(); i++)
             //for( unsigned px=0; px<dims[0]; px++)
-                for( unsigned j=0; j<g.n()*g.Nx(); j++)
+                for( unsigned j=0; j<l.n()*l.Nx(); j++)
                 {
-                    unsigned idx1 = i*g.n()*g.Nx() + j;
-                    unsigned idx2 = ((coords[1]*g.n()*g.Ny()+i)*dims[0] + coords[0])*g.n()*g.Nx() + j;
+                    unsigned idx1 = i*l.n()*l.Nx() + j;
+                    unsigned idx2 = ((coords[1]*l.n()*l.Ny()+i)*dims[0] + coords[0])*l.n()*l.Nx() + j;
                     temp[idx1] = global[idx2];
                 }
     return MPI_Vector<thrust::host_vector<double> >(temp, g.communicator()); 
diff --git a/inc/dg/backend/mpi_grid.h b/inc/dg/backend/mpi_grid.h
index c103589e9..05bd2438e 100644
--- a/inc/dg/backend/mpi_grid.h
+++ b/inc/dg/backend/mpi_grid.h
@@ -559,9 +559,9 @@ struct aMPITopology3d
         double z1 = g.z0() + g.lz()/(double)dims[2]*(double)(coords[2]+1); 
         if( coords[2] == dims[2]-1) 
             z1 = g.z1();
-        Nx = g.Nx()/dims[0];
-        Ny = g.Ny()/dims[1];
-        Nz = g.Nz()/dims[2];
+        unsigned Nx = g.Nx()/dims[0];
+        unsigned Ny = g.Ny()/dims[1];
+        unsigned Nz = g.Nz()/dims[2];
         
         return Grid3d(x0, x1, y0, y1, z0, z1, g.n(), Nx, Ny, Nz, g.bcx(), g.bcy(), g.bcz());
     }
diff --git a/inc/dg/backend/split_and_join.h b/inc/dg/backend/split_and_join.h
index effd7bd6f..df06ec19a 100644
--- a/inc/dg/backend/split_and_join.h
+++ b/inc/dg/backend/split_and_join.h
@@ -15,15 +15,16 @@ namespace dg
 * @tparam thrust_vector1 either thrust::host_vector or thrust::device_vector
 * @tparam thrust_vector2 either thrust::host_vector or thrust::device_vector
 * @param in contiguous 3d vector (must be of size grid.size())
-* @param out contains grid.Nz() 2d vectors of 2d size on output (gets resized if necessary)
+* @param out contains \c grid.Nz() 2d vectors of 2d size on output (gets resized if necessary)
 * @param grid provide dimensions in 3rd and first two dimensions
 */
 template<class thrust_vector1, class thrust_vector2>
 void split( const thrust_vector1& in, std::vector<thrust_vector2>& out, const aTopology3d& grid)
 {
-    unsigned size2d=grid.n()*grid.n()*grid.Nx()*grid.Ny();
-    out.resize( grid.Nz());
-    for(unsigned i=0; i<grid.Nz(); i++)
+    Grid3d l( grid);
+    unsigned size2d=l.n()*l.n()*l.Nx()*l.Ny();
+    out.resize( l.Nz());
+    for(unsigned i=0; i<l.Nz(); i++)
         out[i].assign( in.begin() + i*size2d, in.begin()+(i+1)*size2d);
 }
 #ifdef MPI_VERSION
@@ -41,9 +42,10 @@ void split( const MPI_Vector<thrust_vector1>& in, std::vector<MPI_Vector<thrust_
     int remain_dims[] = {true,true,false}; 
     MPI_Cart_sub( in.communicator(), remain_dims, &planeComm);
     //local size2d
-    unsigned size2d=grid.n()*grid.n()*grid.Nx()*grid.Ny();
-    out.resize( grid.Nz());
-    for(unsigned i=0; i<grid.Nz(); i++)
+    Grid3d l = grid.local();
+    unsigned size2d=l.n()*l.n()*l.Nx()*l.Ny();
+    out.resize( l.Nz());
+    for(unsigned i=0; i<l.Nz(); i++)
     {
         out[i].data().assign( in.data().begin() + i*size2d, in.data().begin()+(i+1)*size2d);
         out[i].communicator() = planeComm;
@@ -55,7 +57,7 @@ void split( const MPI_Vector<thrust_vector1>& in, std::vector<MPI_Vector<thrust_
 *
 * @tparam thrust_vector1 either thrust::host_vector or thrust::device_vector
 * @tparam thrust_vector2 either thrust::host_vector or thrust::device_vector
-* @param in grid.Nz() 2d vectors of 2d size 
+* @param in \c grid.Nz() 2d vectors of 2d size 
 * @param out contiguous 3d vector (gets resized if necessary) 
 * @param grid provide dimensions in 3rd and first two dimensions
 * @note split followed by join restores the original vector
@@ -75,10 +77,11 @@ void join( const std::vector<thrust_vector1>& in, thrust_vector2& out, const aTo
 template<class thrust_vector1, class thrust_vector2>
 void join( const std::vector<MPI_Vector<thrust_vector1> >& in, MPI_Vector<thrust_vector2 >& out, const aMPITopology3d& grid)
 {
-    unsigned size2d=grid.n()*grid.n()*grid.Nx()*grid.Ny();
-    out.data().resize( size2d*grid.Nz());
+    Grid2d l(grid);
+    unsigned size2d=l.n()*l.n()*l.Nx()*l.Ny();
+    out.data().resize( size2d*l.Nz());
     out.communicator() = grid.communicator();
-    for(unsigned i=0; i<grid.Nz(); i++)
+    for(unsigned i=0; i<l.Nz(); i++)
         thrust::copy( in[i].data().begin(), in[i].data().end(), out.data().begin()+i*size2d);
 }
 #endif //MPI_VERSION
diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index 7f36fd60a..76b8559c5 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -69,6 +69,7 @@
  *
  *             High level matrix creation functions
  *         @defgroup interpolation Interpolation and projection
+ *         @defgroup utilities Averaging
  *         @defgroup scatter Scatter and Gather
  *     @}
  *     @defgroup geometry Geometric grids and operations
@@ -79,7 +80,6 @@
  *        @defgroup basicgeometry Geometry base classes
  *        @defgroup pullback pullback and pushforward
  *        @defgroup metric create volume
- *        @defgroup utilities Averaging
  *        @defgroup generators Grid Generator classes
  *            The classes to perform field line integration for DS and averaging classes
  *    @}
-- 
GitLab


From 2d0557bb608acf35ab9f00ce6ccd644bc7e2359c Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Thu, 26 Oct 2017 15:37:25 +0200
Subject: [PATCH 393/453] reworked and documented config system

---
 config/README.md          | 50 +++++++++++++++++++++++++++++++++++++++
 config/default.mk         | 16 +++++++++----
 config/devices/devices.mk | 34 ++++++++------------------
 3 files changed, 71 insertions(+), 29 deletions(-)
 create mode 100644 config/README.md

diff --git a/config/README.md b/config/README.md
new file mode 100644
index 000000000..cb62dc1ee
--- /dev/null
+++ b/config/README.md
@@ -0,0 +1,50 @@
+The Makefiles in FELTOR are configured by hardware specific *.mk files in the config folder. Every Makefile includes these files in the order  
+
+```shell
+feltor/config/default.mk #Sets the default variables
+feltor/config/*.mk       #overwrite variables if machine is recognized
+feltor/config/devices/devices.mk    #recombine variables depending on device
+```
+
+The machine specific config files (e.g. vsc3.mk) should have an include guard and can overwrite or add to any of the following variables:
+
+| variable  | default value                            | description                              |
+| :-------: | :--------------------------------------- | :--------------------------------------- |
+|    CC     | g++                                      | C++ compiler                             |
+|  CFLAGS   | -std=c++11 -Wall -x c++                  | flags for the C++ compiler               |
+|   MPICC   | mpic++                                   | the mpi wrapper for the c++ compiler     |
+| MPICFLAGS | -std=c++11 -Wall -x c++                  | flags for MPI compilation                |
+|   NVCC    | nvcc                                     | CUDA compiler                            |
+| NVCCFLAGS | -std=c++11                               | flags for nvcc                           |
+| NVCCARCH  | -arch sm_20                              | specify the **gpu** compute capability  https://developer.nvidia.com/cuda-gpus (can be overwritten on the command line) |
+|    OPT    | -O3                                      | optimization flags for the **host** code (can be overwritten on the command line, CUDA kernel code is always compiled with -O3) |
+|  OMPFLAG  | -fopenmp                                 | The compiler flag activating the OpenMP support |
+|           |                                          |                                          |
+|  INCLUDE  | -I$(HOME)/include                        | cusp, thrust, json and the draw libraries. The default expects to find (symbolic links to ) these libraries in your home folder |
+|   LIBS    | -lnetcdf -lhdf5 -ldhf5_hl                | netcdf library                           |
+|  JSONLIB  | -L$(HOME)/include/json/../../src/lib_json -ljsoncpp | the JSONCPP library                      |
+|  GLFLAGS  | $$(pkg-config --static --libs glfw3)     | glfw3 installation (if glfw3 was installed correctly the default should work) |
+
+
+The file `feltor/config/devices/devices.mk` defines device specific configurations and MACROS that essentially steer the behaviour of the cusp and thrust libraries and serve as include guards. These are activated by setting the variable **device**, which for now can take one of the following values:
+
+| value | description                              | flags                                    |
+| ----- | ---------------------------------------- | ---------------------------------------- |
+| gpu   | combines CC and NVCC  into CC, CFLAGS, NVCCFLAGS and NVCCARCH  into CFLAGS and analogously the MPI flags MPICC and NVCC into MPICC, MPICFLAGS, NVCCFLAGS and NVCCARCH into MPICFLAGS using nvcc's --compiler-bindir and -Xcompiler options | -D_FORCE_INLINES added to CFLAGS and MPICFLAGS |
+| !gpu  | if device != gpu all thrust device calls redirect to OpenMP using THRUST_DEVICE_SYSTEM macro | -DTHRUST_DEVICE_SYSTEM= THRUST_DEVICE_SYSTEM_OMP $(OMPFLAG) added to both CFLAGS and MPICFLAGS |
+| mic   | specify OPT for Intel Xeon Phi architecture | OPT = -O3 -xMIC-AVX512                   |
+| skl   | specify OPT for Intel Skylake processors | OPT = -xCORE-AVX512 -mtune=skylake -O3   |
+
+### Examples
+
+The **device** variable should be, the **OPT** and the **NVCCARCH** variables are intended to be specified on the command line: 
+
+```shell
+make blas_b device=gpu NVCCARCH=-arch sm_35 #Compile using nvcc for a Tesla K40
+make blas_b device=omp OPT=-O2              #Compile for OpenMP using -O2 
+```
+
+```shell
+make blas_mpib device=mic #Compile an MPI program for the Xeon Phi architecture 
+```
+
diff --git a/config/default.mk b/config/default.mk
index b15f57d5b..db46091f5 100644
--- a/config/default.mk
+++ b/config/default.mk
@@ -1,14 +1,20 @@
 ifndef INCLUDED #include guard
 INCLUDED=1
 
-#default machine values
-INCLUDE = -I$(HOME)/include#  # cusp and thrust and the draw libraries
-GLFLAGS =$$(pkg-config --static --libs glfw3) #glfw3 installation
-CC=g++ -std=c++14 #C++ compiler
+#default compilation values
+CC=g++ #C++ compiler
+CFLAGS=-Wall -x c++ -std=c++11 
 MPICC=mpic++  #mpi compiler
-OPT=-O3 # optimization flag
+MPICFLAGS=-Wall -x c++ -std=c++11
+NVCC=nvcc #CUDA compiler
 NVCCARCH=-arch sm_20 #nvcc gpu compute capability
+NVCCFLAGS=-std=c++11 #
+OPT=-O3 # optimization flag to be used in Makefile
 OMPFLAG=-fopenmp #openmp flag for CC and MPICC
+
+#external libraries
+INCLUDE = -I$(HOME)/include# cusp, thrust, jsoncpp and the draw libraries
 LIBS=-lnetcdf -lhdf5 -lhdf5_hl # netcdf library for file output
 JSONLIB=-L$(HOME)/include/json/../../src/lib_json -ljsoncpp # json library for input parameters
+GLFLAGS =$$(pkg-config --static --libs glfw3) #glfw3 installation
 endif # INCLUDED
diff --git a/config/devices/devices.mk b/config/devices/devices.mk
index 64837b888..f757f51c6 100644
--- a/config/devices/devices.mk
+++ b/config/devices/devices.mk
@@ -1,39 +1,25 @@
 ifeq ($(strip $(device)),gpu)
 ccc_:=$(CC)
-CC = nvcc --compiler-bindir $(ccc_)
-OPT=-O3
+CC = $(NVCC) --compiler-bindir $(ccc_)
+flags_:=$(CFLAGS)
+CFLAGS = -Xcompiler "$(flags_)"
+CFLAGS+= $(NVCCARCH) $(NVCCFLAGS) 
 CFLAGS+=-D_FORCE_INLINES
-CFLAGS+= --compiler-options -Wall $(NVCCARCH)
-#CFLAGS+= -Xcompiler $(OMPFLAG)
-#CFLAGS+= -DTHRUST_HOST_SYSTEM=THRUST_HOST_SYSTEM_OMP
-#CFLAGS+= -DCUSP_DEVICE_BLAS_SYSTEM=CUSP_DEVICE_BLAS_CUBLAS -lcublas
-#CFLAGS+= -DCUSP_USE_TEXTURE_MEMORY
+############################################
 mpiccc_:=$(MPICC)
 MPICC=nvcc --compiler-bindir $(mpiccc_)
+mpiflags_:=$(MPICFLAGS)
+MPICFLAGS = -Xcompiler "$(mpiflags_)" $(NVCCARCH) $(NVCCFLAGS)
 MPICFLAGS+= -D_FORCE_INLINES
-#MPICFLAGS+= -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_CUDA
-#MPICFLAGS+= -DCUSP_DEVICE_BLAS_SYSTEM=CUSP_DEVICE_BLAS_CUBLAS -lcublas
-#MPICFLAGS+= -DCUSP_USE_TEXTURE_MEMORY
-MPICFLAGS+= --compiler-options -Wall $(NVCCARCH)
-MPICFLAGS+= --compiler-options $(OPT)
+else
+CFLAGS+= -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_OMP $(OMPFLAG)
+MPICFLAGS+= -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_OMP $(OMPFLAG)
 endif #device=gpu
 ifeq ($(strip $(device)),omp)
-CFLAGS+=-Wall -x c++
-CFLAGS+= -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_OMP
-CFLAGS+= $(OMPFLAG) 
-MPICFLAGS+=$(CFLAGS) #includes values in CFLAGS defined later
 endif #device=omp
 ifeq ($(strip $(device)),mic)
-CFLAGS+=-Wall -x c++ 
-CFLAGS+= -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_OMP
-CFLAGS+= $(OMPFLAG) #-mmic 
-MPICFLAGS+=$(CFLAGS) #includes values in CFLAGS defined later
 OPT=-O3 -xMIC-AVX512 
 endif #device=mic
 ifeq ($(strip $(device)),skl)
-CFLAGS+=-Wall -x c++ 
-CFLAGS+= -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_OMP
-CFLAGS+= $(OMPFLAG) #-mmic 
-MPICFLAGS+=$(CFLAGS) #includes values in CFLAGS defined later
 OPT=-xCORE-AVX512 -mtune=skylake -O3 
 endif #device=mic
-- 
GitLab


From 4eafeac3fac5de09e3abcdc7aa8c65e34e4c3e18 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Thu, 26 Oct 2017 15:40:54 +0200
Subject: [PATCH 394/453] update documentation on config

---
 config/README.md | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/config/README.md b/config/README.md
index cb62dc1ee..c50d95949 100644
--- a/config/README.md
+++ b/config/README.md
@@ -1,7 +1,9 @@
-The Makefiles in FELTOR are configured by hardware specific *.mk files in the config folder. Every Makefile includes these files in the order  
+###Configuration
+
+The Makefiles for programs in FELTOR are configured by hardware specific *.mk files in the config folder. Every Makefile includes these files in the order  
 
 ```shell
-feltor/config/default.mk #Sets the default variables
+feltor/config/default.mk #Defines the default variables
 feltor/config/*.mk       #overwrite variables if machine is recognized
 feltor/config/devices/devices.mk    #recombine variables depending on device
 ```
@@ -40,11 +42,18 @@ The file `feltor/config/devices/devices.mk` defines device specific configuratio
 The **device** variable should be, the **OPT** and the **NVCCARCH** variables are intended to be specified on the command line: 
 
 ```shell
-make blas_b device=gpu NVCCARCH=-arch sm_35 #Compile using nvcc for a Tesla K40
-make blas_b device=omp OPT=-O2              #Compile for OpenMP using -O2 
-```
+#Compile using nvcc for a Tesla K40:
+make blas_b device=gpu NVCCARCH='-arch sm_35'
 
-```shell
-make blas_mpib device=mic #Compile an MPI program for the Xeon Phi architecture 
+#Compile for OpenMP using -O2:
+make blas_b device=omp OPT=-O2     
+
+#Hybrid MPI+OpenMP program for the Xeon Phi architecture:
+make blas_mpib device=mic 
+
+#Hybrid MPI+GPU program for the Tesla P100 GPU, host code with -O2:
+make blas_mpib device=gpu NVCCARCH='-arch sm_60' OPT=-O2
 ```
 
+ 
+
-- 
GitLab


From 40d708f3e1d14fca5938956c99a62d7621c6f5d9 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Thu, 26 Oct 2017 16:17:41 +0200
Subject: [PATCH 395/453] update README.md with info on using FELTOR as a
 library

---
 README.md        | 33 ++++++++++++++++++++++++---------
 config/README.md |  2 +-
 2 files changed, 25 insertions(+), 10 deletions(-)

diff --git a/README.md b/README.md
index 1afb84104..885831102 100644
--- a/README.md
+++ b/README.md
@@ -30,11 +30,11 @@ Now you need to tell the feltor configuration where these external libraries are
  $ cd include
  $ ln -s path/to/thrust/thrust thrust
  $ ln -s path/to/cusplibrary/cusp cusp
-```
+ ```
 > If you do not like this, you can also create your own config file as discribed [here](https://github.com/feltor-dev/feltor/wiki/Configuration).
 
 Now let us compile the first benchmark program. 
- 
+
 
  ```sh
  $ cd path/to/feltor/inc/dg
@@ -58,7 +58,7 @@ and when prompted for input vector sizes type for example
 `3 100 100 10`
 which makes a grid with 3 polynomial coefficients, 100 cells in x, 100 cells in y and 10 in z. If you compiled for OpenMP, you can set the number of threads with e.g. `export OMP_NUM_THREADS=4`.
 > This is a benchmark program to benchmark various elemental functions the library is built on. Go ahead and vary the input parameters and
-see how your hardware performs. You can compile and run any other program that ends in `_t.cu` (test programs) or `_b.cu` (benchmark programs) in `feltor/inc/dg` in this way. 
+> see how your hardware performs. You can compile and run any other program that ends in `_t.cu` (test programs) or `_b.cu` (benchmark programs) in `feltor/inc/dg` in this way. 
 
 Now, let us test the mpi setup 
 > You can of course skip this if you don't have mpi installed on your computer.
@@ -70,7 +70,7 @@ Now, let us test the mpi setup
  $ make blas_mpib device=omp  # (for MPI+OpenMP)
  # or
  $ make blas_mpib device=gpu # (for MPI+GPU)
- ```
+```
 Run the code with
 `$ mpirun -n '# of procs' ./blas_mpib `
 then tell how many process you want to use in the x-, y- and z- direction, for example:
@@ -84,7 +84,7 @@ For data output we use the [NetCDF](http://www.unidata.ucar.edu/software/netcdf/
 Our JSON input files are parsed by [JsonCpp](https://www.github.com/open-source-parsers/jsoncpp) distributed under the MIT license (the 0.y.x branch to avoid C++-11 support).     
 > Some desktop applications in FELTOR use the [draw library]( https://github.com/mwiesenberger/draw) (developed by us also under MIT), which depends on OpenGL (s.a. [installation guide](http://en.wikibooks.org/wiki/OpenGL_Programming)) and [glfw](http://www.glfw.org), an OpenGL development library under a BSD-like license. You don't need these when you are on a cluster. 
 
- 
+
 As in Step 3 you need to create links to the jsoncpp library include path (and optionally the draw library) in your include folder or provide the paths in your config file. We are ready to compile now
 
 ```sh
@@ -98,7 +98,7 @@ As in Step 3 you need to create links to the jsoncpp library include path (and o
  # or
  $ make toefl_mpi device=omp  # (compile on gpu or omp)
  $ export OMP_NUM_THREADS=2   # (set OpenMP thread number to 1 for pure MPI) 
- $ echo 2 2 | mpirun -n 4 ./toefl_mpi <inputfile.json> <outputfile.json>
+ $ echo 2 2 | mpirun -n 4 ./toefl_mpi <inputfile.json> <outputfile.nc>
  $ # (a multi node simulation with now in total 8 threads with output stored in a file)
  $ # The mpi program will wait for you to type the number of processes in x and y direction before
  $ # running. That is why the echo is there. 
@@ -108,7 +108,22 @@ The technical documentation on what equations are discretized,
 input/output parameters, etc. can be generated as a pdf with 
 `make doc ` in the `path/to/feltor/src/toefl` directory.
 
-## 2. Further reading
+##2. Using FELTOR as a library
+
+It is possible to use FELTOR as a library in your own code project. Just include
+
+```C++
+#include "mpi.h" #optional; activates MPI in FELTOR 
+#define THRUST_DEVICE_SYSTEM THRUST_DEVICE_SYSTEM_OMP # optional; redirect CUDA calls to OpenMP functions; Note that you have to activate OpenMP  
+#include "dg/algorithms.h"
+#include "geometries/geometries.h"
+```
+
+and provide the `path/to/feltor/inc` to the include path of your compiler. Note that you also have to specify the `path/to/thrust/thrust` and `path/to/cusplibrary/cusp`.
+
+> If you want to activate the MPI backend of FELTOR you have to include `mpi.h` before any FELTOR header. If you want to use OpenMP instead of CUDA for the device functions you have to define the `THRUST_DEVICE_SYSTEM` macro and activate OpenMP in your compiler (e.g `g++ -fopenmp`).
+
+## 3. Further reading
 Please check out our [wiki pages](https://github.com/feltor-dev/feltor/wiki) for some general information, user oriented documentation and Troubleshooting. 
 Moreover, we maintain tex files in every src folder for technical documentation, 
  which can be compiled using pdflatex with 
@@ -118,7 +133,7 @@ You can generate a local version from source code.
 This depends on the `doxygen`, `libjs-mathjax` and `graphviz` packages.
 Type `make doc` in the folder `path/to/feltor/doc` and open `index.html` (a symbolic link to `dg/html/modules.html`) with your favorite browser. 
 
-## 3. Contributions and Acknowledgements
+## 4. Contributions and Acknowledgements
 For instructions on how to contribute read the [wiki page](https://github.com/feltor-dev/feltor/wiki/Contributions).
 We gratefully acknowledge contributions from 
 - Ralph Kube
@@ -132,7 +147,7 @@ We further acknowledge support on the Knights landing architecture from the High
 
 and from Intel Barcelona
 - Harald Servat
-## 4. License 
+## 5. License 
 FELTOR is free software and licensed under the very permissive MIT license. It was originally developed by Matthias Wiesenberger and Markus Held.
 
 ## Official releases 
diff --git a/config/README.md b/config/README.md
index c50d95949..ca1c62c91 100644
--- a/config/README.md
+++ b/config/README.md
@@ -1,4 +1,4 @@
-###Configuration
+### Configuration
 
 The Makefiles for programs in FELTOR are configured by hardware specific *.mk files in the config folder. Every Makefile includes these files in the order  
 
-- 
GitLab


From 782c7937827d696c6a6987100a4b49b72a5b45ae Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Thu, 26 Oct 2017 16:28:32 +0200
Subject: [PATCH 396/453] refine C++ example in README.md

---
 README.md | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 885831102..fee059a99 100644
--- a/README.md
+++ b/README.md
@@ -110,16 +110,19 @@ input/output parameters, etc. can be generated as a pdf with
 
 ##2. Using FELTOR as a library
 
-It is possible to use FELTOR as a library in your own code project. Just include
+It is possible to use FELTOR as a library in your own code project. Note that this library is **header-only**, which means that you just have to include the relevant header and you're good to go:
 
 ```C++
-#include "mpi.h" #optional; activates MPI in FELTOR 
-#define THRUST_DEVICE_SYSTEM THRUST_DEVICE_SYSTEM_OMP # optional; redirect CUDA calls to OpenMP functions; Note that you have to activate OpenMP  
+//optional: activate MPI in FELTOR
+#include "mpi.h"  
+// optional: redirect CUDA calls to OpenMP functions; 
+// Note that you then also have to specify the OpenMP flag when compiling
+#define THRUST_DEVICE_SYSTEM THRUST_DEVICE_SYSTEM_OMP  
 #include "dg/algorithms.h"
 #include "geometries/geometries.h"
 ```
 
-and provide the `path/to/feltor/inc` to the include path of your compiler. Note that you also have to specify the `path/to/thrust/thrust` and `path/to/cusplibrary/cusp`.
+and add `path/to/feltor/inc` as well as  `path/to/thrust/thrust` and  `path/to/cusplibrary/cusp`to the include path of your compiler. 
 
 > If you want to activate the MPI backend of FELTOR you have to include `mpi.h` before any FELTOR header. If you want to use OpenMP instead of CUDA for the device functions you have to define the `THRUST_DEVICE_SYSTEM` macro and activate OpenMP in your compiler (e.g `g++ -fopenmp`).
 
-- 
GitLab


From 07ff27513bf22ecdf57df40510c8ca9b8bb19cbd Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Thu, 26 Oct 2017 16:38:28 +0200
Subject: [PATCH 397/453] further adds

---
 README.md | 49 +++++++++++++++++++++++++++----------------------
 1 file changed, 27 insertions(+), 22 deletions(-)

diff --git a/README.md b/README.md
index fee059a99..c36f16d82 100644
--- a/README.md
+++ b/README.md
@@ -23,7 +23,30 @@ $ git clone https://www.github.com/cusplibrary/cusplibrary
 ```
 > Our code only depends on external libraries that are themselves openly available. We note here that we do not distribute copies of these libraries.
 
-Now you need to tell the feltor configuration where these external libraries are located on your computer. The default way to do this is to go in your `HOME` directory, make an include directory and link the paths in this directory:
+####  Using FELTOR as a library
+
+It is possible to use FELTOR as a library in your own code project. Note that the library is **header-only**, which means that you just have to include the relevant header(s) and you're good to go:
+
+```C++
+//optional: activate MPI in FELTOR
+#include "mpi.h"  
+// optional: redirect CUDA calls to OpenMP functions; 
+// note that you then also have to specify an OpenMP flag when compiling
+#define THRUST_DEVICE_SYSTEM THRUST_DEVICE_SYSTEM_OMP 
+//include the basic dg-library
+#include "dg/algorithms.h"
+//include the geometries expansion
+#include "geometries/geometries.h"
+```
+
+and add `path/to/feltor/inc` as well as  `path/to/thrust/thrust` and  `path/to/cusplibrary/cusp` to the include path of your compiler. 
+
+> If you want to activate the MPI backend of FELTOR you have to include `mpi.h` before any FELTOR header. If you want to use OpenMP instead of CUDA for the device functions you have to define the `THRUST_DEVICE_SYSTEM` macro and activate OpenMP in your compiler (e.g `g++ -fopenmp`). I you want to use CUDA then you have to compile with nvcc. 
+
+####  Using FELTOR's code projects
+
+In order to compile one of the many codes inside FELTOR you need to tell the feltor configuration where the external libraries are located on your computer. The default way to do this is to go in your `HOME` directory, make an include directory and link the paths in this directory:
+
  ```sh
  $ cd ~
  $ mkdir include
@@ -108,25 +131,7 @@ The technical documentation on what equations are discretized,
 input/output parameters, etc. can be generated as a pdf with 
 `make doc ` in the `path/to/feltor/src/toefl` directory.
 
-##2. Using FELTOR as a library
-
-It is possible to use FELTOR as a library in your own code project. Note that this library is **header-only**, which means that you just have to include the relevant header and you're good to go:
-
-```C++
-//optional: activate MPI in FELTOR
-#include "mpi.h"  
-// optional: redirect CUDA calls to OpenMP functions; 
-// Note that you then also have to specify the OpenMP flag when compiling
-#define THRUST_DEVICE_SYSTEM THRUST_DEVICE_SYSTEM_OMP  
-#include "dg/algorithms.h"
-#include "geometries/geometries.h"
-```
-
-and add `path/to/feltor/inc` as well as  `path/to/thrust/thrust` and  `path/to/cusplibrary/cusp`to the include path of your compiler. 
-
-> If you want to activate the MPI backend of FELTOR you have to include `mpi.h` before any FELTOR header. If you want to use OpenMP instead of CUDA for the device functions you have to define the `THRUST_DEVICE_SYSTEM` macro and activate OpenMP in your compiler (e.g `g++ -fopenmp`).
-
-## 3. Further reading
+## 2. Further reading
 Please check out our [wiki pages](https://github.com/feltor-dev/feltor/wiki) for some general information, user oriented documentation and Troubleshooting. 
 Moreover, we maintain tex files in every src folder for technical documentation, 
  which can be compiled using pdflatex with 
@@ -136,7 +141,7 @@ You can generate a local version from source code.
 This depends on the `doxygen`, `libjs-mathjax` and `graphviz` packages.
 Type `make doc` in the folder `path/to/feltor/doc` and open `index.html` (a symbolic link to `dg/html/modules.html`) with your favorite browser. 
 
-## 4. Contributions and Acknowledgements
+## 3. Contributions and Acknowledgements
 For instructions on how to contribute read the [wiki page](https://github.com/feltor-dev/feltor/wiki/Contributions).
 We gratefully acknowledge contributions from 
 - Ralph Kube
@@ -150,7 +155,7 @@ We further acknowledge support on the Knights landing architecture from the High
 
 and from Intel Barcelona
 - Harald Servat
-## 5. License 
+## 4. License 
 FELTOR is free software and licensed under the very permissive MIT license. It was originally developed by Matthias Wiesenberger and Markus Held.
 
 ## Official releases 
-- 
GitLab


From 5190685d5f1494980b8f46d15a26e98bbb0ecd9e Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 26 Oct 2017 22:19:49 +0200
Subject: [PATCH 398/453] debugged configuration for gpu + std=c++11 support

---
 README.md                 |  1 +
 config/README.md          | 28 ++++++++++++++++------------
 config/default.mk         |  6 +++---
 config/devices/devices.mk | 17 ++++++++---------
 config/marconi.mk         |  4 ++--
 5 files changed, 30 insertions(+), 26 deletions(-)

diff --git a/README.md b/README.md
index c36f16d82..518d4a2a3 100644
--- a/README.md
+++ b/README.md
@@ -42,6 +42,7 @@ It is possible to use FELTOR as a library in your own code project. Note that th
 and add `path/to/feltor/inc` as well as  `path/to/thrust/thrust` and  `path/to/cusplibrary/cusp` to the include path of your compiler. 
 
 > If you want to activate the MPI backend of FELTOR you have to include `mpi.h` before any FELTOR header. If you want to use OpenMP instead of CUDA for the device functions you have to define the `THRUST_DEVICE_SYSTEM` macro and activate OpenMP in your compiler (e.g `g++ -fopenmp`). I you want to use CUDA then you have to compile with nvcc. 
+See `path/to/feltor/config/README.md` for further details on configurating the library and how to compile.
 
 ####  Using FELTOR's code projects
 
diff --git a/config/README.md b/config/README.md
index ca1c62c91..9e61310c8 100644
--- a/config/README.md
+++ b/config/README.md
@@ -3,8 +3,8 @@
 The Makefiles for programs in FELTOR are configured by hardware specific *.mk files in the config folder. Every Makefile includes these files in the order  
 
 ```shell
-feltor/config/default.mk #Defines the default variables
-feltor/config/*.mk       #overwrite variables if machine is recognized
+feltor/config/default.mk            #Defines the default variables
+feltor/config/*.mk                  #overwrite variables if machine is recognized
 feltor/config/devices/devices.mk    #recombine variables depending on device
 ```
 
@@ -16,37 +16,38 @@ The machine specific config files (e.g. vsc3.mk) should have an include guard an
 |  CFLAGS   | -std=c++11 -Wall -x c++                  | flags for the C++ compiler               |
 |   MPICC   | mpic++                                   | the mpi wrapper for the c++ compiler     |
 | MPICFLAGS | -std=c++11 -Wall -x c++                  | flags for MPI compilation                |
+   OPT    | -O3                                      | optimization flags for the **host** code (can be overwritten on the command line, CUDA kernel code is always compiled with -O3) |
+|  OMPFLAG  | -fopenmp                                 | The compiler flag activating the OpenMP support |
 |   NVCC    | nvcc                                     | CUDA compiler                            |
-| NVCCFLAGS | -std=c++11                               | flags for nvcc                           |
+| NVCCFLAGS | -std=c++11  -Xcompiler -Wall                             | flags for nvcc                           |
 | NVCCARCH  | -arch sm_20                              | specify the **gpu** compute capability  https://developer.nvidia.com/cuda-gpus (can be overwritten on the command line) |
-|    OPT    | -O3                                      | optimization flags for the **host** code (can be overwritten on the command line, CUDA kernel code is always compiled with -O3) |
-|  OMPFLAG  | -fopenmp                                 | The compiler flag activating the OpenMP support |
-|           |                                          |                                          |
+        |                                          |                                          |
 |  INCLUDE  | -I$(HOME)/include                        | cusp, thrust, json and the draw libraries. The default expects to find (symbolic links to ) these libraries in your home folder |
 |   LIBS    | -lnetcdf -lhdf5 -ldhf5_hl                | netcdf library                           |
 |  JSONLIB  | -L$(HOME)/include/json/../../src/lib_json -ljsoncpp | the JSONCPP library                      |
 |  GLFLAGS  | $$(pkg-config --static --libs glfw3)     | glfw3 installation (if glfw3 was installed correctly the default should work) |
 
 
-The file `feltor/config/devices/devices.mk` defines device specific configurations and MACROS that essentially steer the behaviour of the cusp and thrust libraries and serve as include guards. These are activated by setting the variable **device**, which for now can take one of the following values:
+The main purpose of the file `feltor/config/devices/devices.mk` is to configure the nvcc + X compilation but it can also be used to specifiy optimizations for specific hardware. These are activated by setting the variable **device**, which for now can take one of the following values:
 
 | value | description                              | flags                                    |
 | ----- | ---------------------------------------- | ---------------------------------------- |
-| gpu   | combines CC and NVCC  into CC, CFLAGS, NVCCFLAGS and NVCCARCH  into CFLAGS and analogously the MPI flags MPICC and NVCC into MPICC, MPICFLAGS, NVCCFLAGS and NVCCARCH into MPICFLAGS using nvcc's --compiler-bindir and -Xcompiler options | -D_FORCE_INLINES added to CFLAGS and MPICFLAGS |
-| !gpu  | if device != gpu all thrust device calls redirect to OpenMP using THRUST_DEVICE_SYSTEM macro | -DTHRUST_DEVICE_SYSTEM= THRUST_DEVICE_SYSTEM_OMP $(OMPFLAG) added to both CFLAGS and MPICFLAGS |
+| gpu   | replaces the CC and CFLAGS variables with the nvcc versions and analogously MPICC and MPICFLAGS  | `CC = $(NVCC) --compiler-bindir $(CC)` `CFLAGS = $(NVCCARCH) $(NVCCFLAGS)` `MPICC = $(NVCC) --compiler-bindir $(MPICC)` `MPICFLAGS = $(NVCCARCH) $(NVCCFLAGS)` |
+| !gpu  | if device != gpu all thrust device calls redirect to OpenMP using the THRUST_DEVICE_SYSTEM macro | `-DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_OMP $(OMPFLAG)` added to both CFLAGS and MPICFLAGS |
+| omp  | specify OPT for OpenMP | OPT = -O3                   |
 | mic   | specify OPT for Intel Xeon Phi architecture | OPT = -O3 -xMIC-AVX512                   |
-| skl   | specify OPT for Intel Skylake processors | OPT = -xCORE-AVX512 -mtune=skylake -O3   |
+| skl   | specify OPT for Intel Skylake processors (icc only) | OPT = -xCORE-AVX512 -mtune=skylake -O3   |
 
 ### Examples
 
-The **device** variable should be, the **OPT** and the **NVCCARCH** variables are intended to be specified on the command line: 
+The **device** variable should be, the **OPT** and the **NVCCARCH** variables can be specified on the command line: 
 
 ```shell
 #Compile using nvcc for a Tesla K40:
 make blas_b device=gpu NVCCARCH='-arch sm_35'
 
 #Compile for OpenMP using -O2:
-make blas_b device=omp OPT=-O2     
+make blas_b device=omp OPT=-O2
 
 #Hybrid MPI+OpenMP program for the Xeon Phi architecture:
 make blas_mpib device=mic 
@@ -55,5 +56,8 @@ make blas_mpib device=mic
 make blas_mpib device=gpu NVCCARCH='-arch sm_60' OPT=-O2
 ```
 
+### General Remarks
+ - If MPI is used in connection with the gpu backend, the mpi installation needs to be **cuda-aware**
+ - when `icc` is used as the C++ compiler the `-restrict` option has to be used to enable the recognition of the restrict keyword
  
 
diff --git a/config/default.mk b/config/default.mk
index db46091f5..8d54305d8 100644
--- a/config/default.mk
+++ b/config/default.mk
@@ -1,15 +1,15 @@
 ifndef INCLUDED #include guard
 INCLUDED=1
 
-#default compilation values
+#compiler and compiler options
 CC=g++ #C++ compiler
 CFLAGS=-Wall -x c++ -std=c++11 
 MPICC=mpic++  #mpi compiler
 MPICFLAGS=-Wall -x c++ -std=c++11
 NVCC=nvcc #CUDA compiler
 NVCCARCH=-arch sm_20 #nvcc gpu compute capability
-NVCCFLAGS=-std=c++11 #
-OPT=-O3 # optimization flag to be used in Makefile
+NVCCFLAGS= -std=c++11 -Xcompiler -Wall#
+OPT=-O3 # optimization flags for host code
 OMPFLAG=-fopenmp #openmp flag for CC and MPICC
 
 #external libraries
diff --git a/config/devices/devices.mk b/config/devices/devices.mk
index f757f51c6..76168f4b2 100644
--- a/config/devices/devices.mk
+++ b/config/devices/devices.mk
@@ -1,19 +1,18 @@
 ifeq ($(strip $(device)),gpu)
 ccc_:=$(CC)
 CC = $(NVCC) --compiler-bindir $(ccc_)
-flags_:=$(CFLAGS)
-CFLAGS = -Xcompiler "$(flags_)"
-CFLAGS+= $(NVCCARCH) $(NVCCFLAGS) 
-CFLAGS+=-D_FORCE_INLINES
+CFLAGS = $(NVCCARCH) $(NVCCFLAGS) 
+CFLAGS+=-D_FORCE_INLINES # solves issue with std=c++11
+CFLAGS+=-D_MWAITXINTRIN_H_INCLUDED # solves issue with std=c++11
 ############################################
 mpiccc_:=$(MPICC)
 MPICC=nvcc --compiler-bindir $(mpiccc_)
-mpiflags_:=$(MPICFLAGS)
-MPICFLAGS = -Xcompiler "$(mpiflags_)" $(NVCCARCH) $(NVCCFLAGS)
-MPICFLAGS+= -D_FORCE_INLINES
+MPICFLAGS = $(NVCCARCH) $(NVCCFLAGS)
+MPICFLAGS+=-D_FORCE_INLINES # solves issue with std=c++11
+MPICFLAGS+=-D_MWAITXINTRIN_H_INCLUDED # solves issue with std=c++11
 else
-CFLAGS+= -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_OMP $(OMPFLAG)
-MPICFLAGS+= -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_OMP $(OMPFLAG)
+CFLAGS+=-DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_OMP $(OMPFLAG)
+MPICFLAGS+=-DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_OMP $(OMPFLAG)
 endif #device=gpu
 ifeq ($(strip $(device)),omp)
 endif #device=omp
diff --git a/config/marconi.mk b/config/marconi.mk
index 8c2ff4cbe..0bb297749 100644
--- a/config/marconi.mk
+++ b/config/marconi.mk
@@ -3,12 +3,12 @@ ifeq ($(strip $(HPC_SYSTEM)),marconi)
 INCLUDE += -I$(HOME)/include # cusp, thrust
 INCLUDE += -I$(NETCDF_INC) -I$(HDF5_INC)
 GLFLAGS  = -lm 
-CC=icc -std=c++14
+CC=icc
 MPICC=mpiicc
 OPT=-O3 -xHost  # overwritten for mic in devices.mk
 #MPICFLAGS+= -DMPICH_IGNORE_CXX_SEEK
 OMPFLAG=-qopenmp
-CFLAGS=-restrict
+CFLAGS=-std=c++11 -Wall -restrict
 JSONLIB=-L$(HOME)/include/json/../../src/lib_json -ljsoncpp # json library for input parameters
 LIBS    +=-L$(HDF5_LIB) -lhdf5 -lhdf5_hl
 LIBS    +=-L$(NETCDF_LIB) -lnetcdf -lcurl
-- 
GitLab


From 1c5f47c0a11b6d7b338ce5332c242ec791106e91 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 26 Oct 2017 22:46:12 +0200
Subject: [PATCH 399/453] debugging configuration for marconi

---
 README.md                 |  4 ++--
 config/README.md          |  6 +++---
 config/default.mk         |  3 +--
 config/devices/devices.mk |  6 +++---
 config/marconi.mk         | 10 +++++-----
 5 files changed, 14 insertions(+), 15 deletions(-)

diff --git a/README.md b/README.md
index 518d4a2a3..6fc6c1d84 100644
--- a/README.md
+++ b/README.md
@@ -23,7 +23,7 @@ $ git clone https://www.github.com/cusplibrary/cusplibrary
 ```
 > Our code only depends on external libraries that are themselves openly available. We note here that we do not distribute copies of these libraries.
 
-####  Using FELTOR as a library
+### Using FELTOR as a library
 
 It is possible to use FELTOR as a library in your own code project. Note that the library is **header-only**, which means that you just have to include the relevant header(s) and you're good to go:
 
@@ -44,7 +44,7 @@ and add `path/to/feltor/inc` as well as  `path/to/thrust/thrust` and  `path/to/c
 > If you want to activate the MPI backend of FELTOR you have to include `mpi.h` before any FELTOR header. If you want to use OpenMP instead of CUDA for the device functions you have to define the `THRUST_DEVICE_SYSTEM` macro and activate OpenMP in your compiler (e.g `g++ -fopenmp`). I you want to use CUDA then you have to compile with nvcc. 
 See `path/to/feltor/config/README.md` for further details on configurating the library and how to compile.
 
-####  Using FELTOR's code projects
+### Using FELTOR's code projects
 
 In order to compile one of the many codes inside FELTOR you need to tell the feltor configuration where the external libraries are located on your computer. The default way to do this is to go in your `HOME` directory, make an include directory and link the paths in this directory:
 
diff --git a/config/README.md b/config/README.md
index 9e61310c8..858fbc4ee 100644
--- a/config/README.md
+++ b/config/README.md
@@ -1,6 +1,6 @@
 ### Configuration
 
-The Makefiles for programs in FELTOR are configured by hardware specific *.mk files in the config folder. Every Makefile includes these files in the order  
+The Makefiles for programs in FELTOR are configured by hardware specific *.mk files in the config folder. Every Makefile includes these files in the order
 
 ```shell
 feltor/config/default.mk            #Defines the default variables
@@ -33,7 +33,7 @@ The main purpose of the file `feltor/config/devices/devices.mk` is to configure
 | value | description                              | flags                                    |
 | ----- | ---------------------------------------- | ---------------------------------------- |
 | gpu   | replaces the CC and CFLAGS variables with the nvcc versions and analogously MPICC and MPICFLAGS  | `CC = $(NVCC) --compiler-bindir $(CC)` `CFLAGS = $(NVCCARCH) $(NVCCFLAGS)` `MPICC = $(NVCC) --compiler-bindir $(MPICC)` `MPICFLAGS = $(NVCCARCH) $(NVCCFLAGS)` |
-| !gpu  | if device != gpu all thrust device calls redirect to OpenMP using the THRUST_DEVICE_SYSTEM macro | `-DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_OMP $(OMPFLAG)` added to both CFLAGS and MPICFLAGS |
+| !gpu  | if device != gpu all thrust device calls redirect to OpenMP using the THRUST_DEVICE_SYSTEM macro | `-x c++` `-DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_OMP` and `$(OMPFLAG)` added to both CFLAGS and MPICFLAGS |
 | omp  | specify OPT for OpenMP | OPT = -O3                   |
 | mic   | specify OPT for Intel Xeon Phi architecture | OPT = -O3 -xMIC-AVX512                   |
 | skl   | specify OPT for Intel Skylake processors (icc only) | OPT = -xCORE-AVX512 -mtune=skylake -O3   |
@@ -58,6 +58,6 @@ make blas_mpib device=gpu NVCCARCH='-arch sm_60' OPT=-O2
 
 ### General Remarks
  - If MPI is used in connection with the gpu backend, the mpi installation needs to be **cuda-aware**
- - when `icc` is used as the C++ compiler the `-restrict` option has to be used to enable the recognition of the restrict keyword
+ - if `icc` is used as the C++ compiler the `-restrict` option has to be used to enable the recognition of the restrict keyword
  
 
diff --git a/config/default.mk b/config/default.mk
index 8d54305d8..e1acb389d 100644
--- a/config/default.mk
+++ b/config/default.mk
@@ -3,9 +3,8 @@ INCLUDED=1
 
 #compiler and compiler options
 CC=g++ #C++ compiler
-CFLAGS=-Wall -x c++ -std=c++11 
 MPICC=mpic++  #mpi compiler
-MPICFLAGS=-Wall -x c++ -std=c++11
+CFLAGS=-Wall -std=c++11 
 NVCC=nvcc #CUDA compiler
 NVCCARCH=-arch sm_20 #nvcc gpu compute capability
 NVCCFLAGS= -std=c++11 -Xcompiler -Wall#
diff --git a/config/devices/devices.mk b/config/devices/devices.mk
index 76168f4b2..09b635e50 100644
--- a/config/devices/devices.mk
+++ b/config/devices/devices.mk
@@ -7,12 +7,12 @@ CFLAGS+=-D_MWAITXINTRIN_H_INCLUDED # solves issue with std=c++11
 ############################################
 mpiccc_:=$(MPICC)
 MPICC=nvcc --compiler-bindir $(mpiccc_)
-MPICFLAGS = $(NVCCARCH) $(NVCCFLAGS)
+MPICFLAGS+= $(NVCCARCH) $(NVCCFLAGS)
 MPICFLAGS+=-D_FORCE_INLINES # solves issue with std=c++11
 MPICFLAGS+=-D_MWAITXINTRIN_H_INCLUDED # solves issue with std=c++11
 else
-CFLAGS+=-DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_OMP $(OMPFLAG)
-MPICFLAGS+=-DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_OMP $(OMPFLAG)
+CFLAGS+=-x c++ -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_OMP $(OMPFLAG)
+MPICFLAGS+=$(CFLAGS)
 endif #device=gpu
 ifeq ($(strip $(device)),omp)
 endif #device=omp
diff --git a/config/marconi.mk b/config/marconi.mk
index 0bb297749..6f86ce841 100644
--- a/config/marconi.mk
+++ b/config/marconi.mk
@@ -1,17 +1,17 @@
 
 ifeq ($(strip $(HPC_SYSTEM)),marconi)
-INCLUDE += -I$(HOME)/include # cusp, thrust
-INCLUDE += -I$(NETCDF_INC) -I$(HDF5_INC)
-GLFLAGS  = -lm 
 CC=icc
 MPICC=mpiicc
 OPT=-O3 -xHost  # overwritten for mic in devices.mk
 #MPICFLAGS+= -DMPICH_IGNORE_CXX_SEEK
 OMPFLAG=-qopenmp
-CFLAGS=-std=c++11 -Wall -restrict
-JSONLIB=-L$(HOME)/include/json/../../src/lib_json -ljsoncpp # json library for input parameters
+CFLAGS+= -restrict
+
+INCLUDE += -I$(HOME)/include # cusp, thrust
+INCLUDE += -I$(NETCDF_INC) -I$(HDF5_INC)
 LIBS    +=-L$(HDF5_LIB) -lhdf5 -lhdf5_hl
 LIBS    +=-L$(NETCDF_LIB) -lnetcdf -lcurl
+GLFLAGS  = -lm 
 endif
 #############################modules to load in .bashrc#######################
 #module load profile/base                         
-- 
GitLab


From b43c59a5490cac68e468ef58a396e176dd021428 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 26 Oct 2017 22:49:58 +0200
Subject: [PATCH 400/453] update README

---
 config/README.md | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/config/README.md b/config/README.md
index 858fbc4ee..caa7903c3 100644
--- a/config/README.md
+++ b/config/README.md
@@ -12,16 +12,16 @@ The machine specific config files (e.g. vsc3.mk) should have an include guard an
 
 | variable  | default value                            | description                              |
 | :-------: | :--------------------------------------- | :--------------------------------------- |
-|    CC     | g++                                      | C++ compiler                             |
+|    CC     | g++                                      | C++ compiler                             |+
+|   MPICC   | mpic++                                   | the corresponding mpi wrapper for the c++ compiler     |
 |  CFLAGS   | -std=c++11 -Wall -x c++                  | flags for the C++ compiler               |
-|   MPICC   | mpic++                                   | the mpi wrapper for the c++ compiler     |
-| MPICFLAGS | -std=c++11 -Wall -x c++                  | flags for MPI compilation                |
+| MPICFLAGS |                  | flags specific to the MPI compilation                |
    OPT    | -O3                                      | optimization flags for the **host** code (can be overwritten on the command line, CUDA kernel code is always compiled with -O3) |
 |  OMPFLAG  | -fopenmp                                 | The compiler flag activating the OpenMP support |
 |   NVCC    | nvcc                                     | CUDA compiler                            |
 | NVCCFLAGS | -std=c++11  -Xcompiler -Wall                             | flags for nvcc                           |
 | NVCCARCH  | -arch sm_20                              | specify the **gpu** compute capability  https://developer.nvidia.com/cuda-gpus (can be overwritten on the command line) |
-        |                                          |                                          |
+|                                          |                                          |     |
 |  INCLUDE  | -I$(HOME)/include                        | cusp, thrust, json and the draw libraries. The default expects to find (symbolic links to ) these libraries in your home folder |
 |   LIBS    | -lnetcdf -lhdf5 -ldhf5_hl                | netcdf library                           |
 |  JSONLIB  | -L$(HOME)/include/json/../../src/lib_json -ljsoncpp | the JSONCPP library                      |
@@ -32,7 +32,7 @@ The main purpose of the file `feltor/config/devices/devices.mk` is to configure
 
 | value | description                              | flags                                    |
 | ----- | ---------------------------------------- | ---------------------------------------- |
-| gpu   | replaces the CC and CFLAGS variables with the nvcc versions and analogously MPICC and MPICFLAGS  | `CC = $(NVCC) --compiler-bindir $(CC)` `CFLAGS = $(NVCCARCH) $(NVCCFLAGS)` `MPICC = $(NVCC) --compiler-bindir $(MPICC)` `MPICFLAGS = $(NVCCARCH) $(NVCCFLAGS)` |
+| gpu   | replaces the CC and CFLAGS variables with the nvcc versions and analogously MPICC and MPICFLAGS  | `CC = $(NVCC) --compiler-bindir $(CC)` `CFLAGS = $(NVCCARCH) $(NVCCFLAGS)` `MPICC = $(NVCC) --compiler-bindir $(MPICC)` `MPICFLAGS+= $(NVCCARCH) $(NVCCFLAGS)` |
 | !gpu  | if device != gpu all thrust device calls redirect to OpenMP using the THRUST_DEVICE_SYSTEM macro | `-x c++` `-DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_OMP` and `$(OMPFLAG)` added to both CFLAGS and MPICFLAGS |
 | omp  | specify OPT for OpenMP | OPT = -O3                   |
 | mic   | specify OPT for Intel Xeon Phi architecture | OPT = -O3 -xMIC-AVX512                   |
-- 
GitLab


From ec61d7a595039bd2594f9c3631ef7ef1e105bd3c Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 26 Oct 2017 23:46:24 +0200
Subject: [PATCH 401/453] compile w/o warnings for g++-4.8

---
 config/default.mk                     |  4 ++--
 inc/dg/backend/fast_interpolation.h   |  2 +-
 inc/dg/backend/operator.h             |  2 +-
 inc/dg/backend/sparseblockmat.cuh     |  8 ++++----
 inc/dg/backend/sparseblockmat.h       |  8 ++++----
 inc/dg/backend/thrust_matrix_blas.cuh |  4 ++--
 inc/dg/backend/thrust_vector_blas.cuh | 14 +++++++-------
 inc/dg/backend/vector_categories.h    | 25 ++++++++++++++++++++++++-
 8 files changed, 45 insertions(+), 22 deletions(-)

diff --git a/config/default.mk b/config/default.mk
index e1acb389d..32f35b024 100644
--- a/config/default.mk
+++ b/config/default.mk
@@ -4,10 +4,10 @@ INCLUDED=1
 #compiler and compiler options
 CC=g++ #C++ compiler
 MPICC=mpic++  #mpi compiler
-CFLAGS=-Wall -std=c++11 
+CFLAGS=-Wall -std=c++11  #flags for CC
 NVCC=nvcc #CUDA compiler
 NVCCARCH=-arch sm_20 #nvcc gpu compute capability
-NVCCFLAGS= -std=c++11 -Xcompiler -Wall#
+NVCCFLAGS= -std=c++11 -Xcompiler -Wall#flags for NVCC
 OPT=-O3 # optimization flags for host code
 OMPFLAG=-fopenmp #openmp flag for CC and MPICC
 
diff --git a/inc/dg/backend/fast_interpolation.h b/inc/dg/backend/fast_interpolation.h
index f8a194d3d..8a63cf7ef 100644
--- a/inc/dg/backend/fast_interpolation.h
+++ b/inc/dg/backend/fast_interpolation.h
@@ -120,7 +120,7 @@ MultiMatrix< EllSparseBlockMat<double>, thrust::host_vector<double> > fast_inter
 MultiMatrix< EllSparseBlockMat<double>, thrust::host_vector<double> > fast_projection( const Grid1d& t, unsigned divide, enum dg::norm no = normed)
 {
     unsigned n=t.n();
-    if( t.N()%divide != 0) throw Error( Message(_ping_)<< "Nx and divide don't match: Nx: " << t.N()<< " divide "<< divide);
+    if( t.N()%divide != 0) throw Error( Message(_ping_)<< "Nx and divide don't match: Nx: " << t.N()<< " divide "<< (unsigned)divide);
     dg::Grid1d g_oldX( -1., 1., n, divide);
     dg::Grid1d g_new(  -1., 1., n, 1);
     dg::IHMatrix projectX;
diff --git a/inc/dg/backend/operator.h b/inc/dg/backend/operator.h
index fc417d495..0c6f4179b 100644
--- a/inc/dg/backend/operator.h
+++ b/inc/dg/backend/operator.h
@@ -34,7 +34,7 @@ class Operator
      *
      * @param n size
      */
-    Operator( const unsigned n): n_(n), data_(n_*n_){}
+    explicit Operator( const unsigned n): n_(n), data_(n_*n_){}
     /**
     * @brief Initialize elements.
     *
diff --git a/inc/dg/backend/sparseblockmat.cuh b/inc/dg/backend/sparseblockmat.cuh
index 8681521e3..e39150fb2 100644
--- a/inc/dg/backend/sparseblockmat.cuh
+++ b/inc/dg/backend/sparseblockmat.cuh
@@ -189,10 +189,10 @@ template<class DeviceContainer>
 inline void EllSparseBlockMatDevice<value_type>::symv( value_type alpha, const DeviceContainer& x, value_type beta, DeviceContainer& y) const
 {
     if( y.size() != (unsigned)num_rows*n*left_size*right_size) {
-        throw Error( Message(_ping_)<<"y has the wrong size "<<y.size()<<" and not "<<(unsigned)num_rows*n*left_size*right_size);
+        throw Error( Message(_ping_)<<"y has the wrong size "<<(unsigned)y.size()<<" and not "<<(unsigned)num_rows*n*left_size*right_size);
     }
     if( x.size() != (unsigned)num_cols*n*left_size*right_size) {
-        throw Error( Message(_ping_)<<"x has the wrong size "<<x.size()<<" and not "<<(unsigned)num_cols*n*left_size*right_size);
+        throw Error( Message(_ping_)<<"x has the wrong size "<<(unsigned)x.size()<<" and not "<<(unsigned)num_cols*n*left_size*right_size);
     }
     launch_multiply_kernel( alpha, x, beta, y);
 }
@@ -201,10 +201,10 @@ template<class DeviceContainer>
 inline void CooSparseBlockMatDevice<value_type>::symv( value_type alpha, const DeviceContainer& x, value_type beta, DeviceContainer& y) const
 {
     if( y.size() != (unsigned)num_rows*n*left_size*right_size) {
-        throw Error( Message(_ping_)<<"y has the wrong size "<<y.size()<<" and not "<<(unsigned)num_rows*n*left_size*right_size);
+        throw Error( Message(_ping_)<<"y has the wrong size "<<(unsigned)y.size()<<" and not "<<(unsigned)num_rows*n*left_size*right_size);
     }
     if( x.size() != (unsigned)num_cols*n*left_size*right_size) {
-        throw Error( Message(_ping_)<<"x has the wrong size "<<x.size()<<" and not "<<(unsigned)num_cols*n*left_size*right_size);
+        throw Error( Message(_ping_)<<"x has the wrong size "<<(unsigned)x.size()<<" and not "<<(unsigned)num_cols*n*left_size*right_size);
     }
     launch_multiply_kernel(alpha, x, beta, y);
 }
diff --git a/inc/dg/backend/sparseblockmat.h b/inc/dg/backend/sparseblockmat.h
index 7141f00ac..a9075a4de 100644
--- a/inc/dg/backend/sparseblockmat.h
+++ b/inc/dg/backend/sparseblockmat.h
@@ -197,10 +197,10 @@ template<class value_type>
 void EllSparseBlockMat<value_type>::symv(value_type alpha, const thrust::host_vector<value_type>& x, value_type beta, thrust::host_vector<value_type>& y) const
 {
     if( y.size() != (unsigned)num_rows*n*left_size*right_size) {
-        throw Error( Message(_ping_)<<"y has the wrong size "<<y.size()<<" and not "<<(unsigned)num_rows*n*left_size*right_size);
+        throw Error( Message(_ping_)<<"y has the wrong size "<<(unsigned)y.size()<<" and not "<<(unsigned)num_rows*n*left_size*right_size);
     }
     if( x.size() != (unsigned)num_cols*n*left_size*right_size) {
-        throw Error( Message(_ping_)<<"x has the wrong size "<<x.size()<<" and not "<<(unsigned)num_cols*n*left_size*right_size);
+        throw Error( Message(_ping_)<<"x has the wrong size "<<(unsigned)x.size()<<" and not "<<(unsigned)num_cols*n*left_size*right_size);
     }
 
 
@@ -275,10 +275,10 @@ template<class value_type>
 void CooSparseBlockMat<value_type>::symv( value_type alpha, const thrust::host_vector<value_type>& x, value_type beta, thrust::host_vector<value_type>& y) const
 {
     if( y.size() != (unsigned)num_rows*n*left_size*right_size) {
-        throw Error( Message(_ping_)<<"y has the wrong size "<<y.size()<<" and not "<<(unsigned)num_rows*n*left_size*right_size);
+        throw Error( Message(_ping_)<<"y has the wrong size "<<(unsigned)y.size()<<" and not "<<(unsigned)num_rows*n*left_size*right_size);
     }
     if( x.size() != (unsigned)num_cols*n*left_size*right_size) {
-        throw Error( Message(_ping_)<<"x has the wrong size "<<x.size()<<" and not "<<(unsigned)num_cols*n*left_size*right_size);
+        throw Error( Message(_ping_)<<"x has the wrong size "<<(unsigned)x.size()<<" and not "<<(unsigned)num_cols*n*left_size*right_size);
     }
 
     //simplest implementation
diff --git a/inc/dg/backend/thrust_matrix_blas.cuh b/inc/dg/backend/thrust_matrix_blas.cuh
index 5429a5035..77bf76bb6 100644
--- a/inc/dg/backend/thrust_matrix_blas.cuh
+++ b/inc/dg/backend/thrust_matrix_blas.cuh
@@ -51,7 +51,7 @@ inline typename MatrixTraits<Matrix>::value_type doDot( const Vector& x, const M
 #if THRUST_DEVICE_SYSTEM!=THRUST_DEVICE_SYSTEM_CUDA
     value_type sum = 0;
     unsigned size=x.size();
-    #pragma omp parallel for simd reduction(+:sum) 
+    #pragma omp parallel for SIMD reduction(+:sum)
     for( unsigned i=0; i<size; i++)
         sum += x[i]*m[i]*y[i];
     return sum;
@@ -75,7 +75,7 @@ inline typename MatrixTraits<Matrix>::value_type doDot( const Matrix& m, const V
 #if THRUST_DEVICE_SYSTEM!=THRUST_DEVICE_SYSTEM_CUDA
     value_type sum = 0;
     unsigned size=x.size();
-    #pragma omp parallel for simd reduction(+:sum)
+    #pragma omp parallel for SIMD reduction(+:sum)
     for( unsigned i=0; i<size; i++)
         sum += x[i]*x[i]*m[i];
     return sum;
diff --git a/inc/dg/backend/thrust_vector_blas.cuh b/inc/dg/backend/thrust_vector_blas.cuh
index c3161abae..3c8d2e4b2 100644
--- a/inc/dg/backend/thrust_vector_blas.cuh
+++ b/inc/dg/backend/thrust_vector_blas.cuh
@@ -71,7 +71,7 @@ typename Vector::value_type doDot( const Vector& x, const Vector& y, ThrustVecto
 #if THRUST_DEVICE_SYSTEM!=THRUST_DEVICE_SYSTEM_CUDA
     value_type sum = 0;
     unsigned size=x.size();
-    #pragma omp parallel for simd reduction(+:sum) 
+    #pragma omp parallel for SIMD reduction(+:sum)
     for( unsigned i=0; i<size; i++)
         sum += x[i]*y[i];
     return sum;
@@ -97,7 +97,7 @@ inline void doScal(  Vector& x,
         return;
 #if THRUST_DEVICE_SYSTEM!=THRUST_DEVICE_SYSTEM_CUDA
     unsigned size=x.size();
-    #pragma omp parallel for simd
+    #pragma omp parallel for SIMD
     for( unsigned i=0; i<size; i++)
         x[i]*=alpha;
 #else
@@ -112,7 +112,7 @@ inline void doPlus(  Vector& x,
 {
 #if THRUST_DEVICE_SYSTEM!=THRUST_DEVICE_SYSTEM_CUDA
     unsigned size=x.size();
-    #pragma omp parallel for simd
+    #pragma omp parallel for SIMD
     for( unsigned i=0; i<size; i++)
         x[i]+=alpha;
 #else
@@ -150,7 +150,7 @@ inline void doAxpby( typename Vector::value_type alpha,
     const typename Vector::value_type * RESTRICT x_ptr = thrust::raw_pointer_cast( &x.data()[0]);
     typename Vector::value_type * RESTRICT y_ptr = thrust::raw_pointer_cast( &y.data()[0]);
     unsigned size = x.size();
-    #pragma omp parallel for simd
+    #pragma omp parallel for SIMD
     for( unsigned i=0; i<size; i++)
         y_ptr[i] = alpha*x_ptr[i] + beta*y_ptr[i];
 #else
@@ -206,7 +206,7 @@ inline void doAxpby( typename Vector::value_type alpha,
     const typename Vector::value_type * RESTRICT y_ptr = thrust::raw_pointer_cast( &y.data()[0]);
     typename Vector::value_type * RESTRICT z_ptr = thrust::raw_pointer_cast( &z.data()[0]);
     unsigned size = x.size();
-    #pragma omp parallel for simd
+    #pragma omp parallel for SIMD
     for( unsigned i=0; i<size; i++)
         z_ptr[i] = alpha*x_ptr[i] + beta*y_ptr[i] + gamma*z_ptr[i];
 #else
@@ -275,7 +275,7 @@ inline void doPointwiseDot(
     const typename Vector::value_type * x2_ptr = thrust::raw_pointer_cast( &(x2.data()[0]));
      typename Vector::value_type * y_ptr = thrust::raw_pointer_cast( &(y.data()[0]));
     unsigned size = x1.size();
-    #pragma omp parallel for simd
+    #pragma omp parallel for SIMD
     for( unsigned i=0; i<size; i++)
     {
         y_ptr[i] = alpha*x1_ptr[i]*x2_ptr[i]+beta*y_ptr[i];
@@ -353,7 +353,7 @@ inline void doPointwiseDot(
           value_type * z_ptr = thrust::raw_pointer_cast( z.data());
     unsigned size = x1.size();
 #if THRUST_DEVICE_SYSTEM!=THRUST_DEVICE_SYSTEM_CUDA
-    #pragma omp parallel for simd
+    #pragma omp parallel for SIMD
     for( unsigned i=0; i<size; i++)
     {
         z_ptr[i] = alpha*x1_ptr[i]*y1_ptr[i] 
diff --git a/inc/dg/backend/vector_categories.h b/inc/dg/backend/vector_categories.h
index 3529a8880..d42c90a7e 100644
--- a/inc/dg/backend/vector_categories.h
+++ b/inc/dg/backend/vector_categories.h
@@ -2,7 +2,7 @@
 #define _DG_VECTOR_CATEGORIES_
 
 #if defined(__INTEL_COMPILER)
-  // On Intel compiler, you need to pass the -restrict compiler flag in addition to your own compiler flags.
+// On Intel compiler, you need to pass the -restrict compiler flag in addition to your own compiler flags.
 # define RESTRICT restrict
 #elif defined(__GNUG__)
 # define RESTRICT __restrict__
@@ -11,6 +11,29 @@
 # define RESTRICT
 #endif
 
+#if THRUST_DEVICE_SYSTEM==THRUST_DEVICE_SYSTEM_OMP
+#if defined(__INTEL_COMPILER)
+
+#if __INTEL_COMPILER < 1500
+#warning icc version >= 15.0 recommended to activate OpenMP 4 support
+#define SIMD
+#else//>1500
+#define SIMD simd
+#endif//__INTEL_COMPILER
+
+#elif defined(__GNUG__)
+
+#define GCC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__)
+#if GCC_VERSION < 40900
+#warning gcc version >= 4.9 recommended to activate OpenMP 4 support
+#define SIMD
+#else
+#define SIMD simd
+#endif //GCC_VERSION
+
+#endif //__GNUG__
+#endif //THRUST_DEVICE_SYSTEM
+
 namespace dg{
 
 struct AnyVectorTag{};
-- 
GitLab


From 83cceed1631ca1dc8ba3f84aab3ffb34865222c5 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 26 Oct 2017 23:53:19 +0200
Subject: [PATCH 402/453] added remark about OpenMP 4 support

---
 config/README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/config/README.md b/config/README.md
index caa7903c3..226ecf148 100644
--- a/config/README.md
+++ b/config/README.md
@@ -59,5 +59,6 @@ make blas_mpib device=gpu NVCCARCH='-arch sm_60' OPT=-O2
 ### General Remarks
  - If MPI is used in connection with the gpu backend, the mpi installation needs to be **cuda-aware**
  - if `icc` is used as the C++ compiler the `-restrict` option has to be used to enable the recognition of the restrict keyword
+ - support for OpenMP-4 is recommended (at least gcc-4.9 or icc-15), but not mandatory
  
 
-- 
GitLab


From db49cbe77160bfc9474f0814139e12551baa282d Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sat, 28 Oct 2017 12:48:39 +0200
Subject: [PATCH 403/453] some more Doku on createField functions and made
 cluster_mpib compile again

---
 inc/dg/algorithm.h                |  1 +
 inc/dg/backend/split_and_join.h   |  2 +-
 inc/dg/cluster_mpib.cu            | 13 +++++--------
 inc/dg/multigrid.h                |  5 ++++-
 inc/geometries/ds.h               | 23 ++++++++++++-----------
 inc/geometries/fieldaligned.h     | 13 ++++++++++---
 inc/geometries/generatorX.h       |  3 +++
 inc/geometries/geometries.h       |  6 +++++-
 inc/geometries/guenther.h         |  2 ++
 inc/geometries/mpi_fieldaligned.h | 10 ++++++++--
 inc/geometries/solovev.h          |  2 ++
 inc/geometries/taylor.h           |  9 ++++++---
 inc/geometries/toroidal.h         |  4 ++++
 13 files changed, 63 insertions(+), 30 deletions(-)

diff --git a/inc/dg/algorithm.h b/inc/dg/algorithm.h
index 75d829621..93387ddf0 100644
--- a/inc/dg/algorithm.h
+++ b/inc/dg/algorithm.h
@@ -15,3 +15,4 @@
 #include "elliptic.h"
 #include "runge_kutta.h"
 #include "multigrid.h"
+#include "backend/timer.cuh"
diff --git a/inc/dg/backend/split_and_join.h b/inc/dg/backend/split_and_join.h
index df06ec19a..3e3b387bf 100644
--- a/inc/dg/backend/split_and_join.h
+++ b/inc/dg/backend/split_and_join.h
@@ -77,7 +77,7 @@ void join( const std::vector<thrust_vector1>& in, thrust_vector2& out, const aTo
 template<class thrust_vector1, class thrust_vector2>
 void join( const std::vector<MPI_Vector<thrust_vector1> >& in, MPI_Vector<thrust_vector2 >& out, const aMPITopology3d& grid)
 {
-    Grid2d l(grid);
+    Grid3d l(grid.local());
     unsigned size2d=l.n()*l.n()*l.Nx()*l.Ny();
     out.data().resize( size2d*l.Nz());
     out.communicator() = grid.communicator();
diff --git a/inc/dg/cluster_mpib.cu b/inc/dg/cluster_mpib.cu
index 3d8e3a906..bbd54465c 100644
--- a/inc/dg/cluster_mpib.cu
+++ b/inc/dg/cluster_mpib.cu
@@ -6,10 +6,7 @@
 #include <omp.h>
 #endif//_OPENMP
 #include "algorithm.h"
-
-#include "backend/timer.cuh"
-#include "../geometries/guenther.h"
-#include "../geometries/magnetic_field.h"
+#include "../geometries/geometries.h"
 
 
 const double lx = 2*M_PI;
@@ -34,6 +31,7 @@ double laplace_fct( double x, double y, double z) { return -1./x*cos(x-R_0)*sin(
 double initial( double x, double y, double z) {return sin(0);}
 
 typedef dg::MDMatrix Matrix;
+typedef dg::MIDMatrix IMatrix;
 typedef dg::MDVec Vector;
 
 
@@ -181,10 +179,9 @@ int main(int argc, char* argv[])
         double Rmax=gpR0+1.0*gpa; 
         double Zmax=1.0*gpa*1.00;
         dg::CylindricalMPIGrid3d g3d( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, n, Nx ,Ny, Nz,dg::DIR, dg::DIR, dg::PER,commEll);
-        dg::geo::TokamakMagneticField magfield = dg::geo::guenther::createMagField(gpR0, gpI0);
-        dg::geo::Field<dg::geo::guenther::MagneticField> field(magfield, gpR0);
-        dg::MDDS::FieldAligned dsFA( field, g3d, 1e-4, dg::DefaultLimiter(), dg::DIR);
-        dg::MDDS ds ( dsFA, field, dg::not_normed, dg::centered);
+        dg::geo::TokamakMagneticField magfield = dg::geo::createGuentherField(gpR0, gpI0);
+        dg::geo::Fieldaligned<dg::aProductMPIGeometry3d, IMatrix, Vector> dsFA( magfield, g3d, 5, 5, true, true, 1e-4);
+        dg::geo::DS<dg::aProductMPIGeometry3d, IMatrix, Matrix, Vector>  ds ( dsFA, dg::not_normed, dg::centered);
         dg::geo::guenther::FuncNeu funcNEU(gpR0,gpI0);
         Vector function = dg::evaluate( funcNEU, g3d) , dsTdsfb(function);
 
diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index 7d520643e..77fe5a71d 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -9,6 +9,9 @@
 #ifdef DG_BENCHMARK
 #include "backend/timer.cuh"
 #endif //DG_BENCHMARK
+#ifdef MPI_VERSION
+#include "backend/mpi_projection.h"
+#endif
 
 namespace dg
 {
@@ -229,7 +232,7 @@ struct MultigridCG2d
     
     ///After a call to a solution method returns the maximum number of iterations allowed at stage  0
     ///(if the solution method returns this number, failure is indicated)
-    unsigned max_iter() const{return cg_[0].get_max();} //could this be a problem in MPI?
+    unsigned max_iter() const{return cg_[0].get_max();} 
 
 private:
 
diff --git a/inc/geometries/ds.h b/inc/geometries/ds.h
index 68f7261af..134f62d0f 100644
--- a/inc/geometries/ds.h
+++ b/inc/geometries/ds.h
@@ -24,8 +24,8 @@ namespace geo{
 *
 * This class discretizes the operators 
 \f$ \nabla_\parallel = \mathbf{v}\cdot \nabla = v^\zeta\partial_\zeta + v^\eta\partial_\eta + v^\varphi\partial_\varphi \f$, 
-\f$\nabla_\parallel^\dagger\f$ and 
-\f$\Delta_\parallel=\nabla_\parallel^\dagger\cdot\nabla_\parallel\f$
+\f$\nabla_\parallel^\dagger = -\nabla\cdot(\vec v .)\f$ and 
+\f$\Delta_\parallel=-\nabla_\parallel^\dagger\cdot\nabla_\parallel\f$
 in arbitrary coordinates
 @snippet ds_t.cu doxygen
 * @ingroup fieldaligned
@@ -41,6 +41,7 @@ in arbitrary coordinates
 template< class ProductGeometry, class IMatrix, class Matrix, class container >
 struct DS
 {
+    typedef dg::geo::Fieldaligned<ProductGeometry, IMatrix, container> FA; //!< conveniently abbreviates underlying \c Fieldaligned type
     ///@brief No memory allocation; all member calls except construct are invalid
     DS(){}
     
@@ -55,7 +56,7 @@ struct DS
         construct( m_fa, no, dir);
     }
     /**
-     * @brief Create a Fieldaligned object and construct
+     * @brief Create a \c Fieldaligned object and construct
      *
      * @param vec The vector field to integrate
      * @param grid The grid on which to operate defines the parallel boundary condition in case there is a limiter.
@@ -67,7 +68,7 @@ struct DS
      * @param no indicate if the symv function should be symmetric (not_normed) or not
      * @param dir indicate the direction in the bracket operator and in symv
      *@note globalbcx and globalbcy  as well as bcz are taken from grid with full limter 
-     * @sa Fieldaligned
+     * @sa \c Fieldaligned
      */
     DS(const dg::geo::BinaryVectorLvl0& vec, const ProductGeometry& grid, unsigned multiplyX=1, unsigned multiplyY=1, bool dependsOnX = true, bool dependsOnY=true, double eps = 1e-5, dg::norm no=dg::normed, dg::direction dir = dg::centered)
     {
@@ -75,18 +76,18 @@ struct DS
         construct( m_fa, no, dir);
     }
     ///@copydoc construct
-    DS(const dg::geo::Fieldaligned<ProductGeometry, IMatrix, container>& fa, dg::norm no=dg::normed, dg::direction dir = dg::centered)
+    DS(const FA& fieldaligned, dg::norm no=dg::normed, dg::direction dir = dg::centered)
     {
-        construct( fa, no, dir);
+        construct( fieldaligned, no, dir);
     }
     /**
-     * @brief Re-construct from a given Fieldaligned object
+     * @brief Re-construct from a given \c Fieldaligned object
      *
-     * @param fa this object will be used in all further member calls
+     * @param fieldaligned this object will be used in all further member calls
      * @param no indicate if the symv function should be symmetric (not_normed) or not
      * @param dir indicate the direction in the bracket operator and in symv
      */
-    void construct(const dg::geo::Fieldaligned<ProductGeometry, IMatrix, container>& fa, dg::norm no=dg::normed, dg::direction dir = dg::centered);
+    void construct(const FA& fieldaligned, dg::norm no=dg::normed, dg::direction dir = dg::centered);
 
     /**
     * @brief forward derivative \f$ g_i = \alpha \frac{1}{h_z^+}(f_{i+1} - f_{i}) + \beta g_i\f$
@@ -169,11 +170,11 @@ struct DS
     const container& precond()const {return m_inv3d;}
 
     /**
-    * @brief access the underlying Fielaligned object for evaluate
+    * @brief access the underlying Fieldaligned object for evaluate
     *
     * @return acces to fieldaligned object
     */
-    const Fieldaligned<ProductGeometry, IMatrix, container>& fieldaligned() const{return m_fa;}
+    const FA& fieldaligned() const{return m_fa;}
     private:
     void do_forward(double alpha, const container& f, double beta, container& dsf);
     void do_backward(double alpha, const container& f, double beta, container& dsf);
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index b8c8e8b45..79f2d042b 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -334,9 +334,16 @@ struct Fieldaligned
 
     ///@brief do not allocate memory; no member call except construct is valid
     Fieldaligned(){}
+    ///@copydoc construct()
+    template <class Limiter = FullLimiter>
+    Fieldaligned(const dg::geo::TokamakMagneticField& vec, const ProductGeometry& grid, unsigned multiplyX, unsigned multiplyY, bool dependsOnX, bool dependsOnY, double eps = 1e-5, dg::bc globalbcx = dg::NEU, dg::bc globalbcy = dg::NEU, Limiter limit = FullLimiter(), double deltaPhi = -1)
+    {
+        dg::geo::BinaryVectorLvl0 bhat( (dg::geo::BHatR)(vec), (dg::geo::BHatZ)(vec), (dg::geo::BHatP)(vec));
+        construct( bhat, grid, multiplyX, multiplyY, dependsOnX, dependsOnY, eps, globalbcx, globalbcy, limit, deltaPhi);
+    }
 
     ///@copydoc construct()
-    template <class Limiter>
+    template <class Limiter = FullLimiter>
     Fieldaligned(const dg::geo::BinaryVectorLvl0& vec, const ProductGeometry& grid, unsigned multiplyX, unsigned multiplyY, bool dependsOnX, bool dependsOnY, double eps = 1e-5, dg::bc globalbcx = dg::NEU, dg::bc globalbcy = dg::NEU, Limiter limit = FullLimiter(), double deltaPhi = -1)
     {
         construct( vec, grid, multiplyX, multiplyY, dependsOnX, dependsOnY, eps, globalbcx, globalbcy, limit, deltaPhi);
@@ -363,7 +370,7 @@ struct Fieldaligned
         by the grid.bcz() variable and can be changed by the set_boundaries function. 
         If there is no limiter, the boundary condition is periodic.
     */
-    template <class Limiter>
+    template <class Limiter = FullLimiter>
     void construct(const dg::geo::BinaryVectorLvl0& vec, const ProductGeometry& grid, unsigned multiplyX, unsigned multiplyY, bool dependsOnX, bool dependsOnY, double eps = 1e-5, dg::bc globalbcx = dg::NEU, dg::bc globalbcy = dg::NEU, Limiter limit = FullLimiter(), double deltaPhi = -1);
 
     bool dependsOnX()const{return m_dependsOnX;}
@@ -504,7 +511,7 @@ template <class Limiter>
 void Fieldaligned<Geometry, IMatrix, container>::construct(const dg::geo::BinaryVectorLvl0& vec, const Geometry& grid, unsigned mx, unsigned my, bool bx, bool by, double eps, dg::bc globalbcx, dg::bc globalbcy, Limiter limit, double deltaPhi)
 {
     m_dependsOnX=bx, m_dependsOnY=by;
-    m_Nz=grid.Nz(), m_bcz=grid.bcz(); 
+    m_Nz=grid.local().Nz(), m_bcz=grid.bcz(); 
     m_g.reset(grid);
     dg::blas1::transfer( dg::evaluate( dg::zero, grid), m_hz_inv), m_hp_inv= m_hz_inv, m_hm_inv= m_hz_inv;
     dg::split( m_hz_inv, m_temp, grid);
diff --git a/inc/geometries/generatorX.h b/inc/geometries/generatorX.h
index af483a6af..085fe0931 100644
--- a/inc/geometries/generatorX.h
+++ b/inc/geometries/generatorX.h
@@ -2,6 +2,8 @@
 
 namespace dg
 {
+namespace geo
+{
 
 /**
 * @brief The abstract generator base class 
@@ -88,4 +90,5 @@ struct aGeneratorX2d
 
 };
 
+}//namespace geo
 }//namespace dg
diff --git a/inc/geometries/geometries.h b/inc/geometries/geometries.h
index ea705983f..0379e612d 100644
--- a/inc/geometries/geometries.h
+++ b/inc/geometries/geometries.h
@@ -15,11 +15,15 @@
 
 //include magnetic field geometries
 #include "solovev.h"
-#include "solovev_parameters.h"
+#include "guenther.h"
+#include "toroidal.h"
 
 #include "init.h"
+#include "fluxfunctions.h"
 #include "magnetic_field.h"
 #include "adaption.h"
 
 //include average
 #include "average.h"
+//include ds and fieldaligned
+#include "ds.h"
diff --git a/inc/geometries/guenther.h b/inc/geometries/guenther.h
index 83209f6a6..253794a13 100644
--- a/inc/geometries/guenther.h
+++ b/inc/geometries/guenther.h
@@ -509,6 +509,8 @@ struct Divb
 
 /**
  * @brief Create a Guenther Magnetic field
+
+ * \f[\psi_p(R,Z) = \cos(\pi(R-R_0)/2)\cos(\pi Z/2),\quad I(\psi_p) = I_0\f]
  * @param R_0 the major radius
  * @param I_0 the current
  * @return A magnetic field object
diff --git a/inc/geometries/mpi_fieldaligned.h b/inc/geometries/mpi_fieldaligned.h
index 7f1df1f5b..618a656cd 100644
--- a/inc/geometries/mpi_fieldaligned.h
+++ b/inc/geometries/mpi_fieldaligned.h
@@ -58,12 +58,18 @@ template <class ProductMPIGeometry, class LocalIMatrix, class CommunicatorXY, cl
 struct Fieldaligned< ProductMPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vector<LocalContainer> > 
 {
     Fieldaligned(){}
-    template <class Limiter>
+    template <class Limiter = FullLimiter>
+    Fieldaligned(const dg::geo::TokamakMagneticField& vec, const ProductMPIGeometry& grid, unsigned multiplyX, unsigned multiplyY, bool dependsOnX, bool dependsOnY, double eps = 1e-5, dg::bc globalbcx = dg::NEU, dg::bc globalbcy = dg::NEU, Limiter limit = FullLimiter(), double deltaPhi = -1)
+    {
+        dg::geo::BinaryVectorLvl0 bhat( (dg::geo::BHatR)(vec), (dg::geo::BHatZ)(vec), (dg::geo::BHatP)(vec));
+        construct( bhat, grid, multiplyX, multiplyY, dependsOnX, dependsOnY, eps, globalbcx, globalbcy, limit, deltaPhi);
+    }
+    template <class Limiter = FullLimiter>
     Fieldaligned(const dg::geo::BinaryVectorLvl0& vec, const ProductMPIGeometry& grid, unsigned multiplyX, unsigned multiplyY, bool dependsOnX, bool dependsOnY, double eps = 1e-5, dg::bc globalbcx = dg::NEU, dg::bc globalbcy = dg::NEU, Limiter limit = FullLimiter(), double deltaPhi = -1)
     {
         construct( vec, grid, multiplyX, multiplyY, dependsOnX, dependsOnY, eps, globalbcx, globalbcy, limit, deltaPhi);
     }
-    template <class Limiter>
+    template <class Limiter = FullLimiter>
     void construct(const dg::geo::BinaryVectorLvl0& vec, const ProductMPIGeometry& grid, unsigned multiplyX, unsigned multiplyY, bool dependsOnX, bool dependsOnY, double eps = 1e-5, dg::bc globalbcx = dg::NEU, dg::bc globalbcy = dg::NEU, Limiter limit = FullLimiter(), double deltaPhi = -1);
 
     bool dependsOnX()const{return m_dependsOnX;}
diff --git a/inc/geometries/solovev.h b/inc/geometries/solovev.h
index 4e9046d95..86814a044 100644
--- a/inc/geometries/solovev.h
+++ b/inc/geometries/solovev.h
@@ -556,6 +556,8 @@ struct PsipRZ: public aCloneableBinaryFunctor<PsipRZ>
 
 /**
  * @brief Create a Solovev Magnetic field
+ *
+ * Based on \c dg::geo::solovev::Psip(gp) and \c dg::geo::solovev::Ipol(gp)
  * @param gp Solovev parameters
  * @return A magnetic field object
  * @ingroup geom
diff --git a/inc/geometries/taylor.h b/inc/geometries/taylor.h
index 1d0a7ca87..5ccb8d6b3 100644
--- a/inc/geometries/taylor.h
+++ b/inc/geometries/taylor.h
@@ -17,7 +17,7 @@
 /*!@file
  *
  * MagneticField objects 
- * @attention When the taylor field is used we need the boost library for special functions
+ * @attention When the taylor field is used we need the <a href="http://www.boost.org"> boost</a> library for special functions
  */
 namespace dg
 {
@@ -27,7 +27,7 @@ namespace geo
  * @brief Contains the Cerfon Taylor state type flux functions (using boost)
  *
  * This is taken from A. J. Cerfon and M. O'Neil: Exact axisymmetric Taylor states for shaped plasmas, Physics of Plasmas 21, 064501 (2014)
- * @attention When the taylor field is used we need the boost library for special functions
+ * @attention When the taylor field is used we need the <a href="http://www.boost.org"> boost</a> library for special functions
  */
 namespace taylor
 {
@@ -39,7 +39,7 @@ typedef dg::geo::solovev::Parameters Parameters; //!< bring Parameters into the
  * @brief \f[ \psi \f]
  *
  * This is taken from A. J. Cerfon and M. O'Neil: Exact axisymmetric Taylor states for shaped plasmas, Physics of Plasmas 21, 064501 (2014)
- * @attention When the taylor field is used we need the boost library for special functions
+ * @attention When the taylor field is used we need the <a href="http://www.boost.org"> boost</a> library for special functions
  */
 struct Psip : public aCloneableBinaryFunctor<Psip>
 { /**
@@ -313,9 +313,12 @@ dg::geo::TokamakMagneticField createMagField( solovev::Parameters gp)
 } //namespace taylor
 /**
  * @brief Create a Taylor Magnetic field
+
+ * Based on \c dg::geo::taylor::Psip(gp) and \c dg::geo::taylor::Ipol(gp)
  * @param gp Solovev parameters
  * @return A magnetic field object
  * @ingroup geom
+ * @attention The header \c taylor.h needs to be included seperately and depends on <a href="http://www.boost.org">boost</a>
  */
 TokamakMagneticField createTaylorField( solovev::Parameters gp)
 {
diff --git a/inc/geometries/toroidal.h b/inc/geometries/toroidal.h
index 0bf27cb8b..a72900508 100644
--- a/inc/geometries/toroidal.h
+++ b/inc/geometries/toroidal.h
@@ -108,6 +108,8 @@ BinaryFunctorsLvl1 createIpol( double I0 )
 
 /**
  * @brief Create a Toroidal Magnetic field
+ *
+ * \f[ \psi_p(R,Z) = 1, \quad I(\psi_p) = 1\f]
  * @param R0 the major radius
  * @return A magnetic field object
  * @ingroup geom
@@ -119,6 +121,8 @@ TokamakMagneticField createToroidalField( double R0)
 }
 /**
  * @brief Create a Magnetic field with circular flux surfaces and constant current
+
+ * \f[ \psi_p(R,Z) = \frac{1}{2}\left((R-R_0)^2 + Z^2 \right), \quad I(\psi_p) = I_0 \f]
  * @param R0 the major radius
  * @param I0 the current
  * @return A magnetic field object
-- 
GitLab


From b52b622b07e7926919397e2ffc7f3d2c18aa3334 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sat, 28 Oct 2017 15:19:20 +0200
Subject: [PATCH 404/453] updated MPI base grid

---
 inc/dg/backend/mpi_grid.h                 | 116 +++++++++++-----------
 inc/dg/backend/projection_mpit.cu         |   2 +-
 inc/dg/backend/typedefs.cuh               |   9 ++
 inc/dg/geometry/geometry_t.cu             |   4 +-
 inc/dg/geometry/mpi_base.h                |   2 +-
 inc/dg/geometry/transform.h               |   8 +-
 inc/geometries/ds_mpit.cu                 |   3 +-
 inc/geometries/ds_t.cu                    |   2 +-
 inc/geometries/fieldaligned.h             |   2 +-
 inc/geometries/geometry_advection_mpib.cu |   6 +-
 inc/geometries/mpi_curvilinear.h          |  14 +--
 inc/geometries/mpi_fieldaligned.h         | 103 +++++++++----------
 12 files changed, 138 insertions(+), 133 deletions(-)

diff --git a/inc/dg/backend/mpi_grid.h b/inc/dg/backend/mpi_grid.h
index 05bd2438e..2f9289390 100644
--- a/inc/dg/backend/mpi_grid.h
+++ b/inc/dg/backend/mpi_grid.h
@@ -147,10 +147,7 @@ struct aMPITopology2d
         os << "GLOBAL GRID \n";
         g.display();
         os << "LOCAL GRID \n";
-
-        Grid2d grid = local();
-        grid.display();
-
+        l.display();
     }
 
     /**
@@ -190,7 +187,6 @@ struct aMPITopology2d
     */
     bool local2globalIdx( int localIdx, int PID, int& globalIdx)const
     {
-        Grid2d l = local();
         if( localIdx < 0 || localIdx >= (int)size()) return -1;
         int coords[2];
         if( MPI_Cart_coords( comm, PID, 2, coords) != MPI_SUCCESS)
@@ -212,7 +208,6 @@ struct aMPITopology2d
     */
     bool global2localIdx( int globalIdx, int& localIdx, int& PID)const
     {
-        Grid2d l = local();
         if( globalIdx < 0 || globalIdx >= (int)g.size()) return -1;
         int coords[2];
         int gIdx0 = globalIdx%(g.n()*g.Nx());
@@ -237,21 +232,7 @@ struct aMPITopology2d
      * @return Grid object
      * @note the boundary conditions in the local grid are not well defined since there might not actually be any boundaries
      */
-    Grid2d local() const {
-        int dims[2], periods[2], coords[2];
-        MPI_Cart_get( comm, 2, dims, periods, coords);
-        double x0 = g.x0() + g.lx()/(double)dims[0]*(double)coords[0]; 
-        double x1 = g.x0() + g.lx()/(double)dims[0]*(double)(coords[0]+1); 
-        if( coords[0] == dims[0]-1) 
-            x1 = g.x1();
-        double y0 = g.y0() + g.ly()/(double)dims[1]*(double)coords[1]; 
-        double y1 = g.y0() + g.ly()/(double)dims[1]*(double)(coords[1]+1); 
-        if( coords[1] == dims[1]-1) 
-            y1 = g.y1();
-        unsigned Nx = g.Nx()/dims[0];
-        unsigned Ny = g.Ny()/dims[1];
-        return Grid2d(x0, x1, y0, y1, g.n(), Nx, Ny, g.bcx(), g.bcy());
-    }
+    const Grid2d& local() const {return l;}
     /**
      * @brief Return the global non-MPI grid 
      *
@@ -261,7 +242,7 @@ struct aMPITopology2d
      * class itself 
      * @return non-MPI Grid object
      */
-    Grid2d global() const {return g;}
+    const Grid2d& global() const {return g;}
     protected:
     ///disallow deletion through base class pointer
     ~aMPITopology2d(){}
@@ -272,15 +253,16 @@ struct aMPITopology2d
      * @copydoc hide_comm_parameters2d
      */
     aMPITopology2d( double x0, double x1, double y0, double y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx, bc bcy, MPI_Comm comm):
-        g( x0, x1, y0, y1, n, Nx, Ny, bcx, bcy), comm( comm)
+        g( x0, x1, y0, y1, n, Nx, Ny, bcx, bcy), l(g), comm( comm)
     {
+        update_local();
         check_division( Nx, Ny, bcx, bcy);
     }
     ///copydoc aTopology2d::aTopology2d(const aTopology2d&)
-    aMPITopology2d(const aMPITopology2d& src):g(src.g),comm(src.comm){}
+    aMPITopology2d(const aMPITopology2d& src):g(src.g),l(src.l),comm(src.comm){ }
     ///copydoc aTopology2d::operator()(const aTopology2d&)
     aMPITopology2d& operator=(const aMPITopology2d& src){
-        g = src.g; comm = src.comm;
+        g = src.g; l = src.l; comm = src.comm;
         return *this;
     }
     ///This function has an implementation 
@@ -305,7 +287,22 @@ struct aMPITopology2d
             else assert( periods[1] == false);
         }
     }
-    Grid2d g; //global grid
+    void update_local(){
+        int dims[2], periods[2], coords[2];
+        MPI_Cart_get( comm, 2, dims, periods, coords);
+        double x0 = g.x0() + g.lx()/(double)dims[0]*(double)coords[0]; 
+        double x1 = g.x0() + g.lx()/(double)dims[0]*(double)(coords[0]+1); 
+        if( coords[0] == dims[0]-1) 
+            x1 = g.x1();
+        double y0 = g.y0() + g.ly()/(double)dims[1]*(double)coords[1]; 
+        double y1 = g.y0() + g.ly()/(double)dims[1]*(double)(coords[1]+1); 
+        if( coords[1] == dims[1]-1) 
+            y1 = g.y1();
+        unsigned Nx = g.Nx()/dims[0];
+        unsigned Ny = g.Ny()/dims[1];
+        l = Grid2d(x0, x1, y0, y1, g.n(), Nx, Ny, g.bcx(), g.bcy());
+    }
+    Grid2d g, l; //global and local grid
     MPI_Comm comm; //just an integer...
 };
 
@@ -476,10 +473,7 @@ struct aMPITopology3d
         os << "GLOBAL GRID \n";
         g.display();
         os << "LOCAL GRID \n";
-
-        Grid3d grid = local();
-        grid.display();
-
+        l.display();
     }
     /**
      * @brief Returns the pid of the process that holds the local grid surrounding the given point
@@ -506,7 +500,6 @@ struct aMPITopology3d
     ///@copydoc aMPITopology2d::local2globalIdx(int,int,int&)const
     bool local2globalIdx( int localIdx, int PID, int& globalIdx)const
     {
-        Grid3d l = local();
         if( localIdx < 0 || localIdx >= (int)size()) return false;
         int coords[3];
         if( MPI_Cart_coords( comm, PID, 3, coords) != MPI_SUCCESS)
@@ -523,7 +516,6 @@ struct aMPITopology3d
     ///@copydoc aMPITopology2d::global2localIdx(int,int&,int&)const
     bool global2localIdx( int globalIdx, int& localIdx, int& PID)const
     {
-        Grid3d l = local();
         if( globalIdx < 0 || globalIdx >= (int)g.size()) return false;
         int coords[3];
         int gIdx0 = globalIdx%(g.n()*g.Nx());
@@ -542,31 +534,9 @@ struct aMPITopology3d
             return false;
     }
     ///@copydoc aMPITopology2d::local()const
-    Grid3d local() const {
-        int dims[3], periods[3], coords[3];
-        MPI_Cart_get( comm, 3, dims, periods, coords);
-        double x0 = g.x0() + g.lx()/(double)dims[0]*(double)coords[0]; 
-        double x1 = g.x0() + g.lx()/(double)dims[0]*(double)(coords[0]+1); 
-        if( coords[0] == dims[0]-1) 
-            x1 = g.x1();
-
-        double y0 = g.y0() + g.ly()/(double)dims[1]*(double)coords[1]; 
-        double y1 = g.y0() + g.ly()/(double)dims[1]*(double)(coords[1]+1); 
-        if( coords[1] == dims[1]-1) 
-            y1 = g.y1();
-
-        double z0 = g.z0() + g.lz()/(double)dims[2]*(double)coords[2]; 
-        double z1 = g.z0() + g.lz()/(double)dims[2]*(double)(coords[2]+1); 
-        if( coords[2] == dims[2]-1) 
-            z1 = g.z1();
-        unsigned Nx = g.Nx()/dims[0];
-        unsigned Ny = g.Ny()/dims[1];
-        unsigned Nz = g.Nz()/dims[2];
-        
-        return Grid3d(x0, x1, y0, y1, z0, z1, g.n(), Nx, Ny, Nz, g.bcx(), g.bcy(), g.bcz());
-    }
+    const Grid3d& local() const {return l;}
      ///@copydoc aMPITopology2d::global()const
-    Grid3d global() const {return g;}
+    const Grid3d& global() const {return g;}
     protected:
     ///disallow deletion through base class pointer
     ~aMPITopology3d(){}
@@ -575,17 +545,18 @@ struct aMPITopology3d
     ///@copydoc hide_bc_parameters3d
     ///@copydoc hide_comm_parameters3d
     aMPITopology3d( double x0, double x1, double y0, double y1, double z0, double z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz, MPI_Comm comm):
-        g( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, bcx, bcy, bcz), comm( comm)
+        g( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, bcx, bcy, bcz), l(g), comm( comm)
     {
+        update_local();
         check_division( Nx, Ny, Nz, bcx, bcy, bcz);
     }
     ///explicit copy constructor (default)
     ///@param src source
-    aMPITopology3d(const aMPITopology3d& src):g(src.g),comm(src.comm){}
+    aMPITopology3d(const aMPITopology3d& src):g(src.g),l(src.l),comm(src.comm){ }
     ///explicit assignment operator (default)
     ///@param src source
     aMPITopology3d& operator=(const aMPITopology3d& src){
-        g = src.g; comm = src.comm;
+        g = src.g; l = src.l; comm = src.comm;
         return *this;
     }
     virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz)=0; 
@@ -614,7 +585,30 @@ struct aMPITopology3d
             else assert( periods[2] == false);
         }
     }
-    Grid3d g; //global grid
+    void update_local(){
+        int dims[3], periods[3], coords[3];
+        MPI_Cart_get( comm, 3, dims, periods, coords);
+        double x0 = g.x0() + g.lx()/(double)dims[0]*(double)coords[0]; 
+        double x1 = g.x0() + g.lx()/(double)dims[0]*(double)(coords[0]+1); 
+        if( coords[0] == dims[0]-1) 
+            x1 = g.x1();
+
+        double y0 = g.y0() + g.ly()/(double)dims[1]*(double)coords[1]; 
+        double y1 = g.y0() + g.ly()/(double)dims[1]*(double)(coords[1]+1); 
+        if( coords[1] == dims[1]-1) 
+            y1 = g.y1();
+
+        double z0 = g.z0() + g.lz()/(double)dims[2]*(double)coords[2]; 
+        double z1 = g.z0() + g.lz()/(double)dims[2]*(double)(coords[2]+1); 
+        if( coords[2] == dims[2]-1) 
+            z1 = g.z1();
+        unsigned Nx = g.Nx()/dims[0];
+        unsigned Ny = g.Ny()/dims[1];
+        unsigned Nz = g.Nz()/dims[2];
+        
+        l = Grid3d(x0, x1, y0, y1, z0, z1, g.n(), Nx, Ny, Nz, g.bcx(), g.bcy(), g.bcz());
+    }
+    Grid3d g, l; //global grid
     MPI_Comm comm; //just an integer...
 };
 ///@cond
@@ -652,9 +646,11 @@ int aMPITopology3d::pidOf( double x, double y, double z) const
 }
 void aMPITopology2d::do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny) {
     g.set(new_n,new_Nx,new_Ny);
+    update_local();
 }
 void aMPITopology3d::do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz) {
     g.set(new_n,new_Nx,new_Ny,new_Nz);
+    update_local();
 }
 ///@endcond
 
diff --git a/inc/dg/backend/projection_mpit.cu b/inc/dg/backend/projection_mpit.cu
index b189d4589..27fe5b6f6 100644
--- a/inc/dg/backend/projection_mpit.cu
+++ b/inc/dg/backend/projection_mpit.cu
@@ -4,7 +4,7 @@
 #include "mpi_projection.h"
 #include "mpi_init.h"
 #include "mpi_evaluation.h"
-#includ2 "transpose.h"
+#include "transpose.h"
 
 
 double shift = 0.2;
diff --git a/inc/dg/backend/typedefs.cuh b/inc/dg/backend/typedefs.cuh
index 810fcd970..3f5be7f5e 100644
--- a/inc/dg/backend/typedefs.cuh
+++ b/inc/dg/backend/typedefs.cuh
@@ -27,6 +27,15 @@ typedef EllSparseBlockMat<double> HMatrix; //!< Host Matrix for derivatives
 #ifdef MPI_VERSION
 #include "mpi_vector.h"
 #include "mpi_matrix.h"
+#if THRUST_DEVICE_SYSTEM==THRUST_DEVICE_SYSTEM_CUDA
+#include "mpi-ext.h"
+#if defined(MPIX_CUDA_AWARE_SUPPORT) && !MPIX_CUDA_AWARE_SUPPORT
+#error "CUDA aware MPI installation required!"
+#else
+#warning "Cannot determine CUDA-aware MPI support!";
+#endif
+#endif
+
 //typedef MPI_Vector<thrust::device_vector<double> >  MDVec; //!< MPI Device Vector s.a. dg::DVec
 typedef MPI_Vector<dg::DVec >  MDVec; //!< MPI Device Vector s.a. dg::DVec
 typedef MPI_Vector<dg::HVec >  MHVec; //!< MPI Host Vector s.a. dg::HVec
diff --git a/inc/dg/geometry/geometry_t.cu b/inc/dg/geometry/geometry_t.cu
index 1640cde5c..05d846c92 100644
--- a/inc/dg/geometry/geometry_t.cu
+++ b/inc/dg/geometry/geometry_t.cu
@@ -22,9 +22,7 @@ int main()
     //std::cin >> n>> Nx>>Ny>>Nz;
     //dg::CylindricalGrid3d<dg::DVec> grid( R_0 , R_0+ 2.*M_PI, 0.,2.*M_PI, 0., 2.*M_PI,  n, Nx, Ny, Nz, dg::DIR, dg::DIR, dg::PER);
     dg::CylindricalGrid3d grid( R_0 , R_0+ 2.*M_PI, 0.,2.*M_PI, 0., 2.*M_PI,  3,32,24,16, dg::DIR, dg::DIR, dg::PER);
-    dg::SparseElement<dg::DVec> vol = dg::tensor::determinant(grid.metric());
-    dg::tensor::sqrt(vol);
-    dg::tensor::invert(vol);
+    dg::SparseElement<dg::DVec> vol = dg::tensor::volume(grid.metric());
 
     dg::DVec b = dg::evaluate( sine, grid);
     dg::DVec vol3d = dg::create::volume( grid);
diff --git a/inc/dg/geometry/mpi_base.h b/inc/dg/geometry/mpi_base.h
index bb7c7a5b6..673479bb4 100644
--- a/inc/dg/geometry/mpi_base.h
+++ b/inc/dg/geometry/mpi_base.h
@@ -250,7 +250,7 @@ struct CylindricalMPIGrid3d: public aProductMPIGeometry3d
     virtual SparseTensor<host_vector > do_compute_metric()const{
         SparseTensor<host_vector> metric(1);
         host_vector R = dg::evaluate(dg::cooX3d, *this);
-        for( unsigned i = 0; i<size(); i++)
+        for( unsigned i = 0; i<local().size(); i++)
             R.data()[i] = 1./R.data()[i]/R.data()[i];
         metric.idx(2,2)=0;
         metric.value(0) = R;
diff --git a/inc/dg/geometry/transform.h b/inc/dg/geometry/transform.h
index cadf88c26..4ae2a1734 100644
--- a/inc/dg/geometry/transform.h
+++ b/inc/dg/geometry/transform.h
@@ -76,8 +76,8 @@ template< class Functor>
 MPI_Vector<thrust::host_vector<double> > pullback( const Functor& f, const aMPIGeometry2d& g)
 {
     std::vector<MPI_Vector<thrust::host_vector<double> > > map = g.map();
-    thrust::host_vector<double> vec( g.size());
-    for( unsigned i=0; i<g.size(); i++)
+    thrust::host_vector<double> vec( g.local().size());
+    for( unsigned i=0; i<g.local().size(); i++)
         vec[i] = f( map[0].data()[i], map[1].data()[i]);
     return MPI_Vector<thrust::host_vector<double> >( vec, g.communicator());
 }
@@ -88,8 +88,8 @@ template< class Functor>
 MPI_Vector<thrust::host_vector<double> > pullback( const Functor& f, const aMPIGeometry3d& g)
 {
     std::vector<MPI_Vector<thrust::host_vector<double> > > map = g.map();
-    thrust::host_vector<double> vec( g.size());
-    for( unsigned i=0; i<g.size(); i++)
+    thrust::host_vector<double> vec( g.local().size());
+    for( unsigned i=0; i<g.local().size(); i++)
         vec[i] = f( map[0].data()[i], map[1].data()[i], map[2].data()[i]);
     return MPI_Vector<thrust::host_vector<double> >( vec, g.communicator());
 }
diff --git a/inc/geometries/ds_mpit.cu b/inc/geometries/ds_mpit.cu
index 08bfdd68b..d5372a6e5 100644
--- a/inc/geometries/ds_mpit.cu
+++ b/inc/geometries/ds_mpit.cu
@@ -34,13 +34,14 @@ int main(int argc, char* argv[])
     dg::mpi_init3d( dg::NEU, dg::NEU, dg::PER, n, Nx, Ny, Nz, comm);
     MPI_Comm_rank( MPI_COMM_WORLD, &rank);
 
-    dg::CylindricalMPIGrid3d g3d( R_0 - 1, R_0+1, -1, 1, 0, 2.*M_PI, n, Nx, Ny, Nz, dg::NEU, dg::NEU, dg::PER, comm);
+    dg::CylindricalMPIGrid3d g3d( R_0 - a, R_0+a, -a, a, 0, 2.*M_PI, n, Nx, Ny, Nz, dg::NEU, dg::NEU, dg::PER, comm);
     const dg::MDVec vol3d = dg::create::volume( g3d);
     if(rank==0)std::cout << "Create parallel Derivative!\n";
     dg::geo::TokamakMagneticField mag = dg::geo::createCircularField( R_0, I_0);
     dg::geo::BinaryVectorLvl0 bhat( (dg::geo::BHatR)(mag), (dg::geo::BHatZ)(mag), (dg::geo::BHatP)(mag));
     dg::geo::Fieldaligned<dg::aProductMPIGeometry3d,dg::MIDMatrix,dg::MDVec>  dsFA( bhat, g3d, 2,2,true,true,1e-10, dg::NEU, dg::NEU, dg::geo::NoLimiter());
     dg::geo::DS<dg::aProductMPIGeometry3d, dg::MIDMatrix, dg::MDMatrix, dg::MDVec> ds( dsFA, dg::not_normed, dg::centered);
+    if(rank==0)std::cout << "Ready!\n";
 
     dg::MDVec function = dg::evaluate( func, g3d), derivative(function);
     const dg::MDVec solution = dg::evaluate( deri, g3d);
diff --git a/inc/geometries/ds_t.cu b/inc/geometries/ds_t.cu
index 1e44a83f5..46c2bb4be 100644
--- a/inc/geometries/ds_t.cu
+++ b/inc/geometries/ds_t.cu
@@ -40,7 +40,7 @@ int main(int argc, char * argv[])
     std::cout << "Create parallel Derivative!\n";
 
     //![doxygen]
-    const dg::CylindricalGrid3d g3d( R_0 - 1, R_0+1, -1, 1, 0, 2.*M_PI, n, Nx, Ny, Nz, dg::NEU, dg::NEU, dg::PER);
+    const dg::CylindricalGrid3d g3d( R_0 - a, R_0+a, -a, a, 0, 2.*M_PI, n, Nx, Ny, Nz, dg::NEU, dg::NEU, dg::PER);
     //create magnetic field
     const dg::geo::TokamakMagneticField mag = dg::geo::createCircularField( R_0, I_0);
     const dg::geo::BinaryVectorLvl0 bhat( (dg::geo::BHatR)(mag), (dg::geo::BHatZ)(mag), (dg::geo::BHatP)(mag));
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index 79f2d042b..188010608 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -511,7 +511,7 @@ template <class Limiter>
 void Fieldaligned<Geometry, IMatrix, container>::construct(const dg::geo::BinaryVectorLvl0& vec, const Geometry& grid, unsigned mx, unsigned my, bool bx, bool by, double eps, dg::bc globalbcx, dg::bc globalbcy, Limiter limit, double deltaPhi)
 {
     m_dependsOnX=bx, m_dependsOnY=by;
-    m_Nz=grid.local().Nz(), m_bcz=grid.bcz(); 
+    m_Nz=grid.Nz(), m_bcz=grid.bcz(); 
     m_g.reset(grid);
     dg::blas1::transfer( dg::evaluate( dg::zero, grid), m_hz_inv), m_hp_inv= m_hz_inv, m_hm_inv= m_hz_inv;
     dg::split( m_hz_inv, m_temp, grid);
diff --git a/inc/geometries/geometry_advection_mpib.cu b/inc/geometries/geometry_advection_mpib.cu
index 9eef02050..d47ff999e 100644
--- a/inc/geometries/geometry_advection_mpib.cu
+++ b/inc/geometries/geometry_advection_mpib.cu
@@ -102,7 +102,7 @@ struct CurvatureDirPer
 };
 
 
-typedef  dg::CurvilinearMPIGrid2d Geometry;
+typedef  dg::geo::CurvilinearMPIGrid2d Geometry;
 
 int main(int argc, char** argv)
 {
@@ -140,8 +140,8 @@ int main(int argc, char** argv)
         MPI_Comm planeComm;
         int remain_dims[] = {true,true,false}; //true true false
         MPI_Cart_sub( comm, remain_dims, &planeComm);
-    dg::geo::RibeiroFluxGenerator generator( c.get_psip(), psi_0, psi_1, gp.R_0, 0., 1);
-    //dg::geo::SimpleOrthogonal generator( c.get_psip(), psi_0, psi_1, gp.R_0, 0., 1);
+    //dg::geo::RibeiroFluxGenerator generator( c.get_psip(), psi_0, psi_1, gp.R_0, 0., 1);
+    dg::geo::SimpleOrthogonal generator( c.get_psip(), psi_0, psi_1, gp.R_0, 0., 1);
     Geometry grid(generator, n, Nx, Ny, dg::DIR, dg::PER, planeComm); //2d
     t.toc();
     if(rank==0)std::cout << "Construction took "<<t.diff()<<"s\n";
diff --git a/inc/geometries/mpi_curvilinear.h b/inc/geometries/mpi_curvilinear.h
index 1200d65bc..74d4523bb 100644
--- a/inc/geometries/mpi_curvilinear.h
+++ b/inc/geometries/mpi_curvilinear.h
@@ -106,7 +106,7 @@ struct CurvilinearProductMPIGrid3d : public dg::aProductMPIGeometry3d
         map_.resize(3);
         CurvilinearMPIGrid2d g(generator,n,Nx,Ny, bcx, bcy, get_perp_comm());
         constructPerp( g);
-        constructParallel(this->Nz());
+        constructParallel(this->local().Nz());
     }
 
 
@@ -129,7 +129,7 @@ struct CurvilinearProductMPIGrid3d : public dg::aProductMPIGeometry3d
             CurvilinearMPIGrid2d g(handle_.get(),new_n,new_Nx,new_Ny, this->bcx(), this->bcy(), get_perp_comm());
             constructPerp( g);
         }
-        constructParallel(this->Nz());
+        constructParallel(this->local().Nz());
     }
     void constructPerp( CurvilinearMPIGrid2d& g2d)
     {
@@ -139,8 +139,8 @@ struct CurvilinearProductMPIGrid3d : public dg::aProductMPIGeometry3d
     void constructParallel( unsigned localNz )
     {
         map_[2]=dg::evaluate(dg::cooZ3d, *this);
-        unsigned size = this->size();
-        unsigned size2d = this->n()*this->n()*this->Nx()*this->Ny();
+        unsigned size = this->local().size();
+        unsigned size2d = this->n()*this->n()*this->local().Nx()*this->local().Ny();
         //resize for 3d values
         for( unsigned r=0; r<4;r++)
         {
@@ -165,8 +165,8 @@ struct CurvilinearProductMPIGrid3d : public dg::aProductMPIGeometry3d
         return jac_;
     }
     virtual SparseTensor<host_vector> do_compute_metric( ) const {
-        thrust::host_vector<double> tempxx( size()), tempxy(size()), tempyy(size()), temppp(size());
-        for( unsigned i=0; i<size(); i++)
+        thrust::host_vector<double> tempxx( local().size()), tempxy(local().size()), tempyy(local().size()), temppp(local().size());
+        for( unsigned i=0; i<local().size(); i++)
         {
             tempxx[i] = (jac_.value(0,0).data()[i]*jac_.value(0,0).data()[i]+jac_.value(0,1).data()[i]*jac_.value(0,1).data()[i]);
             tempxy[i] = (jac_.value(0,0).data()[i]*jac_.value(1,0).data()[i]+jac_.value(0,1).data()[i]*jac_.value(1,1).data()[i]);
@@ -198,7 +198,7 @@ CurvilinearMPIGrid2d::CurvilinearMPIGrid2d( const CurvilinearProductMPIGrid3d& g
     jac_=g.jacobian();
     metric_=g.metric();
     //now resize to 2d
-    unsigned s = this->size();
+    unsigned s = this->local().size();
     for( unsigned i=0; i<jac_.values().size(); i++)
         jac_.value(i).data().resize(s);
     for( unsigned i=0; i<metric_.values().size(); i++)
diff --git a/inc/geometries/mpi_fieldaligned.h b/inc/geometries/mpi_fieldaligned.h
index 618a656cd..b7e925393 100644
--- a/inc/geometries/mpi_fieldaligned.h
+++ b/inc/geometries/mpi_fieldaligned.h
@@ -136,7 +136,7 @@ void Fieldaligned<MPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vec
     const dg::geo::BinaryVectorLvl0& vec, const MPIGeometry& grid, unsigned mx, unsigned my, bool bx, bool by, double eps, dg::bc globalbcx, dg::bc globalbcy, Limiter limit, double deltaPhi)
 {
     m_dependsOnX=bx, m_dependsOnY=by;
-    m_Nz=grid.Nz(), m_bcz=grid.bcz(); 
+    m_Nz=grid.local().Nz(), m_bcz=grid.bcz(); 
     m_g.reset(grid);
     dg::blas1::transfer( dg::evaluate( dg::zero, grid), m_hz_inv), m_hp_inv= m_hz_inv, m_hm_inv= m_hz_inv;
     dg::split( m_hz_inv, m_temp, grid);
@@ -209,56 +209,57 @@ MPI_Vector<container> Fieldaligned<G,MPIDistMat<M,C>, MPI_Vector<container> >::e
 
     MPI_Vector<container> temp(init2d), tempP(init2d), tempM(init2d);
     MPI_Vector<container> vec3d = dg::evaluate( dg::zero, m_g.get());
-    std::vector<MPI_Vector<container> >  plus2d(Nz, zero2d), minus2d(plus2d), result(plus2d);
-    unsigned turns = rounds; 
-    if( turns ==0) turns++;
-    //first apply Interpolation many times, scale and store results
-    for( unsigned r=0; r<turns; r++)
-        for( unsigned i0=0; i0<Nz; i0++)
-        {
-            dg::blas1::copy( init2d, tempP);
-            dg::blas1::copy( init2d, tempM);
-            unsigned rep = r*Nz + i0; 
-            for(unsigned k=0; k<rep; k++)
-            {
-                dg::blas2::symv( m_plus, tempP, temp);
-                temp.swap( tempP);
-                dg::blas2::symv( m_minus, tempM, temp);
-                temp.swap( tempM);
-            }
-            dg::blas1::scal( tempP, unary(  (double)rep*m_g.get().hz() ) );
-            dg::blas1::scal( tempM, unary( -(double)rep*m_g.get().hz() ) );
-            dg::blas1::axpby( 1., tempP, 1., plus2d[i0]);
-            dg::blas1::axpby( 1., tempM, 1., minus2d[i0]);
-        }
-    //now we have the plus and the minus filaments
-    if( rounds == 0) //there is a limiter
-    {
-        for( unsigned i0=0; i0<m_Nz; i0++)
-        {
-            int idx = (int)(i0+m_coords2*m_Nz)  - (int)p0;
-            if(idx>=0)
-                result[i0] = plus2d[idx];
-            else
-                result[i0] = minus2d[abs(idx)];
-            thrust::copy( result[i0].data().begin(), result[i0].data().end(), vec3d.data().begin() + i0*m_perp_size);
-        }
-    }
-    else //sum up plus2d and minus2d
-    {
-        for( unsigned i0=0; i0<Nz; i0++)
-        {
-            unsigned revi0 = (Nz - i0)%Nz; //reverted index
-            dg::blas1::axpby( 1., plus2d[i0], 0., result[i0]);
-            dg::blas1::axpby( 1., minus2d[revi0], 1., result[i0]);
-        }
-        dg::blas1::axpby( -1., init2d, 1., result[0]);
-        for(unsigned i0=0; i0<m_Nz; i0++)
-        {
-            int idx = ((int)i0 + m_coords2*m_Nz -(int)p0 + Nz)%Nz; //shift index
-            thrust::copy( result[idx].data().begin(), result[idx].data().end(), vec3d.data().begin() + i0*m_perp_size);
-        }
-    }
+    //std::vector<MPI_Vector<container> >  plus2d(Nz, zero2d); 
+    //std::vector<MPI_Vector<container> >  minus2d(plus2d), result(plus2d);
+    //unsigned turns = rounds; 
+    //if( turns ==0) turns++;
+    ////first apply Interpolation many times, scale and store results
+    //for( unsigned r=0; r<turns; r++)
+    //    for( unsigned i0=0; i0<Nz; i0++)
+    //    {
+    //        dg::blas1::copy( init2d, tempP);
+    //        dg::blas1::copy( init2d, tempM);
+    //        unsigned rep = r*Nz + i0; 
+    //        for(unsigned k=0; k<rep; k++)
+    //        {
+    //            dg::blas2::symv( m_plus, tempP, temp);
+    //            temp.swap( tempP);
+    //            dg::blas2::symv( m_minus, tempM, temp);
+    //            temp.swap( tempM);
+    //        }
+    //        dg::blas1::scal( tempP, unary(  (double)rep*m_g.get().hz() ) );
+    //        dg::blas1::scal( tempM, unary( -(double)rep*m_g.get().hz() ) );
+    //        dg::blas1::axpby( 1., tempP, 1., plus2d[i0]);
+    //        dg::blas1::axpby( 1., tempM, 1., minus2d[i0]);
+    //    }
+    ////now we have the plus and the minus filaments
+    //if( rounds == 0) //there is a limiter
+    //{
+    //    for( unsigned i0=0; i0<m_Nz; i0++)
+    //    {
+    //        int idx = (int)(i0+m_coords2*m_Nz)  - (int)p0;
+    //        if(idx>=0)
+    //            result[i0] = plus2d[idx];
+    //        else
+    //            result[i0] = minus2d[abs(idx)];
+    //        thrust::copy( result[i0].data().begin(), result[i0].data().end(), vec3d.data().begin() + i0*m_perp_size);
+    //    }
+    //}
+    //else //sum up plus2d and minus2d
+    //{
+    //    for( unsigned i0=0; i0<Nz; i0++)
+    //    {
+    //        unsigned revi0 = (Nz - i0)%Nz; //reverted index
+    //        dg::blas1::axpby( 1., plus2d[i0], 0., result[i0]);
+    //        dg::blas1::axpby( 1., minus2d[revi0], 1., result[i0]);
+    //    }
+    //    dg::blas1::axpby( -1., init2d, 1., result[0]);
+    //    for(unsigned i0=0; i0<m_Nz; i0++)
+    //    {
+    //        int idx = ((int)i0 + m_coords2*m_Nz -(int)p0 + Nz)%Nz; //shift index
+    //        thrust::copy( result[idx].data().begin(), result[idx].data().end(), vec3d.data().begin() + i0*m_perp_size);
+    //    }
+    //}
     return vec3d;
 }
 
-- 
GitLab


From 236d1e2e03382c0f48cc1c6ded7badfb5a253db7 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sat, 28 Oct 2017 16:43:13 +0200
Subject: [PATCH 405/453] debuggin mpi programs in geometries

---
 inc/geometries/ds_curv_mpit.cu    |   2 +
 inc/geometries/ds_curv_t.cu       |   6 +-
 inc/geometries/mpi_curvilinear.h  |   2 +
 inc/geometries/mpi_fieldaligned.h | 109 +++++++++++++++---------------
 4 files changed, 62 insertions(+), 57 deletions(-)

diff --git a/inc/geometries/ds_curv_mpit.cu b/inc/geometries/ds_curv_mpit.cu
index 6e8f40962..454cbe244 100644
--- a/inc/geometries/ds_curv_mpit.cu
+++ b/inc/geometries/ds_curv_mpit.cu
@@ -44,7 +44,9 @@ int main(int argc, char * argv[])
     unsigned mx=1, my=10;
     double psi_0 = -20, psi_1 = -4;
     dg::geo::FluxGenerator flux( mag.get_psip(), mag.get_ipol(), psi_0, psi_1, gp.R_0, 0., 1);
+    std::cout << "Constructing Grid...\n";
     dg::geo::CurvilinearProductMPIGrid3d g3d(flux, n, Nx, Ny,Nz, dg::DIR, dg::PER, dg::PER, comm);
+    std::cout << "Constructing Fieldlines...\n";
     dg::geo::DS<dg::aProductMPIGeometry3d, dg::MIHMatrix, dg::MHMatrix, dg::MHVec> ds( mag, g3d, mx, my, false, true, 1e-8, dg::normed, dg::centered);
     
     t.toc();
diff --git a/inc/geometries/ds_curv_t.cu b/inc/geometries/ds_curv_t.cu
index bfb74b4e8..772a3fcfb 100644
--- a/inc/geometries/ds_curv_t.cu
+++ b/inc/geometries/ds_curv_t.cu
@@ -29,8 +29,6 @@ int main(int argc, char * argv[])
     }
     dg::geo::solovev::Parameters gp(js);
     dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField( gp);
-    dg::Timer t;
-    t.tic();
     std::cout << "Type n(3), Nx(8), Ny(80), Nz(20)\n";
     unsigned n,Nx,Ny,Nz;
     std::cin >> n>> Nx>>Ny>>Nz;   
@@ -39,9 +37,13 @@ int main(int argc, char * argv[])
     std::cin >> mx >> my;
 
     double psi_0 = -20, psi_1 = -4;
+    dg::Timer t;
+    t.tic();
     dg::geo::FluxGenerator flux( mag.get_psip(), mag.get_ipol(), psi_0, psi_1, gp.R_0, 0., 1);
+    std::cout << "Constructing Grid...\n";
     dg::geo::CurvilinearProductGrid3d g3d(flux, n, Nx, Ny,Nz, dg::DIR);
     //dg::geo::Fieldaligned<dg::aGeometry3d, dg::IHMatrix, dg::HVec> fieldaligned( bhat, g3d, 1, 4, gp.rk4eps, dg::NoLimiter() ); 
+    std::cout << "Constructing Fieldlines...\n";
     dg::geo::DS<dg::aProductGeometry3d, dg::IHMatrix, dg::HMatrix, dg::HVec> ds( mag, g3d, mx, my, false, true, 1e-8, dg::normed, dg::centered);
     
     t.toc();
diff --git a/inc/geometries/mpi_curvilinear.h b/inc/geometries/mpi_curvilinear.h
index 74d4523bb..9cee121bf 100644
--- a/inc/geometries/mpi_curvilinear.h
+++ b/inc/geometries/mpi_curvilinear.h
@@ -138,6 +138,7 @@ struct CurvilinearProductMPIGrid3d : public dg::aProductMPIGeometry3d
     }
     void constructParallel( unsigned localNz )
     {
+        map_.resize(3);
         map_[2]=dg::evaluate(dg::cooZ3d, *this);
         unsigned size = this->local().size();
         unsigned size2d = this->n()*this->n()*this->local().Nx()*this->local().Ny();
@@ -198,6 +199,7 @@ CurvilinearMPIGrid2d::CurvilinearMPIGrid2d( const CurvilinearProductMPIGrid3d& g
     jac_=g.jacobian();
     metric_=g.metric();
     //now resize to 2d
+    map_.pop_back();
     unsigned s = this->local().size();
     for( unsigned i=0; i<jac_.values().size(); i++)
         jac_.value(i).data().resize(s);
diff --git a/inc/geometries/mpi_fieldaligned.h b/inc/geometries/mpi_fieldaligned.h
index b7e925393..d13396081 100644
--- a/inc/geometries/mpi_fieldaligned.h
+++ b/inc/geometries/mpi_fieldaligned.h
@@ -92,9 +92,9 @@ struct Fieldaligned< ProductMPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY
 
     void set_boundaries( dg::bc bcz, const MPI_Vector<LocalContainer>& global, double scal_left, double scal_right)
     {
-        dg::split( global, m_temp);
+        dg::split( global, m_temp, m_g.get());
         dg::blas1::axpby( scal_left, m_temp[0],               0., m_left);
-        dg::blas1::axpby( scal_right, m_temp[m_g.get().Nz()], 0., m_left);
+        dg::blas1::axpby( scal_right, m_temp[m_g.get().local().Nz()], 0., m_left);
         m_bcz = bcz;
     }
 
@@ -149,7 +149,7 @@ void Fieldaligned<MPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vec
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     dg::Handle<aMPIGeometry2d> grid_coarse( grid.perp_grid());
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-    m_perp_size = grid_coarse.get().size();
+    m_perp_size = grid_coarse.get().local().size();
     dg::blas1::transfer( dg::pullback(limit, grid_coarse.get()), m_limiter);
     dg::blas1::transfer( dg::evaluate(zero, grid_coarse.get()), m_left);
     m_ghostM = m_ghostP = m_right = m_left;
@@ -205,61 +205,60 @@ MPI_Vector<container> Fieldaligned<G,MPIDistMat<M,C>, MPI_Vector<container> >::e
     const dg::Handle<aMPIGeometry2d> g2d = m_g.get().perp_grid();
     MPI_Vector<container> init2d = dg::pullback( binary, g2d.get()); 
     MPI_Vector<container> zero2d = dg::evaluate( dg::zero, g2d.get()); 
-    unsigned Nz = m_g.get().global().Nz();
+    unsigned globalNz = m_g.get().global().Nz();
 
     MPI_Vector<container> temp(init2d), tempP(init2d), tempM(init2d);
     MPI_Vector<container> vec3d = dg::evaluate( dg::zero, m_g.get());
-    //std::vector<MPI_Vector<container> >  plus2d(Nz, zero2d); 
-    //std::vector<MPI_Vector<container> >  minus2d(plus2d), result(plus2d);
-    //unsigned turns = rounds; 
-    //if( turns ==0) turns++;
-    ////first apply Interpolation many times, scale and store results
-    //for( unsigned r=0; r<turns; r++)
-    //    for( unsigned i0=0; i0<Nz; i0++)
-    //    {
-    //        dg::blas1::copy( init2d, tempP);
-    //        dg::blas1::copy( init2d, tempM);
-    //        unsigned rep = r*Nz + i0; 
-    //        for(unsigned k=0; k<rep; k++)
-    //        {
-    //            dg::blas2::symv( m_plus, tempP, temp);
-    //            temp.swap( tempP);
-    //            dg::blas2::symv( m_minus, tempM, temp);
-    //            temp.swap( tempM);
-    //        }
-    //        dg::blas1::scal( tempP, unary(  (double)rep*m_g.get().hz() ) );
-    //        dg::blas1::scal( tempM, unary( -(double)rep*m_g.get().hz() ) );
-    //        dg::blas1::axpby( 1., tempP, 1., plus2d[i0]);
-    //        dg::blas1::axpby( 1., tempM, 1., minus2d[i0]);
-    //    }
-    ////now we have the plus and the minus filaments
-    //if( rounds == 0) //there is a limiter
-    //{
-    //    for( unsigned i0=0; i0<m_Nz; i0++)
-    //    {
-    //        int idx = (int)(i0+m_coords2*m_Nz)  - (int)p0;
-    //        if(idx>=0)
-    //            result[i0] = plus2d[idx];
-    //        else
-    //            result[i0] = minus2d[abs(idx)];
-    //        thrust::copy( result[i0].data().begin(), result[i0].data().end(), vec3d.data().begin() + i0*m_perp_size);
-    //    }
-    //}
-    //else //sum up plus2d and minus2d
-    //{
-    //    for( unsigned i0=0; i0<Nz; i0++)
-    //    {
-    //        unsigned revi0 = (Nz - i0)%Nz; //reverted index
-    //        dg::blas1::axpby( 1., plus2d[i0], 0., result[i0]);
-    //        dg::blas1::axpby( 1., minus2d[revi0], 1., result[i0]);
-    //    }
-    //    dg::blas1::axpby( -1., init2d, 1., result[0]);
-    //    for(unsigned i0=0; i0<m_Nz; i0++)
-    //    {
-    //        int idx = ((int)i0 + m_coords2*m_Nz -(int)p0 + Nz)%Nz; //shift index
-    //        thrust::copy( result[idx].data().begin(), result[idx].data().end(), vec3d.data().begin() + i0*m_perp_size);
-    //    }
-    //}
+    std::vector<MPI_Vector<container> >  plus2d(globalNz, zero2d), minus2d(plus2d), result(plus2d); 
+    unsigned turns = rounds; 
+    if( turns ==0) turns++;
+    //first apply Interpolation many times, scale and store results
+    for( unsigned r=0; r<turns; r++)
+        for( unsigned i0=0; i0<globalNz; i0++)
+        {
+            dg::blas1::copy( init2d, tempP);
+            dg::blas1::copy( init2d, tempM);
+            unsigned rep = r*globalNz + i0; 
+            for(unsigned k=0; k<rep; k++)
+            {
+                dg::blas2::symv( m_plus, tempP, temp);
+                temp.swap( tempP);
+                dg::blas2::symv( m_minus, tempM, temp);
+                temp.swap( tempM);
+            }
+            dg::blas1::scal( tempP, unary(  (double)rep*m_g.get().hz() ) );
+            dg::blas1::scal( tempM, unary( -(double)rep*m_g.get().hz() ) );
+            dg::blas1::axpby( 1., tempP, 1., plus2d[i0]);
+            dg::blas1::axpby( 1., tempM, 1., minus2d[i0]);
+        }
+    //now we have the plus and the minus filaments
+    if( rounds == 0) //there is a limiter
+    {
+        for( unsigned i0=0; i0<m_Nz; i0++)
+        {
+            int idx = (int)(i0+m_coords2*m_Nz)  - (int)p0;
+            if(idx>=0)
+                result[i0] = plus2d[idx];
+            else
+                result[i0] = minus2d[abs(idx)];
+            thrust::copy( result[i0].data().begin(), result[i0].data().end(), vec3d.data().begin() + i0*m_perp_size);
+        }
+    }
+    else //sum up plus2d and minus2d
+    {
+        for( unsigned i0=0; i0<globalNz; i0++)
+        {
+            unsigned revi0 = (globalNz - i0)%globalNz; //reverted index
+            dg::blas1::axpby( 1., plus2d[i0], 0., result[i0]);
+            dg::blas1::axpby( 1., minus2d[revi0], 1., result[i0]);
+        }
+        dg::blas1::axpby( -1., init2d, 1., result[0]);
+        for(unsigned i0=0; i0<m_Nz; i0++)
+        {
+            int idx = ((int)i0 + m_coords2*m_Nz -(int)p0 + globalNz)%globalNz; //shift index
+            thrust::copy( result[idx].data().begin(), result[idx].data().end(), vec3d.data().begin() + i0*m_perp_size);
+        }
+    }
     return vec3d;
 }
 
-- 
GitLab


From 2c044d313369f7fac8d8f4930289547ad3d17bdd Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sat, 28 Oct 2017 23:11:03 +0200
Subject: [PATCH 406/453] played around with ds_t.cu and ds_mpit.cu

---
 inc/dg/nullstelle.h               |  69 ++++++++++++-------
 inc/dg/runge_kutta.h              |   4 +-
 inc/geometries/ds.h               |  34 ++++++---
 inc/geometries/ds_curv_mpit.cu    |   2 +-
 inc/geometries/ds_curv_t.cu       |   6 +-
 inc/geometries/ds_mpit.cu         |   3 +-
 inc/geometries/ds_t.cu            |   3 +-
 inc/geometries/fieldaligned.h     | 110 ++++++++++++++++++------------
 inc/geometries/mpi_fieldaligned.h |  52 ++++++++++++--
 src/heat/heat.cuh                 |  48 +++++--------
 src/heat/parameters.h             |   7 +-
 11 files changed, 210 insertions(+), 128 deletions(-)

diff --git a/inc/dg/nullstelle.h b/inc/dg/nullstelle.h
index c7242ce4f..ea55fcb2d 100644
--- a/inc/dg/nullstelle.h
+++ b/inc/dg/nullstelle.h
@@ -15,7 +15,7 @@ namespace dg{
  *
  * @ingroup numerical0
  */
-class KeineNST_1D: public std::exception
+class NoRoot1d: public std::exception
 {
   private:
     double x1, x2;
@@ -25,61 +25,80 @@ class KeineNST_1D: public std::exception
      * \param x_min left boundary
      * \param x_max right boundary
      */
-    KeineNST_1D(double x_min, double x_max): x1(x_min), x2(x_max){}
-    /*! @brief display on std::cerr
-     *
-     * meldet %Fehler und gibt linke und rechte Grenze aus
+    NoRoot1d(double x_min, double x_max): x1(x_min), x2(x_max){}
+    /*! @brief display left and right boundary on std::cerr
      */
-    void anzeigen() const
+    void display() const
     {
       std::cerr << "Between " <<x1 << " and "<<x2<<" is no root!\n";
     }
     /**
      * @brief what string
-     *
-     * @return 
      */
     char const* what() const throw(){ return "Failed to find root!";}
 };
 
-/*! @brief Find a root of a 1d function in given boundaries
+/*! @brief Find a root of a 1d function in given boundaries using bisection
  *
  * @ingroup root
  * It is assumed that a sign change occurs at the root.
  * Function jumps closer to the root by checking the sign. 
  * \tparam UnaryOp unary function operator
- * \param funktion Function or Functor
+ * \param op Function or Functor
  * \param x_min left boundary, contains new left boundary on execution
  * \param x_max right boundary, contains new right boundary on execution
- * \param aufloesung accuracy of the root finding	
+ * \param eps accuracy of the root finding	
  * \return number of used steps to reach the desired accuracy
- * \throw KeineNST_1D if no root lies between the given boundaries
+ * \throw NoRoot1d if no root lies between the given boundaries
  * \throw std::runtime_error if after 60 steps the accuracy wasn't reached
  *
- * \code nullstelle_1D(funk, x_min, x_max, aufloesung); \endcode
+ * \code nullstelle_1D(funk, x_min, x_max, eps); \endcode
  * \note If the root is found exactly the x_min = x_max 
  */
 template <typename UnaryOp>           
-int bisection1d (UnaryOp& funktion, double& x_min, double& x_max, const double aufloesung) 
+int bisection1d (UnaryOp& op, double& x_min, double& x_max, const double eps) 
+{
+    double  mitte;
+    double wert_max, wert_mitte, wert_min;
+    wert_max=op(x_max);
+    wert_min=op(x_min);
+
+    if(wert_max*wert_min>=0) 
+        throw NoRoot1d(x_min, x_max);
+    
+    int j_max = 60;
+    for(int j=0; j<j_max; j++)
+    {
+        wert_mitte = op( mitte =(x_min+x_max)/2.0 );
+        if(wert_mitte==0) 			    {x_min=x_max=mitte; return j+3;}
+        else if(wert_mitte*wert_max>0) 	x_max = mitte;
+        else 				            x_min = mitte;
+        if((x_max-x_min)<eps)           return j+3; 
+    }
+    throw std::runtime_error("Too many steps in root finding!");
+}
+
+template <typename UnaryOp>           
+int false_position (UnaryOp& op, double& x_min, double& x_max, const double eps) 
 {
     double  mitte;
-    double wert_oben, wert_mitte, wert_unten;
-    wert_oben=funktion(x_max);
-    wert_unten=funktion(x_min);
+    double wert_max, wert_mitte, wert_min;
+    wert_max=op(x_max);
+    wert_min=op(x_min);
 
-    if(wert_oben*wert_unten>=0) 
-        throw KeineNST_1D(x_min, x_max);
+    if(wert_max*wert_min>=0) 
+        throw NoRoot1d(x_min, x_max);
     
     int j_max = 60;
     for(int j=0; j<j_max; j++)
     {
-        wert_mitte = funktion( mitte=(x_min+x_max)/2.0 );
-        if(wert_mitte==0) 			        {x_min=x_max=mitte; return j+3;}
-        else if(wert_mitte*wert_oben>0) 	x_max = mitte;
-        else 				                x_min = mitte;
-        if((x_max-x_min)<aufloesung)        return j+3; 
+        wert_mitte = op( mitte = (x_min*wert_max - x_max*wert_min)/(wert_max-wert_min) );
+        if(wert_mitte==0) 			    {x_min=x_max=mitte; return j+3;}
+        else if(wert_mitte*wert_max>0) 	{x_max = mitte; wert_max = wert_mitte;}
+        else 				            {x_min = mitte; wert_min = wert_mitte;}
+        if((x_max-x_min)<eps)           return j+3; 
     }
-    throw std::runtime_error("Zu viele Schritte bei Nullstellensuche! evtl. j_max aendern");
+    throw std::runtime_error("Too many steps in root finding!");
 }
 
       
diff --git a/inc/dg/runge_kutta.h b/inc/dg/runge_kutta.h
index 04c00bfcc..6eabc9aab 100644
--- a/inc/dg/runge_kutta.h
+++ b/inc/dg/runge_kutta.h
@@ -564,8 +564,8 @@ int integrateRK(RHS& rhs, const container& begin, container& end, double T_max,
     container old_end(begin), temp(begin);
     end = begin;
     if( T_max == 0) return 0;
-    double dt = T_max/10;
-    int NT = 10;
+    double dt = T_max/2;
+    int NT = 2;
     double error = 1e10;
  
     while( error > eps_abs && NT < pow( 2, 18) )
diff --git a/inc/geometries/ds.h b/inc/geometries/ds.h
index 134f62d0f..cef08514b 100644
--- a/inc/geometries/ds.h
+++ b/inc/geometries/ds.h
@@ -49,30 +49,48 @@ struct DS
      * @brief Create the magnetic unit vector field and construct
      * @copydetails DS(const dg::geo::BinaryVectorLvl0&,const ProductGeometry&,unsigned,unsigned,bool,bool,double,dg::norm,dg::direction)
      */
-    DS(const dg::geo::TokamakMagneticField& vec, const ProductGeometry& grid, unsigned multiplyX=1, unsigned multiplyY=1, bool dependsOnX = true, bool dependsOnY=true, double eps = 1e-5, dg::norm no=dg::normed, dg::direction dir = dg::centered)
+    template <class Limiter = FullLimiter>
+    DS(const dg::geo::TokamakMagneticField& vec, const ProductGeometry& grid, 
+        dg::bc bcx = dg::NEU, 
+        dg::bc bcy = dg::NEU, 
+        Limiter limit = FullLimiter(), 
+        dg::norm no=dg::normed, dg::direction dir = dg::centered, 
+        double eps = 1e-5, unsigned multiplyX=5, unsigned multiplyY=5, bool dependsOnX = true, bool dependsOnY=true)
     {
         dg::geo::BinaryVectorLvl0 bhat( (dg::geo::BHatR)(vec), (dg::geo::BHatZ)(vec), (dg::geo::BHatP)(vec));
-        m_fa.construct( bhat, grid, multiplyX, multiplyY, dependsOnX, dependsOnY, eps, grid.bcx(), grid.bcy(), FullLimiter());
+        m_fa.construct( bhat, grid, bcx, bcy, limit, eps, multiplyX, multiplyY, dependsOnX, dependsOnY);
         construct( m_fa, no, dir);
     }
     /**
      * @brief Create a \c Fieldaligned object and construct
      *
+     * @tparam Limiter Class that can be evaluated on a 2d grid, returns 1 if there
+        is a limiter and 0 if there isn't. 
+        If a field line crosses the limiter in the plane \f$ \phi=0\f$ then the limiter boundary conditions apply. 
      * @param vec The vector field to integrate
      * @param grid The grid on which to operate defines the parallel boundary condition in case there is a limiter.
+     * @param bcx Defines the interpolation behaviour when a fieldline intersects the boundary box in the perpendicular direction
+     * @param bcy Defines the interpolation behaviour when a fieldline intersects the boundary box in the perpendicular direction
+     * @param limit Instance of the limiter class (Default is a limiter everywhere, 
+        note that if grid.bcz() is periodic it doesn't matter if there is a limiter or not)
+     * @param no indicate if the symv function should be symmetric (not_normed) or not
+     * @param dir indicate the direction in the bracket operator and in symv
+     * @param eps Desired accuracy of the fieldline integrator
      * @param multiplyX defines the resolution in X of the fine grid relative to grid
      * @param multiplyY defines the resolution in Y of the fine grid relative to grid
      * @param dependsOnX indicates, whether the given vector field vec depends on the first coordinate
      * @param dependsOnY indicates, whether the given vector field vec depends on the second coordinate
-     * @param eps Desired accuracy of the fieldline integrator
-     * @param no indicate if the symv function should be symmetric (not_normed) or not
-     * @param dir indicate the direction in the bracket operator and in symv
-     *@note globalbcx and globalbcy  as well as bcz are taken from grid with full limter 
      * @sa \c Fieldaligned
      */
-    DS(const dg::geo::BinaryVectorLvl0& vec, const ProductGeometry& grid, unsigned multiplyX=1, unsigned multiplyY=1, bool dependsOnX = true, bool dependsOnY=true, double eps = 1e-5, dg::norm no=dg::normed, dg::direction dir = dg::centered)
+    template<class Limiter>
+    DS(const dg::geo::BinaryVectorLvl0& vec, const ProductGeometry& grid, 
+        dg::bc bcx = dg::NEU, 
+        dg::bc bcy = dg::NEU, 
+        Limiter limit = FullLimiter(), 
+        dg::norm no=dg::normed, dg::direction dir = dg::centered, 
+        double eps = 1e-5, unsigned multiplyX=5, unsigned multiplyY=5, bool dependsOnX = true, bool dependsOnY=true)
     {
-        m_fa.construct( vec, grid, multiplyX, multiplyY, dependsOnX, dependsOnY, eps, grid.bcx(), grid.bcy(), FullLimiter());
+        m_fa.construct( vec, grid, bcx, bcy, limit, eps, multiplyX, multiplyY, dependsOnX, dependsOnY);
         construct( m_fa, no, dir);
     }
     ///@copydoc construct
diff --git a/inc/geometries/ds_curv_mpit.cu b/inc/geometries/ds_curv_mpit.cu
index 454cbe244..7abfd5421 100644
--- a/inc/geometries/ds_curv_mpit.cu
+++ b/inc/geometries/ds_curv_mpit.cu
@@ -47,7 +47,7 @@ int main(int argc, char * argv[])
     std::cout << "Constructing Grid...\n";
     dg::geo::CurvilinearProductMPIGrid3d g3d(flux, n, Nx, Ny,Nz, dg::DIR, dg::PER, dg::PER, comm);
     std::cout << "Constructing Fieldlines...\n";
-    dg::geo::DS<dg::aProductMPIGeometry3d, dg::MIHMatrix, dg::MHMatrix, dg::MHVec> ds( mag, g3d, mx, my, false, true, 1e-8, dg::normed, dg::centered);
+    dg::geo::DS<dg::aProductMPIGeometry3d, dg::MIHMatrix, dg::MHMatrix, dg::HVec> ds( mag, g3d, dg::NEU, dg::NEU, dg::FullLimiter(), 1e-8, mx, my, false, true, dg::normed, dg::centered);
     
     t.toc();
     if(rank==0)std::cout << "Construction took "<<t.diff()<<"s\n";
diff --git a/inc/geometries/ds_curv_t.cu b/inc/geometries/ds_curv_t.cu
index 772a3fcfb..97f4e354e 100644
--- a/inc/geometries/ds_curv_t.cu
+++ b/inc/geometries/ds_curv_t.cu
@@ -40,11 +40,11 @@ int main(int argc, char * argv[])
     dg::Timer t;
     t.tic();
     dg::geo::FluxGenerator flux( mag.get_psip(), mag.get_ipol(), psi_0, psi_1, gp.R_0, 0., 1);
-    std::cout << "Constructing Grid...\n";
+    std::cout << "Constructing Grid..."<<std::endl;
     dg::geo::CurvilinearProductGrid3d g3d(flux, n, Nx, Ny,Nz, dg::DIR);
     //dg::geo::Fieldaligned<dg::aGeometry3d, dg::IHMatrix, dg::HVec> fieldaligned( bhat, g3d, 1, 4, gp.rk4eps, dg::NoLimiter() ); 
-    std::cout << "Constructing Fieldlines...\n";
-    dg::geo::DS<dg::aProductGeometry3d, dg::IHMatrix, dg::HMatrix, dg::HVec> ds( mag, g3d, mx, my, false, true, 1e-8, dg::normed, dg::centered);
+    std::cout << "Constructing Fieldlines..."<<std::endl;
+    dg::geo::DS<dg::aProductGeometry3d, dg::IHMatrix, dg::HMatrix, dg::HVec> ds( mag, g3d, dg::NEU, dg::NEU, dg::geo::FullLimiter(), dg::normed, dg::centered, 1e-8, mx, my, false, true);
     
     t.toc();
     std::cout << "Construction took "<<t.diff()<<"s\n";
diff --git a/inc/geometries/ds_mpit.cu b/inc/geometries/ds_mpit.cu
index d5372a6e5..6e89361aa 100644
--- a/inc/geometries/ds_mpit.cu
+++ b/inc/geometries/ds_mpit.cu
@@ -36,10 +36,11 @@ int main(int argc, char* argv[])
 
     dg::CylindricalMPIGrid3d g3d( R_0 - a, R_0+a, -a, a, 0, 2.*M_PI, n, Nx, Ny, Nz, dg::NEU, dg::NEU, dg::PER, comm);
     const dg::MDVec vol3d = dg::create::volume( g3d);
+    if(rank==0)std::cout << "Note that it's faster to compute with OMP_NUM_THREADS=1\n";
     if(rank==0)std::cout << "Create parallel Derivative!\n";
     dg::geo::TokamakMagneticField mag = dg::geo::createCircularField( R_0, I_0);
     dg::geo::BinaryVectorLvl0 bhat( (dg::geo::BHatR)(mag), (dg::geo::BHatZ)(mag), (dg::geo::BHatP)(mag));
-    dg::geo::Fieldaligned<dg::aProductMPIGeometry3d,dg::MIDMatrix,dg::MDVec>  dsFA( bhat, g3d, 2,2,true,true,1e-10, dg::NEU, dg::NEU, dg::geo::NoLimiter());
+    dg::geo::Fieldaligned<dg::aProductMPIGeometry3d,dg::MIDMatrix,dg::MDVec>  dsFA( bhat, g3d, dg::NEU, dg::NEU, dg::geo::NoLimiter(), 1e-6, 5,5,true,true);
     dg::geo::DS<dg::aProductMPIGeometry3d, dg::MIDMatrix, dg::MDMatrix, dg::MDVec> ds( dsFA, dg::not_normed, dg::centered);
     if(rank==0)std::cout << "Ready!\n";
 
diff --git a/inc/geometries/ds_t.cu b/inc/geometries/ds_t.cu
index 46c2bb4be..8eb76e01e 100644
--- a/inc/geometries/ds_t.cu
+++ b/inc/geometries/ds_t.cu
@@ -29,6 +29,7 @@ double deri(double R, double Z, double phi)
 int main(int argc, char * argv[])
 {
     std::cout << "First test the cylindrical version\n";
+    std::cout << "Note that it's faster to compute with OMP_NUM_THREADS=1\n";
     std::cout << "Type n, Nx, Ny, Nz\n";
     unsigned n, Nx, Ny, Nz;
     std::cin >> n>> Nx>>Ny>>Nz;
@@ -45,7 +46,7 @@ int main(int argc, char * argv[])
     const dg::geo::TokamakMagneticField mag = dg::geo::createCircularField( R_0, I_0);
     const dg::geo::BinaryVectorLvl0 bhat( (dg::geo::BHatR)(mag), (dg::geo::BHatZ)(mag), (dg::geo::BHatP)(mag));
     //create Fieldaligned object and construct DS from it
-    const dg::geo::Fieldaligned<dg::aProductGeometry3d,dg::IDMatrix,dg::DVec>  dsFA( bhat, g3d, mx, my, true,true, 1e-6, dg::NEU, dg::NEU, dg::geo::NoLimiter());
+    const dg::geo::Fieldaligned<dg::aProductGeometry3d,dg::IDMatrix,dg::DVec>  dsFA( bhat, g3d, dg::NEU, dg::NEU, dg::geo::NoLimiter(), 1e-6, mx, my, true,true);
     dg::geo::DS<dg::aProductGeometry3d, dg::IDMatrix, dg::DMatrix, dg::DVec> ds( dsFA, dg::not_normed, dg::centered);
     ///##########################################################///
     //apply to function 
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index 188010608..1980c4c65 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -42,29 +42,6 @@ typedef ZERO NoLimiter;
 ///@cond
 namespace detail{
 
-struct DZField
-{
-    void operator()( const dg::HVec& y, dg::HVec& yp)
-    {
-        yp[0] = yp[1] = 0;
-        yp[2] = 1.;
-    }
-    double error( const dg::HVec& x0, const dg::HVec& x1)
-    {
-        return sqrt( (x0[0]-x1[0])*(x0[0]-x1[0]) +(x0[1]-x1[1])*(x0[1]-x1[1])+(x0[2]-x1[2])*(x0[2]-x1[2]));
-    }
-    bool monitor( const dg::HVec& end){ return true;}
-    double operator()( double x, double y) //1/B
-    {
-        return 1.;
-    }
-    double operator()( double x, double y, double z)
-    {
-        return 1.;
-    }
-
-};
-
 struct DSFieldCylindrical
 {
     DSFieldCylindrical( const dg::geo::BinaryVectorLvl0& v):v_(v) { }
@@ -200,29 +177,37 @@ struct BoxIntegrator
      * @param g The 2d or 3d grid
      * @param eps the accuracy of the runge kutta integrator
      */
-    BoxIntegrator( const Field& field, const Topology& g, double eps): field_(field), g_(g), coords_(3), coordsp_(3), eps_(eps) {}
+    BoxIntegrator( const Field& field, const Topology& g, double eps): m_field(field), m_g(g), m_coords0(3), m_coords1(3), m_deltaPhi0(0), m_eps(eps) {}
     /**
      * @brief Set the starting coordinates for next field line integration
      * @param coords the new coords (must have size = 3)
      */
-    void set_coords( const thrust::host_vector<double>& coords){ coords_ = coords;}
+    void set_coords( const thrust::host_vector<double>& coords){ 
+        m_coords0 = coords;
+        m_deltaPhi0 = 0;
+    }
 
     /**
      * @brief Integrate from 0 to deltaPhi
      * @param deltaPhi upper integration boundary
-     * @return 1 if result is inside the box, -1 else
+     * @return >0 if result is inside the box, <0 else
+     * @note changes starting coords!
      */
     double operator()( double deltaPhi)
     {
-        dg::integrateRK4( field_, coords_, coordsp_, deltaPhi, eps_);
-        if( !g_.contains( coordsp_[0], coordsp_[1]) ) return -1;
+        double delta = deltaPhi - m_deltaPhi0;
+        dg::integrateRK4( m_field, m_coords0, m_coords1, delta, m_eps);
+        m_deltaPhi0 = deltaPhi;
+        m_coords0 = m_coords1;
+        if( !m_g.contains( m_coords1[0], m_coords1[1]) ) return -1;
         return +1;
     }
     private:
-    const Field& field_;
-    const Topology& g_;
-    thrust::host_vector<double> coords_, coordsp_;
-    double eps_;
+    const Field& m_field;
+    const Topology& m_g;
+    thrust::host_vector<double> m_coords0, m_coords1;
+    double m_deltaPhi0;
+    double m_eps;
 };
 
 /**
@@ -288,7 +273,8 @@ void integrate_all_fieldlines2d( const dg::geo::BinaryVectorLvl0& vec, const dg:
     dg::geo::detail::DSField field( vec, grid_field);
     //field in case of cartesian grid
     dg::geo::detail::DSFieldCylindrical cyl_field(vec);
-    for( unsigned i=0; i<grid_evaluate.size(); i++)
+    unsigned size = grid_evaluate.size();
+    for( unsigned i=0; i<size; i++)
     {
         thrust::host_vector<double> coords(3), coordsP(3), coordsM(3);
         coords[0] = y[0][i], coords[1] = y[1][i], coords[2] = y[2][i]; //x,y,s
@@ -336,17 +322,33 @@ struct Fieldaligned
     Fieldaligned(){}
     ///@copydoc construct()
     template <class Limiter = FullLimiter>
-    Fieldaligned(const dg::geo::TokamakMagneticField& vec, const ProductGeometry& grid, unsigned multiplyX, unsigned multiplyY, bool dependsOnX, bool dependsOnY, double eps = 1e-5, dg::bc globalbcx = dg::NEU, dg::bc globalbcy = dg::NEU, Limiter limit = FullLimiter(), double deltaPhi = -1)
+    Fieldaligned(const dg::geo::TokamakMagneticField& vec, 
+        const ProductGeometry& grid, 
+        dg::bc globalbcx = dg::NEU, 
+        dg::bc globalbcy = dg::NEU, 
+        Limiter limit = FullLimiter(), 
+        double eps = 1e-5,
+        unsigned multiplyX=5, unsigned multiplyY=5, 
+        bool dependsOnX=true, bool dependsOnY=true, 
+        double deltaPhi = -1)
     {
         dg::geo::BinaryVectorLvl0 bhat( (dg::geo::BHatR)(vec), (dg::geo::BHatZ)(vec), (dg::geo::BHatP)(vec));
-        construct( bhat, grid, multiplyX, multiplyY, dependsOnX, dependsOnY, eps, globalbcx, globalbcy, limit, deltaPhi);
+        construct( bhat, grid, globalbcx, globalbcy, limit, eps, multiplyX, multiplyY, dependsOnX, dependsOnY, deltaPhi);
     }
 
     ///@copydoc construct()
     template <class Limiter = FullLimiter>
-    Fieldaligned(const dg::geo::BinaryVectorLvl0& vec, const ProductGeometry& grid, unsigned multiplyX, unsigned multiplyY, bool dependsOnX, bool dependsOnY, double eps = 1e-5, dg::bc globalbcx = dg::NEU, dg::bc globalbcy = dg::NEU, Limiter limit = FullLimiter(), double deltaPhi = -1)
+    Fieldaligned(const dg::geo::BinaryVectorLvl0& vec, 
+        const ProductGeometry& grid, 
+        dg::bc globalbcx = dg::NEU, 
+        dg::bc globalbcy = dg::NEU, 
+        Limiter limit = FullLimiter(), 
+        double eps = 1e-5,
+        unsigned multiplyX=5, unsigned multiplyY=5, 
+        bool dependsOnX=true, bool dependsOnY=true, 
+        double deltaPhi = -1)
     {
-        construct( vec, grid, multiplyX, multiplyY, dependsOnX, dependsOnY, eps, globalbcx, globalbcy, limit, deltaPhi);
+        construct( vec, grid, globalbcx, globalbcy, limit, eps, multiplyX, multiplyY, dependsOnX, dependsOnY, deltaPhi);
     }
     /**
     * @brief Construct from a field and a grid
@@ -356,22 +358,30 @@ struct Fieldaligned
         If a field line crosses the limiter in the plane \f$ \phi=0\f$ then the limiter boundary conditions apply. 
     * @param vec The field to integrate
     * @param grid The grid on which to operate defines the parallel boundary condition in case there is a limiter.
-    * @param multiplyX defines the resolution in X of the fine grid relative to grid
-    * @param multiplyY defines the resolution in Y of the fine grid relative to grid
-    * @param dependsOnX indicates, whether the given vector field vec depends on the first coordinate
-    * @param dependsOnY indicates, whether the given vector field vec depends on the second coordinate
-    * @param eps Desired accuracy of the fieldline integrator
     * @param globalbcx Defines the interpolation behaviour when a fieldline intersects the boundary box in the perpendicular direction
     * @param globalbcy Defines the interpolation behaviour when a fieldline intersects the boundary box in the perpendicular direction
     * @param limit Instance of the limiter class (Default is a limiter everywhere, 
         note that if grid.bcz() is periodic it doesn't matter if there is a limiter or not)
+    * @param eps Desired accuracy of the fieldline integrator
+    * @param multiplyX defines the resolution in X of the fine grid relative to grid
+    * @param multiplyY defines the resolution in Y of the fine grid relative to grid
+    * @param dependsOnX indicates, whether the given vector field vec depends on the first coordinate
+    * @param dependsOnY indicates, whether the given vector field vec depends on the second coordinate
     * @param deltaPhi Is either <0 (then it's ignored), or may differ from grid.hz() if grid.Nz() == 1
     * @note If there is a limiter, the boundary condition on the first/last plane is set 
         by the grid.bcz() variable and can be changed by the set_boundaries function. 
         If there is no limiter, the boundary condition is periodic.
     */
     template <class Limiter = FullLimiter>
-    void construct(const dg::geo::BinaryVectorLvl0& vec, const ProductGeometry& grid, unsigned multiplyX, unsigned multiplyY, bool dependsOnX, bool dependsOnY, double eps = 1e-5, dg::bc globalbcx = dg::NEU, dg::bc globalbcy = dg::NEU, Limiter limit = FullLimiter(), double deltaPhi = -1);
+    void construct(const dg::geo::BinaryVectorLvl0& vec, 
+        const ProductGeometry& grid, 
+        dg::bc globalbcx = dg::NEU, 
+        dg::bc globalbcy = dg::NEU, 
+        Limiter limit = FullLimiter(), 
+        double eps = 1e-5, 
+        unsigned multiplyX=5, unsigned multiplyY=5, 
+        bool dependsOnX=true, bool dependsOnY=true, 
+        double deltaPhi = -1);
 
     bool dependsOnX()const{return m_dependsOnX;}
     bool dependsOnY()const{return m_dependsOnY;}
@@ -508,8 +518,12 @@ struct Fieldaligned
 
 template<class Geometry, class IMatrix, class container>
 template <class Limiter>
-void Fieldaligned<Geometry, IMatrix, container>::construct(const dg::geo::BinaryVectorLvl0& vec, const Geometry& grid, unsigned mx, unsigned my, bool bx, bool by, double eps, dg::bc globalbcx, dg::bc globalbcy, Limiter limit, double deltaPhi)
+void Fieldaligned<Geometry, IMatrix, container>::construct(
+    const dg::geo::BinaryVectorLvl0& vec, const Geometry& grid, 
+    dg::bc globalbcx, dg::bc globalbcy, Limiter limit, double eps, 
+    unsigned mx, unsigned my, bool bx, bool by, double deltaPhi)
 {
+    dg::Timer t;
     m_dependsOnX=bx, m_dependsOnY=by;
     m_Nz=grid.Nz(), m_bcz=grid.bcz(); 
     m_g.reset(grid);
@@ -528,20 +542,26 @@ void Fieldaligned<Geometry, IMatrix, container>::construct(const dg::geo::Binary
     //%%%%%%%%%%%%%%%%%%%%%%%%%%Set starting points and integrate field lines%%%%%%%%%%%%%%
     std::vector<thrust::host_vector<double> > yp_coarse( 3), ym_coarse(yp_coarse), yp, ym; 
     
+    t.tic();
     dg::Handle<dg::aGeometry2d> grid_magnetic = grid_coarse;//INTEGRATE HIGH ORDER GRID
     //grid_magnetic.set( 7, grid_magnetic.Nx(), grid_magnetic.Ny());
     detail::integrate_all_fieldlines2d( vec, grid_magnetic.get(), grid_coarse.get(), yp_coarse, ym_coarse, deltaPhi, eps);
+    t.toc();
+    std::cout << "Fieldline integration took: "<<t.diff()<<"\n";
 
+    //%%%%%%%%%%%%%%%%%%Create interpolation and projection%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+    t.tic();
     dg::Grid2d grid_fine( grid_coarse.get() );//FINE GRID
     grid_fine.multiplyCellNumbers((double)mx, (double)my);
     dg::IHMatrix interpolate = dg::create::interpolation( grid_fine, grid_coarse.get());  //INTERPOLATE TO FINE GRID
     dg::geo::detail::interpolate_and_clip( interpolate, grid_fine, grid_fine, yp_coarse, ym_coarse, yp, ym);
-    //%%%%%%%%%%%%%%%%%%Create interpolation and projection%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     dg::IHMatrix plusFine  = dg::create::interpolation( yp[0], yp[1], grid_coarse.get(), globalbcx, globalbcy), plus, plusT;
     dg::IHMatrix minusFine = dg::create::interpolation( ym[0], ym[1], grid_coarse.get(), globalbcx, globalbcy), minus, minusT;
     dg::IHMatrix projection = dg::create::projection( grid_coarse.get(), grid_fine);
     cusp::multiply( projection, plusFine, plus);
     cusp::multiply( projection, minusFine, minus);
+    t.toc();
+    std::cout << "Multiplication        took: "<<t.diff()<<"\n";
     plusT = dg::transpose( plus);
     minusT = dg::transpose( minus);     
     dg::blas2::transfer( plus, m_plus);
diff --git a/inc/geometries/mpi_fieldaligned.h b/inc/geometries/mpi_fieldaligned.h
index d13396081..441bee38a 100644
--- a/inc/geometries/mpi_fieldaligned.h
+++ b/inc/geometries/mpi_fieldaligned.h
@@ -59,18 +59,42 @@ struct Fieldaligned< ProductMPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY
 {
     Fieldaligned(){}
     template <class Limiter = FullLimiter>
-    Fieldaligned(const dg::geo::TokamakMagneticField& vec, const ProductMPIGeometry& grid, unsigned multiplyX, unsigned multiplyY, bool dependsOnX, bool dependsOnY, double eps = 1e-5, dg::bc globalbcx = dg::NEU, dg::bc globalbcy = dg::NEU, Limiter limit = FullLimiter(), double deltaPhi = -1)
+    Fieldaligned(const dg::geo::TokamakMagneticField& vec, 
+        const ProductMPIGeometry& grid, 
+        dg::bc globalbcx = dg::NEU, 
+        dg::bc globalbcy = dg::NEU, 
+        Limiter limit = FullLimiter(), 
+        double eps = 1e-5,
+        unsigned multiplyX=5, unsigned multiplyY=5, 
+        bool dependsOnX=true, bool dependsOnY=true, 
+        double deltaPhi = -1)
     {
         dg::geo::BinaryVectorLvl0 bhat( (dg::geo::BHatR)(vec), (dg::geo::BHatZ)(vec), (dg::geo::BHatP)(vec));
-        construct( bhat, grid, multiplyX, multiplyY, dependsOnX, dependsOnY, eps, globalbcx, globalbcy, limit, deltaPhi);
+        construct( bhat, grid, globalbcx, globalbcy, limit, eps, multiplyX, multiplyY, dependsOnX, dependsOnY, deltaPhi);
     }
     template <class Limiter = FullLimiter>
-    Fieldaligned(const dg::geo::BinaryVectorLvl0& vec, const ProductMPIGeometry& grid, unsigned multiplyX, unsigned multiplyY, bool dependsOnX, bool dependsOnY, double eps = 1e-5, dg::bc globalbcx = dg::NEU, dg::bc globalbcy = dg::NEU, Limiter limit = FullLimiter(), double deltaPhi = -1)
+    Fieldaligned(const dg::geo::BinaryVectorLvl0& vec, 
+        const ProductMPIGeometry& grid, 
+        dg::bc globalbcx = dg::NEU, 
+        dg::bc globalbcy = dg::NEU, 
+        Limiter limit = FullLimiter(), 
+        double eps = 1e-5,
+        unsigned multiplyX=5, unsigned multiplyY=5, 
+        bool dependsOnX=true, bool dependsOnY=true, 
+        double deltaPhi = -1)
     {
-        construct( vec, grid, multiplyX, multiplyY, dependsOnX, dependsOnY, eps, globalbcx, globalbcy, limit, deltaPhi);
+        construct( vec, grid, globalbcx, globalbcy, limit, eps, multiplyX, multiplyY, dependsOnX, dependsOnY, deltaPhi);
     }
     template <class Limiter = FullLimiter>
-    void construct(const dg::geo::BinaryVectorLvl0& vec, const ProductMPIGeometry& grid, unsigned multiplyX, unsigned multiplyY, bool dependsOnX, bool dependsOnY, double eps = 1e-5, dg::bc globalbcx = dg::NEU, dg::bc globalbcy = dg::NEU, Limiter limit = FullLimiter(), double deltaPhi = -1);
+    void construct(const dg::geo::BinaryVectorLvl0& vec, 
+        const ProductMPIGeometry& grid, 
+        dg::bc globalbcx = dg::NEU, 
+        dg::bc globalbcy = dg::NEU, 
+        Limiter limit = FullLimiter(), 
+        double eps = 1e-5, 
+        unsigned multiplyX=5, unsigned multiplyY=5, 
+        bool dependsOnX=true, bool dependsOnY=true, 
+        double deltaPhi = -1);
 
     bool dependsOnX()const{return m_dependsOnX;}
     bool dependsOnY()const{return m_dependsOnY;}
@@ -133,8 +157,13 @@ struct Fieldaligned< ProductMPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY
 template<class MPIGeometry, class LocalIMatrix, class CommunicatorXY, class LocalContainer>
 template <class Limiter>
 void Fieldaligned<MPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vector<LocalContainer> >::construct(
-    const dg::geo::BinaryVectorLvl0& vec, const MPIGeometry& grid, unsigned mx, unsigned my, bool bx, bool by, double eps, dg::bc globalbcx, dg::bc globalbcy, Limiter limit, double deltaPhi)
+    const dg::geo::BinaryVectorLvl0& vec, const MPIGeometry& grid, 
+    dg::bc globalbcx, dg::bc globalbcy, Limiter limit, double eps, 
+    unsigned mx, unsigned my, bool bx, bool by, double deltaPhi)
 {
+    dg::Timer t;
+    int rank;
+    MPI_Comm_rank( grid.communicator(), &rank);
     m_dependsOnX=bx, m_dependsOnY=by;
     m_Nz=grid.local().Nz(), m_bcz=grid.bcz(); 
     m_g.reset(grid);
@@ -156,21 +185,28 @@ void Fieldaligned<MPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vec
     //%%%%%%%%%%%%%%%%%%%%%%%%%%Set starting points and integrate field lines%%%%%%%%%%%%%%
     std::vector<thrust::host_vector<double> > yp_coarse( 3), ym_coarse(yp_coarse), yp, ym; 
     
+    t.tic();
     dg::Handle<dg::aMPIGeometry2d> grid_magnetic = grid_coarse;//INTEGRATE HIGH ORDER GRID
     //grid_magnetic->set( 7, grid_magnetic->global().Nx(), grid_magnetic->global().Ny());
     dg::Handle<dg::aGeometry2d> global_grid_magnetic = grid_magnetic.get().global_geometry();
     detail::integrate_all_fieldlines2d( vec, global_grid_magnetic.get(), grid_coarse.get().local(), yp_coarse, ym_coarse, deltaPhi, eps);
+    t.toc();
+    if(rank==0) std::cout << "Fieldline integration took: "<<t.diff()<<"\n";
 
+    //%%%%%%%%%%%%%%%%%%Create interpolation and projection%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+    t.tic();
     dg::MPIGrid2d grid_fine( grid_coarse.get() );//FINE GRID
     grid_fine.multiplyCellNumbers((double)mx, (double)my);
     dg::IHMatrix interpolate = dg::create::interpolation( grid_fine.local(), grid_coarse.get().local());  //INTERPOLATE TO FINE GRID
     dg::geo::detail::interpolate_and_clip( interpolate, grid_fine.local(), grid_fine.global(), yp_coarse, ym_coarse, yp, ym);
-    //%%%%%%%%%%%%%%%%%%Create interpolation and projection%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     dg::IHMatrix plusFine  = dg::create::interpolation( yp[0], yp[1], grid_coarse.get().global(), globalbcx, globalbcy), plus;
     dg::IHMatrix minusFine = dg::create::interpolation( ym[0], ym[1], grid_coarse.get().global(), globalbcx, globalbcy), minus;
     dg::IHMatrix projection = dg::create::projection( grid_coarse.get().local(), grid_fine.local());
     cusp::multiply( projection, plusFine, plus);
     cusp::multiply( projection, minusFine, minus);
+    t.toc();
+    if(rank==0) std::cout << "Multiplication        took: "<<t.diff()<<"\n";
+    t.tic();
     dg::MIHMatrix temp = dg::convert( plus, grid_coarse.get()), tempT;
     tempT  = dg::transpose( temp);
     dg::blas2::transfer( temp, m_plus);
@@ -179,6 +215,8 @@ void Fieldaligned<MPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vec
     tempT  = dg::transpose( temp);
     dg::blas2::transfer( temp, m_minus);
     dg::blas2::transfer( tempT, m_minusT);
+    t.toc();
+    if(rank==0) std::cout << "Conversion            took: "<<t.diff()<<"\n";
     //%%%%%%%%%%%%%%%%%%%%%%%project h and copy into h vectors%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     dg::MHVec hp( dg::evaluate( dg::zero, grid_coarse.get())), hm(hp), hz(hp);
     dg::blas2::symv( projection, yp[2], hp.data());
diff --git a/src/heat/heat.cuh b/src/heat/heat.cuh
index 7ddde0afe..e2e027a61 100644
--- a/src/heat/heat.cuh
+++ b/src/heat/heat.cuh
@@ -12,35 +12,22 @@
   Contains the solvers 
   */
 
-namespace eule
+namespace heat
 {
 ///@addtogroup solver
 ///@{
-/**
- * @brief Diffusive terms for Feltor solver
- *
- * @tparam Matrix The Matrix class
- * @tparam container The Vector class 
- * @tparam container The container class
- */
-template< class Geometry, class DS, class Matrix, class container>
-struct Rolkar
+template< class Geometry, class IMatrix, class Matrix, class container>
+struct Implicit
 {
-    Rolkar( const Geometry& g, eule::Parameters p, dg::geo::solovev::GeomParameters gp):
+    Implicit( const Geometry& g, Parameters p, dg::geo::solovev::Parameters gp, dg::geo::Fieldaligned<Geometry, IMatrix, Matrix, container>& fa):
         p(p),
         gp(gp),
-        dsNU_( typename DS::FieldAligned(
-                dg::geo::Field<dg::geo::solovev::MagneticField>(
-                    dg::geo::solovev::MagneticField(gp), gp.R_0), 
-                g, gp.rk4eps, 
-                dg::geo::PsiLimiter<dg::geo::solovev::Psip>(
-                    dg::geo::solovev::Psip(gp), gp.psipmaxlim
+        dsNU_( fa, dg::geo::createSolovevField( gp), g, 1,1, true, true, gp.rk4eps, g.bcx(), g.bcy() 
+                dg::geo::PsiLimiter( dg::geo::solovev::Psip(gp), gp.psipmaxlim
                     ), 
                 g.bcx()
                 ), 
-            dg::geo::Field<dg::geo::solovev::MagneticField>(
-              dg::geo::solovev::MagneticField(gp), gp.R_0
-              ), dg::normed, dg::forward ),
+              , dg::normed, dg::forward ),
         elliptic( g, dg::normed, dg::forward)
     {
         using namespace dg::geo::solovev;
@@ -68,18 +55,20 @@ struct Rolkar
     }
     const container& damping(){return dampprof_;}
     const container& weights(){return elliptic.weights();}
+    const container& inv_weights(){return elliptic.inv_weights();}
     const container& precond(){return elliptic.precond();}
   private:
     const eule::Parameters p;
     const dg::geo::solovev::GeomParameters gp;
     container dampprof_;
-    DS dsNU_;
+    dg::geo::Fieldaligned<Geometry, IMatrix, Matrix, container> dsNU_;
+    dg::geo::DS<Geometry, IMatrix, Matrix, container> dsNU_;
     dg::GeneralEllipticSym<Geometry, Matrix, container> elliptic;
 
 };
 
 template< class DS, class Matrix, class container >
-struct Feltor
+struct Explicit
 {
     //typedef std::vector<container> Vector;
     typedef typename dg::VectorTraits<container>::value_type value_type;
@@ -88,7 +77,7 @@ struct Feltor
     //typedef dg::DMatrix Matrix; //fastest device Matrix (does this conflict with 
 
     template<class Grid3d>
-    Feltor( const Grid3d& g, eule::Parameters p,dg::geo::solovev::GeomParameters gp);
+    Explicit( const Grid3d& g, eule::Parameters p,dg::geo::solovev::GeomParameters gp);
 
     const DS ds(){return dsNU_;}
 
@@ -126,7 +115,7 @@ struct Feltor
 
 template<class DS, class Matrix, class container>
 template<class Grid>
-Feltor<DS, Matrix, container>::Feltor( const Grid& g, eule::Parameters p, dg::geo::solovev::GeomParameters gp): 
+Explicit<DS, Matrix, container>::Explicit( const Grid& g, eule::Parameters p, dg::geo::solovev::GeomParameters gp): 
     chi( dg::evaluate( dg::one, g)), omega(chi),  lambda(chi), tmo(chi),
     one( dg::evaluate( dg::one, g)),    
     w3d( dg::create::volume(g)), v3d( dg::create::inv_volume(g)),      
@@ -170,16 +159,15 @@ Feltor<DS, Matrix, container>::Feltor( const Grid& g, eule::Parameters p, dg::ge
 //         bfield = dg::evaluate( solovev::bPhi( gp.R_0, gp.I_0),g);
 //         elliptic.set_z( bfield);
     //////////////////////////////init fields /////////////////////
-    using namespace dg::geo::solovev;
-    MagneticField mf(gp);
-    dg::blas1::transfer(  dg::pullback(dg::geo::Field<MagneticField>(mf, gp.R_0),                     g), binv);
-    dg::blas1::transfer(  dg::pullback(dg::geo::GradLnB<MagneticField>(mf, gp.R_0),                   g), gradlnB);
+    TokamakMagneticField mf = dg::geo::createSolovevField(gp);
+    dg::blas1::transfer(  dg::pullback(dg::geo::InvB(mf),    g), binv);
+    dg::blas1::transfer(  dg::pullback(dg::geo::GradLnB(mf), g), gradlnB);
 }
 
 
 
 template<class DS, class M, class V>
-void Feltor<DS, M, V>::energies( std::vector<V>& y)
+void Explicit<DS, M, V>::energies( std::vector<V>& y)
 {
     double S[1]    = {0.0};    
     double Dpar[1] = {0.0};
@@ -254,7 +242,7 @@ void Feltor<DS, M, V>::energies( std::vector<V>& y)
 
 //do not overwrite y
 template<class DS, class Matrix, class container>
-void Feltor<DS, Matrix, container>::operator()( std::vector<container>& y, std::vector<container>& yp)
+void Explicit<DS, Matrix, container>::operator()( std::vector<container>& y, std::vector<container>& yp)
 {
     /* y[0] := T - 1 or T
     */
diff --git a/src/heat/parameters.h b/src/heat/parameters.h
index 8b50f5021..a0fd5184d 100644
--- a/src/heat/parameters.h
+++ b/src/heat/parameters.h
@@ -1,11 +1,8 @@
 #pragma once
 #include "dg/enums.h"
-#include "json.h"
+#include "json/json.h"
 
-namespace eule{
-/**
- * @brief Provide a mapping between input file and named parameters
- */
+namespace heat{
 struct Parameters
 {
     unsigned n, Nx, Ny, Nz; 
-- 
GitLab


From d5d86dbc7a45d81097a1ce665fd71c030e3375fb Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sun, 29 Oct 2017 01:21:02 +0200
Subject: [PATCH 407/453] reduced execution time of grid constructors by
 introducing minimum parallelization size in thrust_vector_blas

---
 inc/dg/backend/functions.h            | 12 ++++-----
 inc/dg/backend/thrust_vector_blas.cuh | 37 +++++++++++++++++++++++++++
 inc/dg/cluster_mpib.cu                |  2 +-
 inc/dg/runge_kutta.h                  | 27 +++++++++----------
 inc/geometries/Makefile               |  2 +-
 inc/geometries/ds_curv_mpit.cu        |  8 +++---
 inc/geometries/ds_curv_t.cu           |  2 --
 inc/geometries/fieldaligned.h         |  7 ++---
 inc/geometries/mpi_fieldaligned.h     |  4 +--
 9 files changed, 68 insertions(+), 33 deletions(-)

diff --git a/inc/dg/backend/functions.h b/inc/dg/backend/functions.h
index 64c4d0013..dc697eff9 100644
--- a/inc/dg/backend/functions.h
+++ b/inc/dg/backend/functions.h
@@ -216,7 +216,7 @@ inline double zero( double x, double y, double z) {return 0.;}
  *
  * @return 
  */
-double cooX1d( double x) {return x;}
+inline double cooX1d( double x) {return x;}
 /**
  * @brief
  * \f[ f(x,y) = x\f]
@@ -226,7 +226,7 @@ double cooX1d( double x) {return x;}
  *
  * @return 
  */
-double cooX2d( double x, double y) {return x;}
+inline double cooX2d( double x, double y) {return x;}
 /**
  * @brief
  * \f[ f(x,y,z) = x\f]
@@ -237,7 +237,7 @@ double cooX2d( double x, double y) {return x;}
  *
  * @return 
  */
-double cooX3d( double x, double y, double z) {return x;}
+inline double cooX3d( double x, double y, double z) {return x;}
 
 /**
  * @brief 
@@ -248,7 +248,7 @@ double cooX3d( double x, double y, double z) {return x;}
  *
  * @return 
  */
-double cooY2d( double x, double y) {return y;}
+inline double cooY2d( double x, double y) {return y;}
 /**
  * @brief
  * \f[ f(x,y,z) = y\f]
@@ -259,7 +259,7 @@ double cooY2d( double x, double y) {return y;}
  *
  * @return 
  */
-double cooY3d( double x, double y, double z) {return y;}
+inline double cooY3d( double x, double y, double z) {return y;}
 /**
  * @brief
  * \f[ f(x,y,z) = z\f]
@@ -270,7 +270,7 @@ double cooY3d( double x, double y, double z) {return y;}
  *
  * @return 
  */
-double cooZ3d( double x, double y, double z) {return z;}
+inline double cooZ3d( double x, double y, double z) {return z;}
 } //namespace dg
 
 ///@}
diff --git a/inc/dg/backend/thrust_vector_blas.cuh b/inc/dg/backend/thrust_vector_blas.cuh
index 3c8d2e4b2..91ba04d8f 100644
--- a/inc/dg/backend/thrust_vector_blas.cuh
+++ b/inc/dg/backend/thrust_vector_blas.cuh
@@ -20,6 +20,7 @@ namespace blas1
     ///@cond
 namespace detail
 {
+const unsigned MIN_SIZE=100;//don't parallelize if work is too small 
 
 
 template< typename value_type>
@@ -71,6 +72,12 @@ typename Vector::value_type doDot( const Vector& x, const Vector& y, ThrustVecto
 #if THRUST_DEVICE_SYSTEM!=THRUST_DEVICE_SYSTEM_CUDA
     value_type sum = 0;
     unsigned size=x.size();
+    if(size<MIN_SIZE)
+    {
+        for( unsigned i=0; i<size; i++)
+            sum += x[i]*y[i];
+        return sum;
+    }
     #pragma omp parallel for SIMD reduction(+:sum)
     for( unsigned i=0; i<size; i++)
         sum += x[i]*y[i];
@@ -97,6 +104,12 @@ inline void doScal(  Vector& x,
         return;
 #if THRUST_DEVICE_SYSTEM!=THRUST_DEVICE_SYSTEM_CUDA
     unsigned size=x.size();
+    if(size<MIN_SIZE)
+    {
+        for( unsigned i=0; i<size; i++)
+            x[i]*=alpha;
+        return;
+    }
     #pragma omp parallel for SIMD
     for( unsigned i=0; i<size; i++)
         x[i]*=alpha;
@@ -112,6 +125,12 @@ inline void doPlus(  Vector& x,
 {
 #if THRUST_DEVICE_SYSTEM!=THRUST_DEVICE_SYSTEM_CUDA
     unsigned size=x.size();
+    if(size<MIN_SIZE)
+    {
+        for( unsigned i=0; i<size; i++)
+            x[i]+=alpha;
+        return;
+    }
     #pragma omp parallel for SIMD
     for( unsigned i=0; i<size; i++)
         x[i]+=alpha;
@@ -150,6 +169,12 @@ inline void doAxpby( typename Vector::value_type alpha,
     const typename Vector::value_type * RESTRICT x_ptr = thrust::raw_pointer_cast( &x.data()[0]);
     typename Vector::value_type * RESTRICT y_ptr = thrust::raw_pointer_cast( &y.data()[0]);
     unsigned size = x.size();
+    if(size<MIN_SIZE) 
+    {
+        for( unsigned i=0; i<size; i++)
+            y_ptr[i] = alpha*x_ptr[i] + beta*y_ptr[i];
+        return;
+    }
     #pragma omp parallel for SIMD
     for( unsigned i=0; i<size; i++)
         y_ptr[i] = alpha*x_ptr[i] + beta*y_ptr[i];
@@ -206,6 +231,12 @@ inline void doAxpby( typename Vector::value_type alpha,
     const typename Vector::value_type * RESTRICT y_ptr = thrust::raw_pointer_cast( &y.data()[0]);
     typename Vector::value_type * RESTRICT z_ptr = thrust::raw_pointer_cast( &z.data()[0]);
     unsigned size = x.size();
+    if(size<MIN_SIZE)
+    {
+        for( unsigned i=0; i<size; i++)
+            z_ptr[i] = alpha*x_ptr[i] + beta*y_ptr[i] + gamma*z_ptr[i];
+        return;
+    }
     #pragma omp parallel for SIMD
     for( unsigned i=0; i<size; i++)
         z_ptr[i] = alpha*x_ptr[i] + beta*y_ptr[i] + gamma*z_ptr[i];
@@ -275,6 +306,12 @@ inline void doPointwiseDot(
     const typename Vector::value_type * x2_ptr = thrust::raw_pointer_cast( &(x2.data()[0]));
      typename Vector::value_type * y_ptr = thrust::raw_pointer_cast( &(y.data()[0]));
     unsigned size = x1.size();
+    if(size<MIN_SIZE)
+    {
+        for( unsigned i=0; i<size; i++)
+            y_ptr[i] = alpha*x1_ptr[i]*x2_ptr[i]+beta*y_ptr[i];
+        return;
+    }
     #pragma omp parallel for SIMD
     for( unsigned i=0; i<size; i++)
     {
diff --git a/inc/dg/cluster_mpib.cu b/inc/dg/cluster_mpib.cu
index bbd54465c..753cefb1b 100644
--- a/inc/dg/cluster_mpib.cu
+++ b/inc/dg/cluster_mpib.cu
@@ -180,7 +180,7 @@ int main(int argc, char* argv[])
         double Zmax=1.0*gpa*1.00;
         dg::CylindricalMPIGrid3d g3d( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, n, Nx ,Ny, Nz,dg::DIR, dg::DIR, dg::PER,commEll);
         dg::geo::TokamakMagneticField magfield = dg::geo::createGuentherField(gpR0, gpI0);
-        dg::geo::Fieldaligned<dg::aProductMPIGeometry3d, IMatrix, Vector> dsFA( magfield, g3d, 5, 5, true, true, 1e-4);
+        dg::geo::Fieldaligned<dg::aProductMPIGeometry3d, IMatrix, Vector> dsFA( magfield, g3d);
         dg::geo::DS<dg::aProductMPIGeometry3d, IMatrix, Matrix, Vector>  ds ( dsFA, dg::not_normed, dg::centered);
         dg::geo::guenther::FuncNeu funcNEU(gpR0,gpI0);
         Vector function = dg::evaluate( funcNEU, g3d) , dsTdsfb(function);
diff --git a/inc/dg/runge_kutta.h b/inc/dg/runge_kutta.h
index 6eabc9aab..6cddc2c2c 100644
--- a/inc/dg/runge_kutta.h
+++ b/inc/dg/runge_kutta.h
@@ -555,23 +555,22 @@ void stepperRK17(RHS& rhs, const container& begin, container& end, double T_min,
  * @param end (write-only) contains solution on output
  * @param T_max time difference
  * @param eps_abs desired accuracy in the error function between end and end_old
- * @return 0 if converged, -1 and a warning to std::cerr when isnan appears, -2 if failed to reach eps_abs
+ * @param NT_init initial number of steps
+ * @return number of iterations if converged, -1 and a warning to std::cerr when isnan appears, -2 if failed to reach eps_abs
  */
 template< class RHS, class container, unsigned s>
-int integrateRK(RHS& rhs, const container& begin, container& end, double T_max, double eps_abs )
+int integrateRK(RHS& rhs, const container& begin, container& end, double T_max, double eps_abs, unsigned NT_init = 2 )
 {
     RK_classic<s, container > rk( begin); 
     container old_end(begin), temp(begin);
     end = begin;
     if( T_max == 0) return 0;
-    double dt = T_max/2;
-    int NT = 2;
+    int NT = NT_init;
+    double dt = T_max/(double)NT;
     double error = 1e10;
  
     while( error > eps_abs && NT < pow( 2, 18) )
     {
-        dt /= 2.;
-        NT *= 2;
         end = begin;
 
         int i=0;
@@ -590,6 +589,8 @@ int integrateRK(RHS& rhs, const container& begin, container& end, double T_max,
         }  
         error = rhs.error( end, old_end);
         old_end = end;
+        dt /= 2.;
+        NT *= 2;
     }
     if( std::isnan( error) )
     {
@@ -601,7 +602,7 @@ int integrateRK(RHS& rhs, const container& begin, container& end, double T_max,
         std::cerr << "ATTENTION: Runge Kutta failed to converge. Error is "<<error<<std::endl;
         return -2;
     }
-    return 0;
+    return NT;
 
 
 }
@@ -609,24 +610,24 @@ int integrateRK(RHS& rhs, const container& begin, container& end, double T_max,
 /// @brief Integrates the differential equation using a stage 4 Runge-Kutta scheme, a rudimentary stepsize-control and monitoring the sanity of integration
 ///@copydetails integrateRK()
 template< class RHS, class container>
-int integrateRK4(RHS& rhs, const container& begin, container& end, double T_max, double eps_abs )
+int integrateRK4(RHS& rhs, const container& begin, container& end, double T_max, double eps_abs, unsigned NT_init = 2 )
 {
-    return integrateRK<RHS, container, 4>( rhs, begin, end, T_max, eps_abs);
+    return integrateRK<RHS, container, 4>( rhs, begin, end, T_max, eps_abs, NT_init);
 }
 
 /// @brief Integrates the differential equation using a stage 6 Runge-Kutta scheme, a rudimentary stepsize-control and monitoring the sanity of integration
 /// @copydetails integrateRK()
 template< class RHS, class container>
-int integrateRK6(RHS& rhs, const container& begin, container& end, double T_max, double eps_abs )
+int integrateRK6(RHS& rhs, const container& begin, container& end, double T_max, double eps_abs, unsigned NT_init = 2 )
 {
-    return integrateRK<RHS, container, 6>( rhs, begin, end, T_max, eps_abs);
+    return integrateRK<RHS, container, 6>( rhs, begin, end, T_max, eps_abs, NT_init);
 }
 /// @brief Integrates the differential equation using a stage 17 Runge-Kutta scheme, a rudimentary stepsize-control and monitoring the sanity of integration
 ///@copydetails integrateRK()
 template< class RHS, class container>
-int integrateRK17(RHS& rhs, const container& begin, container& end, double T_max, double eps_abs )
+int integrateRK17(RHS& rhs, const container& begin, container& end, double T_max, double eps_abs, unsigned NT_init = 2 )
 {
-    return integrateRK<RHS, container, 17>( rhs, begin, end, T_max, eps_abs);
+    return integrateRK<RHS, container, 17>( rhs, begin, end, T_max, eps_abs, NT_init);
 }
 
 ///@}
diff --git a/inc/geometries/Makefile b/inc/geometries/Makefile
index ecbcfadc0..8694cb8c9 100644
--- a/inc/geometries/Makefile
+++ b/inc/geometries/Makefile
@@ -18,7 +18,7 @@ geometry_diag: geometry_diag.cu solovev.h
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(LIBS) $(INCLUDE) $(JSONLIB) -g
 
 %_t: %_t.cu 
-	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -DDG_DEBUG -g 
+	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -DDG_DEBUG -pg
 
 %_mpit: %_mpit.cu 
 	$(MPICC) $(OPT) $(INCLUDE) $(MPICFLAGS)  $< -o $@ -g $(LIBS) $(JSONLIB) -DDG_DEBUG
diff --git a/inc/geometries/ds_curv_mpit.cu b/inc/geometries/ds_curv_mpit.cu
index 7abfd5421..48ea0f620 100644
--- a/inc/geometries/ds_curv_mpit.cu
+++ b/inc/geometries/ds_curv_mpit.cu
@@ -44,10 +44,10 @@ int main(int argc, char * argv[])
     unsigned mx=1, my=10;
     double psi_0 = -20, psi_1 = -4;
     dg::geo::FluxGenerator flux( mag.get_psip(), mag.get_ipol(), psi_0, psi_1, gp.R_0, 0., 1);
-    std::cout << "Constructing Grid...\n";
+    if(rank==0)std::cout << "Constructing Grid...\n";
     dg::geo::CurvilinearProductMPIGrid3d g3d(flux, n, Nx, Ny,Nz, dg::DIR, dg::PER, dg::PER, comm);
-    std::cout << "Constructing Fieldlines...\n";
-    dg::geo::DS<dg::aProductMPIGeometry3d, dg::MIHMatrix, dg::MHMatrix, dg::HVec> ds( mag, g3d, dg::NEU, dg::NEU, dg::FullLimiter(), 1e-8, mx, my, false, true, dg::normed, dg::centered);
+    if(rank==0)std::cout << "Constructing Fieldlines...\n";
+    dg::geo::DS<dg::aProductMPIGeometry3d, dg::MIHMatrix, dg::MHMatrix, dg::MHVec> ds( mag, g3d, dg::NEU, dg::NEU, dg::geo::FullLimiter(), dg::normed, dg::centered, 1e-8, mx, my, false, true);
     
     t.toc();
     if(rank==0)std::cout << "Construction took "<<t.diff()<<"s\n";
@@ -65,9 +65,7 @@ int main(int argc, char * argv[])
     if(rank==0)std::cout << "Divergence of B is "<<norm<<"\n";
 
     ds.centered( 1., lnB, 0., gradB);
-    if(rank==0)std::cout << "num. norm of gradLnB is "<<sqrt( dg::blas2::dot( gradB,vol3d, gradB))<<"\n";
     norm = sqrt( dg::blas2::dot( gradLnB, vol3d, gradLnB) );
-    if(rank==0)std::cout << "ana. norm of gradLnB is "<<norm<<"\n";
     dg::blas1::axpby( 1., gradB, -1., gradLnB, gradLnB);
     double norm2 = sqrt(dg::blas2::dot(gradLnB, vol3d, gradLnB));
     if(rank==0)std::cout << "rel. error of lnB is    "<<norm2/norm<<"\n";
diff --git a/inc/geometries/ds_curv_t.cu b/inc/geometries/ds_curv_t.cu
index 97f4e354e..195b39317 100644
--- a/inc/geometries/ds_curv_t.cu
+++ b/inc/geometries/ds_curv_t.cu
@@ -62,9 +62,7 @@ int main(int argc, char * argv[])
     std::cout << "Divergence of B is "<<norm<<"\n";
 
     ds.centered( 1., lnB, 0., gradB);
-    std::cout << "num. norm of gradLnB is "<<sqrt( dg::blas2::dot( gradB,vol3d, gradB))<<"\n";
     norm = sqrt( dg::blas2::dot( gradLnB, vol3d, gradLnB) );
-    std::cout << "ana. norm of gradLnB is "<<norm<<"\n";
     dg::blas1::axpby( 1., gradB, -1., gradLnB, gradLnB);
     double norm2 = sqrt(dg::blas2::dot(gradLnB, vol3d, gradLnB));
     std::cout << "rel. error of lnB is    "<<norm2/norm<<"\n";
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index 1980c4c65..ed5a60e79 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -196,7 +196,7 @@ struct BoxIntegrator
     double operator()( double deltaPhi)
     {
         double delta = deltaPhi - m_deltaPhi0;
-        dg::integrateRK4( m_field, m_coords0, m_coords1, delta, m_eps);
+        dg::integrateRK4( m_field, m_coords0, m_coords1, delta, m_eps, 2);
         m_deltaPhi0 = deltaPhi;
         m_coords0 = m_coords1;
         if( !m_g.contains( m_coords1[0], m_coords1[1]) ) return -1;
@@ -228,7 +228,7 @@ void boxintegrator( const Field& field, const Topology& grid,
         thrust::host_vector<double>& coords1, 
         double& phi1, double eps)
 {
-    dg::integrateRK4( field, coords0, coords1, phi1, eps); //integration
+    dg::integrateRK6( field, coords0, coords1, phi1, eps, 2); //integration
     double R = coords1[0], Z=coords1[1];
     //First, catch periodic domain
     grid.shift_topologic( coords0[0], coords0[1], R, Z);
@@ -544,7 +544,8 @@ void Fieldaligned<Geometry, IMatrix, container>::construct(
     
     t.tic();
     dg::Handle<dg::aGeometry2d> grid_magnetic = grid_coarse;//INTEGRATE HIGH ORDER GRID
-    //grid_magnetic.set( 7, grid_magnetic.Nx(), grid_magnetic.Ny());
+    grid_magnetic.get().set( 7, grid_magnetic.get().Nx(), grid_magnetic.get().Ny());
+    std::cout << "Integrating fieldlines...\n";
     detail::integrate_all_fieldlines2d( vec, grid_magnetic.get(), grid_coarse.get(), yp_coarse, ym_coarse, deltaPhi, eps);
     t.toc();
     std::cout << "Fieldline integration took: "<<t.diff()<<"\n";
diff --git a/inc/geometries/mpi_fieldaligned.h b/inc/geometries/mpi_fieldaligned.h
index 441bee38a..e39f1e4a5 100644
--- a/inc/geometries/mpi_fieldaligned.h
+++ b/inc/geometries/mpi_fieldaligned.h
@@ -185,10 +185,10 @@ void Fieldaligned<MPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vec
     //%%%%%%%%%%%%%%%%%%%%%%%%%%Set starting points and integrate field lines%%%%%%%%%%%%%%
     std::vector<thrust::host_vector<double> > yp_coarse( 3), ym_coarse(yp_coarse), yp, ym; 
     
-    t.tic();
     dg::Handle<dg::aMPIGeometry2d> grid_magnetic = grid_coarse;//INTEGRATE HIGH ORDER GRID
-    //grid_magnetic->set( 7, grid_magnetic->global().Nx(), grid_magnetic->global().Ny());
+    grid_magnetic.get().set( 7, grid_magnetic.get().Nx(), grid_magnetic.get().Ny());
     dg::Handle<dg::aGeometry2d> global_grid_magnetic = grid_magnetic.get().global_geometry();
+    t.tic();
     detail::integrate_all_fieldlines2d( vec, global_grid_magnetic.get(), grid_coarse.get().local(), yp_coarse, ym_coarse, deltaPhi, eps);
     t.toc();
     if(rank==0) std::cout << "Fieldline integration took: "<<t.diff()<<"\n";
-- 
GitLab


From b970fce0638af2ec85249bd4b903abb9536f6c75 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sun, 29 Oct 2017 11:34:29 +0100
Subject: [PATCH 408/453] corrected bug in non-communicating convert function

---
 inc/dg/backend/mpi_grid.h         | 17 ++++++++++++-----
 inc/dg/backend/mpi_projection.h   | 13 +++++++++----
 inc/dg/backend/transpose.h        |  2 ++
 inc/geometries/ds_curv_mpit.cu    |  4 ++--
 inc/geometries/mpi_fieldaligned.h |  2 ++
 5 files changed, 27 insertions(+), 11 deletions(-)

diff --git a/inc/dg/backend/mpi_grid.h b/inc/dg/backend/mpi_grid.h
index 2f9289390..7cbdd9f6a 100644
--- a/inc/dg/backend/mpi_grid.h
+++ b/inc/dg/backend/mpi_grid.h
@@ -132,11 +132,14 @@ struct aMPITopology2d
     const DLT<double>& dlt() const{return g.dlt();}
     /**
      * @brief The total global number of points
-     *
-     * @note for the total local number of points call grid.local().size()
-     * @return n()*n()*Nx()*Ny()
+     * @return equivalent of \c n()*n()*Nx()*Ny()
      */
     unsigned size() const { return g.size();}
+    /**
+     * @brief The total local number of points
+     * @return equivalent of \c local.size()
+     */
+    unsigned local_size() const { return l.size();}
     /**
      * @brief Display global and local grid
      *
@@ -459,10 +462,14 @@ struct aMPITopology3d
     const DLT<double>& dlt() const{return g.dlt();}
     /**
      * @brief The total global number of points
-     *
-     * @return n()*n()*Nx()*Ny()*Nz()
+     * @return equivalent of \c n()*n()*Nx()*Ny()*Nz()
      */
     unsigned size() const { return g.size();}
+    /**
+     * @brief The total local number of points
+     * @return equivalent of \c local.size()
+     */
+    unsigned local_size() const { return l.size();}
     /**
      * @brief Display global and local grid paramters 
      *
diff --git a/inc/dg/backend/mpi_projection.h b/inc/dg/backend/mpi_projection.h
index d4b15b499..b3c0c0cc5 100644
--- a/inc/dg/backend/mpi_projection.h
+++ b/inc/dg/backend/mpi_projection.h
@@ -57,11 +57,12 @@ void global2bufferIdx( const cusp::array1d<int, cusp::host_memory>& global_idx,
  * @brief Convert a matrix with local row and global column indices to a row distributed MPI matrix
  *
  * @tparam ConversionPolicy has to have the members: 
- *  - global2localIdx(unsigned,unsigned,unsigned) const;
+ *  - \c global2localIdx(unsigned,unsigned,unsigned) \c const;
  * where the first parameter is the global index and the 
  * other two are the pair (local idx, rank). 
- *  - MPI_Comm %communicator() const;  returns the communicator to use in the gather/scatter
- * @param global the column indices need to be global, the row indices local
+ *  - \c MPI_Comm \c %communicator() \c const;  returns the communicator to use in the gather/scatter
+ *  - \c local_size(); return the local vector size
+ * @param global the column indices and num_cols need to be global, the row indices and num_rows local
  * @param policy the conversion object
  *
  * @return a row distributed MPI matrix. If no MPI communication is needed the collective communicator will have zero size. 
@@ -84,8 +85,12 @@ dg::MIHMatrix convert( const dg::IHMatrix& global, const ConversionPolicy& polic
         for(unsigned i=0; i<local_idx.size(); i++)
             if( !policy.global2localIdx(global.column_indices[i], local_idx[i], pids[i]) ) success = false;
         assert( success);
+        dg::IHMatrix local( global.num_rows, policy.local_size(), global.values.size());
         comm = dg::GeneralComm< dg::iHVec, dg::HVec>();
-        return dg::MIHMatrix( global, comm, dg::row_dist);
+        local.row_offsets=global.row_offsets;
+        local.column_indices=local_idx;
+        local.values=global.values;
+        return dg::MIHMatrix( local, comm, dg::row_dist);
     }
     dg::IHMatrix local( global.num_rows, comm.size(), global.values.size());
     local.row_offsets=global.row_offsets;
diff --git a/inc/dg/backend/transpose.h b/inc/dg/backend/transpose.h
index ae990dd9e..25f3ec3fe 100644
--- a/inc/dg/backend/transpose.h
+++ b/inc/dg/backend/transpose.h
@@ -23,6 +23,8 @@ MPIDistMat<LocalMatrix, Collective> doTranspose( const MPIDistMat<LocalMatrix, C
 {
     LocalMatrix tr = doTranspose( src.matrix(), typename MatrixTraits<LocalMatrix>::matrix_category());
     MPIDistMat<LocalMatrix, Collective> out( tr, src.collective());
+    if( src.get_dist() == dg::row_dist) out.set_dist( dg::col_dist);
+    if( src.get_dist() == dg::col_dist) out.set_dist( dg::row_dist);
     return out;
 }
 #endif// MPI_VERSION
diff --git a/inc/geometries/ds_curv_mpit.cu b/inc/geometries/ds_curv_mpit.cu
index 48ea0f620..4366d883e 100644
--- a/inc/geometries/ds_curv_mpit.cu
+++ b/inc/geometries/ds_curv_mpit.cu
@@ -57,8 +57,8 @@ int main(int argc, char * argv[])
     dg::MHVec ones3d = dg::evaluate( dg::one, g3d);
     dg::MHVec vol3d = dg::create::volume( g3d);
     dg::blas1::pointwiseDivide( ones3d, B, B);
-    dg::MHVec function = dg::pullback( dg::geo::FuncNeu(mag), g3d), derivative(function);
-    ds( function, derivative);
+    //dg::MHVec function = dg::pullback( dg::geo::FuncNeu(mag), g3d), derivative(function);
+    //ds( function, derivative);
 
     ds.centeredAdj( 1., B, 0., divB);
     double norm =  sqrt( dg::blas2::dot(divB, vol3d, divB));
diff --git a/inc/geometries/mpi_fieldaligned.h b/inc/geometries/mpi_fieldaligned.h
index e39f1e4a5..f07053fda 100644
--- a/inc/geometries/mpi_fieldaligned.h
+++ b/inc/geometries/mpi_fieldaligned.h
@@ -348,6 +348,8 @@ void Fieldaligned<G,MPIDistMat<M,C>, MPI_Vector<container> >::ePlus( enum whichM
 template<class G, class M, class C, class container>
 void Fieldaligned<G,MPIDistMat<M,C>, MPI_Vector<container> >::eMinus( enum whichMatrix which, const MPI_Vector<container>& f, MPI_Vector<container>& fme ) 
 {
+    int rank; 
+    MPI_Comm_rank(m_g.get().communicator(), &rank);
     dg::split( f, m_f, m_g.get());
     //1. compute 2d interpolation in every plane and store in m_temp
     for( unsigned i0=0; i0<m_Nz; i0++)
-- 
GitLab


From 2a3284fdf4aa57166ed1ad5e1755b42d4116eeb5 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sun, 29 Oct 2017 12:35:31 +0100
Subject: [PATCH 409/453] update the FELTOR README.md with more elaborate
 examples

---
 README.md                         | 171 ++++++++++++++++++++----------
 inc/geometries/ds.h               |   2 +-
 inc/geometries/fieldaligned.h     |   6 +-
 inc/geometries/mpi_fieldaligned.h |   6 +-
 4 files changed, 121 insertions(+), 64 deletions(-)

diff --git a/README.md b/README.md
index 6fc6c1d84..fbf09eee8 100644
--- a/README.md
+++ b/README.md
@@ -14,46 +14,26 @@ Our core level functions are parallelized for a variety of hardware from multi-c
 ## 1. Quick start guide
 Go ahead and clone our library into any folder you like 
 ```sh
-$ git clone https://www.github.com/feltor-dev/feltor
+git clone https://www.github.com/feltor-dev/feltor
 ```
 You also need to clone  [thrust]( https://github.com/thrust/thrust) and [cusp](https://github.com/cusplibrary/cusplibrary) distributed under the Apache-2.0 license. So again in a folder of your choice
 ```sh
-$ git clone https://www.github.com/thrust/thrust
-$ git clone https://www.github.com/cusplibrary/cusplibrary
+git clone https://www.github.com/thrust/thrust
+git clone https://www.github.com/cusplibrary/cusplibrary
 ```
 > Our code only depends on external libraries that are themselves openly available. We note here that we do not distribute copies of these libraries.
 
-### Using FELTOR as a library
-
-It is possible to use FELTOR as a library in your own code project. Note that the library is **header-only**, which means that you just have to include the relevant header(s) and you're good to go:
-
-```C++
-//optional: activate MPI in FELTOR
-#include "mpi.h"  
-// optional: redirect CUDA calls to OpenMP functions; 
-// note that you then also have to specify an OpenMP flag when compiling
-#define THRUST_DEVICE_SYSTEM THRUST_DEVICE_SYSTEM_OMP 
-//include the basic dg-library
-#include "dg/algorithms.h"
-//include the geometries expansion
-#include "geometries/geometries.h"
-```
-
-and add `path/to/feltor/inc` as well as  `path/to/thrust/thrust` and  `path/to/cusplibrary/cusp` to the include path of your compiler. 
-
-> If you want to activate the MPI backend of FELTOR you have to include `mpi.h` before any FELTOR header. If you want to use OpenMP instead of CUDA for the device functions you have to define the `THRUST_DEVICE_SYSTEM` macro and activate OpenMP in your compiler (e.g `g++ -fopenmp`). I you want to use CUDA then you have to compile with nvcc. 
-See `path/to/feltor/config/README.md` for further details on configurating the library and how to compile.
 
 ### Using FELTOR's code projects
 
 In order to compile one of the many codes inside FELTOR you need to tell the feltor configuration where the external libraries are located on your computer. The default way to do this is to go in your `HOME` directory, make an include directory and link the paths in this directory:
 
  ```sh
- $ cd ~
- $ mkdir include
- $ cd include
- $ ln -s path/to/thrust/thrust thrust
- $ ln -s path/to/cusplibrary/cusp cusp
+cd ~
+mkdir include
+cd include
+ln -s path/to/thrust/thrust thrust
+ln -s path/to/cusplibrary/cusp cusp
  ```
 > If you do not like this, you can also create your own config file as discribed [here](https://github.com/feltor-dev/feltor/wiki/Configuration).
 
@@ -61,11 +41,11 @@ Now let us compile the first benchmark program.
 
 
  ```sh
- $ cd path/to/feltor/inc/dg
+cd path/to/feltor/inc/dg
  
- $ make blas_b device=omp #(for an OpenMP version)
+make blas_b device=omp #(for an OpenMP version)
  #or
- $ make blas_b device=gpu #(if you have a gpu and nvcc )
+make blas_b device=gpu #(if you have a gpu and nvcc )
  ```
 > The minimum requirement to compile and run an application is a working C++ compiler (g++ per default) and a CPU. 
 > To simplify the compilation process we use the GNU Make utility, a standard build automation tool that automatically builds the executable program. 
@@ -76,7 +56,7 @@ Now let us compile the first benchmark program.
 
 Run the code with 
 ```sh
-$ ./blas_b 
+./blas_b 
 ```
 and when prompted for input vector sizes type for example
 `3 100 100 10`
@@ -89,11 +69,11 @@ Now, let us test the mpi setup
 > If you intend to use the MPI backend, an implementation library of the mpi standard is required. Per default `mpic++` is used for compilation.
 
 ```sh
- $ cd path/to/feltor/inc/dg
+cd path/to/feltor/inc/dg
  
- $ make blas_mpib device=omp  # (for MPI+OpenMP)
- # or
- $ make blas_mpib device=gpu # (for MPI+GPU)
+make blas_mpib device=omp  # (for MPI+OpenMP)
+# or
+make blas_mpib device=gpu # (for MPI+GPU)
 ```
 Run the code with
 `$ mpirun -n '# of procs' ./blas_mpib `
@@ -112,35 +92,112 @@ Our JSON input files are parsed by [JsonCpp](https://www.github.com/open-source-
 As in Step 3 you need to create links to the jsoncpp library include path (and optionally the draw library) in your include folder or provide the paths in your config file. We are ready to compile now
 
 ```sh
- $ cd path/to/feltor/src/toefl # or any other project in the src folder
+cd path/to/feltor/src/toefl # or any other project in the src folder
  
- $ make toeflR device=gpu     # (compile on gpu or omp)
- $ ./toeflR <inputfile.json>  # (behold a live simulation with glfw output on screen)
- # or
- $ make toefl_hpc device=gpu  # (compile on gpu or omp)
- $ ./toefl_hpc <inputfile.json> <outputfile.nc> # (a single node simulation with output stored in a file)
- # or
- $ make toefl_mpi device=omp  # (compile on gpu or omp)
- $ export OMP_NUM_THREADS=2   # (set OpenMP thread number to 1 for pure MPI) 
- $ echo 2 2 | mpirun -n 4 ./toefl_mpi <inputfile.json> <outputfile.nc>
- $ # (a multi node simulation with now in total 8 threads with output stored in a file)
- $ # The mpi program will wait for you to type the number of processes in x and y direction before
- $ # running. That is why the echo is there. 
+make toeflR device=gpu     # (compile on gpu or omp)
+./toeflR <inputfile.json>  # (behold a live simulation with glfw output on screen)
+# or
+make toefl_hpc device=gpu  # (compile on gpu or omp)
+./toefl_hpc <inputfile.json> <outputfile.nc> # (a single node simulation with output stored in a file)
+# or
+make toefl_mpi device=omp  # (compile on gpu or omp)
+export OMP_NUM_THREADS=2   # (set OpenMP thread number to 1 for pure MPI) 
+echo 2 2 | mpirun -n 4 ./toefl_mpi <inputfile.json> <outputfile.nc>
+# (a multi node simulation with now in total 8 threads with output stored in a file)
+# The mpi program will wait for you to type the number of processes in x and y direction before
+# running. That is why the echo is there. 
 ```
 A default input file is located in `path/to/feltor/src/toefl/input`. All three programs solve the same equations. 
 The technical documentation on what equations are discretized, 
 input/output parameters, etc. can be generated as a pdf with 
 `make doc ` in the `path/to/feltor/src/toefl` directory.
 
+### Using FELTOR as a library
+
+It is possible to use FELTOR as a library in your own code project. Note that the library is **header-only**, which means that you just have to include the relevant header(s) and you're good to go. For example in the following program "test.cpp" we compute the square L2 norm of a function:
+
+```C++
+#include <iostream>
+//include the basic dg-library
+#include "dg/algorithm.h"
+//optional: include the geometries expansion
+#include "geometries/geometries.h"
+
+double function(double x, double y){return exp(x)*exp(y);}
+int main()
+{ 
+    //create a 2d grid with 3 polynomial coefficients
+    dg::CartesianGrid2d g2d( 0, 2, 0, 2, 3, 20, 20);
+    //discretize a function on this grid
+    const dg::DVec x = dg::evaluate( function, g2d);
+    //create the volume element
+    const dg::DVec vol2d = dg::create::volume( g2d); 
+    //compute the square L2 norm on the device
+    double norm = dg::blas2::dot( x, vol2d, x);
+    // norm is now: (exp(4)-exp(0))^2/4
+    std::cou << norm <<std::endl;
+    return 0;
+}
+```
+
+To compile and run this code for a GPU use
+```sh
+nvcc -x cu -Ipath/to/feltor/inc -Ipath/to/thrust/thrust -Ipath/to/cusplibrary/cusp test.cpp -o test
+./test
+```
+
+Or if you want to use OpenMP and gcc instead of CUDA for the device functions you can also use
+
+```shell
+g++ -fopenmp -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_OMP -Ipath/to/feltor/inc -Ipath/to/thrust/thrust -Ipath/to/cusplibrary/cusp test.cpp -o test
+export OMP_NUM_THREADS=4
+./test
+```
+
+If you want to use mpi, just include the MPI header before any other FELTOR header and use our convenient typedefs like so:
+
+```C++
+#include <iostream>
+//activate MPI in FELTOR
+#include "mpi.h" 
+#include "dg/algorithm.h"
+
+double function(double x, double y){return exp(x)*exp(y);}
+int main(int argc, char* argv[])
+{ 
+    //init MPI and create a 2d Cartesian Communicator assuming 4 MPI threads
+    MPI_Init( &argc, &argv);
+    int periods[2] = {true, true}, np[2] = {2,2};
+    MPI_Comm comm;
+    MPI_Cart_create( MPI_COMM_WORLD, 2, np, periods, true, &comm);
+    //create a 2d grid with 3 polynomial coefficients
+    dg::CartesianMPIGrid2d g2d( 0, 2, 0, 2, 3, 20, 20, comm);
+    //discretize a function on this grid
+    const dg::MDVec x = dg::evaluate( function, g2d);
+    //create the volume element
+    const dg::MDVec vol2d = dg::create::volume( g2d); 
+    //compute the square L2 norm 
+    double norm = dg::blas2::dot( x, vol2d, x);
+    //on every thread norm is now: (exp(4)-exp(0))^2/4 
+    //be a good MPI citizen and clean up
+    MPI_Finalize();
+    return 0;
+}
+```
+
+Compile e.g. for a hybrid MPI + OpenMP hardware platform with 
+
+```shell
+mpic++ -fopenmp -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_OMP -Ipath/to/feltor/inc -Ipath/to/thrust/thrust -Ipath/to/cusplibrary/cusp test.cpp -o test
+mpirun -n 4 ./test
+```
+
+Note the striking similarity to the previous program. Especially the line calling the dot function did not change at all. The compiler chooses the correct implementation for you! This is a first example of a *container free numerical algorithm*.
+
 ## 2. Further reading
-Please check out our [wiki pages](https://github.com/feltor-dev/feltor/wiki) for some general information, user oriented documentation and Troubleshooting. 
-Moreover, we maintain tex files in every src folder for technical documentation, 
- which can be compiled using pdflatex with 
-`make doc ` in the respective src folder.
-The [developer oriented documentation](http://feltor-dev.github.io/feltor/inc/dg/html/modules.html) of the dG library was generated with [Doxygen](http://www.doxygen.org). 
-You can generate a local version from source code.
-This depends on the `doxygen`, `libjs-mathjax` and `graphviz` packages.
-Type `make doc` in the folder `path/to/feltor/doc` and open `index.html` (a symbolic link to `dg/html/modules.html`) with your favorite browser. 
+
+Please check out our [wiki pages](https://github.com/feltor-dev/feltor/wiki) for some general information, user oriented documentation and Troubleshooting. Moreover, we maintain tex files in every src folder for technical documentation, which can be compiled using pdflatex with `make doc ` in the respective src folder.
+The [developer oriented documentation](http://feltor-dev.github.io/feltor/inc/dg/html/modules.html) of the dG library was generated with [Doxygen](http://www.doxygen.org). You can generate a local version from source code. This depends on the `doxygen`, `libjs-mathjax` and `graphviz` packages. Type `make doc` in the folder `path/to/feltor/doc` and open `index.html` (a symbolic link to `dg/html/modules.html`) with your favorite browser. 
 
 ## 3. Contributions and Acknowledgements
 For instructions on how to contribute read the [wiki page](https://github.com/feltor-dev/feltor/wiki/Contributions).
diff --git a/inc/geometries/ds.h b/inc/geometries/ds.h
index cef08514b..dd8ba2fab 100644
--- a/inc/geometries/ds.h
+++ b/inc/geometries/ds.h
@@ -49,7 +49,7 @@ struct DS
      * @brief Create the magnetic unit vector field and construct
      * @copydetails DS(const dg::geo::BinaryVectorLvl0&,const ProductGeometry&,unsigned,unsigned,bool,bool,double,dg::norm,dg::direction)
      */
-    template <class Limiter = FullLimiter>
+    template <class Limiter>
     DS(const dg::geo::TokamakMagneticField& vec, const ProductGeometry& grid, 
         dg::bc bcx = dg::NEU, 
         dg::bc bcy = dg::NEU, 
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index ed5a60e79..009991b49 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -321,7 +321,7 @@ struct Fieldaligned
     ///@brief do not allocate memory; no member call except construct is valid
     Fieldaligned(){}
     ///@copydoc construct()
-    template <class Limiter = FullLimiter>
+    template <class Limiter>
     Fieldaligned(const dg::geo::TokamakMagneticField& vec, 
         const ProductGeometry& grid, 
         dg::bc globalbcx = dg::NEU, 
@@ -337,7 +337,7 @@ struct Fieldaligned
     }
 
     ///@copydoc construct()
-    template <class Limiter = FullLimiter>
+    template <class Limiter>
     Fieldaligned(const dg::geo::BinaryVectorLvl0& vec, 
         const ProductGeometry& grid, 
         dg::bc globalbcx = dg::NEU, 
@@ -372,7 +372,7 @@ struct Fieldaligned
         by the grid.bcz() variable and can be changed by the set_boundaries function. 
         If there is no limiter, the boundary condition is periodic.
     */
-    template <class Limiter = FullLimiter>
+    template <class Limiter>
     void construct(const dg::geo::BinaryVectorLvl0& vec, 
         const ProductGeometry& grid, 
         dg::bc globalbcx = dg::NEU, 
diff --git a/inc/geometries/mpi_fieldaligned.h b/inc/geometries/mpi_fieldaligned.h
index f07053fda..a9336af3e 100644
--- a/inc/geometries/mpi_fieldaligned.h
+++ b/inc/geometries/mpi_fieldaligned.h
@@ -58,7 +58,7 @@ template <class ProductMPIGeometry, class LocalIMatrix, class CommunicatorXY, cl
 struct Fieldaligned< ProductMPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vector<LocalContainer> > 
 {
     Fieldaligned(){}
-    template <class Limiter = FullLimiter>
+    template <class Limiter>
     Fieldaligned(const dg::geo::TokamakMagneticField& vec, 
         const ProductMPIGeometry& grid, 
         dg::bc globalbcx = dg::NEU, 
@@ -72,7 +72,7 @@ struct Fieldaligned< ProductMPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY
         dg::geo::BinaryVectorLvl0 bhat( (dg::geo::BHatR)(vec), (dg::geo::BHatZ)(vec), (dg::geo::BHatP)(vec));
         construct( bhat, grid, globalbcx, globalbcy, limit, eps, multiplyX, multiplyY, dependsOnX, dependsOnY, deltaPhi);
     }
-    template <class Limiter = FullLimiter>
+    template <class Limiter>
     Fieldaligned(const dg::geo::BinaryVectorLvl0& vec, 
         const ProductMPIGeometry& grid, 
         dg::bc globalbcx = dg::NEU, 
@@ -85,7 +85,7 @@ struct Fieldaligned< ProductMPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY
     {
         construct( vec, grid, globalbcx, globalbcy, limit, eps, multiplyX, multiplyY, dependsOnX, dependsOnY, deltaPhi);
     }
-    template <class Limiter = FullLimiter>
+    template <class Limiter>
     void construct(const dg::geo::BinaryVectorLvl0& vec, 
         const ProductMPIGeometry& grid, 
         dg::bc globalbcx = dg::NEU, 
-- 
GitLab


From dac462fd8c5152b9dba18aab069bcc8844fe2de1 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sun, 29 Oct 2017 12:44:04 +0100
Subject: [PATCH 410/453] small corrections in README

---
 README.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index fbf09eee8..513324f71 100644
--- a/README.md
+++ b/README.md
@@ -135,7 +135,7 @@ int main()
     //compute the square L2 norm on the device
     double norm = dg::blas2::dot( x, vol2d, x);
     // norm is now: (exp(4)-exp(0))^2/4
-    std::cou << norm <<std::endl;
+    std::cout << norm <<std::endl;
     return 0;
 }
 ```
@@ -189,6 +189,7 @@ Compile e.g. for a hybrid MPI + OpenMP hardware platform with
 
 ```shell
 mpic++ -fopenmp -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_OMP -Ipath/to/feltor/inc -Ipath/to/thrust/thrust -Ipath/to/cusplibrary/cusp test.cpp -o test
+export OMP_NUM_THREADS=2
 mpirun -n 4 ./test
 ```
 
@@ -197,7 +198,7 @@ Note the striking similarity to the previous program. Especially the line callin
 ## 2. Further reading
 
 Please check out our [wiki pages](https://github.com/feltor-dev/feltor/wiki) for some general information, user oriented documentation and Troubleshooting. Moreover, we maintain tex files in every src folder for technical documentation, which can be compiled using pdflatex with `make doc ` in the respective src folder.
-The [developer oriented documentation](http://feltor-dev.github.io/feltor/inc/dg/html/modules.html) of the dG library was generated with [Doxygen](http://www.doxygen.org). You can generate a local version from source code. This depends on the `doxygen`, `libjs-mathjax` and `graphviz` packages. Type `make doc` in the folder `path/to/feltor/doc` and open `index.html` (a symbolic link to `dg/html/modules.html`) with your favorite browser. 
+The [developer oriented documentation](http://feltor-dev.github.io/feltor/inc/dg/html/modules.html) of the dG library was generated with [Doxygen](http://www.doxygen.org) and LateX. You can generate a local version including informative pdf writeups on implemented numerical methods directly from source code. This depends on the `doxygen`, `libjs-mathjax` and `graphviz` packages and LateX. Type `make doc` in the folder `path/to/feltor/doc` and open `index.html` (a symbolic link to `dg/html/modules.html`) with your favorite browser. Links to the pdf writeups can be found among other places on the Mainpage. 
 
 ## 3. Contributions and Acknowledgements
 For instructions on how to contribute read the [wiki page](https://github.com/feltor-dev/feltor/wiki/Contributions).
-- 
GitLab


From e8b53a34cf2a1dd2d53d3a81b436b13cccfe652c Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sun, 29 Oct 2017 12:49:49 +0100
Subject: [PATCH 411/453] change link in README

---
 README.md        | 2 +-
 config/README.md | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 513324f71..c0ccb8658 100644
--- a/README.md
+++ b/README.md
@@ -35,7 +35,7 @@ cd include
 ln -s path/to/thrust/thrust thrust
 ln -s path/to/cusplibrary/cusp cusp
  ```
-> If you do not like this, you can also create your own config file as discribed [here](https://github.com/feltor-dev/feltor/wiki/Configuration).
+> If you do not like this, you can also create your own config file as discribed [here](config/README.md).
 
 Now let us compile the first benchmark program. 
 
diff --git a/config/README.md b/config/README.md
index 226ecf148..c6b266fb9 100644
--- a/config/README.md
+++ b/config/README.md
@@ -12,7 +12,7 @@ The machine specific config files (e.g. vsc3.mk) should have an include guard an
 
 | variable  | default value                            | description                              |
 | :-------: | :--------------------------------------- | :--------------------------------------- |
-|    CC     | g++                                      | C++ compiler                             |+
+|    CC     | g++                                      | C++ compiler                             |
 |   MPICC   | mpic++                                   | the corresponding mpi wrapper for the c++ compiler     |
 |  CFLAGS   | -std=c++11 -Wall -x c++                  | flags for the C++ compiler               |
 | MPICFLAGS |                  | flags specific to the MPI compilation                |
-- 
GitLab


From d2e3bafa422eae39b96874d5783cc1a71fbd7406 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sun, 29 Oct 2017 15:13:58 +0100
Subject: [PATCH 412/453] begin to debug heat src files

---
 README.md                         |  12 ++-
 inc/dg/algorithm.h                |   1 +
 inc/dg/cluster_mpib.cu            |   2 +-
 inc/geometries/ds.h               |  26 ++++-
 inc/geometries/ds_mpit.cu         |   2 +-
 inc/geometries/ds_t.cu            |   6 +-
 inc/geometries/fieldaligned.h     |  15 ++-
 inc/geometries/mpi_fieldaligned.h |  23 +++-
 src/feltor/feltor.cu              |   2 +-
 src/feltor/feltor.cuh             |  23 ++--
 src/feltor/feltor_hpc.cu          |   2 +-
 src/heat/heat.cu                  |  24 ++---
 src/heat/heat.cuh                 | 171 +++++++++++-------------------
 13 files changed, 151 insertions(+), 158 deletions(-)

diff --git a/README.md b/README.md
index c0ccb8658..231fd34eb 100644
--- a/README.md
+++ b/README.md
@@ -82,10 +82,12 @@ then tell how many process you want to use in the x-, y- and z- direction, for e
 when prompted for input vector sizes type for example
 `3 100 100 10` (number of cells divided by number of procs must be an integer number). If you compiled for MPI+OpenMP, you can set the number of OpenMP threads with e.g. `export OMP_NUM_THREADS=2`.
 
-Now, we want to compile a simulation program. First, we have to download and install some libraries for I/O-operations.
+### Running a simulation
 
-For data output we use the [NetCDF](http://www.unidata.ucar.edu/software/netcdf/) library under an MIT - like license. The underlying [HDF5](https://www.hdfgroup.org/HDF5/) library also uses a very permissive license. Note that for the mpi versions of applications you need to build hdf5 and netcdf with the --enable-parallel flag. Do NOT use the pnetcdf library, which uses the classic netcdf file format.  
-Our JSON input files are parsed by [JsonCpp](https://www.github.com/open-source-parsers/jsoncpp) distributed under the MIT license (the 0.y.x branch to avoid C++-11 support).     
+Now, we want to compile and run a simulation program. First, we have to download and install some libraries for I/O-operations.
+
+For data output we use the [NetCDF](http://www.unidata.ucar.edu/software/netcdf/) library under an MIT - like license. The underlying [HDF5](https://www.hdfgroup.org/HDF5/) library also uses a very permissive license. Note that for the mpi versions of applications you need to build hdf5 and netcdf with the --enable-parallel flag. Do NOT use the pnetcdf library, which uses the classic netcdf file format. 
+Our JSON input files are parsed by [JsonCpp](https://www.github.com/open-source-parsers/jsoncpp) distributed under the MIT license (use the 0.y.x branch to avoid C++-11 support).     
 > Some desktop applications in FELTOR use the [draw library]( https://github.com/mwiesenberger/draw) (developed by us also under MIT), which depends on OpenGL (s.a. [installation guide](http://en.wikibooks.org/wiki/OpenGL_Programming)) and [glfw](http://www.glfw.org), an OpenGL development library under a BSD-like license. You don't need these when you are on a cluster. 
 
 
@@ -126,7 +128,7 @@ It is possible to use FELTOR as a library in your own code project. Note that th
 double function(double x, double y){return exp(x)*exp(y);}
 int main()
 { 
-    //create a 2d grid with 3 polynomial coefficients
+    //create a 2d discretization of [0,2]x[0,2] with 3 polynomial coefficients
     dg::CartesianGrid2d g2d( 0, 2, 0, 2, 3, 20, 20);
     //discretize a function on this grid
     const dg::DVec x = dg::evaluate( function, g2d);
@@ -170,7 +172,7 @@ int main(int argc, char* argv[])
     int periods[2] = {true, true}, np[2] = {2,2};
     MPI_Comm comm;
     MPI_Cart_create( MPI_COMM_WORLD, 2, np, periods, true, &comm);
-    //create a 2d grid with 3 polynomial coefficients
+    //create a 2d discretization of [0,2]x[0,2] with 3 polynomial coefficients
     dg::CartesianMPIGrid2d g2d( 0, 2, 0, 2, 3, 20, 20, comm);
     //discretize a function on this grid
     const dg::MDVec x = dg::evaluate( function, g2d);
diff --git a/inc/dg/algorithm.h b/inc/dg/algorithm.h
index 93387ddf0..fbfb44603 100644
--- a/inc/dg/algorithm.h
+++ b/inc/dg/algorithm.h
@@ -16,3 +16,4 @@
 #include "runge_kutta.h"
 #include "multigrid.h"
 #include "backend/timer.cuh"
+#include "backend/xspacelib.cuh"
diff --git a/inc/dg/cluster_mpib.cu b/inc/dg/cluster_mpib.cu
index 753cefb1b..8bb5fa55b 100644
--- a/inc/dg/cluster_mpib.cu
+++ b/inc/dg/cluster_mpib.cu
@@ -180,7 +180,7 @@ int main(int argc, char* argv[])
         double Zmax=1.0*gpa*1.00;
         dg::CylindricalMPIGrid3d g3d( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, n, Nx ,Ny, Nz,dg::DIR, dg::DIR, dg::PER,commEll);
         dg::geo::TokamakMagneticField magfield = dg::geo::createGuentherField(gpR0, gpI0);
-        dg::geo::Fieldaligned<dg::aProductMPIGeometry3d, IMatrix, Vector> dsFA( magfield, g3d);
+        dg::geo::Fieldaligned<dg::aProductMPIGeometry3d, IMatrix, Vector> dsFA( magfield, g3d, dg::NEU, dg::NEU, dg::geo::FullLimiter());
         dg::geo::DS<dg::aProductMPIGeometry3d, IMatrix, Matrix, Vector>  ds ( dsFA, dg::not_normed, dg::centered);
         dg::geo::guenther::FuncNeu funcNEU(gpR0,gpI0);
         Vector function = dg::evaluate( funcNEU, g3d) , dsTdsfb(function);
diff --git a/inc/geometries/ds.h b/inc/geometries/ds.h
index dd8ba2fab..9887e42c3 100644
--- a/inc/geometries/ds.h
+++ b/inc/geometries/ds.h
@@ -119,16 +119,31 @@ struct DS
     void forward( double alpha, const container& f, double beta, container& g){
         do_forward( alpha, f, beta, g);
     }
+    /**
+    * @brief forward derivative \f$ g_i = \frac{1}{h_z^+}(f_{i+1} - f_{i}) \f$
+    *
+    * @param f The vector to derive
+    * @param g contains result on output (write only)
+    * @note the vector sizes need to equal the grid size in the constructor
+    */
+    void forward( const container& f, container& g){do_forward(1.,f,0.,g);}
+
     ///@brief backward derivative \f$ g_i = \alpha \frac{1}{2h_z^-}(f_{i} - f_{i-1}) + \beta g_i \f$
     ///@copydetails forward
     void backward( double alpha, const container& f, double beta, container& g){
         do_backward( alpha, f, beta, g);
     }
+    ///@brief backward derivative \f$ g_i = \frac{1}{2h_z^-}(f_{i} - f_{i-1}) \f$
+    ///@copydetails forward
+    void backward( const container& f, container& g){do_backward(1.,f,0.,g);}
     ///@brief centered derivative \f$ g_i = \alpha \frac{1}{2h_z}(f_{i+1} - f_{i-1}) + \beta g_i\f$
     ///@copydetails forward
     void centered( double alpha, const container& f, double beta, container& g){
         do_centered( alpha, f, beta, g);
     }
+    ///@brief centered derivative \f$ g_i = \frac{1}{2h_z}(f_{i+1} - f_{i-1})\f$
+    ///@copydetails forward
+    void centered( const container& f, container& g){do_centered(1.,f,0.,g);}
 
     ///@brief forward adjoint \f$ g = -\alpha \nabla\cdot(\vec v f) + \beta g\f$
     ///@copydetails forward
@@ -145,6 +160,15 @@ struct DS
     void centeredAdj(double alpha, const container& f, double beta, container& g){
         do_centeredAdj( alpha, f, beta, g, dg::normed);
     }
+    void forwardAdj(const container& f, container& g){
+        do_forwardAdj( 1.,f,0.,g, dg::normed);
+    }
+    void backwardAdj(const container& f, container& g){
+        do_backwardAdj( 1.,f,0.,g, dg::normed);
+    }
+    void centeredAdj(const container& f, container& g){
+        do_centeredAdj( 1.,f,0.,g, dg::normed);
+    }
 
     /**
     * @brief compute parallel derivative
@@ -306,7 +330,7 @@ void DS<G, I,M,container>::do_centeredAdj( double alpha, const container& f, dou
 }
 
 template<class G,class I, class M, class container>
-void DS<G,I,M,container>::do_symv( const container& f, container& dsTdsf)
+void DS<G,I,M,container>::do_symv( double alpha, const container& f, double beta, container& dsTdsf)
 {
     if(m_dir == dg::centered)
     {
diff --git a/inc/geometries/ds_mpit.cu b/inc/geometries/ds_mpit.cu
index 6e89361aa..f3f09f9c1 100644
--- a/inc/geometries/ds_mpit.cu
+++ b/inc/geometries/ds_mpit.cu
@@ -1,7 +1,7 @@
 #include <iostream>
 
 #include <mpi.h>
-
+#define DG_BENCHMARK
 #include "dg/backend/timer.cuh"
 #include "dg/backend/mpi_evaluation.h"
 #include "dg/backend/mpi_init.h"
diff --git a/inc/geometries/ds_t.cu b/inc/geometries/ds_t.cu
index 8eb76e01e..57e301272 100644
--- a/inc/geometries/ds_t.cu
+++ b/inc/geometries/ds_t.cu
@@ -1,7 +1,7 @@
 #include <iostream>
 
 #include <cusp/print.h>
-
+#define DG_BENCHMARK
 #include "dg/backend/functions.h"
 #include "dg/backend/timer.cuh"
 #include "dg/blas.h"
@@ -30,11 +30,11 @@ int main(int argc, char * argv[])
 {
     std::cout << "First test the cylindrical version\n";
     std::cout << "Note that it's faster to compute with OMP_NUM_THREADS=1\n";
-    std::cout << "Type n, Nx, Ny, Nz\n";
+    std::cout << "Type n (3), Nx(20), Ny(20), Nz(20)\n";
     unsigned n, Nx, Ny, Nz;
     std::cin >> n>> Nx>>Ny>>Nz;
     std::cout << "You typed "<<n<<" "<<Nx<<" "<<Ny<<" "<<Nz<<std::endl;
-    std::cout << "Type mx and my\n";
+    std::cout << "Type mx (5) and my (5)\n";
     unsigned mx, my;
     std::cin >> mx>> my;
     std::cout << "You typed "<<mx<<" "<<my<<std::endl;
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index 009991b49..225226414 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -523,7 +523,6 @@ void Fieldaligned<Geometry, IMatrix, container>::construct(
     dg::bc globalbcx, dg::bc globalbcy, Limiter limit, double eps, 
     unsigned mx, unsigned my, bool bx, bool by, double deltaPhi)
 {
-    dg::Timer t;
     m_dependsOnX=bx, m_dependsOnY=by;
     m_Nz=grid.Nz(), m_bcz=grid.bcz(); 
     m_g.reset(grid);
@@ -542,16 +541,26 @@ void Fieldaligned<Geometry, IMatrix, container>::construct(
     //%%%%%%%%%%%%%%%%%%%%%%%%%%Set starting points and integrate field lines%%%%%%%%%%%%%%
     std::vector<thrust::host_vector<double> > yp_coarse( 3), ym_coarse(yp_coarse), yp, ym; 
     
+#ifdef DG_BENCHMARK
+    dg::Timer t;
     t.tic();
+    std::cout << "Generate high order grid...\n";
+#endif
     dg::Handle<dg::aGeometry2d> grid_magnetic = grid_coarse;//INTEGRATE HIGH ORDER GRID
     grid_magnetic.get().set( 7, grid_magnetic.get().Nx(), grid_magnetic.get().Ny());
-    std::cout << "Integrating fieldlines...\n";
+#ifdef DG_BENCHMARK
+    t.toc();
+     std::cout << "High order grid gen   took: "<<t.diff()<<"\n";
+    t.tic();
+#endif
     detail::integrate_all_fieldlines2d( vec, grid_magnetic.get(), grid_coarse.get(), yp_coarse, ym_coarse, deltaPhi, eps);
+#ifdef DG_BENCHMARK
     t.toc();
     std::cout << "Fieldline integration took: "<<t.diff()<<"\n";
 
     //%%%%%%%%%%%%%%%%%%Create interpolation and projection%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     t.tic();
+#endif
     dg::Grid2d grid_fine( grid_coarse.get() );//FINE GRID
     grid_fine.multiplyCellNumbers((double)mx, (double)my);
     dg::IHMatrix interpolate = dg::create::interpolation( grid_fine, grid_coarse.get());  //INTERPOLATE TO FINE GRID
@@ -561,8 +570,10 @@ void Fieldaligned<Geometry, IMatrix, container>::construct(
     dg::IHMatrix projection = dg::create::projection( grid_coarse.get(), grid_fine);
     cusp::multiply( projection, plusFine, plus);
     cusp::multiply( projection, minusFine, minus);
+#ifdef DG_BENCHMARK
     t.toc();
     std::cout << "Multiplication        took: "<<t.diff()<<"\n";
+#endif
     plusT = dg::transpose( plus);
     minusT = dg::transpose( minus);     
     dg::blas2::transfer( plus, m_plus);
diff --git a/inc/geometries/mpi_fieldaligned.h b/inc/geometries/mpi_fieldaligned.h
index a9336af3e..683357b47 100644
--- a/inc/geometries/mpi_fieldaligned.h
+++ b/inc/geometries/mpi_fieldaligned.h
@@ -11,6 +11,9 @@
 #include "dg/backend/interpolation.cuh"
 #include "dg/backend/functions.h"
 #include "dg/runge_kutta.h"
+#ifdef DG_BENCHMARK
+#include "dg/backend/timer.cuh"
+#endif 
 
 namespace dg{
 namespace geo{
@@ -161,9 +164,6 @@ void Fieldaligned<MPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vec
     dg::bc globalbcx, dg::bc globalbcy, Limiter limit, double eps, 
     unsigned mx, unsigned my, bool bx, bool by, double deltaPhi)
 {
-    dg::Timer t;
-    int rank;
-    MPI_Comm_rank( grid.communicator(), &rank);
     m_dependsOnX=bx, m_dependsOnY=by;
     m_Nz=grid.local().Nz(), m_bcz=grid.bcz(); 
     m_g.reset(grid);
@@ -185,16 +185,29 @@ void Fieldaligned<MPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vec
     //%%%%%%%%%%%%%%%%%%%%%%%%%%Set starting points and integrate field lines%%%%%%%%%%%%%%
     std::vector<thrust::host_vector<double> > yp_coarse( 3), ym_coarse(yp_coarse), yp, ym; 
     
+#ifdef DG_BENCHMARK
+    dg::Timer t;
+    int rank;
+    MPI_Comm_rank( grid.communicator(), &rank);
+    t.tic();
+    if(rank==0)std::cout << "Generate high order grid...\n";
+#endif
     dg::Handle<dg::aMPIGeometry2d> grid_magnetic = grid_coarse;//INTEGRATE HIGH ORDER GRID
     grid_magnetic.get().set( 7, grid_magnetic.get().Nx(), grid_magnetic.get().Ny());
     dg::Handle<dg::aGeometry2d> global_grid_magnetic = grid_magnetic.get().global_geometry();
+#ifdef DG_BENCHMARK
+    t.toc();
+    if(rank==0) std::cout << "High order grid gen   took: "<<t.diff()<<"\n";
     t.tic();
+#endif
     detail::integrate_all_fieldlines2d( vec, global_grid_magnetic.get(), grid_coarse.get().local(), yp_coarse, ym_coarse, deltaPhi, eps);
+#ifdef DG_BENCHMARK
     t.toc();
     if(rank==0) std::cout << "Fieldline integration took: "<<t.diff()<<"\n";
 
     //%%%%%%%%%%%%%%%%%%Create interpolation and projection%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     t.tic();
+#endif
     dg::MPIGrid2d grid_fine( grid_coarse.get() );//FINE GRID
     grid_fine.multiplyCellNumbers((double)mx, (double)my);
     dg::IHMatrix interpolate = dg::create::interpolation( grid_fine.local(), grid_coarse.get().local());  //INTERPOLATE TO FINE GRID
@@ -204,9 +217,11 @@ void Fieldaligned<MPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vec
     dg::IHMatrix projection = dg::create::projection( grid_coarse.get().local(), grid_fine.local());
     cusp::multiply( projection, plusFine, plus);
     cusp::multiply( projection, minusFine, minus);
+#ifdef DG_BENCHMARK
     t.toc();
     if(rank==0) std::cout << "Multiplication        took: "<<t.diff()<<"\n";
     t.tic();
+#endif
     dg::MIHMatrix temp = dg::convert( plus, grid_coarse.get()), tempT;
     tempT  = dg::transpose( temp);
     dg::blas2::transfer( temp, m_plus);
@@ -215,8 +230,10 @@ void Fieldaligned<MPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vec
     tempT  = dg::transpose( temp);
     dg::blas2::transfer( temp, m_minus);
     dg::blas2::transfer( tempT, m_minusT);
+#ifdef DG_BENCHMARK
     t.toc();
     if(rank==0) std::cout << "Conversion            took: "<<t.diff()<<"\n";
+#endif
     //%%%%%%%%%%%%%%%%%%%%%%%project h and copy into h vectors%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     dg::MHVec hp( dg::evaluate( dg::zero, grid_coarse.get())), hm(hp), hz(hp);
     dg::blas2::symv( projection, yp[2], hp.data());
diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index 4a72261b3..d18f79842 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -46,7 +46,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     const feltor::Parameters p( js);
-    const dg::geo::solovev::GeomParameters gp(gs);
+    const dg::geo::solovev::Parameters gp(gs);
     p.display( std::cout);
     gp.display( std::cout);
     /////////glfw initialisation ////////////////////////////////////////////
diff --git a/src/feltor/feltor.cuh b/src/feltor/feltor.cuh
index 28fcde5db..a0fe0350a 100644
--- a/src/feltor/feltor.cuh
+++ b/src/feltor/feltor.cuh
@@ -1,18 +1,9 @@
 #pragma once
 
 #include "dg/algorithm.h"
-#include "dg/poisson.h"
 #include "parameters.h"
 #include "geometries/geometries.h"
 
-#ifdef DG_BENCHMARK
-#include "dg/backend/timer.cuh"
-#endif //DG_BENCHMARK
-/*!@file
-
-  Contains the solvers 
-  */
-
 namespace feltor
 {
 ///@addtogroup solver
@@ -31,7 +22,7 @@ template<class Geometry, class IMatrix, class Matrix, class container>
 struct Implicit
 {
 
-    Implicit( const Geometry& g, feltor::Parameters p, dg::geo::solovev::GeomParameters gp, DS& dsN, DS& dsDIR):
+    Implicit( const Geometry& g, feltor::Parameters p, dg::geo::solovev::Parameters gp, DS& dsN, DS& dsDIR):
         p(p),
         gp(gp),
         LaplacianM_perpN  ( g, g.bcx(), g.bcy(), dg::normed, dg::centered),
@@ -89,7 +80,7 @@ struct Implicit
     const container& precond(){return LaplacianM_perpDIR.precond();}
   private:
     const feltor::Parameters p;
-    const dg::geo::solovev::GeomParameters gp;
+    const dg::geo::solovev::Parameters gp;
     container temp;
     container dampgauss_;
     dg::Elliptic<Geometry, Matrix, container> LaplacianM_perpN,LaplacianM_perpDIR;
@@ -99,7 +90,7 @@ struct Implicit
 template< class Geometry, class IMatrix, class Matrix, class container >
 struct Explicit
 {
-    Explicit( const Geometry& g, const dg::geo::TokamakMagneticField& mag, feltor::Parameters p, dg::geo::solovev::GeomParameters gp);
+    Explicit( const Geometry& g, const dg::geo::TokamakMagneticField& mag, feltor::Parameters p, dg::geo::solovev::Parameters gp);
 
 
     dg::DS<Geometry, IMatrix, Matrix, container>& ds(){return dsN_;}
@@ -198,14 +189,14 @@ struct Explicit
 
     //matrices and solvers
     dg::DS<Geometry, IMatrix, Matrix, container> dsDIR_, dsN_;
-    dg::Poisson<   Geometry, Matrix, container> poissonN,poissonDIR; 
-    dg::Elliptic<  Geometry, Matrix, container > pol,lapperpN,lapperpDIR;
+    dg::Poisson<    Geometry, Matrix, container> poissonN,poissonDIR; 
+    dg::Elliptic<   Geometry, Matrix, container> pol,lapperpN,lapperpDIR;
     dg::Helmholtz< Geometry, Matrix, container > invgammaDIR, invgammaN;
 
     dg::Invert<container> invert_pol,invert_invgammaN,invert_invgammaPhi;
 
     const feltor::Parameters p;
-    const dg::geo::solovev::GeomParameters gp;
+    const dg::geo::solovev::Parameters gp;
 
     double mass_, energy_, diff_, ediff_, aligned_;
     std::vector<double> evec;
@@ -214,7 +205,7 @@ struct Explicit
 
 ///@cond
 template<class Grid, class IMatrix, class Matrix, class container>
-Explicit<Grid, IMatrix, Matrix, container>::Explicit( const Grid& g, const TokamakMagneticField& mag, feltor::Parameters p, dg::geo::solovev::GeomParameters gp): 
+Explicit<Grid, IMatrix, Matrix, container>::Explicit( const Grid& g, const TokamakMagneticField& mag, feltor::Parameters p, dg::geo::solovev::Parameters gp): 
     dsDIR_( dg::FieldAligned<Grid, IMatrix, container>( mag, g, gp.rk4eps, 
                 dg::geo::PsiLimiter(mag.psip(), gp.psipmaxlim), 
                 dg::DIR, (2*M_PI)/((double)p.Nz)
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index a9079b613..ce6e6140d 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -44,7 +44,7 @@ int main( int argc, char* argv[])
         reader.parse(ks,gs,false);
     }
     const feltor::Parameters p( js);
-    const dg::geo::solovev::GeomParameters gp(gs);
+    const dg::geo::solovev::Parameters gp(gs);
     p.display( std::cout);
     gp.display( std::cout);
     std::string input = js.toStyledString(), geom = gs.toStyledString();
diff --git a/src/heat/heat.cu b/src/heat/heat.cu
index 1af95be9d..379cea13c 100644
--- a/src/heat/heat.cu
+++ b/src/heat/heat.cu
@@ -7,22 +7,12 @@
 
 #include "draw/host_window.h"
 //#include "draw/device_window.cuh"
-#include "dg/backend/xspacelib.cuh"
-#include "dg/backend/timer.cuh"
-#include "dg/runge_kutta.h"
-#include "dg/multistep.h"
-#include "dg/elliptic.h"
-#include "dg/cg.h"
-
+#include "dg/algorithm.h"
 #include "geometries/geometries.h"
-#include "heat/parameters.h"
-
 
+#include "parameters.h"
 #include "heat.cuh"
 
-typedef dg::FieldAligned< dg::CylindricalGrid3d, dg::IDMatrix, dg::DVec> DFA;
-using namespace dg::geo::solovev;
-
 int main( int argc, char* argv[])
 {
     ////Parameter initialisation ////////////////////////////////////////////
@@ -48,8 +38,8 @@ int main( int argc, char* argv[])
         std::cerr << "ERROR: Wrong number of arguments!\nUsage: "<< argv[0]<<" [inputfile] [geomfile] \n";
         return -1;
     }
-    const eule::Parameters p( js); p.display( std::cout);
-    const GeomParameters gp(gs); gp.display( std::cout);
+    const heat::Parameters p( js); p.display( std::cout);
+    const dg::geo::solovev::Parameters gp(gs); gp.display( std::cout);
     /////////glfw initialisation ////////////////////////////////////////////
     std::ifstream is( "window_params.js");
     reader.parse( is, js, false);
@@ -113,9 +103,9 @@ int main( int argc, char* argv[])
     
     //create RHS     
     std::cout << "initialize feltor" << std::endl;
-    eule::Feltor<dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec > feltor( grid, p,gp); //initialize before rolkar!
+    heat::Explicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec> feltor( grid, p,gp); //initialize before rolkar!
     std::cout << "initialize rolkar" << std::endl;
-    eule::Rolkar<dg::CylindricalGrid3d , dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec > rolkar( grid, p,gp);
+    heat::Implicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec > rolkar( grid, p,gp);
 
     ////////////////////////////////The initial field////////////////////////////////
  //initial perturbation
@@ -130,7 +120,7 @@ int main( int argc, char* argv[])
     
     //background profile
     std::cout << "T background" << std::endl;
-    dg::geo::Nprofile<Psip> prof(p.bgprofamp, p.nprofileamp, gp, Psip(gp)); //initial background profile
+    dg::geo::Nprofile prof(p.bgprofamp, p.nprofileamp, gp, dg::geo::solovev::Psip(gp)); //initial background profile
     std::vector<dg::DVec> y0(1, dg::evaluate( prof, grid)), y1(y0); 
     
 //     //field aligning
diff --git a/src/heat/heat.cuh b/src/heat/heat.cuh
index e2e027a61..3f74a16bb 100644
--- a/src/heat/heat.cuh
+++ b/src/heat/heat.cuh
@@ -19,31 +19,29 @@ namespace heat
 template< class Geometry, class IMatrix, class Matrix, class container>
 struct Implicit
 {
-    Implicit( const Geometry& g, Parameters p, dg::geo::solovev::Parameters gp, dg::geo::Fieldaligned<Geometry, IMatrix, Matrix, container>& fa):
+    Implicit( const Geometry& g, Parameters p, dg::geo::solovev::Parameters gp):
         p(p),
         gp(gp),
-        dsNU_( fa, dg::geo::createSolovevField( gp), g, 1,1, true, true, gp.rk4eps, g.bcx(), g.bcy() 
-                dg::geo::PsiLimiter( dg::geo::solovev::Psip(gp), gp.psipmaxlim
-                    ), 
-                g.bcx()
-                ), 
-              , dg::normed, dg::forward ),
+        dsNU_(dg::geo::createSolovevField( gp), g g.bcx(), g.bcy(),
+              dg::geo::PsiLimiter( dg::geo::solovev::Psip(gp), 
+              dg::normed, dg::forward,
+              gp.psipmaxlim),gp.rk4eps, 1,1, true, true), 
         elliptic( g, dg::normed, dg::forward)
     {
         using namespace dg::geo::solovev;
         MagneticField c(gp);
-        container bfield = dg::evaluate( dg::geo::FieldR<MagneticField>(c, gp.R_0),g);
+        container bfield = dg::pullback( dg::geo::FieldR(c),g);
         elliptic.set_x( bfield);
-        bfield = dg::evaluate( dg::geo::FieldZ<MagneticField>(c, gp.R_0),g);
+        bfield = dg::pullback( dg::geo::FieldZ(c),g);
         elliptic.set_y( bfield);
-        bfield = dg::evaluate( dg::geo::FieldP<MagneticField>(c, gp.R_0),g);
+        bfield = dg::pullback( dg::geo::FieldP(c),g);
         elliptic.set_z( bfield);
 
-        dg::blas1::transfer( dg::pullback( dg::geo::GaussianProfDamping<Psip>(Psip(gp), gp.psipmax, gp.alpha), g), dampprof_);
+        dg::blas1::transfer( dg::pullback( dg::geo::GaussianProfDamping(c.psip(), gp.psipmax, gp.alpha), g), dampprof_);
     }
-    void operator()( std::vector<container>& x, std::vector<container>& y)
+    void operator()( const std::vector<container>& x, std::vector<container>& y)
     {
-        dg::blas1::axpby( 0., x, 0, y);
+        dg::blas1::scal( y, 0.);
         if (p.p_diff ==0)    {
           dg::blas2::gemv( dsNU_, x[0], y[0]); 
           dg::blas1::scal(y[0], p.nu_parallel );  
@@ -58,30 +56,22 @@ struct Implicit
     const container& inv_weights(){return elliptic.inv_weights();}
     const container& precond(){return elliptic.precond();}
   private:
-    const eule::Parameters p;
-    const dg::geo::solovev::GeomParameters gp;
+    const heat::Parameters p;
+    const dg::geo::solovev::Parameters gp;
     container dampprof_;
-    dg::geo::Fieldaligned<Geometry, IMatrix, Matrix, container> dsNU_;
     dg::geo::DS<Geometry, IMatrix, Matrix, container> dsNU_;
     dg::GeneralEllipticSym<Geometry, Matrix, container> elliptic;
 
 };
 
-template< class DS, class Matrix, class container >
+template< class Geometry, class IMatrix, class Matrix, class container >
 struct Explicit
 {
-    //typedef std::vector<container> Vector;
-    typedef typename dg::VectorTraits<container>::value_type value_type;
-    //typedef typename thrust::iterator_system<typename container::iterator>::type MemorySpace;
-    //typedef cusp::ell_matrix<int, value_type, MemorySpace> Matrix;
-    //typedef dg::DMatrix Matrix; //fastest device Matrix (does this conflict with 
+    Explicit( const Geometry& g, heat::Parameters p, dg::geo::solovev::Parameters gp);
 
-    template<class Grid3d>
-    Explicit( const Grid3d& g, eule::Parameters p,dg::geo::solovev::GeomParameters gp);
+    const dg::geo::DS<Geometry,IMatrix,Matrix,container>& ds(){return dsNU_;}
 
-    const DS ds(){return dsNU_;}
-
-    void operator()( std::vector<container>& y, std::vector<container>& yp);
+    void operator()( const std::vector<container>& y, std::vector<container>& yp);
 
     double mass( ) {return mass_;}
     double mass_diffusion( ) {return diff_;}
@@ -94,80 +84,43 @@ struct Explicit
 
 
     container chi, omega, lambda,tmo; //!!Attention: chi and omega are helper variables and may be changed at any time and by any method!!
-    container binv, gradlnB;
+    container m_invB, gradlnB;
 //     ,pupil;
     const container  one;
     const container w3d, v3d;
 
     //matrices and solvers
-    DS dsDIR_, dsNU_;
-
-//     dg::Elliptic< Matrix, container > lapperp; 
-//     dg::GeneralEllipticSym<Matrix, container> elliptic;
+    dg::geo::DS<Geometry,IMatrix,Matrix,container> dsDIR_, dsNU_;
 
-    const eule::Parameters p;
-    const dg::geo::solovev::GeomParameters gp;
+    const heat::Parameters p;
+    const dg::geo::solovev::Parameters gp;
     std::vector<double> evec;
 
     double mass_, energy_, diff_, ediff_;
 
 };
 
-template<class DS, class Matrix, class container>
-template<class Grid>
-Explicit<DS, Matrix, container>::Explicit( const Grid& g, eule::Parameters p, dg::geo::solovev::GeomParameters gp): 
+template<class Geometry, class IMatrix, class Matrix, class container>
+Explicit<Geometry,IMatrix,Matrix,container>::Explicit( const Geometry& g, heat::Parameters p, dg::geo::solovev::Parameters gp): 
     chi( dg::evaluate( dg::one, g)), omega(chi),  lambda(chi), tmo(chi),
     one( dg::evaluate( dg::one, g)),    
     w3d( dg::create::volume(g)), v3d( dg::create::inv_volume(g)),      
-    dsDIR_( typename DS::FieldAligned( 
-                dg::geo::Field<dg::geo::solovev::MagneticField>(
-                    dg::geo::solovev::MagneticField(gp), gp.R_0
-                    ), 
-                g, gp.rk4eps, 
-                dg::geo::PsiLimiter<dg::geo::solovev::Psip>(
-                    dg::geo::solovev::Psip(gp), gp.psipmaxlim
-                    ), 
-                dg::DIR
-                ), 
-            dg::geo::Field<dg::geo::solovev::MagneticField>(
-                dg::geo::solovev::MagneticField(gp), gp.R_0
-                ), 
-            dg::normed, dg::centered ),
-    dsNU_( typename DS::FieldAligned(
-                dg::geo::Field<dg::geo::solovev::MagneticField>(
-                    dg::geo::solovev::MagneticField(gp), gp.R_0), 
-                g, gp.rk4eps, 
-                dg::geo::PsiLimiter<dg::geo::solovev::Psip>(
-                    dg::geo::solovev::Psip(gp), gp.psipmaxlim
-                    ), 
-                g.bcx()
-                ), 
-          dg::geo::Field<dg::geo::solovev::MagneticField>(
-              dg::geo::solovev::MagneticField(gp), gp.R_0
-              ), 
-          dg::normed, dg::centered ),
-//     lapperp ( g,g.bcx(), g.bcy(),     dg::normed,  dg::centered),
-//         elliptic( g, dg::normed, dg::forward),
+    dsDIR_( dg::geo::createSolovevField(gp), g, dg::DIR,dg::DIR,dg::geo::PsiLimiter( dg::geo::solovev::Psip(gp), gp.psipmaxlim), dg::normed, dg::centered), 
+    dsNU_( dg::geo::createSolovevField(gp), g, dg::NEU,dg::NEU,dg::geo::PsiLimiter( dg::geo::solovev::Psip(gp), gp.psipmaxlim), dg::normed, dg::centered), 
     p(p),
     gp(gp),
     evec(1)
 {
-//         container bfield = dg::evaluate( solovev::bR( gp.R_0, gp.I_0),g);
-//         elliptic.set_x( bfield);
-//         bfield = dg::evaluate( solovev::bZ( gp.R_0, gp.I_0),g);
-//         elliptic.set_y( bfield);
-//         bfield = dg::evaluate( solovev::bPhi( gp.R_0, gp.I_0),g);
-//         elliptic.set_z( bfield);
     //////////////////////////////init fields /////////////////////
-    TokamakMagneticField mf = dg::geo::createSolovevField(gp);
-    dg::blas1::transfer(  dg::pullback(dg::geo::InvB(mf),    g), binv);
+    dg::geo::TokamakMagneticField mf = dg::geo::createSolovevField(gp);
+    dg::blas1::transfer(  dg::pullback(dg::geo::InvB(mf),    g), m_invB);
     dg::blas1::transfer(  dg::pullback(dg::geo::GradLnB(mf), g), gradlnB);
 }
 
 
 
-template<class DS, class M, class V>
-void Explicit<DS, M, V>::energies( std::vector<V>& y)
+template<class G,class I, class M, class V>
+void Explicit<G,I,M,V>::energies( std::vector<V>& y)
 {
     double S[1]    = {0.0};    
     double Dpar[1] = {0.0};
@@ -195,20 +148,20 @@ void Explicit<DS, M, V>::energies( std::vector<V>& y)
     if (p.p_diff ==0)    {
     //     (A) adjoint
 //         dsNU_( y[0], omega); 
-//         dsNU_.centeredT(omega,lambda);
+//         dsNU_.centeredAdj(omega,lambda);
         dsNU_.forward( y[0], omega); 
-        dsNU_.forwardT(omega,lambda);
+        dsNU_.forwardAdj(omega,lambda);
         dg::blas1::axpby( 0.5, lambda, 0.,chi,chi); 
 // 
         dsNU_.backward( y[0], omega); 
-        dsNU_.backwardT(omega,lambda);
+        dsNU_.backwardAdj(omega,lambda);
         dg::blas1::axpby( 0.5, lambda, 1., chi,chi); 
         Dpar[0]= p.nu_parallel*dg::blas2::dot(y[0], w3d, chi);
     }  
     if (p.p_diff ==1)    {
         // (B) nonadjoint
-        dsNU_( binv, lambda); //gradpar 1/B
-        dg::blas1::pointwiseDivide(lambda,  binv, lambda); //-ds lnB
+        dsNU_( m_invB, lambda); //gradpar 1/B
+        dg::blas1::pointwiseDivide(lambda,  m_invB, lambda); //-ds lnB
         dsNU_(y[0],omega); //ds T
         dg::blas1::pointwiseDot(omega, lambda, omega);            // -ds lnB ds T
         dsNU_.dss(y[0],lambda);                                          //ds^2 T 
@@ -241,8 +194,8 @@ void Explicit<DS, M, V>::energies( std::vector<V>& y)
 
 
 //do not overwrite y
-template<class DS, class Matrix, class container>
-void Explicit<DS, Matrix, container>::operator()( std::vector<container>& y, std::vector<container>& yp)
+template<class G, class I, class Matrix, class container>
+void Explicit<G,I,Matrix,container>::operator()(const std::vector<container>& y, std::vector<container>& yp)
 {
     /* y[0] := T - 1 or T
     */
@@ -269,38 +222,38 @@ void Explicit<DS, Matrix, container>::operator()( std::vector<container>& y, std
 //         // (A) adjoint
 //         //U=v_parallel gradlnB
 //     //     dg::blas1::pointwiseDot(y[0], gradlnB, lambda);
-//     //     dsNU_.centeredT(lambda,omega);    
+//     //     dsNU_.centeredAdj(lambda,omega);    
 //     //     dg::blas1::axpby( p.nu_parallel, omega, 1., yp[0]); 
 // 
 //         //U=1.  
 //         //centered
 //         dg::blas1::pointwiseDot(y[0],pupil,lambda);    //U*T  
-//         dsNU_.centeredT(lambda,omega);    // dsT UT
-// //         dsNU_.backwardT(lambda,omega);
-// //         dsNU_.forwardT(lambda,omega);   
+//         dsNU_.centeredAdj(lambda,omega);    // dsT UT
+// //         dsNU_.backwardAdj(lambda,omega);
+// //         dsNU_.forwardAdj(lambda,omega);   
 //         dg::blas1::axpby( -1.0, omega, 1., yp[0]); //dsT (UT)
 // 
 //         //corr(1): div(UB) 
-// //         dg::blas1::pointwiseDivide(pupil,binv,omega); //= U B
-// //         dsNU_.centeredT(omega,lambda);     //div UB
+// //         dg::blas1::pointwiseDivide(pupil,m_invB,omega); //= U B
+// //         dsNU_.centeredAdj(omega,lambda);     //div UB
 // //         dg::blas1::axpby( 1.0, lambda, 1., yp[0]); //+div UB
 //         
 //         //corr(2): div(B) 
-// //         dg::blas1::pointwiseDivide(one,binv,omega); //= U B
-// //         dsNU_.centeredT(omega,lambda);     //div UB
+// //         dg::blas1::pointwiseDivide(one,m_invB,omega); //= U B
+// //         dsNU_.centeredAdj(omega,lambda);     //div UB
 // //         dg::blas1::axpby( 1.0, lambda, 1., yp[0]); //+div UB
 //         
 //         //corr(3): UT/B divB 
-//     //     dg::blas1::pointwiseDivide(one,binv,omega); //=  B
-//     //     dsNU_.centeredT(omega,lambda);     //div B
-//     //     dg::blas1::pointwiseDot(y[0],binv,omega); //T/B
+//     //     dg::blas1::pointwiseDivide(one,m_invB,omega); //=  B
+//     //     dsNU_.centeredAdj(omega,lambda);     //div B
+//     //     dg::blas1::pointwiseDot(y[0],m_invB,omega); //T/B
 //     //     dg::blas1::pointwiseDot(omega,pupil,omega); // U T/B
 //     //     dg::blas1::pointwiseDot(omega,lambda,lambda); //  UT/B divB
 //     //     dg::blas1::axpby( 1.0, lambda, 1., yp[0]); //+ UT/B div UB
 // 
 //         //corr(4): U  divB
-// //         dg::blas1::pointwiseDivide(one,binv,omega); //=  B
-// //         dsNU_.centeredT(omega,lambda);     //div B
+// //         dg::blas1::pointwiseDivide(one,m_invB,omega); //=  B
+// //         dsNU_.centeredAdj(omega,lambda);     //div B
 // //         dg::blas1::pointwiseDot(pupil,lambda,lambda); //  U divB
 // //         dg::blas1::axpby( 1.0, lambda, 1., yp[0]); //+ U div UB
 //     }
@@ -309,8 +262,8 @@ void Explicit<DS, Matrix, container>::operator()( std::vector<container>& y, std
 //         dg::blas1::pointwiseDot(y[0],pupil,lambda);    //UT
 //         dsNU_(lambda,omega);    //Dz UT
 //         dg::blas1::axpby( -1.0, omega, 1., yp[0]); //-  ds U T
-//         dsNU_( binv, lambda); //gradpar 1/B
-//         dg::blas1::pointwiseDivide(lambda,  binv, lambda); //-ds lnB  
+//         dsNU_( m_invB, lambda); //gradpar 1/B
+//         dg::blas1::pointwiseDivide(lambda,  m_invB, lambda); //-ds lnB  
 //         dg::blas1::pointwiseDot(y[0],pupil,omega);    //=  U T  
 //         dg::blas1::pointwiseDot(omega,  lambda, omega); //-U T ds lnB  
 //         dg::blas1::axpby( -1.0, omega, 1., yp[0]); //UT dslnB
@@ -327,16 +280,16 @@ void Explicit<DS, Matrix, container>::operator()( std::vector<container>& y, std
     if (p.p_diff ==0)    {
 // //         centered
 // //         dsNU_( y[0], omega); 
-// //         dsNU_.centeredT(omega,lambda);
+// //         dsNU_.centeredAdj(omega,lambda);
 // //         dg::blas1::axpby( p.nu_parallel, lambda, 1., yp[0]); 
 // 
         //forward, backward (stegi) without jump
 //         dsNU_.forward( y[0], omega); 
-//         dsNU_.forwardT(omega,lambda);
+//         dsNU_.forwardAdj(omega,lambda);
 //         dg::blas1::axpby( 0.5*p.nu_parallel, lambda, 1., yp[0]); 
 // 
 //         dsNU_.backward( y[0], omega); 
-//         dsNU_.backwardT(omega,lambda);
+//         dsNU_.backwardAdj(omega,lambda);
 //         dg::blas1::axpby( 0.5*p.nu_parallel, lambda, 1., yp[0]); 
 //         //with jump
 //        dsNU_.symv(y[0],lambda);
@@ -344,7 +297,7 @@ void Explicit<DS, Matrix, container>::operator()( std::vector<container>& y, std
 //        dg::blas1::axpby( p.nu_parallel, lambda, 1., yp[0]); 
 // 
 //         dsNU_.backward( y[0], omega); 
-//         dsNU_.backwardT(omega,lambda);
+//         dsNU_.backwardAdj(omega,lambda);
 //         dg::blas1::axpby( 0.5*p.nu_parallel, lambda, 1., yp[0]); 
         //with jump
 //        dsNU_.symv(y[0],lambda);
@@ -356,8 +309,8 @@ void Explicit<DS, Matrix, container>::operator()( std::vector<container>& y, std
 
     if (p.p_diff ==1)    {
         // (B) nonadjoint
-        dsNU_( binv, lambda); //gradpar 1/B
-        dg::blas1::pointwiseDivide(lambda,  binv, lambda); //-ds lnB
+        dsNU_( m_invB, lambda); //gradpar 1/B
+        dg::blas1::pointwiseDivide(lambda,  m_invB, lambda); //-ds lnB
         dsNU_(y[0],omega); //ds T
         dg::blas1::pointwiseDot(omega, lambda, omega);            //- ds lnB ds T
         dg::blas1::axpby(p.nu_parallel, omega, 1., yp[0]);    
@@ -375,11 +328,15 @@ void Explicit<DS, Matrix, container>::operator()( std::vector<container>& y, std
     if (p.p_diff ==3)    {
         // (D) nonadjoint with direct method
         dsNU_.forward( y[0], omega); 
-        dsDIR_.forwardTD(omega,lambda);
+        dg::blas1::pointwiseDot(omega, m_invB, omega);
+        dsDIR_.backward(omega,lambda);
+        dg::blas1::pointwiseDivide(lambda, m_invB, lambda);
         dg::blas1::axpby( 0.5*p.nu_parallel, lambda, 1., yp[0]); 
 
         dsNU_.backward( y[0], omega); 
-        dsDIR_.backwardTD(omega,lambda);
+        dg::blas1::pointwiseDot(omega, m_invB, omega);
+        dsDIR_.forward(omega,lambda);
+        dg::blas1::pointwiseDivide(lambda, m_invB, lambda);
 
         dg::blas1::axpby( 0.5*p.nu_parallel, lambda, 1., yp[0]); 
     }
@@ -399,5 +356,5 @@ void Explicit<DS, Matrix, container>::operator()( std::vector<container>& y, std
 
 ///@}
 
-} //namespace eule
+} //namespace heat
 
-- 
GitLab


From 45e181f7d78b387ac73be8d8cdd47d628e732d03 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Sun, 29 Oct 2017 18:11:07 +0100
Subject: [PATCH 413/453] added dss function to ds and debugged heat.cuh

---
 inc/geometries/ds.h | 68 +++++++++++++++++++++++++++++++++------------
 src/heat/heat.cuh   | 16 +++++++----
 2 files changed, 60 insertions(+), 24 deletions(-)

diff --git a/inc/geometries/ds.h b/inc/geometries/ds.h
index 9887e42c3..2513c62e4 100644
--- a/inc/geometries/ds.h
+++ b/inc/geometries/ds.h
@@ -15,7 +15,6 @@
  * This file includes the appropriate headers for parallel derivatives
  */
 
-//TODO: use buffers to make symv const
 namespace dg{
 namespace geo{
 
@@ -171,7 +170,7 @@ struct DS
     }
 
     /**
-    * @brief compute parallel derivative
+    * @brief \f$ \vec v\cdot \nabla f \f$
     *
     * dependent on dir redirects to either forward(), backward() or centered()
     * @param f The vector to derive
@@ -189,7 +188,17 @@ struct DS
      * @param dsTdsf contains result on output (write only)
      * @note if dependsOnX is false then no jump terms will be added in the x-direction and similar in y
      */
-    void symv( const container& f, container& dsTdsf){ do_symv( f, dsTdsf);}
+    void symv( const container& f, container& dsTdsf){ do_symv( 1., f, 0., dsTdsf);}
+    void symv( double alpha, const container& f, double beta, container& dsTdsf){ do_symv( alpha, f, beta, dsTdsf);}
+    /**
+     * @brief Discretizes \f$ \nabla_\parallel^2 f \f$ 
+     *
+     * The formula used is \f[ \nabla_\parallel^2 f = 2\left(\frac{f^+}{h^+h^0} - \frac{f^0}{h^+h^-} + \frac{f^-}{h^-h^0}\right)
+     * @param f The vector to derive
+     * @param dsTdsf contains result on output (write only)
+     */
+    void dss( const container& f, container& dsTdsf){ do_dss( 1., f, 0., dsTdsf);}
+    void dss( double alpha, const container& f, double beta, container& dsTdsf){ do_symv( alpha, f, beta, dsTdsf);}
 
     ///@copydoc Fieldaligned::set_boundaries(dg::bc,double,double)
     void set_boundaries( dg::bc bcz, double left, double right)
@@ -224,7 +233,8 @@ struct DS
     void do_forwardAdj(double alpha, const container& f, double beta, container& dsf, dg::norm no);
     void do_backwardAdj(double alpha, const container& f, double beta, container& dsf, dg::norm no);
     void do_centeredAdj(double alpha, const container& f, double beta, container& dsf, dg::norm no);
-    void do_symv(const container& f, container& dsf);
+    void do_symv(double alpha, const container& f, double beta, container& dsf);
+    void do_dss(double alpha, const container& f, double beta, container& dsf);
 
     Fieldaligned<ProductGeometry, IMatrix, container> m_fa;
     container m_temp;
@@ -294,11 +304,13 @@ void DS<G,I,M,container>::do_forwardAdj( double alpha, const container& f, doubl
     dg::blas1::pointwiseDot( m_vol3d, f, m_temp0);
     dg::blas1::pointwiseDot( m_temp0, m_fa.hp_inv(), m_temp0);
     m_fa(einsPlusT, m_temp0, m_tempP);
-    dg::blas1::axpby( 1., m_tempP, -1., m_temp0, m_temp0);
     if(no == dg::normed) 
+    {
+        dg::blas1::axpby( 1., m_tempP, -1., m_temp0, m_temp0);
         dg::blas1::pointwiseDot( alpha, m_inv3d, m_temp0, beta, dsf); 
+    }
     else
-        dg::blas1::axpby( alpha, m_tempM, beta, dsf);
+        dg::blas1::axpbypgz( alpha, m_tempP, -alpha, m_temp0, beta, dsf);
 }
 template<class G,class I, class M, class container>
 void DS<G,I,M,container>::do_backwardAdj( double alpha, const container& f, double beta, container& dsf, dg::norm no)
@@ -307,11 +319,13 @@ void DS<G,I,M,container>::do_backwardAdj( double alpha, const container& f, doub
     dg::blas1::pointwiseDot( m_vol3d, f, m_temp0);
     dg::blas1::pointwiseDot( m_temp0, m_fa.hm_inv(), m_temp0);
     m_fa(einsMinusT, m_temp0, m_tempM);
-    dg::blas1::axpby( 1., m_temp0, -1., m_tempM, m_temp0);
     if(no == dg::normed) 
+    {
+        dg::blas1::axpby( 1., m_temp0, -1., m_tempM, m_temp0);
         dg::blas1::pointwiseDot( alpha, m_inv3d, m_temp0, beta, dsf); 
+    }
     else
-        dg::blas1::axpby( alpha, m_tempM, beta, dsf);
+        dg::blas1::axpbypgz( alpha, m_temp0, -alpha, m_tempM, beta, dsf);
 }
 template<class G, class I, class M, class container>
 void DS<G, I,M,container>::do_centeredAdj( double alpha, const container& f, double beta, container& dsf, dg::norm no)
@@ -321,11 +335,13 @@ void DS<G, I,M,container>::do_centeredAdj( double alpha, const container& f, dou
     dg::blas1::pointwiseDot( m_temp0, m_fa.hz_inv(), m_temp0);
     m_fa(einsPlusT,  m_temp0, m_tempP);
     m_fa(einsMinusT, m_temp0, m_tempM);
-    dg::blas1::axpby( 1., m_tempP, -1., m_tempM);
     if(no == dg::normed) 
+    {
+        dg::blas1::axpby( 1., m_tempP, -1., m_tempM);
         dg::blas1::pointwiseDot( alpha, m_inv3d, m_tempM, beta, dsf); 
+    }
     else
-        dg::blas1::axpby( alpha, m_tempM, beta, dsf);
+        dg::blas1::axpbypgz( alpha, m_tempP, -alpha, m_tempM, beta, dsf);
 
 }
 
@@ -335,25 +351,41 @@ void DS<G,I,M,container>::do_symv( double alpha, const container& f, double beta
     if(m_dir == dg::centered)
     {
         do_centered( 1., f, 0., m_tempP);
-        do_centeredAdj( 1., m_tempP, 0., dsTdsf, dg::not_normed);
+        do_centeredAdj( 1., m_tempP, 0., m_temp, dg::not_normed);
     }
     else 
     {
         do_forward( 1., f, 0., m_tempP);
         do_forwardAdj( 1., m_tempP, 0., m_temp0, dg::not_normed);
         do_backward( 1., f, 0., m_tempM);
-        do_backwardAdj( 1., m_tempM, 0., dsTdsf, dg::not_normed);
-        dg::blas1::axpby(0.5,m_temp0,0.5,dsTdsf);
+        do_backwardAdj( 1., m_tempM, 0., m_temp, dg::not_normed);
+        dg::blas1::axpby(0.5,m_temp0,0.5,m_temp);
     }
-    dg::blas1::pointwiseDivide( dsTdsf, m_weights_wo_vol, dsTdsf);
+    dg::blas1::pointwiseDivide( m_temp, m_weights_wo_vol, m_temp);
     //     add jump term 
     if(m_fa.dependsOnX())
-        dg::blas2::symv( -1., m_jumpX, f, 1., dsTdsf);
+        dg::blas2::symv( -1., m_jumpX, f, 1., m_temp);
     if(m_fa.dependsOnY())
-        dg::blas2::symv( -1., m_jumpY, f, 1., dsTdsf);
-    dg::blas1::pointwiseDot( m_weights_wo_vol, dsTdsf, dsTdsf); //make it symmetric
+        dg::blas2::symv( -1., m_jumpY, f, 1., m_temp);
+
     if( m_no == dg::normed)
-        dg::blas1::pointwiseDot( m_inv3d, dsTdsf, dsTdsf); //make it symmetric
+    {
+        dg::blas1::pointwiseDot( m_weights_wo_vol, m_temp, m_temp);
+        dg::blas1::pointwiseDot( alpha, m_inv3d, m_temp, beta, dsTdsf); 
+    }
+    else
+        dg::blas1::pointwiseDot( alpha, m_weights_wo_vol, m_temp, beta, dsTdsf);
+}
+
+template<class G,class I, class M, class container>
+void DS<G,I,M,container>::do_dss( double alpha, const container& f, double beta, container& dssf)
+{
+    m_fa(einsPlus,  f, m_tempP);
+    m_fa(einsMinus, f, m_tempM);
+    dg::blas1::pointwiseDot( 1., m_tempP, m_fa.hp_inv(), 1., m_tempM, m_fa.hm_inv(), 0., m_tempM);
+    dg::blas1::pointwiseDot( f,  m_fa.hp_inv(), m_temp0);
+    dg::blas1::pointwiseDot( 2.*alpha, m_fa.hz_inv(), m_tempM, -2.*alpha, m_fa.hm_inv(), m_temp0, beta, dssf);
+
 }
 ///@endcond
 
diff --git a/src/heat/heat.cuh b/src/heat/heat.cuh
index 3f74a16bb..37b0ab08c 100644
--- a/src/heat/heat.cuh
+++ b/src/heat/heat.cuh
@@ -22,14 +22,14 @@ struct Implicit
     Implicit( const Geometry& g, Parameters p, dg::geo::solovev::Parameters gp):
         p(p),
         gp(gp),
-        dsNU_(dg::geo::createSolovevField( gp), g g.bcx(), g.bcy(),
-              dg::geo::PsiLimiter( dg::geo::solovev::Psip(gp), 
+        dsNU_(dg::geo::createSolovevField( gp), g, g.bcx(), g.bcy(),
+              dg::geo::PsiLimiter( dg::geo::solovev::Psip(gp), gp.psipmaxlim), 
               dg::normed, dg::forward,
-              gp.psipmaxlim),gp.rk4eps, 1,1, true, true), 
+              gp.rk4eps, 1,1, true, true), 
         elliptic( g, dg::normed, dg::forward)
     {
         using namespace dg::geo::solovev;
-        MagneticField c(gp);
+        dg::geo::TokamakMagneticField c = dg::geo::createSolovevField(gp);
         container bfield = dg::pullback( dg::geo::FieldR(c),g);
         elliptic.set_x( bfield);
         bfield = dg::pullback( dg::geo::FieldZ(c),g);
@@ -179,11 +179,15 @@ void Explicit<G,I,M,V>::energies( std::vector<V>& y)
     if (p.p_diff ==3)    {
         // (D) nonadjoint with direct method
         dsNU_.forward( y[0], omega); 
-        dsNU_.forwardTD(omega,lambda);
+        dg::blas1::pointwiseDot(omega, m_invB, omega);
+        dsNU_.backward(omega,lambda);
+        dg::blas1::pointwiseDivide(lambda, m_invB, lambda);
         dg::blas1::axpby( 0.5, lambda, 0.,chi,chi); 
 
         dsNU_.backward( y[0], omega); 
-        dsNU_.backwardTD(omega,lambda);
+        dg::blas1::pointwiseDot(omega, m_invB, omega);
+        dsNU_.forward(omega,lambda);
+        dg::blas1::pointwiseDivide(lambda, m_invB, lambda);
         dg::blas1::axpby( 0.5, lambda, 1., chi,chi); 
 //         Dpar[0]= p.nu_parallel*dg::blas2::dot(one, w3d, chi); 
         Dpar[0]= p.nu_parallel*dg::blas2::dot(y[0], w3d, chi); 
-- 
GitLab


From 08e9db71f32a9726ebad7099207ce5425044065f Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Fri, 27 Oct 2017 11:14:33 +0200
Subject: [PATCH 414/453] added include guards in ell_interpolation_t and
 ell_interpolation_b.cu

---
 config/README.md                      | 2 +-
 inc/dg/backend/ell_interpolation_b.cu | 9 +++++++++
 inc/dg/backend/ell_interpolation_t.cu | 9 +++++++++
 3 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/config/README.md b/config/README.md
index c6b266fb9..3495ab634 100644
--- a/config/README.md
+++ b/config/README.md
@@ -33,7 +33,7 @@ The main purpose of the file `feltor/config/devices/devices.mk` is to configure
 | value | description                              | flags                                    |
 | ----- | ---------------------------------------- | ---------------------------------------- |
 | gpu   | replaces the CC and CFLAGS variables with the nvcc versions and analogously MPICC and MPICFLAGS  | `CC = $(NVCC) --compiler-bindir $(CC)` `CFLAGS = $(NVCCARCH) $(NVCCFLAGS)` `MPICC = $(NVCC) --compiler-bindir $(MPICC)` `MPICFLAGS+= $(NVCCARCH) $(NVCCFLAGS)` |
-| !gpu  | if device != gpu all thrust device calls redirect to OpenMP using the THRUST_DEVICE_SYSTEM macro | `-x c++` `-DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_OMP` and `$(OMPFLAG)` added to both CFLAGS and MPICFLAGS |
+| !gpu  | if device != gpu all thrust device calls redirect to OpenMP using the THRUST_DEVICE_SYSTEM macro | `-x c++` `-DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_OMP` and `$(OMPFLAG)` added to CFLAGS, `MPICFLAGS+=$(CFLAGS)` |
 | omp  | specify OPT for OpenMP | OPT = -O3                   |
 | mic   | specify OPT for Intel Xeon Phi architecture | OPT = -O3 -xMIC-AVX512                   |
 | skl   | specify OPT for Intel Skylake processors (icc only) | OPT = -xCORE-AVX512 -mtune=skylake -O3   |
diff --git a/inc/dg/backend/ell_interpolation_b.cu b/inc/dg/backend/ell_interpolation_b.cu
index 60bcdaf42..bd59a075f 100644
--- a/inc/dg/backend/ell_interpolation_b.cu
+++ b/inc/dg/backend/ell_interpolation_b.cu
@@ -1,3 +1,11 @@
+#include <iostream>
+#ifndef __NVCC__
+#warning "This program has to be compiled with nvcc!"
+int main(){
+    std::cout << "This program has to be compiled with nvcc!\n";
+    return 0;
+}
+#else
 #include <cusp/print.h>
 #include "timer.cuh"
 #include "xspacelib.cuh"
@@ -92,3 +100,4 @@ int main()
     
     return 0;
 }
+#endif //__NVCC__
diff --git a/inc/dg/backend/ell_interpolation_t.cu b/inc/dg/backend/ell_interpolation_t.cu
index 16fd84505..5e0edfd97 100644
--- a/inc/dg/backend/ell_interpolation_t.cu
+++ b/inc/dg/backend/ell_interpolation_t.cu
@@ -1,3 +1,11 @@
+#include <iostream>
+#ifndef __NVCC__
+#warning "This program has to be compiled with nvcc!"
+int main(){
+    std::cout << "This program has to be compiled with nvcc!\n";
+    return 0;
+}
+#else
 //#include <cusp/coo_matrix.h>
 //#include <cusp/print.h>
 #include "xspacelib.cuh"
@@ -91,3 +99,4 @@ int main()
     
     return 0;
 }
+#endif //__NVCC__
-- 
GitLab


From ac9355cbe7a0dd399fd70945fe9681e673f71688 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 30 Oct 2017 11:15:36 +0100
Subject: [PATCH 415/453] added weights function in refined_elliptic.h

---
 inc/dg/refined_elliptic.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/inc/dg/refined_elliptic.h b/inc/dg/refined_elliptic.h
index 8850e976f..1fc364444 100644
--- a/inc/dg/refined_elliptic.h
+++ b/inc/dg/refined_elliptic.h
@@ -63,6 +63,7 @@ class RefinedElliptic
      * @return inverse weights
      */
     const container& inv_weights()const {return inv_weights_;}
+    const container& weights()const {return weights_;}
     /**
      * @brief Returns the preconditioner to use in conjugate gradient
      *
-- 
GitLab


From 5d737f1354f10824d0cb338068db287026a34fcd Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Tue, 31 Oct 2017 15:10:25 +0100
Subject: [PATCH 416/453] added TOC in README.md and update asela and feltor a
 bit

---
 README.md                  |  3 +++
 src/asela/asela.cu         | 12 +++++-------
 src/asela/asela.cuh        | 14 +++++++-------
 src/asela/asela_hpc.cu     | 10 +++++-----
 src/asela/asela_mpi.cu     | 10 +++++-----
 src/feltor2D/feltor.cu     | 18 +++++++++---------
 src/feltor2D/feltor_hpc.cu | 16 ++++++++--------
 src/feltor2D/feltor_mpi.cu | 10 +++++-----
 8 files changed, 47 insertions(+), 46 deletions(-)

diff --git a/README.md b/README.md
index 231fd34eb..027ed9df4 100644
--- a/README.md
+++ b/README.md
@@ -11,7 +11,10 @@ Our core level functions are parallelized for a variety of hardware from multi-c
 [![DOI](https://zenodo.org/badge/14143578.svg)](https://zenodo.org/badge/latestdoi/14143578)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
 
+[TOC]
+
 ## 1. Quick start guide
+
 Go ahead and clone our library into any folder you like 
 ```sh
 git clone https://www.github.com/feltor-dev/feltor
diff --git a/src/asela/asela.cu b/src/asela/asela.cu
index a3c81ef5e..59e3cf4e4 100644
--- a/src/asela/asela.cu
+++ b/src/asela/asela.cu
@@ -25,8 +25,6 @@
 */
 typedef dg::FieldAligned< dg::CylindricalGrid3d, dg::IDMatrix, dg::DVec> DFA;
 
-using namespace dg::geo::solovev;
-
 int main( int argc, char* argv[])
 {
     ////Parameter initialisation ////////////////////////////////////////////
@@ -51,8 +49,8 @@ int main( int argc, char* argv[])
         std::cerr << "ERROR: Too many arguments!\nUsage: "<< argv[0]<<" [inputfile] [geomfile] \n";
         return -1;
     }
-    const eule::Parameters p( js);
-    const dg::geo::solovev::GeomParameters gp(gs);
+    const asela::Parameters p( js);
+    const dg::geo::solovev::Parameters gp(gs);
     p.display( std::cout);
     gp.display( std::cout);
     /////////glfw initialisation ////////////////////////////////////////////
@@ -72,9 +70,9 @@ int main( int argc, char* argv[])
 
     dg::CylindricalGrid3d grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n, p.Nx, p.Ny, p.Nz, p.bc, p.bc, dg::PER);      //create RHS 
     std::cout << "Constructing Asela...\n";
-    eule::Asela<dg::CylindricalGrid3d, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec> asela( grid, p,gp); //initialize before rolkar!
-    std::cout << "Constructing Rolkar...\n";
-    eule::Rolkar<dg::CylindricalGrid3d, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec> rolkar(  grid, p, gp, asela.ds(), asela.dsDIR());
+    asela::Asela<dg::CylindricalGrid3d, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec> asela( grid, p,gp); //initialize before rolkar!
+    std::cout << "Constructing Implicit...\n";
+    asela::Implicit<dg::CylindricalGrid3d, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec> rolkar(  grid, p, gp, asela.ds(), asela.dsDIR());
     std::cout << "Done!\n";
 
    /////////////////////The initial field///////////////////////////////////////////
diff --git a/src/asela/asela.cuh b/src/asela/asela.cuh
index b5e9224c7..bd8c51bd1 100644
--- a/src/asela/asela.cuh
+++ b/src/asela/asela.cuh
@@ -11,7 +11,7 @@
 #endif //DG_BENCHMARK
 
 // #define APAR
-namespace eule
+namespace asela
 {
 ///@addtogroup solver
 ///@{
@@ -31,7 +31,7 @@ namespace eule
  * @tparam container The container class
  */
 template<class Geometry, class DS, class Matrix, class container>
-struct Rolkar
+struct Implicit
 {
         /**
      * @brief Construct from parameters
@@ -41,7 +41,7 @@ struct Rolkar
      * @param p the physics parameters
      * @param gp the geometry parameters
      */
-    Rolkar( const Geometry& g, eule::Parameters p, dg::geo::solovev::GeomParameters gp, DS& dsN, DS& dsDIR):
+    Implicit( const Geometry& g, eule::Parameters p, dg::geo::solovev::Parameters gp, DS& dsN, DS& dsDIR):
         p(p),
         gp(gp),
         LaplacianM_perpN  ( g, g.bcx(), g.bcy(), dg::normed, dg::centered),
@@ -119,7 +119,7 @@ struct Rolkar
      */
   private:
     const eule::Parameters p;
-    const dg::geo::solovev::GeomParameters gp;
+    const dg::geo::solovev::Parameters gp;
     container temp;
     container dampgauss_;
     dg::Elliptic<Geometry, Matrix, container> LaplacianM_perpN,LaplacianM_perpDIR;
@@ -145,7 +145,7 @@ struct Asela
      * @param p the physics parameters
      * @param gp the geometry parameters
      */
-    Asela( const Geometry& g, eule::Parameters p, dg::geo::solovev::GeomParameters gp);
+    Asela( const Geometry& g, eule::Parameters p, dg::geo::solovev::Parameters gp);
     /**
      * @brief Return a ds class for evaluation purposes
      *
@@ -272,7 +272,7 @@ struct Asela
     dg::Invert<container> invert_maxwell, invert_pol, invert_invgammaN,invert_invgammaNW,invert_invgammaA, invert_invgammaPhi;
 
     const eule::Parameters p;
-    const dg::geo::solovev::GeomParameters gp;
+    const dg::geo::solovev::Parameters gp;
 
     double mass_, energy_, diff_, ediff_, aligned_;
     std::vector<double> evec;
@@ -281,7 +281,7 @@ struct Asela
 ///@}
 
 template<class Grid, class DS, class Matrix, class container>
-Asela<Grid, DS, Matrix, container>::Asela( const Grid& g, Parameters p, dg::geo::solovev::GeomParameters gp): 
+Asela<Grid, DS, Matrix, container>::Asela( const Grid& g, Parameters p, dg::geo::solovev::Parameters gp): 
     dsDIR_( typename DS::FieldAligned( 
                 dg::geo::Field<dg::geo::solovev::MagneticField>(
                     dg::geo::solovev::MagneticField(gp), gp.R_0
diff --git a/src/asela/asela_hpc.cu b/src/asela/asela_hpc.cu
index 253de85eb..31f06ca7c 100644
--- a/src/asela/asela_hpc.cu
+++ b/src/asela/asela_hpc.cu
@@ -40,8 +40,8 @@ int main( int argc, char* argv[])
         reader.parse(is,js,false);
         reader.parse(ks,gs,false);
     }
-    const eule::Parameters p( js);
-    const dg::geo::solovev::GeomParameters gp(gs);
+    const asela::Parameters p( js);
+    const dg::geo::solovev::Parameters gp(gs);
     p.display( std::cout);
     gp.display( std::cout);
     std::string input = js.toStyledString(), geom = gs.toStyledString();
@@ -58,9 +58,9 @@ int main( int argc, char* argv[])
      
     //create RHS 
     std::cout << "Constructing Asela...\n";
-    eule::Asela<dg::CylindricalGrid3d, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec> asela( grid, p, gp); //initialize before rolkar!
-    std::cout << "Constructing Rolkar...\n";
-    eule::Rolkar< dg::CylindricalGrid3d, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec > rolkar( grid, p, gp, asela.ds(), asela.dsDIR());
+    asela::Asela<dg::CylindricalGrid3d, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec> asela( grid, p, gp); //initialize before rolkar!
+    std::cout << "Constructing Implicit...\n";
+    asela::Implicit< dg::CylindricalGrid3d, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec > rolkar( grid, p, gp, asela.ds(), asela.dsDIR());
     std::cout << "Done!\n";
 
     /////////////////////The initial field//////////////////////////////////////////
diff --git a/src/asela/asela_mpi.cu b/src/asela/asela_mpi.cu
index 5d669187b..579f447ca 100644
--- a/src/asela/asela_mpi.cu
+++ b/src/asela/asela_mpi.cu
@@ -72,8 +72,8 @@ int main( int argc, char* argv[])
         reader.parse(is,js,false);
         reader.parse(ks,gs,false);
     }
-    const eule::Parameters p( js);
-    const dg::geo::solovev::GeomParameters gp(gs);
+    const asela::Parameters p( js);
+    const dg::geo::solovev::Parameters gp(gs);
     if(rank==0)p.display( std::cout);
     if(rank==0)gp.display( std::cout);
     std::string input = js.toStyledString(), geom = gs.toStyledString();
@@ -90,9 +90,9 @@ int main( int argc, char* argv[])
      
     //create RHS 
     if(rank==0)std::cout << "Constructing Asela...\n";
-    eule::Asela<dg::CylindricalMPIGrid3d, dg::DS<DFA, dg::MDMatrix, dg::MDVec>, dg::MDMatrix, dg::MDVec> asela( grid, p, gp); //initialize before rolkar!
-    if(rank==0)std::cout << "Constructing Rolkar...\n";
-    eule::Rolkar< dg::CylindricalMPIGrid3d, dg::DS<DFA, dg::MDMatrix, dg::MDVec>, dg::MDMatrix, dg::MDVec > rolkar( grid, p, gp, asela.ds(), asela.dsDIR());
+    asela::Asela<dg::CylindricalMPIGrid3d, dg::DS<DFA, dg::MDMatrix, dg::MDVec>, dg::MDMatrix, dg::MDVec> asela( grid, p, gp); //initialize before rolkar!
+    if(rank==0)std::cout << "Constructing Implicit...\n";
+    asela::Implicit< dg::CylindricalMPIGrid3d, dg::DS<DFA, dg::MDMatrix, dg::MDVec>, dg::MDMatrix, dg::MDVec > rolkar( grid, p, gp, asela.ds(), asela.dsDIR());
     if(rank==0)std::cout << "Done!\n";
 
     /////////////////////The initial field/////////////////////////////////////////
diff --git a/src/feltor2D/feltor.cu b/src/feltor2D/feltor.cu
index 42d2dcdad..b0cd31577 100644
--- a/src/feltor2D/feltor.cu
+++ b/src/feltor2D/feltor.cu
@@ -47,7 +47,7 @@ int main( int argc, char* argv[])
         std::cerr << "ERROR: Too many arguments!\nUsage: "<< argv[0]<<" [inputfile] [geomfile] \n";
         return -1;
     }
-    const eule::Parameters p( js);
+    const feltor::Parameters p( js);
     const dg::geo::solovev::GeomParameters gp(gs);
     p.display( std::cout);
     gp.display( std::cout);
@@ -67,9 +67,9 @@ int main( int argc, char* argv[])
      dg::CylindricalGrid3d grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n, p.Nx, p.Ny, 1, p.bc, p.bc, dg::PER);  
     //create RHS 
     std::cout << "Constructing Feltor...\n";
-    eule::Feltor<dg::CylindricalGrid3d, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec > feltor( grid, p, gp); //initialize before rolkar!
-    std::cout << "Constructing Rolkar...\n";
-    eule::Rolkar<dg::CylindricalGrid3d, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec> rolkar( grid, p, gp, feltor.ds(), feltor.dsDIR());
+    feltor::Explicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec > feltor (grid, p, gp); //initialize before implicit!
+    std::cout << "Constructing Implicit...\n";
+    feltor::Implicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec> implicit( grid, p, gp, feltor.ds(), feltor.dsDIR());
     std::cout << "Done!\n";
 
     /////////////////////The initial field///////////////////////////////////////////
@@ -107,11 +107,11 @@ int main( int argc, char* argv[])
 
     std::cout << "initialize karniadakis" << std::endl;
     dg::Karniadakis< std::vector<dg::DVec> > karniadakis( y0, y0[0].size(), p.eps_time);
-    karniadakis.init( feltor, rolkar, y0, p.dt);
+    karniadakis.init( feltor, implicit, y0, p.dt);
     std::cout << "Done!\n";
     //std::cout << "first karniadakis" << std::endl;
 
-    //karniadakis( feltor, rolkar, y0); //now energies and potential are at time 0
+    //karniadakis( feltor, implicit, y0); //now energies and potential are at time 0
     //std::cout << "Done!\n";
 
 
@@ -170,7 +170,7 @@ int main( int argc, char* argv[])
         //draw potential
         //transform to Vor
 //         dvisual=feltor.potential()[0];
-//         dg::blas2::gemv( rolkar.laplacianM(), dvisual, y1[1]);
+//         dg::blas2::gemv( implicit.laplacianM(), dvisual, y1[1]);
 //         hvisual = y1[1];
         dg::blas1::transfer( feltor.potential()[0], hvisual);
         dg::blas2::gemv( equi, hvisual, visual);
@@ -208,7 +208,7 @@ int main( int argc, char* argv[])
         //draw VOR
         //transform to Vor
         dvisual=feltor.potential()[0];
-        dg::blas2::gemv( rolkar.laplacianM(), dvisual, y1[1]);
+        dg::blas2::gemv( implicit.laplacianM(), dvisual, y1[1]);
         dg::blas1::transfer( y1[1], hvisual);
         dg::blas2::gemv( equi, hvisual, visual);
         colors.scalemax() = (float)thrust::reduce( visual.begin(),visual.end(), 0.,thrust::maximum<double>()  );
@@ -232,7 +232,7 @@ int main( int argc, char* argv[])
 #endif//DG_BENCHMARK
         for( unsigned i=0; i<p.itstp; i++)
         {
-            try{ karniadakis( feltor, rolkar, y0);}
+            try{ karniadakis( feltor, implicit, y0);}
             catch( dg::Fail& fail) { 
                 std::cerr << "CG failed to converge to "<<fail.epsilon()<<"\n";
                 std::cerr << "Does Simulation respect CFL condition?\n";
diff --git a/src/feltor2D/feltor_hpc.cu b/src/feltor2D/feltor_hpc.cu
index 7e29224a8..7f1d27f29 100644
--- a/src/feltor2D/feltor_hpc.cu
+++ b/src/feltor2D/feltor_hpc.cu
@@ -42,8 +42,8 @@ int main( int argc, char* argv[])
         reader.parse(is,js,false);
         reader.parse(ks,gs,false);
     }
-    const eule::Parameters p( js);
-    const dg::geo::solovev::GeomParameters gp(gs);
+    const feltor::Parameters p( js);
+    const dg::geo::solovev::Parameters gp(gs);
     p.display( std::cout);
     gp.display( std::cout);
     std::string input = js.toStyledString(), geom = gs.toStyledString();
@@ -58,14 +58,14 @@ int main( int argc, char* argv[])
     dg::CylindricalGrid3d grid_out( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n_out, p.Nx_out, p.Ny_out,1,p.bc, p.bc, dg::PER);  
     //create RHS 
     std::cout << "Constructing Feltor...\n";
-    eule::Feltor<dg::CylindricalGrid3d, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec > feltor( grid, p, gp); //initialize before rolkar!
-    std::cout << "Constructing Rolkar...\n";
-    eule::Rolkar<dg::CylindricalGrid3d, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec> rolkar( grid, p, gp, feltor.ds(), feltor.dsDIR());
+    feltor::Explicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec > feltor( grid, p, gp); //initialize before implicit!
+    std::cout << "Constructing Implicit...\n";
+    feltor::Implicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec> implicit( grid, p, gp, feltor.ds(), feltor.dsDIR());
     std::cout << "Done!\n";
 
     /////////////////////The initial field///////////////////////////////////////////
     //background profile
-    dg::geo::Nprofile<Psip> prof(p.bgprofamp, p.nprofileamp, gp, Psip(gp)); //initial background profile
+    dg::geo::Nprofile prof(p.bgprofamp, p.nprofileamp, gp, Psip(gp)); //initial background profile
     std::vector<dg::DVec> y0(4, dg::evaluate( prof, grid)), y1(y0); 
     //initial perturbation
     if (p.mode == 0  || p.mode ==1) 
@@ -97,7 +97,7 @@ int main( int argc, char* argv[])
     
     std::cout << "initialize karniadakis" << std::endl;
     dg::Karniadakis< std::vector<dg::DVec> > karniadakis( y0, y0[0].size(), p.eps_time);
-    karniadakis.init( feltor, rolkar, y0, p.dt);
+    karniadakis.init( feltor, implicit, y0, p.dt);
 //     feltor.energies(y0); //now energies and potential are at time 0
     std::cout << "Done!\n";
     /////////////////////////////set up netcdf/////////////////////////////////////
@@ -196,7 +196,7 @@ int main( int argc, char* argv[])
 #endif//DG_BENCHMARK
         for( unsigned j=0; j<p.itstp; j++)
         {
-            try{ karniadakis( feltor, rolkar, y0);}
+            try{ karniadakis( feltor, implicit, y0);}
             catch( dg::Fail& fail) { 
                 std::cerr << "CG failed to converge to "<<fail.epsilon()<<"\n";
                 std::cerr << "Does Simulation respect CFL condition?\n";
diff --git a/src/feltor2D/feltor_mpi.cu b/src/feltor2D/feltor_mpi.cu
index 4f504d989..f5aa594bd 100644
--- a/src/feltor2D/feltor_mpi.cu
+++ b/src/feltor2D/feltor_mpi.cu
@@ -76,9 +76,9 @@ int main( int argc, char* argv[])
     dg::CylindricalMPIGrid3d grid_out( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n_out, p.Nx_out, p.Ny_out, 1, p.bc, p.bc, dg::PER, comm);  
      
     if(rank==0)std::cout << "Constructing Feltor...\n";
-    eule::Feltor<dg::CylindricalMPIGrid3d, dg::DS<DFA, dg::MDMatrix, dg::MDVec>, dg::MDMatrix, dg::MDVec> feltor( grid, p, gp); //initialize before rolkar!
-    if(rank==0)std::cout << "Constructing Rolkar...\n";
-    eule::Rolkar< dg::CylindricalMPIGrid3d, dg::DS<DFA, dg::MDMatrix, dg::MDVec>, dg::MDMatrix, dg::MDVec > rolkar( grid, p, gp, feltor.ds(), feltor.dsDIR());
+    feltor::Explicit<dg::CylindricalMPIGrid3d, dg::DS<DFA, dg::MDMatrix, dg::MDVec>, dg::MDMatrix, dg::MDVec> feltor( grid, p, gp); //initialize before implicit!
+    if(rank==0)std::cout << "Constructing implicit...\n";
+    feltor::Implicit< dg::CylindricalMPIGrid3d, dg::DS<DFA, dg::MDMatrix, dg::MDVec>, dg::MDMatrix, dg::MDVec > implicit( grid, p, gp, feltor.ds(), feltor.dsDIR());
     if(rank==0)std::cout << "Done!\n";
 
     /////////////////////The initial field///////////////////////////////////////////
@@ -112,7 +112,7 @@ int main( int argc, char* argv[])
     dg::blas1::axpby( 0., y0[3], 0., y0[3]); //set Ui = 0
     
     dg::Karniadakis< std::vector<dg::MDVec> > karniadakis( y0, y0[0].size(), p.eps_time);
-    karniadakis.init( feltor, rolkar, y0, p.dt);
+    karniadakis.init( feltor, implicit, y0, p.dt);
     //feltor.energies(y0); //now energies and potential are at time 0
     /////////////////////////////set up netcdf/////////////////////////////////
     file::NC_Error_Handle err;
@@ -229,7 +229,7 @@ int main( int argc, char* argv[])
 #endif//DG_BENCHMARK
         for( unsigned j=0; j<p.itstp; j++)
         {
-            try{ karniadakis( feltor, rolkar, y0);}
+            try{ karniadakis( feltor, implicit, y0);}
             catch( dg::Fail& fail) { 
                 if(rank==0)std::cerr << "CG failed to converge to "<<fail.epsilon()<<"\n";
                 if(rank==0)std::cerr << "Does Simulation respect CFL condition?\n";
-- 
GitLab


From 8c2efb08cdc19d7c31f9aa9c952d0d05fa63886e Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Tue, 31 Oct 2017 17:32:15 +0100
Subject: [PATCH 417/453] Update README with a table of contents

---
 README.md | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 027ed9df4..a260bed97 100644
--- a/README.md
+++ b/README.md
@@ -11,7 +11,16 @@ Our core level functions are parallelized for a variety of hardware from multi-c
 [![DOI](https://zenodo.org/badge/14143578.svg)](https://zenodo.org/badge/latestdoi/14143578)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
 
-[TOC]
+## Table of contents
+
+* [1. Quick start guide](#1-quick-start-guide)
+    * [Using FELTOR's code projects](#using-feltors-code-projects)
+    * [Running a FELTOR simulation](#running-a-feltor-simulation)
+    * [Using FELTOR as a library](#using-feltor-as-a-library)
+* [2. Further Reading](#2-further-reading)
+* [3. Contributions and Acknowledgements](#3-contributions-and-acknowledgements)
+* [4. License](#4-license)
+* [5. Official Releases](#5-official-releases)
 
 ## 1. Quick start guide
 
@@ -85,7 +94,7 @@ then tell how many process you want to use in the x-, y- and z- direction, for e
 when prompted for input vector sizes type for example
 `3 100 100 10` (number of cells divided by number of procs must be an integer number). If you compiled for MPI+OpenMP, you can set the number of OpenMP threads with e.g. `export OMP_NUM_THREADS=2`.
 
-### Running a simulation
+### Running a FELTOR simulation
 
 Now, we want to compile and run a simulation program. First, we have to download and install some libraries for I/O-operations.
 
@@ -222,7 +231,7 @@ and from Intel Barcelona
 ## 4. License 
 FELTOR is free software and licensed under the very permissive MIT license. It was originally developed by Matthias Wiesenberger and Markus Held.
 
-## Official releases 
+## 5. Official releases 
 Our latest code release has a shiny DOI badge from zenodo
 
 [![DOI](https://zenodo.org/badge/14143578.svg)](https://zenodo.org/badge/latestdoi/14143578)
-- 
GitLab


From f2c9d3f0e47c2be64f4b74f258a22f430a51389c Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Tue, 31 Oct 2017 17:41:03 +0100
Subject: [PATCH 418/453] added remark about c++98 standard in configuration

---
 config/README.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/config/README.md b/config/README.md
index 3495ab634..3b531a0f3 100644
--- a/config/README.md
+++ b/config/README.md
@@ -58,7 +58,8 @@ make blas_mpib device=gpu NVCCARCH='-arch sm_60' OPT=-O2
 
 ### General Remarks
  - If MPI is used in connection with the gpu backend, the mpi installation needs to be **cuda-aware**
- - if `icc` is used as the C++ compiler the `-restrict` option has to be used to enable the recognition of the restrict keyword
- - support for OpenMP-4 is recommended (at least gcc-4.9 or icc-15), but not mandatory
+ - If `icc` is used as the C++ compiler the `-restrict` option has to be used to enable the recognition of the restrict keyword
+ - Support for OpenMP-4 is recommended (at least gcc-4.9 or icc-15), but not mandatory
+ - The library headers are compliant with the c++98 standard but we reserve the right to change that in future updates
  
 
-- 
GitLab


From 124a1641e651950922d880f40cf4b6c9bd684d7b Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Tue, 31 Oct 2017 18:07:37 +0100
Subject: [PATCH 419/453] change acknowledgements and terms of use in README

---
 README.md | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index a260bed97..7b3d354c5 100644
--- a/README.md
+++ b/README.md
@@ -18,8 +18,8 @@ Our core level functions are parallelized for a variety of hardware from multi-c
     * [Running a FELTOR simulation](#running-a-feltor-simulation)
     * [Using FELTOR as a library](#using-feltor-as-a-library)
 * [2. Further Reading](#2-further-reading)
-* [3. Contributions and Acknowledgements](#3-contributions-and-acknowledgements)
-* [4. License](#4-license)
+* [3. Acknowledgements](#3-acknowledgements)
+* [4. Terms of use](#4-terms-of-use)
 * [5. Official Releases](#5-official-releases)
 
 ## 1. Quick start guide
@@ -214,9 +214,10 @@ Note the striking similarity to the previous program. Especially the line callin
 Please check out our [wiki pages](https://github.com/feltor-dev/feltor/wiki) for some general information, user oriented documentation and Troubleshooting. Moreover, we maintain tex files in every src folder for technical documentation, which can be compiled using pdflatex with `make doc ` in the respective src folder.
 The [developer oriented documentation](http://feltor-dev.github.io/feltor/inc/dg/html/modules.html) of the dG library was generated with [Doxygen](http://www.doxygen.org) and LateX. You can generate a local version including informative pdf writeups on implemented numerical methods directly from source code. This depends on the `doxygen`, `libjs-mathjax` and `graphviz` packages and LateX. Type `make doc` in the folder `path/to/feltor/doc` and open `index.html` (a symbolic link to `dg/html/modules.html`) with your favorite browser. Links to the pdf writeups can be found among other places on the Mainpage. 
 
-## 3. Contributions and Acknowledgements
-For instructions on how to contribute read the [wiki page](https://github.com/feltor-dev/feltor/wiki/Contributions).
-We gratefully acknowledge contributions from 
+## 3. Acknowledgements
+
+FELTOR is developed by Matthias Wiesenberger and Markus Held and includes contributions from an increasing number of people.
+We gratefully acknowledge fruitful discussions and code contribution from 
 - Ralph Kube
 - Eduard Reiter
 - Lukas Einkemmer
@@ -228,10 +229,13 @@ We further acknowledge support on the Knights landing architecture from the High
 
 and from Intel Barcelona
 - Harald Servat
-## 4. License 
-FELTOR is free software and licensed under the very permissive MIT license. It was originally developed by Matthias Wiesenberger and Markus Held.
+
+## 4. Terms of use
+
+FELTOR is [fair](https://www.force11.org/fairprinciples) software and licensed under the very permissive [MIT license](LICENSE).  The MIT License gives you great freedom in what you do with the code. It basically allows you to do anything you like as long as you name us (Matthias Wiesenberger and Markus Held) as creators, in particular in publications that use FELTOR to produce results. In this case we suggest to take a snapshot of the used code and create and cite a DOI via e.g. [Zenodo](http://www.zenodo.org) or to cite one of the existing DOIs if you did not alter the contained code in any way. We are happy if you cite our papers, but you don't have to just because you used our code and we certainly do not demand to be coauthors when we do not contribute directly to your results. 
 
 ## 5. Official releases 
+
 Our latest code release has a shiny DOI badge from zenodo
 
 [![DOI](https://zenodo.org/badge/14143578.svg)](https://zenodo.org/badge/latestdoi/14143578)
-- 
GitLab


From 84bb0edde35c2cf56a26d18aa29745873f4f4817 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Tue, 31 Oct 2017 21:08:22 +0100
Subject: [PATCH 420/453] refinements to README

---
 README.md | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/README.md b/README.md
index 7b3d354c5..53689bebd 100644
--- a/README.md
+++ b/README.md
@@ -13,14 +13,14 @@ Our core level functions are parallelized for a variety of hardware from multi-c
 
 ## Table of contents
 
-* [1. Quick start guide](#1-quick-start-guide)
-    * [Using FELTOR's code projects](#using-feltors-code-projects)
-    * [Running a FELTOR simulation](#running-a-feltor-simulation)
-    * [Using FELTOR as a library](#using-feltor-as-a-library)
-* [2. Further Reading](#2-further-reading)
-* [3. Acknowledgements](#3-acknowledgements)
-* [4. Terms of use](#4-terms-of-use)
-* [5. Official Releases](#5-official-releases)
+[1. Quick start guide](#1-quick-start-guide)
+​	[Using FELTOR's code projects](#using-feltors-code-projects)
+​	[Running a FELTOR simulation](#running-a-feltor-simulation)
+​	[Using FELTOR as a library](#using-feltor-as-a-library)
+[2. Further Reading](#2-further-reading)
+[3. Acknowledgements](#3-acknowledgements)
+[4. Terms of use](#4-terms-of-use)
+[5. Official Releases](#5-official-releases)
 
 ## 1. Quick start guide
 
@@ -212,11 +212,11 @@ Note the striking similarity to the previous program. Especially the line callin
 ## 2. Further reading
 
 Please check out our [wiki pages](https://github.com/feltor-dev/feltor/wiki) for some general information, user oriented documentation and Troubleshooting. Moreover, we maintain tex files in every src folder for technical documentation, which can be compiled using pdflatex with `make doc ` in the respective src folder.
-The [developer oriented documentation](http://feltor-dev.github.io/feltor/inc/dg/html/modules.html) of the dG library was generated with [Doxygen](http://www.doxygen.org) and LateX. You can generate a local version including informative pdf writeups on implemented numerical methods directly from source code. This depends on the `doxygen`, `libjs-mathjax` and `graphviz` packages and LateX. Type `make doc` in the folder `path/to/feltor/doc` and open `index.html` (a symbolic link to `dg/html/modules.html`) with your favorite browser. Links to the pdf writeups can be found among other places on the Mainpage. 
+The [developer oriented documentation](http://feltor-dev.github.io/feltor/inc/dg/html/modules.html) of the dG library was generated with [Doxygen](http://www.doxygen.org) and LateX. You can generate a local version including informative pdf writeups on implemented numerical methods directly from source code. This depends on the `doxygen`, `libjs-mathjax` and `graphviz` packages and LateX. Type `make doc` in the folder `path/to/feltor/doc` and open `index.html` (a symbolic link to `dg/html/modules.html`) with your favorite browser. Links to the pdf writeups can be found, among other places, on the Mainpage. 
 
 ## 3. Acknowledgements
 
-FELTOR is developed by Matthias Wiesenberger and Markus Held and includes contributions from an increasing number of people.
+FELTOR is developed by Matthias Wiesenberger and Markus Held and receives contributions from an increasing number of people.
 We gratefully acknowledge fruitful discussions and code contribution from 
 - Ralph Kube
 - Eduard Reiter
@@ -232,7 +232,7 @@ and from Intel Barcelona
 
 ## 4. Terms of use
 
-FELTOR is [fair](https://www.force11.org/fairprinciples) software and licensed under the very permissive [MIT license](LICENSE).  The MIT License gives you great freedom in what you do with the code. It basically allows you to do anything you like as long as you name us (Matthias Wiesenberger and Markus Held) as creators, in particular in publications that use FELTOR to produce results. In this case we suggest to take a snapshot of the used code and create and cite a DOI via e.g. [Zenodo](http://www.zenodo.org) or to cite one of the existing DOIs if you did not alter the contained code in any way. We are happy if you cite our papers, but you don't have to just because you used our code and we certainly do not demand to be coauthors when we do not contribute directly to your results. 
+FELTOR is [fair](https://www.force11.org/fairprinciples) software and licensed under the very permissive [MIT license](LICENSE). The MIT License grants you great freedom in what you do with the code as long as you name us (Matthias Wiesenberger and Markus Held) as creators, in particular in publications that use FELTOR to produce results. In this case we suggest to take a snapshot of the used code and create and cite a DOI via e.g. [Zenodo](http://www.zenodo.org) or to cite one of the existing DOIs if you did not alter the contained code in any way. We are happy if you cite our papers, but you don't have to just because you used our code and we certainly do not demand to be coauthors when we do not contribute directly to your results. 
 
 ## 5. Official releases 
 
-- 
GitLab


From 6170a68e49b39de4d5961d53d54bea5b1908a1cc Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Tue, 31 Oct 2017 23:06:32 +0100
Subject: [PATCH 421/453] moved from markdown README to asciidoc README due to
 richer support for TOC and code

---
 README.md | 361 +++++++++++++++++++++++++++++++++++-------------------
 1 file changed, 236 insertions(+), 125 deletions(-)

diff --git a/README.md b/README.md
index 53689bebd..6d7027d6c 100644
--- a/README.md
+++ b/README.md
@@ -1,111 +1,176 @@
-# Welcome to the FELTOR project!
+= Welcome to the FELTOR project!
+:source-highlighter: pygments
+:toc: macro
 
-![3dsimulation](3dpic.jpg)
+image::3dpic.jpg[3dsimulation]
 
-FELTOR (Full-F ELectromagnetic code in TORoidal geometry) is both a numerical library and a scientific software package built on top of it.
+FELTOR (Full-F ELectromagnetic code in TORoidal geometry) is both a
+numerical library and a scientific software package built on top of it.
 
-Its main physical target are plasma edge and scrape-off layer (gyro-)fluid simulations.
-The numerical methods centre around discontinuous Galerkin methods on structured grids. 
-Our core level functions are parallelized for a variety of hardware from multi-core cpu to hybrid MPI+GPU, which makes the library incredibly fast. 
+Its main physical target are plasma edge and scrape-off layer
+(gyro-)fluid simulations. The numerical methods centre around
+discontinuous Galerkin methods on structured grids. Our core level
+functions are parallelized for a variety of hardware from multi-core cpu
+to hybrid MPI{plus}GPU, which makes the library incredibly fast.
 
-[![DOI](https://zenodo.org/badge/14143578.svg)](https://zenodo.org/badge/latestdoi/14143578)
-[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
+https://zenodo.org/badge/latestdoi/14143578[image:https://zenodo.org/badge/14143578.svg[DOI]]
+link:LICENSE[image:https://img.shields.io/badge/License-MIT-yellow.svg[License:
+MIT]]
 
-## Table of contents
+toc::[]
 
-[1. Quick start guide](#1-quick-start-guide)
-​	[Using FELTOR's code projects](#using-feltors-code-projects)
-​	[Running a FELTOR simulation](#running-a-feltor-simulation)
-​	[Using FELTOR as a library](#using-feltor-as-a-library)
-[2. Further Reading](#2-further-reading)
-[3. Acknowledgements](#3-acknowledgements)
-[4. Terms of use](#4-terms-of-use)
-[5. Official Releases](#5-official-releases)
+== 1. Quick start guide
 
-## 1. Quick start guide
+Go ahead and clone our library into any folder you like
 
-Go ahead and clone our library into any folder you like 
-```sh
+[source,sh]
+----
 git clone https://www.github.com/feltor-dev/feltor
-```
-You also need to clone  [thrust]( https://github.com/thrust/thrust) and [cusp](https://github.com/cusplibrary/cusplibrary) distributed under the Apache-2.0 license. So again in a folder of your choice
-```sh
-git clone https://www.github.com/thrust/thrust
-git clone https://www.github.com/cusplibrary/cusplibrary
-```
-> Our code only depends on external libraries that are themselves openly available. We note here that we do not distribute copies of these libraries.
-
-
-### Using FELTOR's code projects
+----
 
-In order to compile one of the many codes inside FELTOR you need to tell the feltor configuration where the external libraries are located on your computer. The default way to do this is to go in your `HOME` directory, make an include directory and link the paths in this directory:
+You also need to clone https://github.com/thrust/thrust[thrust] and
+https://github.com/cusplibrary/cusplibrary[cusp] distributed under the
+Apache-2.0 license. So again in a folder of your choice
 
- ```sh
-cd ~
-mkdir include
-cd include
-ln -s path/to/thrust/thrust thrust
+[source,sh]
+----
+git clone https://www.github.com/thrust/thrust
+git clone https://www.github.com/cusplibrary/cusplibrary
+----
+
+____
+Our code only depends on external libraries that are themselves openly
+available. We note here that we do not distribute copies of these
+libraries.
+____
+
+=== Using FELTOR's code projects
+
+In order to compile one of the many codes inside FELTOR you need to tell
+the feltor configuration where the external libraries are located on
+your computer. The default way to do this is to go in your `HOME`
+directory, make an include directory and link the paths in this
+directory
+
+[source,sh]
+----
+cd ~ 
+mkdir include 
+cd include 
+ln -s path/to/thrust/thrust thrust 
 ln -s path/to/cusplibrary/cusp cusp
- ```
-> If you do not like this, you can also create your own config file as discribed [here](config/README.md).
+----
 
-Now let us compile the first benchmark program. 
+____
+If you do not like this, you can also create your own config file as
+discribed link:config/README.md[here].
+____
 
+Now let us compile the first benchmark program.
 
- ```sh
+[source,sh]
+----
 cd path/to/feltor/inc/dg
- 
+
 make blas_b device=omp #(for an OpenMP version)
- #or
-make blas_b device=gpu #(if you have a gpu and nvcc )
- ```
-> The minimum requirement to compile and run an application is a working C++ compiler (g++ per default) and a CPU. 
-> To simplify the compilation process we use the GNU Make utility, a standard build automation tool that automatically builds the executable program. 
-> We don't use new C++-11 standard features to avoid complications since some clusters are a bit behind on up-to-date compilers.
-> The OpenMP standard is natively supported by most recent C++ compilers.   
-> Our GPU backend uses the [Nvidia-CUDA](https://developer.nvidia.com/cuda-zone) programming environment and in order to compile and run a program for a GPU a user needs the nvcc CUDA compiler (available free of charge) and a NVidia GPU. However, we explicitly note here that due to the modular design of our software a user does not have to possess a GPU nor the nvcc compiler. The CPU version of the backend is equally valid and provides the same functionality.
-
-
-Run the code with 
-```sh
+#or
+make blas_b
+device=gpu #(if you have a gpu and nvcc )
+----
+
+____
+The minimum requirement
+to compile and run an application is a working C{plus}{plus} compiler (g{plus}{plus} per
+default) and a CPU. To simplify the compilation process we use the GNU
+Make utility, a standard build automation tool that automatically builds
+the executable program. We don't use new C{plus}{plus}-11 standard features to
+avoid complications since some clusters are a bit behind on up-to-date
+compilers. The OpenMP standard is natively supported by most recent
+C{plus}{plus} compilers. 
+Our GPU backend uses the
+https://developer.nvidia.com/cuda-zone[Nvidia-CUDA] programming
+environment and in order to compile and run a program for a GPU a user
+needs the nvcc CUDA compiler (available free of charge) and a NVidia
+GPU. However, we explicitly note here that due to the modular design of
+our software a user does not have to possess a GPU nor the nvcc
+compiler. The CPU version of the backend is equally valid and provides
+the same functionality.
+____
+
+Run the code with
+
+[source,sh]
+----
 ./blas_b 
-```
-and when prompted for input vector sizes type for example
-`3 100 100 10`
-which makes a grid with 3 polynomial coefficients, 100 cells in x, 100 cells in y and 10 in z. If you compiled for OpenMP, you can set the number of threads with e.g. `export OMP_NUM_THREADS=4`.
-> This is a benchmark program to benchmark various elemental functions the library is built on. Go ahead and vary the input parameters and
-> see how your hardware performs. You can compile and run any other program that ends in `_t.cu` (test programs) or `_b.cu` (benchmark programs) in `feltor/inc/dg` in this way. 
+----
+
+and when prompted for input vector sizes type for example `3 100 100 10`
+which makes a grid with 3 polynomial coefficients, 100 cells in x, 100
+cells in y and 10 in z. If you compiled for OpenMP, you can set the
+number of threads with e.g. `export OMP_NUM_THREADS=4`. 
+____
+This is a
+benchmark program to benchmark various elemental functions the library
+is built on. Go ahead and vary the input parameters and see how your
+hardware performs. You can compile and run any other program that ends
+in `_t.cu` (test programs) or `_b.cu` (benchmark programs) in
+`feltor/inc/dg` in this way.
+____
 
 Now, let us test the mpi setup 
-> You can of course skip this if you don't have mpi installed on your computer.
-> If you intend to use the MPI backend, an implementation library of the mpi standard is required. Per default `mpic++` is used for compilation.
-
-```sh
+____
+You can of course skip this if you
+don't have mpi installed on your computer. If you intend to use the
+MPI backend, an implementation library of the mpi standard is required.
+Per default `mpic++` is used for compilation.
+____
+
+[source,sh]
+----
 cd path/to/feltor/inc/dg
  
 make blas_mpib device=omp  # (for MPI+OpenMP)
 # or
 make blas_mpib device=gpu # (for MPI+GPU)
-```
-Run the code with
-`$ mpirun -n '# of procs' ./blas_mpib `
-then tell how many process you want to use in the x-, y- and z- direction, for example:
-`2 2 1` (i.e. 2 procs in x, 2 procs in y and 1 in z; total number of procs is 4)
-when prompted for input vector sizes type for example
-`3 100 100 10` (number of cells divided by number of procs must be an integer number). If you compiled for MPI+OpenMP, you can set the number of OpenMP threads with e.g. `export OMP_NUM_THREADS=2`.
-
-### Running a FELTOR simulation
-
-Now, we want to compile and run a simulation program. First, we have to download and install some libraries for I/O-operations.
-
-For data output we use the [NetCDF](http://www.unidata.ucar.edu/software/netcdf/) library under an MIT - like license. The underlying [HDF5](https://www.hdfgroup.org/HDF5/) library also uses a very permissive license. Note that for the mpi versions of applications you need to build hdf5 and netcdf with the --enable-parallel flag. Do NOT use the pnetcdf library, which uses the classic netcdf file format. 
-Our JSON input files are parsed by [JsonCpp](https://www.github.com/open-source-parsers/jsoncpp) distributed under the MIT license (use the 0.y.x branch to avoid C++-11 support).     
-> Some desktop applications in FELTOR use the [draw library]( https://github.com/mwiesenberger/draw) (developed by us also under MIT), which depends on OpenGL (s.a. [installation guide](http://en.wikibooks.org/wiki/OpenGL_Programming)) and [glfw](http://www.glfw.org), an OpenGL development library under a BSD-like license. You don't need these when you are on a cluster. 
-
-
-As in Step 3 you need to create links to the jsoncpp library include path (and optionally the draw library) in your include folder or provide the paths in your config file. We are ready to compile now
-
-```sh
+----
+
+Run the code with `$ mpirun -n '# of procs' ./blas_mpib` then tell how
+many process you want to use in the x-, y- and z- direction, for
+example: `2 2 1` (i.e. 2 procs in x, 2 procs in y and 1 in z; total
+number of procs is 4) when prompted for input vector sizes type for
+example `3 100 100 10` (number of cells divided by number of procs must
+be an integer number). If you compiled for MPI{plus}OpenMP, you can set the
+number of OpenMP threads with e.g. `export OMP_NUM_THREADS=2`.
+
+=== Running a FELTOR simulation
+
+Now, we want to compile and run a simulation program. First, we have to
+download and install some libraries for I/O-operations.
+
+For data output we use the
+http://www.unidata.ucar.edu/software/netcdf/[NetCDF] library under an
+MIT - like license. The underlying https://www.hdfgroup.org/HDF5/[HDF5]
+library also uses a very permissive license. Note that for the mpi
+versions of applications you need to build hdf5 and netcdf with the
+--enable-parallel flag. Do NOT use the pnetcdf library, which uses the
+classic netcdf file format. Our JSON input files are parsed by
+https://www.github.com/open-source-parsers/jsoncpp[JsonCpp] distributed
+under the MIT license (use the 0.y.x branch to avoid C{plus}{plus}-11 support). 
+____
+Some desktop applications in FELTOR use the
+https://github.com/mwiesenberger/draw[draw library] (developed by us
+also under MIT), which depends on OpenGL (s.a.
+http://en.wikibooks.org/wiki/OpenGL_Programming[installation guide]) and
+http://www.glfw.org[glfw], an OpenGL development library under a
+BSD-like license. You don't need these when you are on a cluster.
+____
+
+As in Step 3 you need to create links to the jsoncpp library include
+path (and optionally the draw library) in your include folder or provide
+the paths in your config file. We are ready to compile now
+
+[source,sh]
+----
 cd path/to/feltor/src/toefl # or any other project in the src folder
  
 make toeflR device=gpu     # (compile on gpu or omp)
@@ -120,17 +185,25 @@ echo 2 2 | mpirun -n 4 ./toefl_mpi <inputfile.json> <outputfile.nc>
 # (a multi node simulation with now in total 8 threads with output stored in a file)
 # The mpi program will wait for you to type the number of processes in x and y direction before
 # running. That is why the echo is there. 
-```
-A default input file is located in `path/to/feltor/src/toefl/input`. All three programs solve the same equations. 
-The technical documentation on what equations are discretized, 
-input/output parameters, etc. can be generated as a pdf with 
-`make doc ` in the `path/to/feltor/src/toefl` directory.
+----
+
+A default input file is located in `path/to/feltor/src/toefl/input`. All
+three programs solve the same equations. The technical documentation on
+what equations are discretized, input/output parameters, etc. can be
+generated as a pdf with `make doc` in the `path/to/feltor/src/toefl`
+directory.
 
-### Using FELTOR as a library
+=== Using FELTOR as a library
 
-It is possible to use FELTOR as a library in your own code project. Note that the library is **header-only**, which means that you just have to include the relevant header(s) and you're good to go. For example in the following program "test.cpp" we compute the square L2 norm of a function:
+It is possible to use FELTOR as a library in your own code project. Note
+that the library is **header-only**, which means that you just have to
+include the relevant header(s) and you're good to go. For example in the
+following program we compute the square L2 norm of a
+function:
 
-```C++
+.test.cpp
+[source,c++]
+----
 #include <iostream>
 //include the basic dg-library
 #include "dg/algorithm.h"
@@ -152,25 +225,32 @@ int main()
     std::cout << norm <<std::endl;
     return 0;
 }
-```
+----
 
 To compile and run this code for a GPU use
-```sh
+
+[source,sh]
+----
 nvcc -x cu -Ipath/to/feltor/inc -Ipath/to/thrust/thrust -Ipath/to/cusplibrary/cusp test.cpp -o test
 ./test
-```
+----
 
-Or if you want to use OpenMP and gcc instead of CUDA for the device functions you can also use
+Or if you want to use OpenMP and gcc instead of CUDA for the device
+functions you can also use
 
-```shell
+[source,sh]
+----
 g++ -fopenmp -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_OMP -Ipath/to/feltor/inc -Ipath/to/thrust/thrust -Ipath/to/cusplibrary/cusp test.cpp -o test
 export OMP_NUM_THREADS=4
 ./test
-```
+----
 
-If you want to use mpi, just include the MPI header before any other FELTOR header and use our convenient typedefs like so:
+If you want to use mpi, just include the MPI header before any other
+FELTOR header and use our convenient typedefs like so:
 
-```C++
+.test_mpi.cpp
+[source,c++]
+----
 #include <iostream>
 //activate MPI in FELTOR
 #include "mpi.h" 
@@ -197,47 +277,78 @@ int main(int argc, char* argv[])
     MPI_Finalize();
     return 0;
 }
-```
+----
 
-Compile e.g. for a hybrid MPI + OpenMP hardware platform with 
+Compile e.g. for a hybrid MPI {plus} OpenMP hardware platform with
 
-```shell
-mpic++ -fopenmp -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_OMP -Ipath/to/feltor/inc -Ipath/to/thrust/thrust -Ipath/to/cusplibrary/cusp test.cpp -o test
+[source,sh]
+----
+mpic++ -fopenmp -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_OMP -Ipath/to/feltor/inc -Ipath/to/thrust/thrust -Ipath/to/cusplibrary/cusp test_mpi.cpp -o test_mpi
 export OMP_NUM_THREADS=2
-mpirun -n 4 ./test
-```
+mpirun -n 4 ./test_mpi
+----
+
+Note the striking similarity to the previous program. Especially the
+line calling the dot function did not change at all. The compiler
+chooses the correct implementation for you! This is a first example of a
+__container free numerical algorithm__.
+
+== 2. Further reading
+
+Please check out our https://github.com/feltor-dev/feltor/wiki[wiki
+pages] for some general information, user oriented documentation and
+Troubleshooting. Moreover, we maintain tex files in every src folder for
+technical documentation, which can be compiled using pdflatex with
+`make doc` in the respective src folder. The
+http://feltor-dev.github.io/feltor/inc/dg/html/modules.html[developer
+oriented documentation] of the dG library was generated with
+http://www.doxygen.org[Doxygen] and LateX. You can generate a local
+version including informative pdf writeups on implemented numerical
+methods directly from source code. This depends on the `doxygen`,
+`libjs-mathjax` and `graphviz` packages and LateX. Type `make doc` in
+the folder `path/to/feltor/doc` and open `index.html` (a symbolic link
+to `dg/html/modules.html`) with your favorite browser. Links to the pdf
+writeups can be found, among other places, on the Mainpage.
+
+== 3. Acknowledgements
+
+FELTOR is developed by Matthias Wiesenberger and Markus Held and
+receives contributions from an increasing number of people. We
+gratefully acknowledge fruitful discussions and code contribution from
 
-Note the striking similarity to the previous program. Especially the line calling the dot function did not change at all. The compiler chooses the correct implementation for you! This is a first example of a *container free numerical algorithm*.
-
-## 2. Further reading
-
-Please check out our [wiki pages](https://github.com/feltor-dev/feltor/wiki) for some general information, user oriented documentation and Troubleshooting. Moreover, we maintain tex files in every src folder for technical documentation, which can be compiled using pdflatex with `make doc ` in the respective src folder.
-The [developer oriented documentation](http://feltor-dev.github.io/feltor/inc/dg/html/modules.html) of the dG library was generated with [Doxygen](http://www.doxygen.org) and LateX. You can generate a local version including informative pdf writeups on implemented numerical methods directly from source code. This depends on the `doxygen`, `libjs-mathjax` and `graphviz` packages and LateX. Type `make doc` in the folder `path/to/feltor/doc` and open `index.html` (a symbolic link to `dg/html/modules.html`) with your favorite browser. Links to the pdf writeups can be found, among other places, on the Mainpage. 
-
-## 3. Acknowledgements
-
-FELTOR is developed by Matthias Wiesenberger and Markus Held and receives contributions from an increasing number of people.
-We gratefully acknowledge fruitful discussions and code contribution from 
 - Ralph Kube
-- Eduard Reiter
-- Lukas Einkemmer
+- Eduard Reiter 
+- Lukas Einkemmer 
 - Jakob Gath
 
-We further acknowledge support on the Knights landing architecture from the High Level Support Team from 
-- Albert Gutiérrez
+We further acknowledge support on the Knights landing architecture from
+the High Level Support Team from 
+
+- Albert Gutiérrez 
 - Xavier Saez
 
-and from Intel Barcelona
+and from Intel Barcelona 
+
 - Harald Servat
 
-## 4. Terms of use
+== 4. Terms of use
 
-FELTOR is [fair](https://www.force11.org/fairprinciples) software and licensed under the very permissive [MIT license](LICENSE). The MIT License grants you great freedom in what you do with the code as long as you name us (Matthias Wiesenberger and Markus Held) as creators, in particular in publications that use FELTOR to produce results. In this case we suggest to take a snapshot of the used code and create and cite a DOI via e.g. [Zenodo](http://www.zenodo.org) or to cite one of the existing DOIs if you did not alter the contained code in any way. We are happy if you cite our papers, but you don't have to just because you used our code and we certainly do not demand to be coauthors when we do not contribute directly to your results. 
+FELTOR is https://www.force11.org/fairprinciples[fair] software and
+licensed under the very permissive link:LICENSE[MIT license]. The MIT
+License grants you great freedom in what you do with the code as long as
+you name us (Matthias Wiesenberger and Markus Held) as creators, in
+particular in publications that use FELTOR to produce results. In this
+case we suggest to take a snapshot of the used code and create and cite
+a DOI via e.g. http://www.zenodo.org[Zenodo] or to cite one of the
+existing DOIs if you did not alter the contained code in any way. We are
+happy if you cite our papers, but you don't have to just because you
+used our code and we certainly do not demand to be coauthors when we do
+not contribute directly to your results.
 
-## 5. Official releases 
+== 5. Official releases
 
 Our latest code release has a shiny DOI badge from zenodo
 
-[![DOI](https://zenodo.org/badge/14143578.svg)](https://zenodo.org/badge/latestdoi/14143578)
+https://zenodo.org/badge/latestdoi/14143578[image:https://zenodo.org/badge/14143578.svg[DOI]]
 
 which makes us officially citable.
-- 
GitLab


From 61885b8c054732d6f504a2ba7eeff8d2ea8e5897 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Tue, 31 Oct 2017 23:07:28 +0100
Subject: [PATCH 422/453] corrected file extension

---
 README.md => README.adoc | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename README.md => README.adoc (100%)

diff --git a/README.md b/README.adoc
similarity index 100%
rename from README.md
rename to README.adoc
-- 
GitLab


From d96de08d0232313ac5d963b3ba8fbaf1061f317e Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Wed, 1 Nov 2017 11:26:23 +0100
Subject: [PATCH 423/453] added triple pointwise dot product to blas1 functions

---
 inc/dg/backend/dx.h                   |  5 +-
 inc/dg/backend/mpi_vector_blas.h      | 10 ++++
 inc/dg/backend/std_vector_blas.cuh    | 15 ++++++
 inc/dg/backend/thrust_vector_blas.cuh | 76 ++++++++++++++++++++++++++-
 inc/dg/blas1.h                        | 26 +++++++++
 inc/dg/blas1_mpit.cu                  | 64 +++++++++++-----------
 inc/dg/blas1_t.cu                     |  8 ++-
 inc/geometries/ds.h                   | 48 ++++++++---------
 8 files changed, 193 insertions(+), 59 deletions(-)

diff --git a/inc/dg/backend/dx.h b/inc/dg/backend/dx.h
index ae49dd34f..48d6bc811 100644
--- a/inc/dg/backend/dx.h
+++ b/inc/dg/backend/dx.h
@@ -404,7 +404,10 @@ EllSparseBlockMat<double> dx_normed( int n, int N, double h, bc bcx, direction d
         return create::dx_minus(n, N, h, bcx);
     return EllSparseBlockMat<double>();
 }
+///@}
 
+///@addtogroup creation
+///@{
 /**
 * @brief Create and assemble a host Matrix for the derivative in 1d
 *
@@ -461,9 +464,9 @@ EllSparseBlockMat<double> jump( const Grid1d& g)
 {
     return jump( g, g.bcx());
 }
+///@}
 
 
-///@}
 } //namespace create
 } //namespace dg
 
diff --git a/inc/dg/backend/mpi_vector_blas.h b/inc/dg/backend/mpi_vector_blas.h
index 1592ea35b..24b267362 100644
--- a/inc/dg/backend/mpi_vector_blas.h
+++ b/inc/dg/backend/mpi_vector_blas.h
@@ -129,6 +129,16 @@ inline void doPointwiseDot( typename VectorTraits<Vector>::value_type alpha,
     doPointwiseDot( alpha, x1.data(), x2.data(), beta, y.data(), typename VectorTraits<container>::vector_category());
 
 }
+template< class Vector>
+inline void doPointwiseDot( typename VectorTraits<Vector>::value_type alpha, 
+        const Vector& x1, const Vector& x2, const Vector& x3,
+        typename VectorTraits<Vector>::value_type beta,
+        Vector& y, 
+        MPIVectorTag)
+{
+    typedef typename Vector::container_type container;
+    doPointwiseDot( alpha, x1.data(), x2.data(), x3.data(), beta, y.data(), typename VectorTraits<container>::vector_category());
+}
 
 template< class Vector>
 inline void doPointwiseDivide( const Vector& x1, const Vector& x2, Vector& y, MPIVectorTag)
diff --git a/inc/dg/backend/std_vector_blas.cuh b/inc/dg/backend/std_vector_blas.cuh
index e2e777108..5cc2d812a 100644
--- a/inc/dg/backend/std_vector_blas.cuh
+++ b/inc/dg/backend/std_vector_blas.cuh
@@ -142,6 +142,21 @@ std::vector<Vector>& y, StdVectorTag)
 }
 template< class Vector>
 inline void doPointwiseDot( typename VectorTraits<Vector>::value_type alpha, 
+const std::vector<Vector>& x1, const std::vector<Vector>& x2, const std::vector<Vector>& x3,
+typename VectorTraits<Vector>::value_type beta, 
+std::vector<Vector>& y, StdVectorTag)
+{
+#ifdef DG_DEBUG
+    assert( !x1.empty());
+    assert( x1.size() == x2.size() );
+    assert( x1.size() == x3.size() );
+    assert( x1.size() == y.size() );
+#endif //DG_DEBUG
+    for( unsigned i=0; i<x1.size(); i++)
+        doPointwiseDot( alpha, x1[i], x2[i],x3[i], beta, y[i], typename VectorTraits<Vector>::vector_category() );
+}
+template< class Vector>
+inline void doPointwiseDot( typename VectorTraits<Vector>::value_type alpha, 
 const std::vector<Vector>& x1, const std::vector<Vector>& y1, 
 typename VectorTraits<Vector>::value_type beta, 
 const std::vector<Vector>& x2, const std::vector<Vector>& y2, 
diff --git a/inc/dg/backend/thrust_vector_blas.cuh b/inc/dg/backend/thrust_vector_blas.cuh
index 91ba04d8f..498aa7dd6 100644
--- a/inc/dg/backend/thrust_vector_blas.cuh
+++ b/inc/dg/backend/thrust_vector_blas.cuh
@@ -96,8 +96,7 @@ inline void doTransform(  const Vector& x, Vector& y,
 }
 
 template< class Vector>
-inline void doScal(  Vector& x, 
-              typename Vector::value_type alpha, 
+inline void doScal(  Vector& x, typename Vector::value_type alpha, 
               ThrustVectorTag)
 {
     if( alpha == 1.) 
@@ -361,6 +360,21 @@ template<class value_type>
         z[row]=alpha*x1[row]*y1[row]+beta*x2[row]*y2[row]+gamma*z[row];
     }
 }
+template<class value_type>
+ __global__ void pointwiseDot_kernel( value_type alpha, value_type beta,
+         const value_type*  x1, const value_type* x2, const value_type* x3, 
+         value_type* y,  
+         const int size
+         )
+{
+    const int thread_id = blockDim.x * blockIdx.x + threadIdx.x;
+    const int grid_size = gridDim.x*blockDim.x;
+    //every thread takes num_rows/grid_size rows
+    for( int row = thread_id; row<size; row += grid_size)
+    {
+        y[row]=alpha*x1[row]*x2[row]*x3[row]+beta*y[row];
+    }
+}
 #endif
 
 template<class value_type>
@@ -424,6 +438,64 @@ inline void doPointwiseDot(
                     +gamma*z[i];
     }
 }
+template<class value_type>
+inline void doPointwiseDot(  
+              value_type alpha, 
+              const thrust::device_vector<value_type>& x1,
+              const thrust::device_vector<value_type>& x2,
+              const thrust::device_vector<value_type>& x3, 
+              value_type beta, 
+              thrust::device_vector<value_type>& y, 
+              ThrustVectorTag)
+{
+    if( alpha==0){ 
+        doScal( y, beta, ThrustVectorTag());
+        return;
+    }
+    const value_type *x1_ptr = thrust::raw_pointer_cast( x1.data());
+    const value_type *x2_ptr = thrust::raw_pointer_cast( x2.data());
+    const value_type *x3_ptr = thrust::raw_pointer_cast( x3.data());
+          value_type * y_ptr = thrust::raw_pointer_cast( y.data());
+    unsigned size = x1.size();
+#if THRUST_DEVICE_SYSTEM!=THRUST_DEVICE_SYSTEM_CUDA
+    if( size <MIN_SIZE)
+    {
+        for( unsigned i=0; i<size; i++)
+        {
+            y_ptr[i] = alpha*x1_ptr[i]*x2_ptr[i]*x3_ptr[i]
+                       +beta*y_ptr[i];
+        }
+        return;
+    }
+    #pragma omp parallel for SIMD
+    for( unsigned i=0; i<size; i++)
+    {
+        y_ptr[i] = alpha*x1_ptr[i]*x2_ptr[i]*x3_ptr[i]
+                   +beta*y_ptr[i];
+    }
+#else
+    //set up kernel parameters
+    const size_t BLOCK_SIZE = 256; 
+    const size_t NUM_BLOCKS = std::min<size_t>((size-1)/BLOCK_SIZE+1, 65000);
+    pointwiseDot_kernel<value_type><<<NUM_BLOCKS, BLOCK_SIZE>>>( alpha, beta, x1_ptr, x2_ptr, x3_ptr, y_ptr, size);
+#endif
+}
+template<class value_type>
+inline void doPointwiseDot(  
+              value_type alpha, 
+              const thrust::host_vector<value_type>& x1,
+              const thrust::host_vector<value_type>& x2,
+              const thrust::host_vector<value_type>& x3,
+              value_type beta, 
+              thrust::host_vector<value_type>& y, 
+              ThrustVectorTag)
+{
+    unsigned size=x1.size();
+    for( unsigned i=0; i<size; i++)
+    {
+        y[i] = alpha*x1[i]*x2[i]*x3[i]+beta*y[i];
+    }
+}
 
 }//namespace detail
 
diff --git a/inc/dg/blas1.h b/inc/dg/blas1.h
index ae318aa34..a17eb6b30 100644
--- a/inc/dg/blas1.h
+++ b/inc/dg/blas1.h
@@ -265,6 +265,32 @@ inline void pointwiseDot( typename VectorTraits<container>::value_type alpha, co
     dg::blas1::detail::doPointwiseDot( alpha, x1, x2, beta, y, typename dg::VectorTraits<container>::vector_category() );
 }
 
+/**
+* @brief \f$ y = \alpha x_1 x_2 x_3 + \beta y\f$ 
+*
+* Multiplies three vectors element by element: \f[ y_i = \alpha x_{1i}x_{2i}x_{3i} + \beta y_i\f]  i iterates over @b all elements inside the container. Specifically for a std::vector<container_type> i includes both the inner and the outer loop. If the container sizes
+ * do not match, the result is undefined.
+
+* @copydoc hide_container
+* @param alpha scalar
+* @param x1 container x1  
+* @param x2 container x2 may alias x1
+* @param x3 container x2 may alias x1 and/or x2
+* @param beta scalar
+* @param y  container y contains result on output ( may alias x1,x2 or x3)
+
+@code
+    dg::DVec two( 100,2), three( 100,3), four(100,4), result(100,6);
+    dg::blas1::pointwiseDot(2., two,  three, four, -4., result ); 
+    //result[i] = 24. (2*2*3*4-4*6)
+@endcode
+*/
+template< class container>
+inline void pointwiseDot( typename VectorTraits<container>::value_type alpha, const container& x1, const container& x2, const container& x3, typename VectorTraits<container>::value_type beta, container& y)
+{
+    dg::blas1::detail::doPointwiseDot( alpha, x1, x2, x3, beta, y, typename dg::VectorTraits<container>::vector_category() );
+}
+
 /**
 * @brief \f$ y = x_1/ x_2\f$ 
 *
diff --git a/inc/dg/blas1_mpit.cu b/inc/dg/blas1_mpit.cu
index 3c40ba5a8..515218bb3 100644
--- a/inc/dg/blas1_mpit.cu
+++ b/inc/dg/blas1_mpit.cu
@@ -10,9 +10,10 @@
 //test program that calls every blas1 function for every specialization
 double two( double x, double y){return 2.;}
 double three( double x, double y){return 3.;}
+double four( double x, double y){return 4.;}
 
-//typedef dg::MPI_Vector<thrust::device_vector<double> > MHVec;
-typedef dg::MPI_Vector<cusp::array1d<double, cusp::device_memory> > MHVec;
+typedef dg::MPI_Vector<thrust::device_vector<double> > MVec;
+//typedef dg::MPI_Vector<cusp::array1d<double, cusp::device_memory> > MVec;
 
 struct EXP{ __host__ __device__ double operator()(double x){return exp(x);}};
 
@@ -44,68 +45,71 @@ int main( int argc, char* argv[])
     MPI_Comm comm;
     MPI_Cart_create( MPI_COMM_WORLD, 2, np, periods, true, &comm);
     dg::MPIGrid2d g( 0,1,0,1, 3,12,12, comm);
-    if( rank == 0)
-        g.display();
-    MHVec v1 = dg::evaluate( two, g);
-    MHVec v2 = dg::evaluate( three, g); 
-    MHVec v3(v1);
+    MVec v1 = dg::evaluate( two, g);
+    MVec v2 = dg::evaluate( three, g); 
+    MVec v3(v1);
+    MVec v4 = dg::evaluate( four, g); 
     unsigned gsize = g.global().n()*g.global().n()*g.global().Nx()*g.global().Ny();
 
     double temp = dg::blas1::dot(v1,v2);
     if(rank==0)std::cout << "(2*3) = "<<temp/gsize << " (6)\n"; 
     dg::blas1::axpby( 2., v1, 3., v2, v3);
-    if(rank==0)std::cout << "2*2+ 3*3 = " << v3[0] <<" (13)\n";
+    if(rank==0)std::cout << "2*2+ 3*3 = " << v3.data()[0] <<" (13)\n";
     dg::blas1::axpby( 0., v1, 3., v2, v3);
-    if(rank==0)std::cout << "0*2+ 3*3 = " << v3[0] <<" (9)\n";
+    if(rank==0)std::cout << "0*2+ 3*3 = " << v3.data()[0] <<" (9)\n";
     dg::blas1::axpby( 2., v1, 0., v2, v3);
-    if(rank==0)std::cout << "2*2+ 0*3 = " << v3[0] <<" (4)\n";
+    if(rank==0)std::cout << "2*2+ 0*3 = " << v3.data()[0] <<" (4)\n";
     dg::blas1::pointwiseDot( v1, v2, v3);
-    if(rank==0)std::cout << "2*3 = "<<v3[0]<<" (6)\n";
+    if(rank==0)std::cout << "2*3 = "<<v3.data()[0]<<" (6)\n";
     dg::blas1::pointwiseDot( 2., v1, v2, -4., v3);
-    if(rank==0)std::cout << "2*2*3 -4*6 = "<<v3[0]<<" (-12)\n";
+    if(rank==0)std::cout << "2*2*3 -4*6 = "<<v3.data()[0]<<" (-12)\n";
+    dg::blas1::pointwiseDot( 2., v1, v2,v4, -4., v3);
+    if(rank==0)std::cout << "2*2*3 -4*(-12) = "<<v3.data()[0]<<" (96)\n";
     dg::blas1::axpby( 2., v1, 3., v2);
-    if(rank==0)std::cout << "2*2+ 3*3 = " << v2[0] <<" (13)\n";
+    if(rank==0)std::cout << "2*2+ 3*3 = " << v2.data()[0] <<" (13)\n";
     dg::blas1::axpby( 2.5, v1, 0., v2);
-    if(rank==0)std::cout << "2.5*2+ 0 = " << v2[0] <<" (5)\n";
+    if(rank==0)std::cout << "2.5*2+ 0 = " << v2.data()[0] <<" (5)\n";
     dg::blas1::copy( v2, v1);
-    if(rank==0)std::cout << "5 = " << v1[0] <<" (5)"<< std::endl;
+    if(rank==0)std::cout << "5 = " << v1.data()[0] <<" (5)"<< std::endl;
     dg::blas1::scal( v1, 0.4);
-    if(rank==0)std::cout << "5*0.4 = " << v1[0] <<" (2)"<< std::endl;
+    if(rank==0)std::cout << "5*0.4 = " << v1.data()[0] <<" (2)"<< std::endl;
     dg::blas1::transform( v1, v3, EXP());
-    if(rank==0)std::cout << "e^2 = " << v3[0] <<" (7.389056...)"<< std::endl;
+    if(rank==0)std::cout << "e^2 = " << v3.data()[0] <<" (7.389056...)"<< std::endl;
     dg::blas1::scal( v2, 0.6);
     dg::blas1::plus( v3, -7.0);
-    if(rank==0)std::cout << "e^2-7 = " << v3[0] <<" (0.389056...)"<< std::endl;
+    if(rank==0)std::cout << "e^2-7 = " << v3.data()[0] <<" (0.389056...)"<< std::endl;
 
     //v1 = 2, v2 = 3
 
     if(rank==0)std::cout << "Test std::vector \n";
-    std::vector<MHVec > w1( 2, v1), w2(2, v2), w3( w2);
+    std::vector<MVec > w1( 2, v1), w2(2, v2), w3( w2), w4( 2, v4);
     temp = dg::blas1::dot( w1, w2);
     if(rank==0)std::cout << "2*(2*3) = "<<temp/gsize << " (12)\n"; 
     dg::blas1::axpby( 2., w1, 3., w2, w3);
-    if(rank==0)std::cout << "2*2+ 3*3 = " << w3[0][0] <<" (13)\n";
+    if(rank==0)std::cout << "2*2+ 3*3 = " << w3[0].data()[0] <<" (13)\n";
     dg::blas1::axpby( 0., w1, 3., w2, w3);
-    if(rank==0)std::cout << "0*2+ 3*3 = " << w3[0][0] <<" (9)\n";
+    if(rank==0)std::cout << "0*2+ 3*3 = " << w3[0].data()[0] <<" (9)\n";
     dg::blas1::axpby( 2., w1, 0., w2, w3);
-    if(rank==0)std::cout << "2*2+ 0*3 = " << w3[0][0] <<" (4)\n";
+    if(rank==0)std::cout << "2*2+ 0*3 = " << w3[0].data()[0] <<" (4)\n";
     dg::blas1::pointwiseDot( w1, w2, w3);
-    if(rank==0)std::cout << "2*3 = "<<w3[0][0]<<" (6)\n";
+    if(rank==0)std::cout << "2*3 = "<<w3[0].data()[0]<<" (6)\n";
     dg::blas1::pointwiseDot( 2., w1, w2, -4., w3);
-    if(rank==0)std::cout << "2*2*3 -4*6 = "<<w3[0][0]<<" (-12)\n";
+    if(rank==0)std::cout << "2*2*3 -4*6 = "<<w3[0].data()[0]<<" (-12)\n";
+    dg::blas1::pointwiseDot( 2., w1, w2,w4, -4., w3);
+    if(rank==0)std::cout << "2*2*3 -4*(-12) = "<<w3[0].data()[0]<<" (96)\n";
     dg::blas1::axpby( 2., w1, 3., w2);
-    if(rank==0)std::cout << "2*2+ 3*3 = " << w2[0][0] <<" (13)\n";
+    if(rank==0)std::cout << "2*2+ 3*3 = " << w2[0].data()[0] <<" (13)\n";
     dg::blas1::axpby( 2.5, w1, 0., w2);
-    if(rank==0)std::cout << "2.5*2+ 0 = " << w2[0][0] <<" (5)\n";
+    if(rank==0)std::cout << "2.5*2+ 0 = " << w2[0].data()[0] <<" (5)\n";
     dg::blas1::copy( w2, w1);
-    if(rank==0)std::cout << "5 = " << w1[0][0] <<" (5)"<< std::endl;
+    if(rank==0)std::cout << "5 = " << w1[0].data()[0] <<" (5)"<< std::endl;
     dg::blas1::scal( w1, 0.4);
-    if(rank==0)std::cout << "5*0.5 = " << w1[0][0] <<" (2)"<< std::endl;
+    if(rank==0)std::cout << "5*0.5 = " << w1[0].data()[0] <<" (2)"<< std::endl;
     dg::blas1::transform( w1, w3, EXP());
-    if(rank==0)std::cout << "e^2 = " << w3[0][0] <<" (7.389056...)"<< std::endl;
+    if(rank==0)std::cout << "e^2 = " << w3[0].data()[0] <<" (7.389056...)"<< std::endl;
     dg::blas1::scal( w2, 0.6);
     dg::blas1::plus( w3, -7.0);
-    if(rank==0)std::cout << "e^2-7 = " << w3[0][0] <<" (0.389056...)"<< std::endl;
+    if(rank==0)std::cout << "e^2-7 = " << w3[0].data()[0] <<" (0.389056...)"<< std::endl;
     if(rank==0)std::cout << "FINISHED\n\n";
 
 
diff --git a/inc/dg/blas1_t.cu b/inc/dg/blas1_t.cu
index 871556347..858058390 100644
--- a/inc/dg/blas1_t.cu
+++ b/inc/dg/blas1_t.cu
@@ -13,7 +13,7 @@ typedef thrust::device_vector<double>  Vector;
 //typedef cusp::array1d<double, cusp::device_memory>  Vector;
 int main()
 {
-    Vector v1( 5, 2), v2( 5, 3), v3(5);
+    Vector v1( 5, 2), v2( 5, 3), v3(5), v4(5,4);
     double temp = dg::blas1::dot(v1,v2);
     std::cout << "5*(2*3) = "<<temp << " (30)\n"; 
     dg::blas1::axpby( 2., v1, 3., v2, v3);
@@ -26,6 +26,8 @@ int main()
     std::cout << "2*3 = "<<v3[0]<<" (6)\n";
     dg::blas1::pointwiseDot( 2., v1, v2, -4., v3);
     std::cout << "2*2*3 -4*6 = "<<v3[0]<<" (-12)\n";
+    dg::blas1::pointwiseDot( 2., v1, v2,v4, -4., v3);
+    std::cout << "2*2*3*4 -4*(-12) = "<<v3[0]<<" (96)\n";
     dg::blas1::axpby( 2., v1, 3., v2);
     std::cout << "2*2+ 3*3 = " << v2[0] <<" (13)\n";
     dg::blas1::axpby( 2.5, v1, 0., v2);
@@ -45,7 +47,7 @@ int main()
     //v1 = 2, v2 = 3
 
     std::cout << "Test std::vector \n";
-    std::vector<Vector > w1( 2, v1), w2(2, v2), w3( w2);
+    std::vector<Vector > w1( 2, v1), w2(2, v2), w3( w2), w4(2,v4);
     temp = dg::blas1::dot( w1, w2);
     std::cout << "2*5*(2*3) = "<<temp << " (60)\n"; 
     dg::blas1::axpby( 2., w1, 3., w2, w3);
@@ -58,6 +60,8 @@ int main()
     std::cout << "2*3 = "<<w3[0][0]<<" (6)\n";
     dg::blas1::pointwiseDot( 2., w1, w2, -4., w3);
     std::cout << "2*2*3 -4*6 = "<<w3[0][0]<<" (-12)\n";
+    dg::blas1::pointwiseDot( 2., w1, w2,w4, -4., w3);
+    std::cout << "2*2*3*4 -4*(-12) = "<<w3[0][0]<<" (96)\n";
     dg::blas1::pointwiseDot( 2., w1[0], w2[0], -4., v1, v2, 0., v2);
     std::cout << "2*2*3 -4*2*3 = "<<v2[0]<<" (-12)\n";
     dg::blas1::axpby( 2., w1, 3., w2);
diff --git a/inc/geometries/ds.h b/inc/geometries/ds.h
index 2513c62e4..df3e20ac2 100644
--- a/inc/geometries/ds.h
+++ b/inc/geometries/ds.h
@@ -146,27 +146,27 @@ struct DS
 
     ///@brief forward adjoint \f$ g = -\alpha \nabla\cdot(\vec v f) + \beta g\f$
     ///@copydetails forward
-    void forwardAdj( double alpha, const container& f, double beta, container& g){
-        do_forwardAdj( alpha, f, beta, g, dg::normed);
+    void forwardDiv( double alpha, const container& f, double beta, container& g){
+        do_forwardDiv( alpha, f, beta, g, dg::normed);
     }
     ///@brief backward adjoint \f$ g = -\alpha \nabla\cdot(\vec v f) + \beta g\f$
     ///@copydetails forward
-    void backwardAdj( double alpha, const container& f, double beta, container& g){
-        do_backwardAdj( alpha, f, beta, g, dg::normed);
+    void backwardDiv( double alpha, const container& f, double beta, container& g){
+        do_backwardDiv( alpha, f, beta, g, dg::normed);
     }
     ///@brief centered adjoint \f$ g = -\alpha \nabla\cdot(\vec v f) + \beta g\f$
     ///@copydetails forward
-    void centeredAdj(double alpha, const container& f, double beta, container& g){
-        do_centeredAdj( alpha, f, beta, g, dg::normed);
+    void centeredDiv(double alpha, const container& f, double beta, container& g){
+        do_centeredDiv( alpha, f, beta, g, dg::normed);
     }
-    void forwardAdj(const container& f, container& g){
-        do_forwardAdj( 1.,f,0.,g, dg::normed);
+    void forwardDiv(const container& f, container& g){
+        do_forwardDiv( 1.,f,0.,g, dg::normed);
     }
-    void backwardAdj(const container& f, container& g){
-        do_backwardAdj( 1.,f,0.,g, dg::normed);
+    void backwardDiv(const container& f, container& g){
+        do_backwardDiv( 1.,f,0.,g, dg::normed);
     }
-    void centeredAdj(const container& f, container& g){
-        do_centeredAdj( 1.,f,0.,g, dg::normed);
+    void centeredDiv(const container& f, container& g){
+        do_centeredDiv( 1.,f,0.,g, dg::normed);
     }
 
     /**
@@ -183,7 +183,7 @@ struct DS
     /**
      * @brief Discretizes \f$ \nabla\cdot ( \vec v \vec v \cdot \nabla . )\f$ as a symmetric matrix
      *
-     * if direction is centered then centered followed by centeredAdj and adding jump terms
+     * if direction is centered then centered followed by centeredDiv and adding jump terms
      * @param f The vector to derive
      * @param dsTdsf contains result on output (write only)
      * @note if dependsOnX is false then no jump terms will be added in the x-direction and similar in y
@@ -230,9 +230,9 @@ struct DS
     void do_forward(double alpha, const container& f, double beta, container& dsf);
     void do_backward(double alpha, const container& f, double beta, container& dsf);
     void do_centered(double alpha, const container& f, double beta, container& dsf);
-    void do_forwardAdj(double alpha, const container& f, double beta, container& dsf, dg::norm no);
-    void do_backwardAdj(double alpha, const container& f, double beta, container& dsf, dg::norm no);
-    void do_centeredAdj(double alpha, const container& f, double beta, container& dsf, dg::norm no);
+    void do_forwardDiv(double alpha, const container& f, double beta, container& dsf, dg::norm no);
+    void do_backwardDiv(double alpha, const container& f, double beta, container& dsf, dg::norm no);
+    void do_centeredDiv(double alpha, const container& f, double beta, container& dsf, dg::norm no);
     void do_symv(double alpha, const container& f, double beta, container& dsf);
     void do_dss(double alpha, const container& f, double beta, container& dsf);
 
@@ -298,7 +298,7 @@ void DS<G, I,M,container>::do_centered( double alpha, const container& f, double
     dg::blas1::pointwiseDot( alpha, m_tempM, m_fa.hz_inv(), beta, dsf);
 }
 template<class G, class I, class M, class container>
-void DS<G,I,M,container>::do_forwardAdj( double alpha, const container& f, double beta, container& dsf, dg::norm no)
+void DS<G,I,M,container>::do_backwardDiv( double alpha, const container& f, double beta, container& dsf, dg::norm no)
 {    
     //adjoint discretisation
     dg::blas1::pointwiseDot( m_vol3d, f, m_temp0);
@@ -306,14 +306,14 @@ void DS<G,I,M,container>::do_forwardAdj( double alpha, const container& f, doubl
     m_fa(einsPlusT, m_temp0, m_tempP);
     if(no == dg::normed) 
     {
-        dg::blas1::axpby( 1., m_tempP, -1., m_temp0, m_temp0);
+        dg::blas1::axpby( 1., m_temp0, -1., m_tempP, m_temp0);
         dg::blas1::pointwiseDot( alpha, m_inv3d, m_temp0, beta, dsf); 
     }
     else
-        dg::blas1::axpbypgz( alpha, m_tempP, -alpha, m_temp0, beta, dsf);
+        dg::blas1::axpbypgz( alpha, m_temp0, -alpha, m_tempP, beta, dsf);
 }
 template<class G,class I, class M, class container>
-void DS<G,I,M,container>::do_backwardAdj( double alpha, const container& f, double beta, container& dsf, dg::norm no)
+void DS<G,I,M,container>::do_forwardDiv( double alpha, const container& f, double beta, container& dsf, dg::norm no)
 {    
     //adjoint discretisation
     dg::blas1::pointwiseDot( m_vol3d, f, m_temp0);
@@ -328,7 +328,7 @@ void DS<G,I,M,container>::do_backwardAdj( double alpha, const container& f, doub
         dg::blas1::axpbypgz( alpha, m_temp0, -alpha, m_tempM, beta, dsf);
 }
 template<class G, class I, class M, class container>
-void DS<G, I,M,container>::do_centeredAdj( double alpha, const container& f, double beta, container& dsf, dg::norm no)
+void DS<G, I,M,container>::do_centeredDiv( double alpha, const container& f, double beta, container& dsf, dg::norm no)
 {               
     //adjoint discretisation
     dg::blas1::pointwiseDot( m_vol3d, f, m_temp0);
@@ -351,14 +351,14 @@ void DS<G,I,M,container>::do_symv( double alpha, const container& f, double beta
     if(m_dir == dg::centered)
     {
         do_centered( 1., f, 0., m_tempP);
-        do_centeredAdj( 1., m_tempP, 0., m_temp, dg::not_normed);
+        do_centeredDiv( 1., m_tempP, 0., m_temp, dg::not_normed);
     }
     else 
     {
         do_forward( 1., f, 0., m_tempP);
-        do_forwardAdj( 1., m_tempP, 0., m_temp0, dg::not_normed);
+        do_forwardDiv( 1., m_tempP, 0., m_temp0, dg::not_normed);
         do_backward( 1., f, 0., m_tempM);
-        do_backwardAdj( 1., m_tempM, 0., m_temp, dg::not_normed);
+        do_backwardDiv( 1., m_tempM, 0., m_temp, dg::not_normed);
         dg::blas1::axpby(0.5,m_temp0,0.5,m_temp);
     }
     dg::blas1::pointwiseDivide( m_temp, m_weights_wo_vol, m_temp);
-- 
GitLab


From 76f6f6ba20eb185f329f70ae939a5079d9a0e801 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Wed, 1 Nov 2017 13:01:26 +0100
Subject: [PATCH 424/453] reworked the naming scheme in ds.h and tests in
 ds_curv

---
 inc/geometries/Makefile        |  2 +-
 inc/geometries/ds.h            | 47 ++++++++++++++++------------------
 inc/geometries/ds_curv_mpit.cu | 19 ++++++++++----
 inc/geometries/ds_curv_t.cu    | 23 ++++++++++++-----
 src/toefl/toeflR.cuh           |  8 ++----
 5 files changed, 55 insertions(+), 44 deletions(-)

diff --git a/inc/geometries/Makefile b/inc/geometries/Makefile
index 8694cb8c9..0a8af7645 100644
--- a/inc/geometries/Makefile
+++ b/inc/geometries/Makefile
@@ -18,7 +18,7 @@ geometry_diag: geometry_diag.cu solovev.h
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(LIBS) $(INCLUDE) $(JSONLIB) -g
 
 %_t: %_t.cu 
-	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -DDG_DEBUG -pg
+	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -DDG_DEBUG -g
 
 %_mpit: %_mpit.cu 
 	$(MPICC) $(OPT) $(INCLUDE) $(MPICFLAGS)  $< -o $@ -g $(LIBS) $(JSONLIB) -DDG_DEBUG
diff --git a/inc/geometries/ds.h b/inc/geometries/ds.h
index df3e20ac2..0f2e81abf 100644
--- a/inc/geometries/ds.h
+++ b/inc/geometries/ds.h
@@ -144,18 +144,21 @@ struct DS
     ///@copydetails forward
     void centered( const container& f, container& g){do_centered(1.,f,0.,g);}
 
-    ///@brief forward adjoint \f$ g = -\alpha \nabla\cdot(\vec v f) + \beta g\f$
+    ///@brief forward divergence \f$ g = \alpha \nabla\cdot(\vec v f) + \beta g\f$
     ///@copydetails forward
+    ///@note forwardDiv is the negative adjoint of backward
     void forwardDiv( double alpha, const container& f, double beta, container& g){
         do_forwardDiv( alpha, f, beta, g, dg::normed);
     }
-    ///@brief backward adjoint \f$ g = -\alpha \nabla\cdot(\vec v f) + \beta g\f$
+    ///@brief backward divergence \f$ g = \alpha \nabla\cdot(\vec v f) + \beta g\f$
     ///@copydetails forward
+    ///@note backwardDiv is the negative adjoint of forward
     void backwardDiv( double alpha, const container& f, double beta, container& g){
         do_backwardDiv( alpha, f, beta, g, dg::normed);
     }
-    ///@brief centered adjoint \f$ g = -\alpha \nabla\cdot(\vec v f) + \beta g\f$
+    ///@brief centered divergence \f$ g = -\alpha \nabla\cdot(\vec v f) + \beta g\f$
     ///@copydetails forward
+    ///@note centeredDiv is the negative adjoint of centered
     void centeredDiv(double alpha, const container& f, double beta, container& g){
         do_centeredDiv( alpha, f, beta, g, dg::normed);
     }
@@ -190,15 +193,15 @@ struct DS
      */
     void symv( const container& f, container& dsTdsf){ do_symv( 1., f, 0., dsTdsf);}
     void symv( double alpha, const container& f, double beta, container& dsTdsf){ do_symv( alpha, f, beta, dsTdsf);}
+    void dss( const container& f, container& g){ do_dss( 1., f, 0., g);}
     /**
-     * @brief Discretizes \f$ \nabla_\parallel^2 f \f$ 
+     * @brief Discretizes \f$ g = \alpha \nabla_\parallel^2 f + \beta g \f$ 
      *
      * The formula used is \f[ \nabla_\parallel^2 f = 2\left(\frac{f^+}{h^+h^0} - \frac{f^0}{h^+h^-} + \frac{f^-}{h^-h^0}\right)
      * @param f The vector to derive
      * @param dsTdsf contains result on output (write only)
      */
-    void dss( const container& f, container& dsTdsf){ do_dss( 1., f, 0., dsTdsf);}
-    void dss( double alpha, const container& f, double beta, container& dsTdsf){ do_symv( alpha, f, beta, dsTdsf);}
+    void dss( double alpha, const container& f, double beta, container& g){ do_symv( alpha, f, beta, g);}
 
     ///@copydoc Fieldaligned::set_boundaries(dg::bc,double,double)
     void set_boundaries( dg::bc bcz, double left, double right)
@@ -301,8 +304,7 @@ template<class G, class I, class M, class container>
 void DS<G,I,M,container>::do_backwardDiv( double alpha, const container& f, double beta, container& dsf, dg::norm no)
 {    
     //adjoint discretisation
-    dg::blas1::pointwiseDot( m_vol3d, f, m_temp0);
-    dg::blas1::pointwiseDot( m_temp0, m_fa.hp_inv(), m_temp0);
+    dg::blas1::pointwiseDot( 1., m_vol3d, f, m_fa.hp_inv(), 0., m_temp0);
     m_fa(einsPlusT, m_temp0, m_tempP);
     if(no == dg::normed) 
     {
@@ -316,32 +318,30 @@ template<class G,class I, class M, class container>
 void DS<G,I,M,container>::do_forwardDiv( double alpha, const container& f, double beta, container& dsf, dg::norm no)
 {    
     //adjoint discretisation
-    dg::blas1::pointwiseDot( m_vol3d, f, m_temp0);
-    dg::blas1::pointwiseDot( m_temp0, m_fa.hm_inv(), m_temp0);
+    dg::blas1::pointwiseDot( 1., m_vol3d, f, m_fa.hm_inv(),0., m_temp0);
     m_fa(einsMinusT, m_temp0, m_tempM);
     if(no == dg::normed) 
     {
-        dg::blas1::axpby( 1., m_temp0, -1., m_tempM, m_temp0);
+        dg::blas1::axpby( 1., m_tempM, -1., m_temp0, m_temp0);
         dg::blas1::pointwiseDot( alpha, m_inv3d, m_temp0, beta, dsf); 
     }
     else
-        dg::blas1::axpbypgz( alpha, m_temp0, -alpha, m_tempM, beta, dsf);
+        dg::blas1::axpbypgz( alpha, m_tempM, -alpha, m_temp0, beta, dsf);
 }
 template<class G, class I, class M, class container>
 void DS<G, I,M,container>::do_centeredDiv( double alpha, const container& f, double beta, container& dsf, dg::norm no)
 {               
     //adjoint discretisation
-    dg::blas1::pointwiseDot( m_vol3d, f, m_temp0);
-    dg::blas1::pointwiseDot( m_temp0, m_fa.hz_inv(), m_temp0);
+    dg::blas1::pointwiseDot( 1., m_vol3d, f, m_fa.hz_inv(), 0.,m_temp0);
     m_fa(einsPlusT,  m_temp0, m_tempP);
     m_fa(einsMinusT, m_temp0, m_tempM);
     if(no == dg::normed) 
     {
-        dg::blas1::axpby( 1., m_tempP, -1., m_tempM);
-        dg::blas1::pointwiseDot( alpha, m_inv3d, m_tempM, beta, dsf); 
+        dg::blas1::axpby( 1., m_tempM, -1., m_tempP);
+        dg::blas1::pointwiseDot( alpha, m_inv3d, m_tempP, beta, dsf); 
     }
     else
-        dg::blas1::axpbypgz( alpha, m_tempP, -alpha, m_tempM, beta, dsf);
+        dg::blas1::axpbypgz( alpha, m_tempM, -alpha, m_tempP, beta, dsf);
 
 }
 
@@ -356,9 +356,9 @@ void DS<G,I,M,container>::do_symv( double alpha, const container& f, double beta
     else 
     {
         do_forward( 1., f, 0., m_tempP);
-        do_forwardDiv( 1., m_tempP, 0., m_temp0, dg::not_normed);
+        do_backwardDiv( 1., m_tempP, 0., m_temp0, dg::not_normed);
         do_backward( 1., f, 0., m_tempM);
-        do_backwardDiv( 1., m_tempM, 0., m_temp, dg::not_normed);
+        do_forwardDiv( 1., m_tempM, 0., m_temp, dg::not_normed);
         dg::blas1::axpby(0.5,m_temp0,0.5,m_temp);
     }
     dg::blas1::pointwiseDivide( m_temp, m_weights_wo_vol, m_temp);
@@ -369,10 +369,7 @@ void DS<G,I,M,container>::do_symv( double alpha, const container& f, double beta
         dg::blas2::symv( -1., m_jumpY, f, 1., m_temp);
 
     if( m_no == dg::normed)
-    {
-        dg::blas1::pointwiseDot( m_weights_wo_vol, m_temp, m_temp);
-        dg::blas1::pointwiseDot( alpha, m_inv3d, m_temp, beta, dsTdsf); 
-    }
+        dg::blas1::pointwiseDot( alpha, m_inv3d, m_weights_wo_vol, m_temp, beta, dsTdsf);
     else
         dg::blas1::pointwiseDot( alpha, m_weights_wo_vol, m_temp, beta, dsTdsf);
 }
@@ -383,8 +380,8 @@ void DS<G,I,M,container>::do_dss( double alpha, const container& f, double beta,
     m_fa(einsPlus,  f, m_tempP);
     m_fa(einsMinus, f, m_tempM);
     dg::blas1::pointwiseDot( 1., m_tempP, m_fa.hp_inv(), 1., m_tempM, m_fa.hm_inv(), 0., m_tempM);
-    dg::blas1::pointwiseDot( f,  m_fa.hp_inv(), m_temp0);
-    dg::blas1::pointwiseDot( 2.*alpha, m_fa.hz_inv(), m_tempM, -2.*alpha, m_fa.hm_inv(), m_temp0, beta, dssf);
+    dg::blas1::pointwiseDot( -2., alpha, f,  m_fa.hp_inv(), m_fa.hm_inv(), beta, dssf);
+    dg::blas1::pointwiseDot( 2.*alpha, m_fa.hz_inv(), m_tempM, 1., dssf);
 
 }
 ///@endcond
diff --git a/inc/geometries/ds_curv_mpit.cu b/inc/geometries/ds_curv_mpit.cu
index 4366d883e..5693d7697 100644
--- a/inc/geometries/ds_curv_mpit.cu
+++ b/inc/geometries/ds_curv_mpit.cu
@@ -53,21 +53,30 @@ int main(int argc, char * argv[])
     if(rank==0)std::cout << "Construction took "<<t.diff()<<"s\n";
     dg::MHVec B = dg::pullback( dg::geo::InvB(mag), g3d), divB(B);
     dg::MHVec lnB = dg::pullback( dg::geo::LnB(mag), g3d), gradB(B);
-    dg::MHVec gradLnB = dg::pullback( dg::geo::GradLnB(mag), g3d);
+    const dg::MHVec gradLnB = dg::pullback( dg::geo::GradLnB(mag), g3d);
     dg::MHVec ones3d = dg::evaluate( dg::one, g3d);
     dg::MHVec vol3d = dg::create::volume( g3d);
     dg::blas1::pointwiseDivide( ones3d, B, B);
     //dg::MHVec function = dg::pullback( dg::geo::FuncNeu(mag), g3d), derivative(function);
     //ds( function, derivative);
 
-    ds.centeredAdj( 1., B, 0., divB);
+    ds.centeredDiv( 1., ones3d, 0., divB);
+    dg::blas1::axpby( 1., gradLnB, 1, divB);
     double norm =  sqrt( dg::blas2::dot(divB, vol3d, divB));
-    if(rank==0)std::cout << "Divergence of B is "<<norm<<"\n";
+    if(rank==0)std::cout << "Centered Divergence of b is "<<norm<<"\n";
+    ds.forwardDiv( 1., ones3d, 0., divB);
+    dg::blas1::axpby( 1., gradLnB, 1, divB);
+    norm =  sqrt( dg::blas2::dot(divB, vol3d, divB));
+    if(rank==0)std::cout << "Forward  Divergence of b is "<<norm<<"\n";
+    ds.backwardDiv( 1., ones3d, 0., divB);
+    dg::blas1::axpby( 1., gradLnB, 1, divB);
+    norm =  sqrt( dg::blas2::dot(divB, vol3d, divB));
+    if(rank==0)std::cout << "Backward Divergence of b is "<<norm<<"\n";
 
     ds.centered( 1., lnB, 0., gradB);
     norm = sqrt( dg::blas2::dot( gradLnB, vol3d, gradLnB) );
-    dg::blas1::axpby( 1., gradB, -1., gradLnB, gradLnB);
-    double norm2 = sqrt(dg::blas2::dot(gradLnB, vol3d, gradLnB));
+    dg::blas1::axpby( 1., gradLnB, -1., gradB);
+    double norm2 = sqrt(dg::blas2::dot(gradB, vol3d, gradB));
     if(rank==0)std::cout << "rel. error of lnB is    "<<norm2/norm<<"\n";
     MPI_Finalize();
     return 0;
diff --git a/inc/geometries/ds_curv_t.cu b/inc/geometries/ds_curv_t.cu
index 195b39317..76f763c60 100644
--- a/inc/geometries/ds_curv_t.cu
+++ b/inc/geometries/ds_curv_t.cu
@@ -50,21 +50,30 @@ int main(int argc, char * argv[])
     std::cout << "Construction took "<<t.diff()<<"s\n";
     dg::HVec B = dg::pullback( dg::geo::InvB(mag), g3d), divB(B);
     dg::HVec lnB = dg::pullback( dg::geo::LnB(mag), g3d), gradB(B);
-    dg::HVec gradLnB = dg::pullback( dg::geo::GradLnB(mag), g3d);
+    const dg::HVec gradLnB = dg::pullback( dg::geo::GradLnB(mag), g3d);
     dg::HVec ones3d = dg::evaluate( dg::one, g3d);
     dg::HVec vol3d = dg::create::volume( g3d);
     dg::blas1::pointwiseDivide( ones3d, B, B);
-    dg::HVec function = dg::pullback( dg::geo::FuncNeu(mag), g3d), derivative(function);
-    ds( function, derivative);
+    //dg::HVec function = dg::pullback( dg::geo::FuncNeu(mag), g3d), derivative(function);
+    //ds( function, derivative);
 
-    ds.centeredAdj( 1., B, 0., divB);
+    ds.centeredDiv( 1., ones3d, 0., divB);
+    dg::blas1::axpby( 1., gradLnB, 1, divB);
     double norm =  sqrt( dg::blas2::dot(divB, vol3d, divB));
-    std::cout << "Divergence of B is "<<norm<<"\n";
+    std::cout << "Centered Divergence of b is "<<norm<<"\n";
+    ds.forwardDiv( 1., ones3d, 0., divB);
+    dg::blas1::axpby( 1., gradLnB, 1, divB);
+    norm =  sqrt( dg::blas2::dot(divB, vol3d, divB));
+    std::cout << "Forward  Divergence of b is "<<norm<<"\n";
+    ds.backwardDiv( 1., ones3d, 0., divB);
+    dg::blas1::axpby( 1., gradLnB, 1, divB);
+    norm =  sqrt( dg::blas2::dot(divB, vol3d, divB));
+    std::cout << "Backward Divergence of b is "<<norm<<"\n";
 
     ds.centered( 1., lnB, 0., gradB);
     norm = sqrt( dg::blas2::dot( gradLnB, vol3d, gradLnB) );
-    dg::blas1::axpby( 1., gradB, -1., gradLnB, gradLnB);
-    double norm2 = sqrt(dg::blas2::dot(gradLnB, vol3d, gradLnB));
+    dg::blas1::axpby( 1., gradLnB, -1., gradB);
+    double norm2 = sqrt(dg::blas2::dot(gradB, vol3d, gradB));
     std::cout << "rel. error of lnB is    "<<norm2/norm<<"\n";
     
     return 0;
diff --git a/src/toefl/toeflR.cuh b/src/toefl/toeflR.cuh
index bfc12433f..bbf14d108 100644
--- a/src/toefl/toeflR.cuh
+++ b/src/toefl/toeflR.cuh
@@ -190,16 +190,12 @@ const container& Explicit<G, M, container>::compute_psi( const container& potent
     //compute psi
     if(equations == "global")
     {
-        dg::blas1::pointwiseDot( binv, omega, omega);
-        dg::blas1::pointwiseDot( binv, omega, omega);
 
-        dg::blas1::axpby( 1., phi[1], -0.5, omega, phi[1]);   //psi  Gamma phi - 0.5 u_E^2
+        dg::blas1::pointwiseDot( -0.5, binv, binv, omega, 1., phi[1]);
     }
     else if ( equations == "drift_global")
     {
-        dg::blas1::pointwiseDot( binv, omega, omega);
-        dg::blas1::pointwiseDot( binv, omega, omega);
-        dg::blas1::axpby( 0.5, omega, 0., phi[1]);
+        dg::blas1::pointwiseDot( 0.5, binv, binv, omega, 0., phi[1]);
     }
     else if( equations == "gravity_global" ) 
         dg::blas1::axpby( 0.5, omega, 0., phi[1]);
-- 
GitLab


From fd85b981eda21df54d4841c7ab7d346b7e33bd6d Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Wed, 1 Nov 2017 13:06:11 +0100
Subject: [PATCH 425/453] changed default my to 100 in ds_curv tests

---
 inc/geometries/ds_curv_mpit.cu | 2 +-
 inc/geometries/ds_curv_t.cu    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/inc/geometries/ds_curv_mpit.cu b/inc/geometries/ds_curv_mpit.cu
index 5693d7697..7b60f863c 100644
--- a/inc/geometries/ds_curv_mpit.cu
+++ b/inc/geometries/ds_curv_mpit.cu
@@ -41,7 +41,7 @@ int main(int argc, char * argv[])
     dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField( gp);
     dg::Timer t;
     t.tic();
-    unsigned mx=1, my=10;
+    unsigned mx=1, my=100;
     double psi_0 = -20, psi_1 = -4;
     dg::geo::FluxGenerator flux( mag.get_psip(), mag.get_ipol(), psi_0, psi_1, gp.R_0, 0., 1);
     if(rank==0)std::cout << "Constructing Grid...\n";
diff --git a/inc/geometries/ds_curv_t.cu b/inc/geometries/ds_curv_t.cu
index 76f763c60..3fd068846 100644
--- a/inc/geometries/ds_curv_t.cu
+++ b/inc/geometries/ds_curv_t.cu
@@ -32,7 +32,7 @@ int main(int argc, char * argv[])
     std::cout << "Type n(3), Nx(8), Ny(80), Nz(20)\n";
     unsigned n,Nx,Ny,Nz;
     std::cin >> n>> Nx>>Ny>>Nz;   
-    std::cout << "Type multipleX (1) and multipleY (10)!\n";
+    std::cout << "Type multipleX (1) and multipleY (100)!\n";
     unsigned mx, my;
     std::cin >> mx >> my;
 
-- 
GitLab


From c01ba14cb44102db3049d5159671cf4426bf0333 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Wed, 1 Nov 2017 21:39:24 +0100
Subject: [PATCH 426/453] in the middle of debugging code

---
 inc/dg/algorithm.h                            | 11 ++-
 inc/dg/geometry/base_geometryX.h              |  1 +
 inc/dg/geometry/geometry.h                    |  2 +
 inc/dg/refined_elliptic.h                     |  4 +
 inc/geometries/ds.h                           | 77 +++++++++++++------
 inc/geometries/ds_mpit.cu                     |  6 +-
 inc/geometries/ds_t.cu                        |  6 +-
 inc/geometries/dz_t.cu                        | 12 +--
 inc/geometries/flux_t.cu                      |  4 +-
 inc/geometries/geometries.h                   | 15 ++--
 .../geometryX_refined_elliptic_b.cu           | 26 ++-----
 inc/geometries/hector.h                       |  3 +-
 inc/geometries/polar.h                        | 28 +++++++
 inc/geometries/ribeiroX.h                     | 19 +----
 inc/geometries/ribeiro_t.cu                   |  4 +-
 inc/geometries/simple_orthogonal_t.cu         | 35 ++++-----
 src/heat/heat.cu                              |  2 +-
 src/heat/heat.cuh                             | 37 ++++-----
 src/heat/heat_hpc.cu                          | 30 +++-----
 19 files changed, 172 insertions(+), 150 deletions(-)

diff --git a/inc/dg/algorithm.h b/inc/dg/algorithm.h
index fbfb44603..010ba2e1d 100644
--- a/inc/dg/algorithm.h
+++ b/inc/dg/algorithm.h
@@ -7,7 +7,6 @@
  */
 #include "blas.h"
 #include "geometry/geometry.h"
-#include "arakawa.h"
 #include "helmholtz.h"
 #include "cg.h"
 #include "functors.h"
@@ -15,5 +14,15 @@
 #include "elliptic.h"
 #include "runge_kutta.h"
 #include "multigrid.h"
+#include "refined_elliptic.h"
 #include "backend/timer.cuh"
+#include "backend/split_and_join.h"
+#include "backend/transpose.h"
 #include "backend/xspacelib.cuh"
+#include "backend/evaluationX.cuh"
+#include "backend/derivativesX.h"
+#include "backend/weightsX.cuh"
+#ifdef MPI_VERSION
+#include "arakawa.h"
+#include "backend/mpi_init.h"
+#endif
diff --git a/inc/dg/geometry/base_geometryX.h b/inc/dg/geometry/base_geometryX.h
index 8ba724d80..43af143de 100644
--- a/inc/dg/geometry/base_geometryX.h
+++ b/inc/dg/geometry/base_geometryX.h
@@ -1,6 +1,7 @@
 #pragma once
 
 #include "../backend/gridX.h"
+#include "../backend/evaluationX.cuh"
 #include "tensor.h"
 
 namespace dg
diff --git a/inc/dg/geometry/geometry.h b/inc/dg/geometry/geometry.h
index 4d2100a7f..5bb524a9f 100644
--- a/inc/dg/geometry/geometry.h
+++ b/inc/dg/geometry/geometry.h
@@ -10,6 +10,8 @@
 #include "../backend/mpi_precon.h"
 #endif//MPI_VERSION
 #include "base_geometry.h"
+#include "base_geometryX.h"
+#include "refined_gridX.h"
 //#include "cartesianX.h"
 #ifdef MPI_VERSION
 #include "mpi_base.h"
diff --git a/inc/dg/refined_elliptic.h b/inc/dg/refined_elliptic.h
index 1fc364444..f53686e40 100644
--- a/inc/dg/refined_elliptic.h
+++ b/inc/dg/refined_elliptic.h
@@ -4,6 +4,10 @@
 #include "backend/projection.cuh"
 #include "elliptic.h"
 #include "geometry/refined_grid.h"
+#ifdef MPI_VERSION
+#include "backend/mpi_projection.h"
+#include "backend/mpi_projection.h"
+#endif
 
 /*! @file 
 
diff --git a/inc/geometries/ds.h b/inc/geometries/ds.h
index 0f2e81abf..0cecdbaf9 100644
--- a/inc/geometries/ds.h
+++ b/inc/geometries/ds.h
@@ -106,6 +106,13 @@ struct DS
      */
     void construct(const FA& fieldaligned, dg::norm no=dg::normed, dg::direction dir = dg::centered);
 
+    ///@copydoc Fieldaligned::set_boundaries(dg::bc,double,double)
+    void set_boundaries( dg::bc bcz, double left, double right){m_fa.set_boundaries( bcz, left, right);}
+    ///@copydoc Fieldaligned::set_boundaries(dg::bc,const container&,const container&)
+    void set_boundaries( dg::bc bcz, const container& left, const container& right){m_fa.set_boundaries( bcz, left, right);}
+    ///@copydoc Fieldaligned::set_boundaries(dg::bc,const container&,double,double)
+    void set_boundaries( dg::bc bcz, const container& global, double scal_left, double scal_right){m_fa.set_boundaries( bcz, global, scal_left, scal_right);}
+
     /**
     * @brief forward derivative \f$ g_i = \alpha \frac{1}{h_z^+}(f_{i+1} - f_{i}) + \beta g_i\f$
     *
@@ -128,78 +135,103 @@ struct DS
     void forward( const container& f, container& g){do_forward(1.,f,0.,g);}
 
     ///@brief backward derivative \f$ g_i = \alpha \frac{1}{2h_z^-}(f_{i} - f_{i-1}) + \beta g_i \f$
-    ///@copydetails forward
+    ///@copydetails forward(double,const container&,double,container&)
     void backward( double alpha, const container& f, double beta, container& g){
         do_backward( alpha, f, beta, g);
     }
     ///@brief backward derivative \f$ g_i = \frac{1}{2h_z^-}(f_{i} - f_{i-1}) \f$
-    ///@copydetails forward
+    ///@copydetails forward(const container&,container&)
     void backward( const container& f, container& g){do_backward(1.,f,0.,g);}
     ///@brief centered derivative \f$ g_i = \alpha \frac{1}{2h_z}(f_{i+1} - f_{i-1}) + \beta g_i\f$
-    ///@copydetails forward
+    ///@copydetails forward(double,const container&,double,container&)
     void centered( double alpha, const container& f, double beta, container& g){
         do_centered( alpha, f, beta, g);
     }
     ///@brief centered derivative \f$ g_i = \frac{1}{2h_z}(f_{i+1} - f_{i-1})\f$
-    ///@copydetails forward
+    ///@copydetails forward(const container&,container&)
     void centered( const container& f, container& g){do_centered(1.,f,0.,g);}
 
     ///@brief forward divergence \f$ g = \alpha \nabla\cdot(\vec v f) + \beta g\f$
-    ///@copydetails forward
+    ///@copydetails forward(double,const container&,double,container&)
     ///@note forwardDiv is the negative adjoint of backward
     void forwardDiv( double alpha, const container& f, double beta, container& g){
         do_forwardDiv( alpha, f, beta, g, dg::normed);
     }
     ///@brief backward divergence \f$ g = \alpha \nabla\cdot(\vec v f) + \beta g\f$
-    ///@copydetails forward
+    ///@copydetails forward(double,const container&,double,container&)
     ///@note backwardDiv is the negative adjoint of forward
     void backwardDiv( double alpha, const container& f, double beta, container& g){
         do_backwardDiv( alpha, f, beta, g, dg::normed);
     }
     ///@brief centered divergence \f$ g = -\alpha \nabla\cdot(\vec v f) + \beta g\f$
-    ///@copydetails forward
+    ///@copydetails forward(double,const container&,double,container&)
     ///@note centeredDiv is the negative adjoint of centered
     void centeredDiv(double alpha, const container& f, double beta, container& g){
         do_centeredDiv( alpha, f, beta, g, dg::normed);
     }
+    ///@brief forward divergence \f$ g = \alpha \nabla\cdot(\vec v f) + \beta g\f$
+    ///@copydetails forward(const container&,container&)
+    ///@note forwardDiv is the negative adjoint of backward
     void forwardDiv(const container& f, container& g){
         do_forwardDiv( 1.,f,0.,g, dg::normed);
     }
+    ///@brief backward divergence \f$ g = \alpha \nabla\cdot(\vec v f) + \beta g\f$
+    ///@copydetails forward(const container&,container&)
+    ///@note backwardDiv is the negative adjoint of forward
     void backwardDiv(const container& f, container& g){
         do_backwardDiv( 1.,f,0.,g, dg::normed);
     }
+    ///@brief centered divergence \f$ g = -\alpha \nabla\cdot(\vec v f) + \beta g\f$
+    ///@copydetails forward(const container&,container&)
+    ///@note centeredDiv is the negative adjoint of centered
     void centeredDiv(const container& f, container& g){
         do_centeredDiv( 1.,f,0.,g, dg::normed);
     }
 
     /**
-    * @brief \f$ \vec v\cdot \nabla f \f$
+    * @brief Discretizes \f$ \vec v\cdot \nabla f \f$
     *
     * dependent on dir redirects to either forward(), backward() or centered()
-    * @param f The vector to derive
-    * @param g contains result on output (write only)
+    * @copydetails forward(const container&,container&)
     */
     void operator()( const container& f, container& g){operator()(1., f, 0., g);}
+    /**
+    * @brief Discretizes \f$ g = \alpha \vec v\cdot \nabla f + \beta g \f$
+    *
+    * dependent on dir redirects to either forward(), backward() or centered()
+    * @copydetails forward(double,const container&,double,container&)
+    */
     void operator()(double alpha, const container& f, double beta, container& g);
 
 
     /**
-     * @brief Discretizes \f$ \nabla\cdot ( \vec v \vec v \cdot \nabla . )\f$ as a symmetric matrix
+     * @brief Discretizes \f$ g = \nabla\cdot ( \vec v \vec v \cdot \nabla f )\f$ as a symmetric matrix
      *
-     * if direction is centered then centered followed by centeredDiv and adding jump terms
-     * @param f The vector to derive
-     * @param dsTdsf contains result on output (write only)
-     * @note if dependsOnX is false then no jump terms will be added in the x-direction and similar in y
+     * if direction is centered then centered followed by centeredDiv and adding jump terms is called, else a symmetric forward/backward discretization is chosen.
+     * @copydetails forward(const container&,container&)
+     * @note if dependsOnX is false then no jump terms will be added in the x-direction; analogous in y
+     */
+    void symv( const container& f, container& g){ do_symv( 1., f, 0., dsTdsf);}
+    /**
+     * @brief Discretizes \f$ g = \alpha \nabla\cdot ( \vec v \vec v \cdot \nabla f ) + \beta g\f$ as a symmetric matrix
+     *
+     * if direction is centered then centered followed by centeredDiv and adding jump terms is called, else a symmetric forward/backward discretization is chosen.
+     * @copydetails forward(double,const container&,double,container&)
+     * @note if dependsOnX is false then no jump terms will be added in the x-direction; analogous in y
      */
-    void symv( const container& f, container& dsTdsf){ do_symv( 1., f, 0., dsTdsf);}
     void symv( double alpha, const container& f, double beta, container& dsTdsf){ do_symv( alpha, f, beta, dsTdsf);}
+    /**
+     * @brief Discretizes \f$ g = \nabla_\parallel^2 f \f$ 
+     *
+     * The formula used is \f[ \nabla_\parallel^2 f = 2\left(\frac{f^+}{h^+h^0} - \frac{f^0}{h^+h^-} + \frac{f^-}{h^-h^0}\right) \f]
+     * @copydetails forward(const container&,container&)
+     */
     void dss( const container& f, container& g){ do_dss( 1., f, 0., g);}
     /**
      * @brief Discretizes \f$ g = \alpha \nabla_\parallel^2 f + \beta g \f$ 
      *
-     * The formula used is \f[ \nabla_\parallel^2 f = 2\left(\frac{f^+}{h^+h^0} - \frac{f^0}{h^+h^-} + \frac{f^-}{h^-h^0}\right)
-     * @param f The vector to derive
-     * @param dsTdsf contains result on output (write only)
+     * The formula used is \f[ \nabla_\parallel^2 f = 2\left(\frac{f^+}{h^+h^0} - \frac{f^0}{h^+h^-} + \frac{f^-}{h^-h^0}\right) \f]
+     * @copydetails forward(double,const container&,double,container&)
      */
     void dss( double alpha, const container& f, double beta, container& g){ do_symv( alpha, f, beta, g);}
 
@@ -356,10 +388,9 @@ void DS<G,I,M,container>::do_symv( double alpha, const container& f, double beta
     else 
     {
         do_forward( 1., f, 0., m_tempP);
-        do_backwardDiv( 1., m_tempP, 0., m_temp0, dg::not_normed);
+        do_backwardDiv( 1., m_tempP, 0., m_temp, dg::not_normed);
         do_backward( 1., f, 0., m_tempM);
-        do_forwardDiv( 1., m_tempM, 0., m_temp, dg::not_normed);
-        dg::blas1::axpby(0.5,m_temp0,0.5,m_temp);
+        do_forwardDiv( 0.5, m_tempM, 0.5, m_temp, dg::not_normed);
     }
     dg::blas1::pointwiseDivide( m_temp, m_weights_wo_vol, m_temp);
     //     add jump term 
@@ -380,7 +411,7 @@ void DS<G,I,M,container>::do_dss( double alpha, const container& f, double beta,
     m_fa(einsPlus,  f, m_tempP);
     m_fa(einsMinus, f, m_tempM);
     dg::blas1::pointwiseDot( 1., m_tempP, m_fa.hp_inv(), 1., m_tempM, m_fa.hm_inv(), 0., m_tempM);
-    dg::blas1::pointwiseDot( -2., alpha, f,  m_fa.hp_inv(), m_fa.hm_inv(), beta, dssf);
+    dg::blas1::pointwiseDot( -2.*alpha, f,  m_fa.hp_inv(), m_fa.hm_inv(), beta, dssf);
     dg::blas1::pointwiseDot( 2.*alpha, m_fa.hz_inv(), m_tempM, 1., dssf);
 
 }
diff --git a/inc/geometries/ds_mpit.cu b/inc/geometries/ds_mpit.cu
index f3f09f9c1..88be045f4 100644
--- a/inc/geometries/ds_mpit.cu
+++ b/inc/geometries/ds_mpit.cu
@@ -2,11 +2,7 @@
 
 #include <mpi.h>
 #define DG_BENCHMARK
-#include "dg/backend/timer.cuh"
-#include "dg/backend/mpi_evaluation.h"
-#include "dg/backend/mpi_init.h"
-#include "dg/backend/functions.h"
-#include "dg/geometry/geometry.h"
+#include "dg/algorithm.h"
 #include "ds.h"
 #include "toroidal.h"
 
diff --git a/inc/geometries/ds_t.cu b/inc/geometries/ds_t.cu
index 57e301272..875241726 100644
--- a/inc/geometries/ds_t.cu
+++ b/inc/geometries/ds_t.cu
@@ -2,11 +2,7 @@
 
 #include <cusp/print.h>
 #define DG_BENCHMARK
-#include "dg/backend/functions.h"
-#include "dg/backend/timer.cuh"
-#include "dg/blas.h"
-#include "dg/functors.h"
-#include "dg/geometry/geometry.h"
+#include "dg/algorithm.h"
 #include "magnetic_field.h"
 #include "ds.h"
 #include "toroidal.h"
diff --git a/inc/geometries/dz_t.cu b/inc/geometries/dz_t.cu
index 9bf7785ca..db4691505 100644
--- a/inc/geometries/dz_t.cu
+++ b/inc/geometries/dz_t.cu
@@ -2,13 +2,9 @@
 
 #include <cusp/print.h>
 
-#include "blas.h"
-#include "ds.h"
-#include "functors.h"
+#include "dg/algorithm.h"
 
-#include "backend/functions.h"
-#include "backend/timer.cuh"
-#include "geometry.h"
+#include "ds.h"
 
 double sine(double x, double y, double z){return sin(z);}
 double cosine(double x, double y, double z){return cos(z);}
@@ -38,9 +34,9 @@ int main()
     const dg::DVec w3d = dg::create::volume( g3d);
     dg::Timer t;
     t.tic();
-    dg::DDS::FieldAligned dsFA( dg::DefaultField(), g3d, 1e-10, dg::DefaultLimiter(), dg::NEU);
+    dg::geo::BinaryVectorLvl0 vec( dg::geo::Constant(0), dg::geo::Constant(0), dg::geo::Constant(1));
 
-    dg::DDS ds ( dsFA, dg::DefaultField(), dg::not_normed, dg::centered);
+    dg::geo::DS<dg::CartesianGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec> ds ( vec, g3d, dg::DIR, dg::DIR, dg::geo::FullLimiter(), dg::not_normed, dg::centered);
     t.toc();
     std::cout << "TEST STRAIGHT FIELD LINES AND BOUNDARIES IN Z\n";
     std::cout << "Creation of parallel Derivative took     "<<t.diff()<<"s\n";
diff --git a/inc/geometries/flux_t.cu b/inc/geometries/flux_t.cu
index 00f55072e..3c8e9fcf5 100644
--- a/inc/geometries/flux_t.cu
+++ b/inc/geometries/flux_t.cu
@@ -5,10 +5,8 @@
 #include <sstream>
 #include <cmath>
 
-#include "dg/backend/xspacelib.cuh"
-#include "dg/functors.h"
+#include "dg/algorithm.h"
 
-#include "dg/backend/timer.cuh"
 #include "curvilinear.h"
 //#include "guenther.h"
 #include "testfunctors.h"
diff --git a/inc/geometries/geometries.h b/inc/geometries/geometries.h
index 0379e612d..22b2d62b5 100644
--- a/inc/geometries/geometries.h
+++ b/inc/geometries/geometries.h
@@ -1,17 +1,20 @@
 #pragma once
 
-//include grids
-#include "curvilinear.h"
-#ifdef MPI_VERSION
-#include "mpi_curvilinear.h"
-#endif
-
 //include grid generators
 #include "simple_orthogonal.h"
+#include "separatrix_orthogonal.h"
 #include "ribeiro.h"
 #include "flux.h"
 #include "hector.h"
 #include "polar.h"
+#include "ribeiroX.h"
+//include grids
+#include "curvilinear.h"
+#include "curvilinearX.h"
+#include "refined_curvilinearX.h"
+#ifdef MPI_VERSION
+#include "mpi_curvilinear.h"
+#endif
 
 //include magnetic field geometries
 #include "solovev.h"
diff --git a/inc/geometries/geometryX_refined_elliptic_b.cu b/inc/geometries/geometryX_refined_elliptic_b.cu
index aa8b4e691..09e616a1b 100644
--- a/inc/geometries/geometryX_refined_elliptic_b.cu
+++ b/inc/geometries/geometryX_refined_elliptic_b.cu
@@ -2,23 +2,11 @@
 
 #include "file/nc_utilities.h"
 
-#include "dg/geometry/refined_gridX.h"
-#include "dg/backend/timer.cuh"
-#include "dg/backend/grid.h"
-#include "dg/backend/gridX.h"
-#include "dg/backend/derivativesX.h"
-#include "dg/backend/evaluationX.cuh"
-#include "dg/refined_elliptic.h"
-#include "dg/cg.h"
+#include "dg/algorithm.h"
 
-#include "solovev.h"
-#include "taylor.h"
-//#include "ribeiroX.h"
-#include "refined_orthogonalX.h"
-#include "separatrix_orthogonal.h"
+#include "geometries.h"
 #include "testfunctors.h"
 
-using namespace dg::geo::taylor;
 const char* parameters = "geometry_params_Xpoint_taylor.js";
 //using namespace dg::geo::solovev;
 //const char* parameters = "geometry_params_Xpoint.js";
@@ -46,7 +34,7 @@ int main(int argc, char**argv)
         std::ifstream is(argv[1]);
         reader.parse(is,js,false);
     }
-    GeomParameters gp(js);
+    dg::geo::taylor::Parameters gp(js);
     gp.display( std::cout);
     dg::Timer t;
     std::cout << "Constructing grid ... \n";
@@ -70,10 +58,10 @@ int main(int argc, char**argv)
     dg::geo::SeparatrixOrthogonal generator(c.get_psip(), psi_0, R_X,Z_X, R0, Z0,0);
     //dg::geo::SimpleOrthogonalX generator(c.get_psip(), psi_0, R_X,Z_X, R0, Z0,0);
     //dg::CurvilinearGridX2d g2d( generator, 0.25, 1./22., n, Nx, Ny, dg::DIR, dg::NEU);
-    dg::EquidistRefinementX equi(add_x, add_y, howmanyX, howmanyY)
-    dg::OrthogonalRefinedGridX2d g2d( equi, generator, 0.25, 1./22., n_ref, n, Nx, Ny, dg::DIR, dg::NEU);
-    dg::Elliptic<dg::OrthogonalRefinedGridX2d, dg::Composite<dg::DMatrix>, dg::DVec> pol( g2d, dg::not_normed, dg::forward);
-    dg::RefinedElliptic<dg::OrthogonalRefinedGridX2d, dg::IDMatrix, dg::Composite<dg::DMatrix>, dg::DVec> pol_refined( g2d, dg::not_normed, dg::forward);
+    dg::EquidistXRefinement equi(add_x, add_y, howmanyX, howmanyY);
+    dg::geo::CurvilinearRefinedGridX2d g2d( equi, generator, 0.25, 1./22., n_ref, n, Nx, Ny, dg::DIR, dg::NEU);
+    dg::Elliptic<dg::geo::CurvilinearRefinedGridX2d, dg::Composite<dg::DMatrix>, dg::DVec> pol( g2d, dg::not_normed, dg::forward);
+    dg::RefinedElliptic<dg::geo::CurvilinearRefinedGridX2d, dg::IDMatrix, dg::Composite<dg::DMatrix>, dg::DVec> pol_refined( g2d, dg::not_normed, dg::forward);
     double fx = 0.25;
     psi_1 = -fx/(1.-fx)*psi_0;
     std::cout << "psi 1 is          "<<psi_1<<"\n";
diff --git a/inc/geometries/hector.h b/inc/geometries/hector.h
index ba5de9b29..4bf7e825e 100644
--- a/inc/geometries/hector.h
+++ b/inc/geometries/hector.h
@@ -5,8 +5,9 @@
 #include "dg/backend/interpolation.cuh"
 #include "dg/geometry/geometry.h"
 #include "dg/elliptic.h"
-#include "fluxfunctions.h"
 #include "dg/cg.h"
+#include "fluxfunctions.h"
+#include "curvilinear.h"
 #include "flux.h"
 #include "adaption.h"
 
diff --git a/inc/geometries/polar.h b/inc/geometries/polar.h
index ec3b355c0..1c9a5a08c 100644
--- a/inc/geometries/polar.h
+++ b/inc/geometries/polar.h
@@ -4,6 +4,14 @@
 namespace dg {
 namespace geo {
 
+/**
+* @brief Polar coordinates
+
+ \f[ x = r \cos(\eta) \\
+     y = r \sin(\eta) \f]
+     with \f$ r = \zeta + r_{min}\f$ 
+ * @ingroup generators_geo
+*/
 struct PolarGenerator : public aGenerator2d
 {
     private:
@@ -11,6 +19,12 @@ struct PolarGenerator : public aGenerator2d
 
     public:
 
+    /**
+    * @brief Construct a ring with minimal and maximal radius
+    *
+    * @param _r_min minimum radius
+    * @param _r_max maximum radius
+    */
     PolarGenerator(double _r_min, double _r_max) : r_min(_r_min), r_max(_r_max) {}
     virtual PolarGenerator* clone() const{return new PolarGenerator(*this); }
 
@@ -56,6 +70,14 @@ struct PolarGenerator : public aGenerator2d
 };
 
 
+/**
+* @brief Log Polar coordinates (conformal) 
+
+ \f[ x = \exp(l) \cos(\eta) \\
+     y = \exp(l) \sin(\eta) \f]
+     with \f$ l = \zeta + \log (r_{min})\f$ 
+ * @ingroup generators_geo
+*/
 struct LogPolarGenerator : public aGenerator2d
 {
     private:
@@ -63,6 +85,12 @@ struct LogPolarGenerator : public aGenerator2d
 
     public:
 
+    /**
+    * @brief Construct a ring with minimal and maximal radius
+    *
+    * @param _r_min minimum radius
+    * @param _r_max maximum radius
+    */
     LogPolarGenerator(double _r_min, double _r_max) : r_min(_r_min), r_max(_r_max) {}
     virtual LogPolarGenerator* clone() const{return new LogPolarGenerator(*this); }
 
diff --git a/inc/geometries/ribeiroX.h b/inc/geometries/ribeiroX.h
index 00aff3310..c3239091a 100644
--- a/inc/geometries/ribeiroX.h
+++ b/inc/geometries/ribeiroX.h
@@ -5,7 +5,7 @@
 #include "dg/backend/evaluationX.cuh"
 #include "dg/backend/weightsX.cuh"
 #include "dg/runge_kutta.h"
-#include "dg/geometry/generatorX.h"
+#include "generatorX.h"
 #include "utilitiesX.h"
 #include "ribeiro.h"
 
@@ -248,21 +248,12 @@ struct RibeiroX : public aGeneratorX2d
     {
         assert( psi_0 < 0 );
         zeta0_ = fpsi_.find_x( psi_0);
-        f0_=zeta0_/psi_0;
         zeta1_= -fx/(1.-fx)*zeta0_;
         x0_=x0, y0_=y0, psi0_=psi_0;
     }
     private:
     bool isConformal()const{return false;}
     bool do_isOrthogonal()const{return false;}
-    double f0() const{return f0_;}
-    /**
-     * @brief The vector f(x)
-     *
-     * @return f(x)
-     */
-    thrust::host_vector<double> fx() const{ return fx_;}
-    double psi1() const {return psi_1_numerical_;}
     void do_generate( 
          const thrust::host_vector<double>& zeta1d, 
          const thrust::host_vector<double>& eta1d, 
@@ -278,8 +269,8 @@ struct RibeiroX : public aGeneratorX2d
         unsigned inside=0;
         for(unsigned i=0; i<zeta1d.size(); i++)
             if( zeta1d[i]< 0) inside++;//how many points are inside
-        thrust::host_vector<double> psi_x;
-        psi_1_numerical_ = dg::geo::detail::construct_psi_values( fpsiMinv_, psi0_, zeta0_, zeta1d, zeta1_, inside, psi_x);
+        thrust::host_vector<double> psi_x, fx_;
+        dg::geo::detail::construct_psi_values( fpsiMinv_, psi0_, zeta0_, zeta1d, zeta1_, inside, psi_x);
 
         //std::cout << "In grid function:\n";
         dg::geo::ribeiro::FieldRZYRYZY fieldRZYRYZYribeiro(psi_);
@@ -309,10 +300,8 @@ struct RibeiroX : public aGeneratorX2d
     virtual double do_eta1(double fy) const { return 2.*M_PI*(1.+fy/(1.-2.*fy));}
     private:
     BinaryFunctorsLvl2 psi_;
-    dg::geo::ribeiro::detail::XFieldFinv fpsiMinv_; 
     dg::geo::ribeiro::detail::FpsiX fpsi_;
-    double f0_, psi_1_numerical_;
-    thrust::host_vector<double> fx_;
+    dg::geo::ribeiro::detail::XFieldFinv fpsiMinv_; 
     double zeta0_, zeta1_;
     double lx_, x0_, y0_, psi0_, psi1_;
     int mode_; //0 = ribeiro, 1 = equalarc
diff --git a/inc/geometries/ribeiro_t.cu b/inc/geometries/ribeiro_t.cu
index 524012d2d..79935d55c 100644
--- a/inc/geometries/ribeiro_t.cu
+++ b/inc/geometries/ribeiro_t.cu
@@ -5,10 +5,8 @@
 #include <sstream>
 #include <cmath>
 
-#include "dg/backend/xspacelib.cuh"
-#include "dg/functors.h"
+#include "dg/algorithm.h"
 
-#include "dg/backend/timer.cuh"
 #include "curvilinear.h"
 //#include "guenther.h"
 #include "solovev.h"
diff --git a/inc/geometries/simple_orthogonal_t.cu b/inc/geometries/simple_orthogonal_t.cu
index a5c194b65..ddce485e4 100644
--- a/inc/geometries/simple_orthogonal_t.cu
+++ b/inc/geometries/simple_orthogonal_t.cu
@@ -4,16 +4,12 @@
 #include <fstream>
 #include <sstream>
 #include <cmath>
+#include <memory>
 
-#include "dg/backend/xspacelib.cuh"
-#include "dg/functors.h"
-
-#include "dg/backend/timer.cuh"
-//#include "guenther.h"
+#include "dg/algorithm.h"
+#include "guenther.h"
 #include "solovev.h"
-
 #include "simple_orthogonal.h"
-//#include "ds.h"
 #include "init.h"
 #include "testfunctors.h"
 #include "curvilinear.h"
@@ -42,7 +38,6 @@ double sineX( double x, double y) {return sin(x)*sin(y);}
 double cosineX( double x, double y) {return cos(x)*sin(y);}
 double sineY( double x, double y) {return sin(x)*sin(y);}
 double cosineY( double x, double y) {return sin(x)*cos(y);}
-//typedef dg::FieldAligned< dg::OrthogonalGrid3d<dg::HVec> , dg::IHMatrix, dg::HVec> DFA;
 
 int main( int argc, char* argv[])
 {
@@ -75,8 +70,8 @@ int main( int argc, char* argv[])
 
     dg::geo::SimpleOrthogonal generator( psip, psi_0, psi_1, gp.R_0, 0., 1);
     dg::geo::CurvilinearProductGrid3d g3d(generator, n, Nx, Ny,Nz, dg::DIR);
-    dg::geo::CurvilinearGrid2d g2d = g3d.perp_grid();
-    dg::Grid2d g2d_periodic(g2d.x0(), g2d.x1(), g2d.y0(), g2d.y1(), g2d.n(), g2d.Nx(), g2d.Ny()+1); 
+    std::unique_ptr<dg::aGeometry2d> g2d( g3d.perp_grid());
+    dg::Grid2d g2d_periodic(g2d->x0(), g2d->x1(), g2d->y0(), g2d->y1(), g2d->n(), g2d->Nx(), g2d->Ny()+1); 
     t.toc();
     std::cout << "Construction took "<<t.diff()<<"s"<<std::endl;
     int ncid;
@@ -94,30 +89,30 @@ int main( int argc, char* argv[])
     err = nc_def_var( ncid, "volume", NC_DOUBLE, 2, dim3d, &volID);
     err = nc_def_var( ncid, "divB", NC_DOUBLE, 2, dim3d, &divBID);
 
-    thrust::host_vector<double> psi_p = dg::pullback( psip.f(), g2d);
+    thrust::host_vector<double> psi_p = dg::pullback( psip.f(), *g2d);
     //g.display();
     err = nc_put_var_double( ncid, onesID, periodify(psi_p, g2d_periodic).data());
-    dg::HVec X( g2d.size()), Y(X); //P = dg::pullback( dg::coo3, g);
-    for( unsigned i=0; i<g2d.size(); i++)
+    dg::HVec X( g2d->size()), Y(X); //P = dg::pullback( dg::coo3, g);
+    for( unsigned i=0; i<g2d->size(); i++)
     {
-        X[i] = g2d.map()[0][i];
-        Y[i] = g2d.map()[1][i];
+        X[i] = g2d->map()[0][i];
+        Y[i] = g2d->map()[1][i];
     }
 
-    dg::HVec temp0( g2d.size()), temp1(temp0);
-    dg::HVec w2d = dg::create::weights( g2d);
+    dg::HVec temp0( g2d->size()), temp1(temp0);
+    dg::HVec w2d = dg::create::weights( *g2d);
 
     err = nc_put_var_double( ncid, coordsID[0], periodify(X, g2d_periodic).data());
     err = nc_put_var_double( ncid, coordsID[1], periodify(Y, g2d_periodic).data());
     //err = nc_put_var_double( ncid, coordsID[2], g.z().data());
 
     //compute and write deformation into netcdf
-    dg::SparseTensor<dg::HVec> metric = g2d.metric();
+    dg::SparseTensor<dg::HVec> metric = g2d->metric();
     dg::HVec g_xx = metric.value(0,0), g_yy=metric.value(1,1);
     dg::SparseElement<dg::HVec> vol_ = dg::tensor::volume(metric);
     dg::HVec vol = vol_.value();
     dg::blas1::pointwiseDivide( g_yy, g_xx, temp0);
-    const dg::HVec ones = dg::evaluate( dg::one, g2d);
+    const dg::HVec ones = dg::evaluate( dg::one, *g2d);
     X=temp0;
     err = nc_put_var_double( ncid, defID, periodify(X, g2d_periodic).data());
     //compute and write conformalratio into netcdf
@@ -184,7 +179,7 @@ int main( int argc, char* argv[])
     std::cout << "Note that the error might also come from the volume in RZP!\n"; //since integration of jacobian is fairly good probably
 
     dg::geo::TokamakMagneticField c=dg::geo::createSolovevField(gp);
-     X = dg::pullback(dg::geo::FuncDirNeu(c, psi_0, psi_1, 550, -150, 30., 1), g2d);
+     X = dg::pullback(dg::geo::FuncDirNeu(c, psi_0, psi_1, 550, -150, 30., 1), *g2d);
      err = nc_put_var_double( ncid, divBID, periodify(X, g2d_periodic).data());
     err = nc_close( ncid);
 
diff --git a/src/heat/heat.cu b/src/heat/heat.cu
index 379cea13c..b44a3510b 100644
--- a/src/heat/heat.cu
+++ b/src/heat/heat.cu
@@ -104,7 +104,7 @@ int main( int argc, char* argv[])
     //create RHS     
     std::cout << "initialize feltor" << std::endl;
     heat::Explicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec> feltor( grid, p,gp); //initialize before rolkar!
-    std::cout << "initialize rolkar" << std::endl;
+    std::cout << "initialize implicit" << std::endl;
     heat::Implicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec > rolkar( grid, p,gp);
 
     ////////////////////////////////The initial field////////////////////////////////
diff --git a/src/heat/heat.cuh b/src/heat/heat.cuh
index 37b0ab08c..5ae601264 100644
--- a/src/heat/heat.cuh
+++ b/src/heat/heat.cuh
@@ -148,14 +148,12 @@ void Explicit<G,I,M,V>::energies( std::vector<V>& y)
     if (p.p_diff ==0)    {
     //     (A) adjoint
 //         dsNU_( y[0], omega); 
-//         dsNU_.centeredAdj(omega,lambda);
+//         dsNU_.centeredDiv(omega,lambda);
         dsNU_.forward( y[0], omega); 
-        dsNU_.forwardAdj(omega,lambda);
-        dg::blas1::axpby( 0.5, lambda, 0.,chi,chi); 
+        dsNU_.backwardDiv(0.5, omega,0., chi);
 // 
         dsNU_.backward( y[0], omega); 
-        dsNU_.backwardAdj(omega,lambda);
-        dg::blas1::axpby( 0.5, lambda, 1., chi,chi); 
+        dsNU_.forwardDiv(0.5, omega,1., chi);
         Dpar[0]= p.nu_parallel*dg::blas2::dot(y[0], w3d, chi);
     }  
     if (p.p_diff ==1)    {
@@ -164,16 +162,15 @@ void Explicit<G,I,M,V>::energies( std::vector<V>& y)
         dg::blas1::pointwiseDivide(lambda,  m_invB, lambda); //-ds lnB
         dsNU_(y[0],omega); //ds T
         dg::blas1::pointwiseDot(omega, lambda, omega);            // -ds lnB ds T
-        dsNU_.dss(y[0],lambda);                                          //ds^2 T 
+        dsNU_.dss(y[0],lambda);                                   //ds^2 T 
         dg::blas1::axpby( 1., omega,  1.,lambda );    
         Dpar[0]= p.nu_parallel*dg::blas2::dot(one, w3d, lambda);  
     }
     if (p.p_diff ==2)    {
         // (C) oldnonadjoint
-        dsNU_.dss(y[0],omega);                                          //ds^2 N 
         dsNU_(y[0],lambda);       
         dg::blas1::pointwiseDot(gradlnB, lambda, lambda);            // ds lnB ds N
-        dg::blas1::axpby( 1., omega, -1., lambda);       
+        dsNU_.dss(1., y[0], -1., lambda);                            // ds^2 N 
         Dpar[0]= p.nu_parallel*dg::blas2::dot(one, w3d, lambda);  
     }
     if (p.p_diff ==3)    {
@@ -226,30 +223,30 @@ void Explicit<G,I,Matrix,container>::operator()(const std::vector<container>& y,
 //         // (A) adjoint
 //         //U=v_parallel gradlnB
 //     //     dg::blas1::pointwiseDot(y[0], gradlnB, lambda);
-//     //     dsNU_.centeredAdj(lambda,omega);    
+//     //     dsNU_.centeredDiv(lambda,omega);    
 //     //     dg::blas1::axpby( p.nu_parallel, omega, 1., yp[0]); 
 // 
 //         //U=1.  
 //         //centered
 //         dg::blas1::pointwiseDot(y[0],pupil,lambda);    //U*T  
-//         dsNU_.centeredAdj(lambda,omega);    // dsT UT
-// //         dsNU_.backwardAdj(lambda,omega);
-// //         dsNU_.forwardAdj(lambda,omega);   
+//         dsNU_.centeredDiv(lambda,omega);    // dsT UT
+// //         dsNU_.forwardDiv(lambda,omega);
+// //         dsNU_.backwardDiv(lambda,omega);   
 //         dg::blas1::axpby( -1.0, omega, 1., yp[0]); //dsT (UT)
 // 
 //         //corr(1): div(UB) 
 // //         dg::blas1::pointwiseDivide(pupil,m_invB,omega); //= U B
-// //         dsNU_.centeredAdj(omega,lambda);     //div UB
+// //         dsNU_.centeredDiv(omega,lambda);     //div UB
 // //         dg::blas1::axpby( 1.0, lambda, 1., yp[0]); //+div UB
 //         
 //         //corr(2): div(B) 
 // //         dg::blas1::pointwiseDivide(one,m_invB,omega); //= U B
-// //         dsNU_.centeredAdj(omega,lambda);     //div UB
+// //         dsNU_.centeredDiv(omega,lambda);     //div UB
 // //         dg::blas1::axpby( 1.0, lambda, 1., yp[0]); //+div UB
 //         
 //         //corr(3): UT/B divB 
 //     //     dg::blas1::pointwiseDivide(one,m_invB,omega); //=  B
-//     //     dsNU_.centeredAdj(omega,lambda);     //div B
+//     //     dsNU_.centeredDiv(omega,lambda);     //div B
 //     //     dg::blas1::pointwiseDot(y[0],m_invB,omega); //T/B
 //     //     dg::blas1::pointwiseDot(omega,pupil,omega); // U T/B
 //     //     dg::blas1::pointwiseDot(omega,lambda,lambda); //  UT/B divB
@@ -257,7 +254,7 @@ void Explicit<G,I,Matrix,container>::operator()(const std::vector<container>& y,
 // 
 //         //corr(4): U  divB
 // //         dg::blas1::pointwiseDivide(one,m_invB,omega); //=  B
-// //         dsNU_.centeredAdj(omega,lambda);     //div B
+// //         dsNU_.centeredDiv(omega,lambda);     //div B
 // //         dg::blas1::pointwiseDot(pupil,lambda,lambda); //  U divB
 // //         dg::blas1::axpby( 1.0, lambda, 1., yp[0]); //+ U div UB
 //     }
@@ -284,16 +281,16 @@ void Explicit<G,I,Matrix,container>::operator()(const std::vector<container>& y,
     if (p.p_diff ==0)    {
 // //         centered
 // //         dsNU_( y[0], omega); 
-// //         dsNU_.centeredAdj(omega,lambda);
+// //         dsNU_.centeredDiv(omega,lambda);
 // //         dg::blas1::axpby( p.nu_parallel, lambda, 1., yp[0]); 
 // 
         //forward, backward (stegi) without jump
 //         dsNU_.forward( y[0], omega); 
-//         dsNU_.forwardAdj(omega,lambda);
+//         dsNU_.backwardDiv(omega,lambda);
 //         dg::blas1::axpby( 0.5*p.nu_parallel, lambda, 1., yp[0]); 
 // 
 //         dsNU_.backward( y[0], omega); 
-//         dsNU_.backwardAdj(omega,lambda);
+//         dsNU_.forwardDiv(omega,lambda);
 //         dg::blas1::axpby( 0.5*p.nu_parallel, lambda, 1., yp[0]); 
 //         //with jump
 //        dsNU_.symv(y[0],lambda);
@@ -301,7 +298,7 @@ void Explicit<G,I,Matrix,container>::operator()(const std::vector<container>& y,
 //        dg::blas1::axpby( p.nu_parallel, lambda, 1., yp[0]); 
 // 
 //         dsNU_.backward( y[0], omega); 
-//         dsNU_.backwardAdj(omega,lambda);
+//         dsNU_.forwardDiv(omega,lambda);
 //         dg::blas1::axpby( 0.5*p.nu_parallel, lambda, 1., yp[0]); 
         //with jump
 //        dsNU_.symv(y[0],lambda);
diff --git a/src/heat/heat_hpc.cu b/src/heat/heat_hpc.cu
index 985c3c225..8d672da6f 100644
--- a/src/heat/heat_hpc.cu
+++ b/src/heat/heat_hpc.cu
@@ -8,23 +8,13 @@
 #include <cusp/coo_matrix.h>
 #include <cusp/print.h>
 
-#include "dg/backend/xspacelib.cuh"
-#include "dg/backend/timer.cuh"
-#include "dg/backend/interpolation.cuh"
-// #include "dg/backend/ell_interpolation.h"
 #include "file/nc_utilities.h"
-#include "dg/runge_kutta.h"
-#include "dg/multistep.h"
-#include "dg/elliptic.h"
-#include "dg/cg.h"
+#include "dg/algorithm.h"
 #include "geometries/geometries.h"
-#include "heat/parameters.h"
 
+#include "parameters.h"
 #include "heat.cuh"
 
-typedef dg::FieldAligned< dg::CylindricalGrid3d, dg::IDMatrix, dg::DVec> DFA;
-using namespace dg::geo::solovev;
-
 int main( int argc, char* argv[])
 {
     ////Parameter initialisation ////////////////////////////////////////////
@@ -44,8 +34,8 @@ int main( int argc, char* argv[])
         reader.parse(is,js,false);
         reader.parse(ks,gs,false);
     }
-    const eule::Parameters p( js); p.display( std::cout);
-    const GeomParameters gp(gs); gp.display( std::cout);
+    const heat::Parameters p( js); p.display( std::cout);
+    const dg::geo::solovev::Parameters gp(gs); gp.display( std::cout);
     ////////////////////////////////set up computations///////////////////////////
 
     double Rmin=gp.R_0-p.boxscaleRm*gp.a;
@@ -86,8 +76,8 @@ int main( int argc, char* argv[])
         std::cout << "geome in"<<geomin <<std::endl;
         reader.parse(inputin,js,false);
         reader.parse(geomin,gs,false);
-        const eule::Parameters pin(js);
-        const dg::geo::solovev::GeomParameters gpin(gs);
+        const heat::Parameters pin(js);
+        const dg::geo::solovev::Parameters gpin(gs);
         double Rminin = gpin.R_0 - pin.boxscaleRm*gpin.a;
         double Zminin =-pin.boxscaleZm*gpin.a*gpin.elongation;
         double Rmaxin = gpin.R_0 + pin.boxscaleRp*gpin.a; 
@@ -104,9 +94,9 @@ int main( int argc, char* argv[])
     }
     // /////////////////////create RHS 
     std::cout << "Constructing Feltor...\n";
-    eule::Feltor<dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec > feltor( grid, p,gp); 
-    std::cout << "Constructing Rolkar...\n";
-    eule::Rolkar<dg::CylindricalGrid3d, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec > rolkar( grid, p,gp);
+    heat::Explicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec> feltor( grid, p,gp); //initialize before rolkar!
+    std::cout << "initialize implicit" << std::endl;
+    heat::Implicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec > rolkar( grid, p,gp);
     std::cout << "Done!\n";
 
     /////////////////////The initial field///////////////////////////////////////////
@@ -118,7 +108,7 @@ int main( int argc, char* argv[])
 //     dg::CONSTANT init0( 0.);
 
     //background profile
-    dg::geo::Nprofile<Psip> prof(p.bgprofamp, p.nprofileamp, gp, Psip(gp)); //initial background profile
+    dg::geo::Nprofile prof(p.bgprofamp, p.nprofileamp, gp, dg::geo::solovev::Psip(gp)); //initial background profile
     std::vector<dg::DVec> y0(1, dg::evaluate( prof, grid)), y1(y0); 
     //field aligning
 //     dg::CONSTANT gaussianZ( 1.);
-- 
GitLab


From 1c476b67d50ff7c532898a188e214e973da43d20 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Wed, 1 Nov 2017 23:57:45 +0100
Subject: [PATCH 427/453] debugging X-point topology programs

---
 inc/dg/algorithm.h                            |  1 +
 inc/dg/refined_elliptic.h                     | 11 ++-
 inc/geometries/ds.h                           | 28 ++----
 .../geometryX_refined_elliptic_b.cu           | 95 +++++++++----------
 4 files changed, 65 insertions(+), 70 deletions(-)

diff --git a/inc/dg/algorithm.h b/inc/dg/algorithm.h
index 010ba2e1d..886a67b7b 100644
--- a/inc/dg/algorithm.h
+++ b/inc/dg/algorithm.h
@@ -22,6 +22,7 @@
 #include "backend/evaluationX.cuh"
 #include "backend/derivativesX.h"
 #include "backend/weightsX.cuh"
+#include "backend/interpolationX.cuh"
 #ifdef MPI_VERSION
 #include "arakawa.h"
 #include "backend/mpi_init.h"
diff --git a/inc/dg/refined_elliptic.h b/inc/dg/refined_elliptic.h
index f53686e40..e2ca8b50d 100644
--- a/inc/dg/refined_elliptic.h
+++ b/inc/dg/refined_elliptic.h
@@ -16,12 +16,19 @@
 namespace dg
 {
 
+ /*!@brief The refined version of \c Elliptic 
+ 
+ * Holds an \c Elliptic object on the fine grid and brackets every call to symv with %interpolation and %projection matrices
+ * @copydoc hide_geometry_matrix_container
+ * @ingroup matrixoperators
+ * @attention This class is still under construction!
+ */
 template < class Geometry,class IMatrix, class Matrix, class container>
 class RefinedElliptic
 {
     public:
     /**
-     * @brief Construct from Grid
+     * @brief Construct from a coarse and a fine grid
      *
      * @param g_coarse The coarse Grid
      * @param g_fine The fine Grid, boundary conditions are taken from here
@@ -70,8 +77,6 @@ class RefinedElliptic
     const container& weights()const {return weights_;}
     /**
      * @brief Returns the preconditioner to use in conjugate gradient
-     *
-     * In this case inverse weights are the best choice
      * @return inverse weights
      */
     const container& precond()const {return inv_weights_;}
diff --git a/inc/geometries/ds.h b/inc/geometries/ds.h
index 0cecdbaf9..e9d3cf9d0 100644
--- a/inc/geometries/ds.h
+++ b/inc/geometries/ds.h
@@ -107,11 +107,17 @@ struct DS
     void construct(const FA& fieldaligned, dg::norm no=dg::normed, dg::direction dir = dg::centered);
 
     ///@copydoc Fieldaligned::set_boundaries(dg::bc,double,double)
-    void set_boundaries( dg::bc bcz, double left, double right){m_fa.set_boundaries( bcz, left, right);}
+    void set_boundaries( dg::bc bcz, double left, double right){
+        m_fa.set_boundaries( bcz, left, right);
+    }
     ///@copydoc Fieldaligned::set_boundaries(dg::bc,const container&,const container&)
-    void set_boundaries( dg::bc bcz, const container& left, const container& right){m_fa.set_boundaries( bcz, left, right);}
+    void set_boundaries( dg::bc bcz, const container& left, const container& right){
+        m_fa.set_boundaries( bcz, left, right);
+    }
     ///@copydoc Fieldaligned::set_boundaries(dg::bc,const container&,double,double)
-    void set_boundaries( dg::bc bcz, const container& global, double scal_left, double scal_right){m_fa.set_boundaries( bcz, global, scal_left, scal_right);}
+    void set_boundaries( dg::bc bcz, const container& global, double scal_left, double scal_right){
+        m_fa.set_boundaries( bcz, global, scal_left, scal_right);
+    }
 
     /**
     * @brief forward derivative \f$ g_i = \alpha \frac{1}{h_z^+}(f_{i+1} - f_{i}) + \beta g_i\f$
@@ -235,22 +241,6 @@ struct DS
      */
     void dss( double alpha, const container& f, double beta, container& g){ do_symv( alpha, f, beta, g);}
 
-    ///@copydoc Fieldaligned::set_boundaries(dg::bc,double,double)
-    void set_boundaries( dg::bc bcz, double left, double right)
-    {
-        m_fa.set_boundaries( bcz, left, right);
-    }
-    ///@copydoc Fieldaligned::set_boundaries(dg::bc,const container&,const container&)
-    void set_boundaries( dg::bc bcz, const container& left, const container& right)
-    {
-        m_fa.set_boundaries( bcz, left, right);
-    }
-    ///@copydoc Fieldaligned::set_boundaries(dg::bc,const container&,double,double)
-    void set_boundaries( dg::bc bcz, const container& global, double scal_left, double scal_right)
-    {
-        m_fa.set_boundaries( bcz, global, scal_left, scal_right);
-    }
-
     const container& weights()const {return m_vol3d;}
     const container& inv_weights()const {return m_inv3d;}
     const container& precond()const {return m_inv3d;}
diff --git a/inc/geometries/geometryX_refined_elliptic_b.cu b/inc/geometries/geometryX_refined_elliptic_b.cu
index 09e616a1b..244d5b0b5 100644
--- a/inc/geometries/geometryX_refined_elliptic_b.cu
+++ b/inc/geometries/geometryX_refined_elliptic_b.cu
@@ -5,6 +5,7 @@
 #include "dg/algorithm.h"
 
 #include "geometries.h"
+#include "taylor.h"
 #include "testfunctors.h"
 
 const char* parameters = "geometry_params_Xpoint_taylor.js";
@@ -19,9 +20,6 @@ int main(int argc, char**argv)
     std::cout << "Type psi_0 (-100)! \n";
     double psi_0, psi_1;
     std::cin >> psi_0;
-    std::cout << "Type n_ref! \n";
-    unsigned n_ref; 
-    std::cin >> n_ref;
     Json::Reader reader;
     Json::Value js;
     if( argc==1)
@@ -59,9 +57,10 @@ int main(int argc, char**argv)
     //dg::geo::SimpleOrthogonalX generator(c.get_psip(), psi_0, R_X,Z_X, R0, Z0,0);
     //dg::CurvilinearGridX2d g2d( generator, 0.25, 1./22., n, Nx, Ny, dg::DIR, dg::NEU);
     dg::EquidistXRefinement equi(add_x, add_y, howmanyX, howmanyY);
-    dg::geo::CurvilinearRefinedGridX2d g2d( equi, generator, 0.25, 1./22., n_ref, n, Nx, Ny, dg::DIR, dg::NEU);
-    dg::Elliptic<dg::geo::CurvilinearRefinedGridX2d, dg::Composite<dg::DMatrix>, dg::DVec> pol( g2d, dg::not_normed, dg::forward);
-    dg::RefinedElliptic<dg::geo::CurvilinearRefinedGridX2d, dg::IDMatrix, dg::Composite<dg::DMatrix>, dg::DVec> pol_refined( g2d, dg::not_normed, dg::forward);
+    dg::geo::CurvilinearGridX2d g2d_coarse( generator, 0.25, 1./22., n, Nx, Ny, dg::DIR, dg::NEU);
+    dg::geo::CurvilinearRefinedGridX2d g2d_fine( equi, generator, 0.25, 1./22., n, Nx, Ny, dg::DIR, dg::NEU);
+    dg::Elliptic<dg::aGeometryX2d, dg::Composite<dg::DMatrix>, dg::DVec> pol( g2d_fine, dg::not_normed, dg::forward);
+    dg::RefinedElliptic<dg::aGeometryX2d, dg::IDMatrix, dg::Composite<dg::DMatrix>, dg::DVec> pol_refined( g2d_coarse, g2d_fine, dg::not_normed, dg::forward);
     double fx = 0.25;
     psi_1 = -fx/(1.-fx)*psi_0;
     std::cout << "psi 1 is          "<<psi_1<<"\n";
@@ -74,7 +73,7 @@ int main(int argc, char**argv)
     file::NC_Error_Handle ncerr;
     ncerr = nc_create( "testX.nc", NC_NETCDF4|NC_CLOBBER, &ncid);
     int dim2d[2];
-    ncerr = file::define_dimensions(  ncid, dim2d, g2d.grid());
+    ncerr = file::define_dimensions(  ncid, dim2d, g2d_fine.grid());
     int coordsID[2], psiID, functionID, function2ID;
     ncerr = nc_def_var( ncid, "x_XYP", NC_DOUBLE, 2, dim2d, &coordsID[0]);
     ncerr = nc_def_var( ncid, "y_XYP", NC_DOUBLE, 2, dim2d, &coordsID[1]);
@@ -82,56 +81,56 @@ int main(int argc, char**argv)
     ncerr = nc_def_var( ncid, "num_solution", NC_DOUBLE, 2, dim2d, &functionID);
     ncerr = nc_def_var( ncid, "ana_solution", NC_DOUBLE, 2, dim2d, &function2ID);
 
-    dg::HVec X( g2d.size()), Y(X); //P = dg::pullback( dg::coo3, g);
-    for( unsigned i=0; i<g2d.size(); i++)
+    dg::HVec X( g2d_fine.size()), Y(X); //P = dg::pullback( dg::coo3, g);
+    for( unsigned i=0; i<g2d_fine.size(); i++)
     {
-        X[i] = g2d.map()[0][i];
-        Y[i] = g2d.map()[1][i];
+        X[i] = g2d_fine.map()[0][i];
+        Y[i] = g2d_fine.map()[1][i];
     }
     ncerr = nc_put_var_double( ncid, coordsID[0], X.data());
     ncerr = nc_put_var_double( ncid, coordsID[1], Y.data());
     //////////////////blob solution////////////////////////////////////////////
-    //const dg::DVec b =        dg::pullback( dg::geo::EllipticBlobDirNeuM<MagneticField>(c,psi_0, psi_1, 450, -340, 40.,1.), g2d.associated());
-    //const dg::DVec bFINE =        dg::pullback( dg::geo::EllipticBlobDirNeuM<MagneticField>(c,psi_0, psi_1, 450, -340, 40.,1.), g2d);
-    //const dg::DVec chi  =  dg::pullback( dg::ONE(), g2d.associated());
-    //const dg::DVec chiFINE  =  dg::pullback( dg::ONE(), g2d);
-    //const dg::DVec solution =     dg::pullback( dg::geo::FuncDirNeu<MagneticField>(c, psi_0, psi_1, 450, -340, 40., 1. ), g2d.associated());
-    //const dg::DVec solutionFINE =     dg::pullback( dg::geo::FuncDirNeu<MagneticField>(c, psi_0, psi_1, 450, -340, 40., 1. ), g2d);
+    //const dg::DVec b =        dg::pullback( dg::geo::EllipticBlobDirNeuM(c,psi_0, psi_1, 450, -340, 40.,1.), g2d_coarse);
+    //const dg::DVec bFINE =        dg::pullback( dg::geo::EllipticBlobDirNeuM(c,psi_0, psi_1, 450, -340, 40.,1.), g2d_fine);
+    //const dg::DVec chi  =  dg::pullback( dg::ONE(), g2d_coarse);
+    //const dg::DVec chiFINE  =  dg::pullback( dg::ONE(), g2d_fine);
+    //const dg::DVec solution =     dg::pullback( dg::geo::FuncDirNeu(c, psi_0, psi_1, 450, -340, 40., 1. ), g2d_coarse);
+    //const dg::DVec solutionFINE =     dg::pullback( dg::geo::FuncDirNeu(c, psi_0, psi_1, 450, -340, 40., 1. ), g2d_fine);
     /////////////////blob on X-point///////////////////////////////////////////
-    const dg::DVec b =        dg::pullback( dg::geo::EllipticBlobDirNeuM<MagneticField>(c,psi_0, psi_1, 480, -420, 40.,1.), g2d.associated());
-    const dg::DVec bFINE =        dg::pullback( dg::geo::EllipticBlobDirNeuM<MagneticField>(c,psi_0, psi_1, 480, -420, 40.,1.), g2d);
-    const dg::DVec chi  =  dg::pullback( dg::ONE(), g2d.associated());
-    const dg::DVec chiFINE  =  dg::pullback( dg::ONE(), g2d);
-    const dg::DVec solution =     dg::pullback( dg::geo::FuncDirNeu<MagneticField>(c, psi_0, psi_1, 480, -420, 40., 1. ), g2d.associated());
-    const dg::DVec solutionFINE =     dg::pullback( dg::geo::FuncDirNeu<MagneticField>(c, psi_0, psi_1, 480, -420, 40., 1. ), g2d);
+    const dg::DVec b =        dg::pullback( dg::geo::EllipticBlobDirNeuM(c,psi_0, psi_1, 480, -420, 40.,1.), g2d_coarse);
+    const dg::DVec bFINE =        dg::pullback( dg::geo::EllipticBlobDirNeuM(c,psi_0, psi_1, 480, -420, 40.,1.), g2d_fine);
+    const dg::DVec chi  =  dg::pullback( dg::ONE(), g2d_coarse);
+    const dg::DVec chiFINE  =  dg::pullback( dg::ONE(), g2d_fine);
+    const dg::DVec solution =     dg::pullback( dg::geo::FuncDirNeu(c, psi_0, psi_1, 480, -420, 40., 1. ), g2d_coarse);
+    const dg::DVec solutionFINE =     dg::pullback( dg::geo::FuncDirNeu(c, psi_0, psi_1, 480, -420, 40., 1. ), g2d_fine);
     ///////////////////////////////////////////////////////////////////////////
-    //const dg::DVec b =        dg::pullback( c.laplacePsip, g2d.associated());
-    //const dg::DVec bFINE =    dg::pullback( c.laplacePsip, g2d);
+    //const dg::DVec b =        dg::pullback( c.laplacePsip, g2d_coarse);
+    //const dg::DVec bFINE =    dg::pullback( c.laplacePsip, g2d_fine);
     //const dg::DVec chi =      dg::evaluate( dg::one, g2d.associated());
     //const dg::DVec chiFINE =  dg::evaluate( dg::one, g2d);
     //const dg::DVec solution =     dg::pullback( c.psip, g2d.associated());
     //const dg::DVec solutionFINE = dg::pullback( c.psip, g2d);
     ///////////////////////////Dir/////FIELALIGNED SIN///////////////////
-    //const dg::DVec b     =    dg::pullback( dg::geo::EllipticXDirNeuM<MagneticField>(c, gp.R_0, psi_0, psi_1), g2d.associated());
-    //const dg::DVec bFINE =    dg::pullback( dg::geo::EllipticXDirNeuM<MagneticField>(c, gp.R_0, psi_0, psi_1), g2d);
-    //dg::DVec chi  =  dg::pullback( dg::geo::Bmodule<MagneticField>(c, gp.R_0), g2d.associated());
-    //dg::DVec chiFINE  =  dg::pullback( dg::geo::Bmodule<MagneticField>(c, gp.R_0), g2d);
+    //const dg::DVec b     =    dg::pullback( dg::geo::EllipticXDirNeuM(c, gp.R_0, psi_0, psi_1), g2d.associated());
+    //const dg::DVec bFINE =    dg::pullback( dg::geo::EllipticXDirNeuM(c, gp.R_0, psi_0, psi_1), g2d);
+    //dg::DVec chi  =  dg::pullback( dg::geo::Bmodule(c, gp.R_0), g2d.associated());
+    //dg::DVec chiFINE  =  dg::pullback( dg::geo::Bmodule(c, gp.R_0), g2d);
     //dg::blas1::plus( chi, 1e5);
     //dg::blas1::plus( chiFINE, 1e5);
     ////const dg::DVec chi      =  dg::pullback( dg::ONE(), g2d.associated());
     ////const dg::DVec chiFINE  =  dg::pullback( dg::ONE(), g2d);
-    //const dg::DVec solution     = dg::pullback( dg::geo::FuncXDirNeu<MagneticField>(c, psi_0, psi_1 ), g2d.associated());
-    //const dg::DVec solutionFINE = dg::pullback( dg::geo::FuncXDirNeu<MagneticField>(c, psi_0, psi_1 ), g2d);
+    //const dg::DVec solution     = dg::pullback( dg::geo::FuncXDirNeu(c, psi_0, psi_1 ), g2d.associated());
+    //const dg::DVec solutionFINE = dg::pullback( dg::geo::FuncXDirNeu(c, psi_0, psi_1 ), g2d);
     //////////////////////////////////////////////////////////////////////////
 
-    const dg::DVec vol3d     = dg::create::volume( g2d.associated());
-    const dg::DVec vol3dFINE = dg::create::volume( g2d);
-    const dg::DVec w3d       = dg::create::weights( g2d.associated());
-    const dg::DVec v3d       = dg::create::inv_weights( g2d.associated());
-    const dg::DVec v3dFINE   = dg::create::inv_weights( g2d);
-    const dg::IDMatrix Q = dg::create::interpolation( g2d);
-    const dg::IDMatrix P = dg::create::projection( g2d);
-    dg::DVec chi_fine = dg::evaluate( dg::zero, g2d), b_fine(chi_fine);
+    const dg::DVec vol3d     = dg::create::volume( g2d_coarse);
+    const dg::DVec vol3dFINE = dg::create::volume( g2d_fine);
+    const dg::DVec w3d       = dg::create::weights( g2d_coarse);
+    const dg::DVec v3d       = dg::create::inv_weights( g2d_coarse);
+    const dg::DVec v3dFINE   = dg::create::inv_weights( g2d_fine);
+    const dg::IDMatrix Q     = dg::create::interpolation( g2d_fine, g2d_coarse);
+    const dg::IDMatrix P     = dg::create::projection( g2d_coarse, g2d_fine);
+    dg::DVec chi_fine = dg::evaluate( dg::zero, g2d_fine), b_fine(chi_fine);
     dg::blas2::gemv( Q, chi, chi_fine);
     dg::blas2::gemv( Q, b, b_fine);
     //pol.set_chi( chi);
@@ -143,10 +142,10 @@ int main(int argc, char**argv)
     std::cout << "eps \t # sandwich \t # direct \t error_sandwich \t error_direct \t hx_max\t hy_max \t time/iteration \n";
     std::cout << eps<<"\t";
     t.tic();
-    dg::DVec x_sandwich    =    dg::evaluate( dg::zero, g2d.associated());
-    dg::DVec x_fine_sw     =    dg::evaluate( dg::zero, g2d);
-    dg::DVec x_direct      =    dg::evaluate( dg::zero, g2d.associated());
-    dg::DVec x_fine_di     =    dg::evaluate( dg::zero, g2d);
+    dg::DVec x_sandwich    =    dg::evaluate( dg::zero, g2d_coarse);
+    dg::DVec x_fine_sw     =    dg::evaluate( dg::zero, g2d_fine);
+    dg::DVec x_direct      =    dg::evaluate( dg::zero, g2d_coarse);
+    dg::DVec x_fine_di     =    dg::evaluate( dg::zero, g2d_fine);
     dg::Invert<dg::DVec > invert1( x_sandwich, n*n*Nx*Ny, eps);
     dg::Invert<dg::DVec > invert2( x_fine_di,  n*n*Nx*Ny, eps);
     dg::DVec bmod(b);
@@ -175,16 +174,16 @@ int main(int argc, char**argv)
     err = dg::blas2::dot( vol3dFINE, error_direct);
     std::cout << sqrt( err/norm) << "\t";//<<sqrt( errFINE/normFINE)<<"\t";
     ///////////////////////////////////metric//////////////////////
-    dg::SparseTensor<dg::DVec> metric = g2d.metric();
+    dg::SparseTensor<dg::DVec> metric = g2d_fine.metric();
     dg::DVec gyy = metric.value(1,1), gxx = metric.value(0,0), vol = dg::tensor::volume(metric).value(); 
     dg::blas1::transform( gxx, gxx, dg::SQRT<double>());
     dg::blas1::transform( gyy, gyy, dg::SQRT<double>());
     dg::blas1::pointwiseDot( gxx, vol, gxx);
     dg::blas1::pointwiseDot( gyy, vol, gyy);
-    dg::blas1::scal( gxx, g2d.hx());
-    dg::blas1::scal( gyy, g2d.hy());
-    double hxX = dg::interpolate( 0, 0, gxx, g2d);
-    double hyX = dg::interpolate( 0, 0, gyy, g2d);
+    dg::blas1::scal( gxx, g2d_fine.hx());
+    dg::blas1::scal( gyy, g2d_fine.hy());
+    double hxX = dg::interpolate( 0, 0, gxx, g2d_fine);
+    double hyX = dg::interpolate( 0, 0, gyy, g2d_fine);
     std::cout << *thrust::max_element( gxx.begin(), gxx.end()) << "\t";
     std::cout << *thrust::max_element( gyy.begin(), gyy.end()) << "\t";
     std::cout << hxX << "\t";
-- 
GitLab


From 9c7a5062eddb26501a3be878a5c7dabac216596b Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Thu, 2 Nov 2017 11:45:30 +0100
Subject: [PATCH 428/453] added projectionX header

---
 inc/dg/backend/interpolationX.cuh |  1 +
 inc/dg/backend/projectionX.h      | 45 +++++++++++++++++++++++++++++++
 inc/geometries/ds.h               |  4 +--
 3 files changed, 48 insertions(+), 2 deletions(-)
 create mode 100644 inc/dg/backend/projectionX.h

diff --git a/inc/dg/backend/interpolationX.cuh b/inc/dg/backend/interpolationX.cuh
index fe8da39b8..cf89f1b4c 100644
--- a/inc/dg/backend/interpolationX.cuh
+++ b/inc/dg/backend/interpolationX.cuh
@@ -2,6 +2,7 @@
 //#include <iomanip>
 
 #include "interpolation.cuh"
+#include "gridX.h"
 
 /*! @file
 
diff --git a/inc/dg/backend/projectionX.h b/inc/dg/backend/projectionX.h
new file mode 100644
index 000000000..236575861
--- /dev/null
+++ b/inc/dg/backend/projectionX.h
@@ -0,0 +1,45 @@
+#pragma once
+#include "projection.cuh"
+#include "gridX.h"
+
+/*!@file 
+  
+  @brief contains creation of projection matrices for X point topology
+ */
+namespace dg{
+///@addtogroup interpolation
+///@{
+namespace create{
+
+///@copydoc interpolationT(const Grid1d&,const Grid1d&)
+cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const GridX1d& g_new, const GridX1d& g_old) {
+    return interpolationT( g_new.grid(), g_old.grid());
+}
+///@copydoc interpolationT(const Grid1d&,const Grid1d&)
+cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const aTopologyX2d& g_new, const aTopologyX2d& g_old) {
+    return interpolationT( g_new.grid(), g_old.grid());
+}
+///@copydoc interpolationT(const Grid1d&,const Grid1d&)
+cusp::coo_matrix<int, double, cusp::host_memory> interpolationT( const aTopologyX3d& g_new, const aTopologyX3d& g_old) {
+    return interpolationT( g_new.grid(), g_old.grid());
+}
+
+///@copydoc projection(const Grid1d&,const Grid1d&)
+cusp::coo_matrix< int, double, cusp::host_memory> projection( const GridX1d& g_new, const GridX1d& g_old) {
+    return projection(g_new, g_old);
+}
+
+///@copydoc projection(const Grid1d&,const Grid1d&)
+cusp::coo_matrix< int, double, cusp::host_memory> projection( const aTopologyX2d& g_new, const aTopologyX2d& g_old) {
+    return projection(g_new, g_old);
+}
+
+///@copydoc projection(const Grid1d&,const Grid1d&)
+cusp::coo_matrix< int, double, cusp::host_memory> projection( const aTopologyX3d& g_new, const aTopologyX3d& g_old) {
+    return projection(g_new, g_old);
+}
+
+///@}
+
+}//namespace create
+}//namespace dg
diff --git a/inc/geometries/ds.h b/inc/geometries/ds.h
index e9d3cf9d0..4b05d6d4d 100644
--- a/inc/geometries/ds.h
+++ b/inc/geometries/ds.h
@@ -217,7 +217,7 @@ struct DS
      * @copydetails forward(const container&,container&)
      * @note if dependsOnX is false then no jump terms will be added in the x-direction; analogous in y
      */
-    void symv( const container& f, container& g){ do_symv( 1., f, 0., dsTdsf);}
+    void symv( const container& f, container& g){ do_symv( 1., f, 0., g);}
     /**
      * @brief Discretizes \f$ g = \alpha \nabla\cdot ( \vec v \vec v \cdot \nabla f ) + \beta g\f$ as a symmetric matrix
      *
@@ -225,7 +225,7 @@ struct DS
      * @copydetails forward(double,const container&,double,container&)
      * @note if dependsOnX is false then no jump terms will be added in the x-direction; analogous in y
      */
-    void symv( double alpha, const container& f, double beta, container& dsTdsf){ do_symv( alpha, f, beta, dsTdsf);}
+    void symv( double alpha, const container& f, double beta, container& g){ do_symv( alpha, f, beta, g);}
     /**
      * @brief Discretizes \f$ g = \nabla_\parallel^2 f \f$ 
      *
-- 
GitLab


From e825811ed5d2a7b192a879d7c4c2f6449489bba9 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Thu, 2 Nov 2017 13:43:43 +0100
Subject: [PATCH 429/453] further documenting and debugging code in geometries
 backend

---
 inc/dg/algorithm.h                            | 17 ++++---
 inc/dg/backend/mpi_init.h                     | 32 ++++++------
 inc/dg/backend/mpi_matrix.h                   |  3 +-
 inc/dg/backend/mpi_matrix_blas.h              |  2 +-
 inc/dg/backend/transpose.h                    |  2 +
 inc/dg/blas2.h                                |  2 +-
 inc/dg/nullstelle.h                           | 26 +---------
 inc/geometries/geometryX_elliptic_b.cu        |  9 ++--
 .../geometryX_refined_elliptic_b.cu           |  6 +--
 inc/geometries/geometry_elliptic_b.cu         | 43 ++++++++--------
 inc/geometries/guenther_ds_b.cu               | 50 ++++++++-----------
 inc/geometries/ribeiro_mpit.cu                |  6 +--
 12 files changed, 83 insertions(+), 115 deletions(-)

diff --git a/inc/dg/algorithm.h b/inc/dg/algorithm.h
index 886a67b7b..84bde4976 100644
--- a/inc/dg/algorithm.h
+++ b/inc/dg/algorithm.h
@@ -5,6 +5,15 @@
  *
  * @note include <mpi.h> before this header to activate mpi support
  */
+#include "backend/timer.cuh"
+#include "backend/split_and_join.h"
+#include "backend/transpose.h"
+#include "backend/xspacelib.cuh"
+#include "backend/evaluationX.cuh"
+#include "backend/derivativesX.h"
+#include "backend/weightsX.cuh"
+#include "backend/interpolationX.cuh"
+#include "backend/projectionX.h"
 #include "blas.h"
 #include "geometry/geometry.h"
 #include "helmholtz.h"
@@ -15,14 +24,6 @@
 #include "runge_kutta.h"
 #include "multigrid.h"
 #include "refined_elliptic.h"
-#include "backend/timer.cuh"
-#include "backend/split_and_join.h"
-#include "backend/transpose.h"
-#include "backend/xspacelib.cuh"
-#include "backend/evaluationX.cuh"
-#include "backend/derivativesX.h"
-#include "backend/weightsX.cuh"
-#include "backend/interpolationX.cuh"
 #ifdef MPI_VERSION
 #include "arakawa.h"
 #include "backend/mpi_init.h"
diff --git a/inc/dg/backend/mpi_init.h b/inc/dg/backend/mpi_init.h
index 7bcc6dcf4..9980abef0 100644
--- a/inc/dg/backend/mpi_init.h
+++ b/inc/dg/backend/mpi_init.h
@@ -15,14 +15,14 @@ namespace dg
 /**
 * @brief Read in number of processses and grid size and create Cartesian MPI communicator
 *
-* Also sets the GPU a process should use in case THRUST_DEVICE_SYSTEM_CUDA
-* @param bcx if bcx==dg::PER then the communicator is periodic in x 
-* @param bcy if bcy==dg::PER then the communicator is periodic in y 
-* @param n read in from is and broadcasted to all processes in MPI_COMM_WORLD
-* @param Nx read in from is and broadcasted to all processes in MPI_COMM_WORLD
-* @param Ny read in from is and broadcasted to all processes in MPI_COMM_WORLD
+* Also sets the GPU via \c rank\% num_devices_per_node a process should use if \c THRUST_DEVICE_SYSTEM==THRUST_DEVICE_SYSTEM_CUDA
+* @param bcx if \c bcx==dg::PER then the communicator is periodic in x 
+* @param bcy if \c bcy==dg::PER then the communicator is periodic in y 
+* @param n  rank 0 reads in from \c is and broadcasts to all processes in \c MPI_COMM_WORLD
+* @param Nx rank 0 reads in from \c is and broadcasts to all processes in \c MPI_COMM_WORLD
+* @param Ny rank 0 reads in from \c is and broadcasts to all processes in \c MPI_COMM_WORLD
 * @param comm (write only) a 2d Cartesian MPI communicator
-* @param is Input stream to read parameters from (npx, npy, n, Nx, Ny)
+* @param is Input stream rank 0 reads parameters (\c npx, \c npy, \c n, \c Nx, \c Ny)
 * @ingroup misc
 */
 void mpi_init2d( dg::bc bcx, dg::bc bcy, unsigned& n, unsigned& Nx, unsigned& Ny, MPI_Comm& comm, std::istream& is = std::cin  )
@@ -71,16 +71,16 @@ void mpi_init2d( dg::bc bcx, dg::bc bcy, unsigned& n, unsigned& Nx, unsigned& Ny
 /**
 * @brief Read in number of processses and grid size and create Cartesian MPI communicator
 *
-* Also sets the GPU a process should use in case THRUST_DEVICE_SYSTEM_CUDA
-* @param bcx if bcx==dg::PER then the communicator is periodic in x 
-* @param bcy if bcy==dg::PER then the communicator is periodic in y 
-* @param bcz if bcz==dg::PER then the communicator is periodic in z 
-* @param n read in from is and broadcasted to all processes in MPI_COMM_WORLD
-* @param Nx read in from is and broadcasted to all processes in MPI_COMM_WORLD
-* @param Ny read in from is and broadcasted to all processes in MPI_COMM_WORLD
-* @param Nz read in from is and broadcasted to all processes in MPI_COMM_WORLD
+* Also sets the GPU via \c rank\% num_devices_per_node a process should use if \c THRUST_DEVICE_SYSTEM==THRUST_DEVICE_SYSTEM_CUDA
+* @param bcx if \c bcx==dg::PER then the communicator is periodic in x 
+* @param bcy if \c bcy==dg::PER then the communicator is periodic in y 
+* @param bcz if \c bcz==dg::PER then the communicator is periodic in z 
+* @param n  rank 0 reads in from \c is and broadcasts to all processes in \c MPI_COMM_WORLD
+* @param Nx rank 0 reads in from \c is and broadcasts to all processes in \c MPI_COMM_WORLD
+* @param Ny rank 0 reads in from \c is and broadcasts to all processes in \c MPI_COMM_WORLD
+* @param Nz rank 0 reads in from \c is and broadcasts to all processes in \c MPI_COMM_WORLD
 * @param comm (write only) a 3d Cartesian MPI communicator
-* @param is Input stream to read parameters from (npx, npy, npz, n, Nx, Ny, Nz)
+* @param is Input stream rank 0 reads parameters (\c npx, \c npy, \c npz, \c n, \c Nx, \c Ny, \c Nz)
 * @ingroup misc
 */
 void mpi_init3d( dg::bc bcx, dg::bc bcy, dg::bc bcz, unsigned& n, unsigned& Nx, unsigned& Ny, unsigned& Nz, MPI_Comm& comm, std::istream& is = std::cin  )
diff --git a/inc/dg/backend/mpi_matrix.h b/inc/dg/backend/mpi_matrix.h
index 5056afcb6..f11dbdcc2 100644
--- a/inc/dg/backend/mpi_matrix.h
+++ b/inc/dg/backend/mpi_matrix.h
@@ -2,7 +2,7 @@
 
 #include "mpi_vector.h"
 #include "memory.h"
-#include "matrix_traits.h"
+#include "mpi_matrix_blas.h"
 
 /*!@file
 
@@ -12,6 +12,7 @@
 */
 namespace dg
 {
+    
 
 ///@addtogroup mpi_structures
 ///@{
diff --git a/inc/dg/backend/mpi_matrix_blas.h b/inc/dg/backend/mpi_matrix_blas.h
index 645e51e41..517af6749 100644
--- a/inc/dg/backend/mpi_matrix_blas.h
+++ b/inc/dg/backend/mpi_matrix_blas.h
@@ -1,6 +1,6 @@
 #pragma once
 
-#include "mpi_matrix.h"
+#include "matrix_traits.h"
 
 ///@cond
 namespace dg
diff --git a/inc/dg/backend/transpose.h b/inc/dg/backend/transpose.h
index 25f3ec3fe..c14396fa2 100644
--- a/inc/dg/backend/transpose.h
+++ b/inc/dg/backend/transpose.h
@@ -1,5 +1,7 @@
 #pragma once
 #include <cusp/transpose.h>
+#include "cusp_matrix_blas.cuh"
+#include "matrix_traits.h"
 #ifdef MPI_VERSION
 #include "mpi_matrix.h"
 #endif //MPI_VERSION
diff --git a/inc/dg/blas2.h b/inc/dg/blas2.h
index df5bf8395..28e55671e 100644
--- a/inc/dg/blas2.h
+++ b/inc/dg/blas2.h
@@ -9,7 +9,7 @@
 #include "backend/sparseblockmat.cuh"
 #include "backend/selfmade_blas.cuh"
 #ifdef MPI_VERSION
-#include "backend/mpi_matrix_blas.h"
+#include "backend/mpi_matrix.h"
 #include "backend/mpi_precon_blas.h"
 #endif //MPI_VERSION
 #include "backend/std_matrix_blas.cuh"
diff --git a/inc/dg/nullstelle.h b/inc/dg/nullstelle.h
index ea55fcb2d..0e240342d 100644
--- a/inc/dg/nullstelle.h
+++ b/inc/dg/nullstelle.h
@@ -13,7 +13,7 @@ namespace dg{
 
 /*! @brief Exception class, that stores boundaries for 1D root finding
  *
- * @ingroup numerical0
+ * @ingroup root
  */
 class NoRoot1d: public std::exception
 {
@@ -78,30 +78,6 @@ int bisection1d (UnaryOp& op, double& x_min, double& x_max, const double eps)
     throw std::runtime_error("Too many steps in root finding!");
 }
 
-template <typename UnaryOp>           
-int false_position (UnaryOp& op, double& x_min, double& x_max, const double eps) 
-{
-    double  mitte;
-    double wert_max, wert_mitte, wert_min;
-    wert_max=op(x_max);
-    wert_min=op(x_min);
-
-    if(wert_max*wert_min>=0) 
-        throw NoRoot1d(x_min, x_max);
-    
-    int j_max = 60;
-    for(int j=0; j<j_max; j++)
-    {
-        wert_mitte = op( mitte = (x_min*wert_max - x_max*wert_min)/(wert_max-wert_min) );
-        if(wert_mitte==0) 			    {x_min=x_max=mitte; return j+3;}
-        else if(wert_mitte*wert_max>0) 	{x_max = mitte; wert_max = wert_mitte;}
-        else 				            {x_min = mitte; wert_min = wert_mitte;}
-        if((x_max-x_min)<eps)           return j+3; 
-    }
-    throw std::runtime_error("Too many steps in root finding!");
-}
-
-      
 //@}
 }//namespace dg
 #endif //_NULLSTELLE_  
diff --git a/inc/geometries/geometryX_elliptic_b.cu b/inc/geometries/geometryX_elliptic_b.cu
index 69b3990b0..d6cb51bfd 100644
--- a/inc/geometries/geometryX_elliptic_b.cu
+++ b/inc/geometries/geometryX_elliptic_b.cu
@@ -18,7 +18,6 @@
 #include "separatrix_orthogonal.h"
 #include "testfunctors.h"
 
-using namespace dg::geo::taylor;
 const char* parameters = "geometry_params_Xpoint_taylor.js";
 
 //using namespace dg::geo::solovev;
@@ -44,7 +43,7 @@ int main(int argc, char**argv)
         std::ifstream is(argv[1]);
         reader.parse(is,js,false);
     }
-    GeomParameters gp(js);
+    dg::geo::taylor::Parameters gp(js);
     gp.display( std::cout);
     dg::Timer t;
     std::cout << "Constructing grid ... \n";
@@ -60,8 +59,8 @@ int main(int argc, char**argv)
     double Z_X = -1.1*gp.elongation*gp.a;
     std::cout << "X-point at "<<R_X <<" "<<Z_X<<"\n";
     dg::geo::SeparatrixOrthogonal generator(c.get_psip(), psi_0, R_X,Z_X, R0, Z0,0);
-    dg::CurvilinearGridX2d g2d( generator, 0.25, 1./22., n, Nx, Ny, dg::DIR, dg::NEU);
-    dg::Elliptic<dg::CurvilinearGridX2d, dg::Composite<dg::DMatrix>, dg::DVec> pol( g2d, dg::not_normed, dg::forward);
+    dg::geo::CurvilinearGridX2d g2d( generator, 0.25, 1./22., n, Nx, Ny, dg::DIR, dg::NEU);
+    dg::Elliptic<dg::geo::CurvilinearGridX2d, dg::Composite<dg::DMatrix>, dg::DVec> pol( g2d, dg::not_normed, dg::forward);
     double fx = 0.25;
     psi_1 = -fx/(1.-fx)*psi_0;
     std::cout << "psi 1 is          "<<psi_1<<"\n";
@@ -123,7 +122,7 @@ int main(int argc, char**argv)
     t.tic();
     dg::Invert<dg::DVec > invert( x, n*n*Nx*Ny, eps);
     //unsigned number = invert(pol, x,b, vol2d, inv_vol2d );
-    unsigned number = invert(pol, x,b, vol2d, v2d ); //inv weights are better preconditioners
+    unsigned number = invert(pol, x,b, vol2d, v2d, v2d ); //inv weights are better preconditioners
     std::cout <<number<<"\t";
     t.toc();
     dg::blas1::axpby( 1.,x,-1., solution, error);
diff --git a/inc/geometries/geometryX_refined_elliptic_b.cu b/inc/geometries/geometryX_refined_elliptic_b.cu
index 244d5b0b5..e0f12e08b 100644
--- a/inc/geometries/geometryX_refined_elliptic_b.cu
+++ b/inc/geometries/geometryX_refined_elliptic_b.cu
@@ -150,8 +150,8 @@ int main(int argc, char**argv)
     dg::Invert<dg::DVec > invert2( x_fine_di,  n*n*Nx*Ny, eps);
     dg::DVec bmod(b);
     pol_refined.compute_rhs( bFINE, bmod);
-    unsigned number_sw = invert1(pol_refined, x_sandwich, bmod, w3d, v3d );
-    unsigned number_di = invert2(pol        , x_fine_di, bFINE, vol3dFINE,v3dFINE);
+    unsigned number_sw = invert1(pol_refined, x_sandwich, bmod, w3d, v3d, v3d );
+    unsigned number_di = invert2(pol        , x_fine_di, bFINE, vol3dFINE,v3dFINE, v3dFINE);
     //unsigned number_di = invert2(pol        , x_direct, bmod, w3d,v3d);
     dg::blas2::gemv( Q, x_sandwich, x_fine_sw);
     //dg::blas2::gemv( Q, x_direct,   x_fine_di);
@@ -188,7 +188,7 @@ int main(int argc, char**argv)
     std::cout << *thrust::max_element( gyy.begin(), gyy.end()) << "\t";
     std::cout << hxX << "\t";
     std::cout << hyX << "\t";
-    std::cout<<t.diff()/(double)number<<"s"<<std::endl;
+    std::cout<<t.diff()/(double)number_di<<"s"<<std::endl;
 
     dg::blas1::transfer( error_direct, X);
     ncerr = nc_put_var_double( ncid, psiID, X.data());
diff --git a/inc/geometries/geometry_elliptic_b.cu b/inc/geometries/geometry_elliptic_b.cu
index e8f47af12..a6e10fd01 100644
--- a/inc/geometries/geometry_elliptic_b.cu
+++ b/inc/geometries/geometry_elliptic_b.cu
@@ -1,4 +1,5 @@
 #include <iostream>
+#include <memory>
 
 #include "file/nc_utilities.h"
 
@@ -45,8 +46,8 @@ int main(int argc, char**argv)
     t.tic();
     dg::geo::SimpleOrthogonal generator( c.get_psip(), psi_0, psi_1, gp.R_0, 0., 1);
     dg::geo::CurvilinearProductGrid3d g3d( generator, n, Nx, Ny,Nz, dg::DIR);
-    dg::geo::CurvilinearGrid2d g2d = g3d.perp_grid();
-    dg::Elliptic<dg::geo::CurvilinearGrid2d, dg::DMatrix, dg::DVec> pol( g2d, dg::not_normed, dg::forward);
+    std::unique_ptr<dg::aGeometry2d> g2d( g3d.perp_grid() );
+    dg::Elliptic<dg::aGeometry2d, dg::DMatrix, dg::DVec> pol( *g2d, dg::not_normed, dg::forward);
     t.toc();
     std::cout << "Construction took "<<t.diff()<<"s\n";
     ///////////////////////////////////////////////////////////////////////////
@@ -54,7 +55,7 @@ int main(int argc, char**argv)
     file::NC_Error_Handle ncerr;
     ncerr = nc_create( "testE.nc", NC_NETCDF4|NC_CLOBBER, &ncid);
     int dim2d[2];
-    ncerr = file::define_dimensions(  ncid, dim2d, g2d);
+    ncerr = file::define_dimensions(  ncid, dim2d, *g2d);
     int coordsID[2], psiID, functionID, function2ID;
     ncerr = nc_def_var( ncid, "x_XYP", NC_DOUBLE, 2, dim2d, &coordsID[0]);
     ncerr = nc_def_var( ncid, "y_XYP", NC_DOUBLE, 2, dim2d, &coordsID[1]);
@@ -62,27 +63,27 @@ int main(int argc, char**argv)
     ncerr = nc_def_var( ncid, "num_solution", NC_DOUBLE, 2, dim2d, &functionID);
     ncerr = nc_def_var( ncid, "ana_solution", NC_DOUBLE, 2, dim2d, &function2ID);
 
-    dg::HVec X( g2d.size()), Y(X); //P = dg::pullback( dg::coo3, g);
-    for( unsigned i=0; i<g2d.size(); i++)
+    dg::HVec X( g2d->size()), Y(X); //P = dg::pullback( dg::coo3, g);
+    for( unsigned i=0; i<g2d->size(); i++)
     {
-        X[i] = g2d.map()[0][i];
-        Y[i] = g2d.map()[1][i];
+        X[i] = g2d->map()[0][i];
+        Y[i] = g2d->map()[1][i];
     }
     ncerr = nc_put_var_double( ncid, coordsID[0], X.data());
     ncerr = nc_put_var_double( ncid, coordsID[1], Y.data());
     ///////////////////////////////////////////////////////////////////////////
-    dg::DVec x = dg::evaluate( dg::zero, g2d);
-    //const dg::DVec b =    dg::pullback( dg::geo::EllipticDirNeuM(c, psi_0, psi_1, 440, -220, 40., 1), g2d);
-    //const dg::DVec chi =  dg::pullback( dg::geo::BmodTheta(c), g2d);
-    //const dg::DVec solution = dg::pullback( dg::geo::FuncDirNeu(c,psi_0, psi_1, 440, -220, 40.,1 ), g2d);
-    const dg::DVec b =    dg::pullback( dg::geo::EllipticDirPerM(c, psi_0, psi_1, 4), g2d);
-    const dg::DVec chi =  dg::pullback( dg::geo::Bmodule(c), g2d);
-    const dg::DVec solution = dg::pullback( dg::geo::FuncDirPer(c, psi_0, psi_1, 4), g2d);
-    //const dg::DVec b =        dg::pullback( dg::geo::LaplacePsi(gp), g2d);
-    //const dg::DVec chi =      dg::pullback( dg::one, g2d);
-    //const dg::DVec solution =     dg::pullback( psip, g2d);
+    dg::DVec x = dg::evaluate( dg::zero, *g2d);
+    //const dg::DVec b =    dg::pullback( dg::geo::EllipticDirNeuM(c, psi_0, psi_1, 440, -220, 40., 1), *g2d);
+    //const dg::DVec chi =  dg::pullback( dg::geo::BmodTheta(c), *g2d);
+    //const dg::DVec solution = dg::pullback( dg::geo::FuncDirNeu(c,psi_0, psi_1, 440, -220, 40.,1 ), *g2d);
+    const dg::DVec b =    dg::pullback( dg::geo::EllipticDirPerM(c, psi_0, psi_1, 4), *g2d);
+    const dg::DVec chi =  dg::pullback( dg::geo::Bmodule(c), *g2d);
+    const dg::DVec solution = dg::pullback( dg::geo::FuncDirPer(c, psi_0, psi_1, 4), *g2d);
+    //const dg::DVec b =        dg::pullback( dg::geo::LaplacePsi(gp), *g2d);
+    //const dg::DVec chi =      dg::pullback( dg::one, *g2d);
+    //const dg::DVec solution =     dg::pullback( psip, *g2d);
 
-    const dg::DVec vol3d = dg::create::volume( g2d);
+    const dg::DVec vol3d = dg::create::volume( *g2d);
     pol.set_chi( chi);
     //compute error
     dg::DVec error( solution);
@@ -99,14 +100,14 @@ int main(int argc, char**argv)
     const double norm = dg::blas2::dot( vol3d, solution);
     std::cout << sqrt( err/norm) << "\t";
 
-    dg::SparseTensor<dg::DVec> metric = g2d.metric();
+    dg::SparseTensor<dg::DVec> metric = g2d->metric();
     dg::DVec gyy = metric.value(1,1), gxx=metric.value(0,0), vol = dg::tensor::volume(metric).value();
     dg::blas1::transform( gxx, gxx, dg::SQRT<double>());
     dg::blas1::transform( gyy, gyy, dg::SQRT<double>());
     dg::blas1::pointwiseDot( gxx, vol, gxx);
     dg::blas1::pointwiseDot( gyy, vol, gyy);
-    dg::blas1::scal( gxx, g2d.hx());
-    dg::blas1::scal( gyy, g2d.hy());
+    dg::blas1::scal( gxx, g2d->hx());
+    dg::blas1::scal( gyy, g2d->hy());
     std::cout << *thrust::max_element( gxx.begin(), gxx.end()) << "\t";
     std::cout << *thrust::max_element( gyy.begin(), gyy.end()) << "\t";
     std::cout<<t.diff()/(double)number<<"s"<<std::endl;
diff --git a/inc/geometries/guenther_ds_b.cu b/inc/geometries/guenther_ds_b.cu
index a34cf3de1..f060f6338 100644
--- a/inc/geometries/guenther_ds_b.cu
+++ b/inc/geometries/guenther_ds_b.cu
@@ -3,21 +3,13 @@
 #include <cusp/print.h>
 #include <cusp/csr_matrix.h>
 
-#include "dg/backend/xspacelib.cuh"
-#include "dg/backend/evaluation.cuh"
-#include "dg/backend/timer.cuh"
-#include "dg/blas.h"
-#include "dg/ds.h"
-#include "dg/backend/functions.h"
-#include "dg/functors.h"
-#include "dg/elliptic.h"
-#include "dg/cg.h"
+#include "dg/algorithm.h"
+#include "ds.h"
 // #include "draw/host_window.h"
 #include "guenther.h"
 #include "magnetic_field.h"
 #include "testfunctors.h"
 
-using namespace dg::geo::guenther;
 
 int main( )
 {
@@ -27,7 +19,7 @@ int main( )
     Json::Value js;
     std::ifstream is("guenther_params.js");
     reader.parse(is,js,false);
-    Parameters gp(js);
+    dg::geo::guenther::Parameters gp(js);
     gp.display( std::cout);
 
     //////////////////////////////////////////////////////////////////////////
@@ -38,21 +30,21 @@ int main( )
     double Zmax=1.0*gp.a*gp.elongation;
     /////////////////////////////////////////////initialze fields /////////////////////
     
-    Field field(gp.R_0, gp.I_0);
-    InvB invb(gp.R_0, gp.I_0);
-    GradLnB gradlnB(gp.R_0, gp.I_0);
-    LnB lnB(gp.R_0, gp.I_0);
-    FieldR bR_(gp.R_0, gp.I_0);
-    FieldZ bZ_(gp.R_0, gp.I_0);
-    FieldP bPhi_(gp.R_0, gp.I_0);
-    FuncNeu funcNEU(gp.R_0,gp.I_0);
-    FuncNeu2 funcNEU2(gp.R_0,gp.I_0);
-    DeriNeu deriNEU(gp.R_0,gp.I_0);
-    DeriNeu2 deriNEU2(gp.R_0,gp.I_0);
-    DeriNeuT2 deriNEUT2(gp.R_0,gp.I_0);
-    DeriNeuT deriNEUT(gp.R_0,gp.I_0);
-    Divb divb(gp.R_0,gp.I_0);
-    B Bfield(gp.R_0, gp.I_0);
+    dg::geo::guenther::Field field(gp.R_0, gp.I_0);
+    dg::geo::guenther::InvB invb(gp.R_0, gp.I_0);
+    dg::geo::guenther::GradLnB gradlnB(gp.R_0, gp.I_0);
+    dg::geo::guenther::LnB lnB(gp.R_0, gp.I_0);
+    dg::geo::guenther::FieldR bR_(gp.R_0, gp.I_0);
+    dg::geo::guenther::FieldZ bZ_(gp.R_0, gp.I_0);
+    dg::geo::guenther::FieldP bPhi_(gp.R_0, gp.I_0);
+    dg::geo::guenther::FuncNeu funcNEU(gp.R_0,gp.I_0);
+    dg::geo::guenther::FuncNeu2 funcNEU2(gp.R_0,gp.I_0);
+    dg::geo::guenther::DeriNeu deriNEU(gp.R_0,gp.I_0);
+    dg::geo::guenther::DeriNeu2 deriNEU2(gp.R_0,gp.I_0);
+    dg::geo::guenther::DeriNeuT2 deriNEUT2(gp.R_0,gp.I_0);
+    dg::geo::guenther::DeriNeuT deriNEUT(gp.R_0,gp.I_0);
+    dg::geo::guenther::Divb divb(gp.R_0,gp.I_0);
+    dg::geo::guenther::B Bfield(gp.R_0, gp.I_0);
     
     std::cout << "Type n, Nx, Ny, Nz\n";
     //std::cout << "Note, that function is resolved exactly in R,Z for n > 2\n";
@@ -90,11 +82,11 @@ int main( )
     const dg::DVec v3d = dg::create::inv_volume( g3d);
 
     std::cout << "computing dsDIR" << std::endl;
-    dg::DDS::FieldAligned dsFA( field, g3d, rk4eps, dg::DefaultLimiter(), dg::DIR);
+    dg::geo::FieldAligned dsFA( field, g3d, dg::DefaultLimiter(), dg::DIR, rk4eps);
     std::cout << "computing dsNEU" << std::endl;
-    dg::DDS::FieldAligned dsNUFA( field, g3d, rk4eps, dg::DefaultLimiter(), dg::NEU);
+    dg::geo::FieldAligned dsNUFA( field, g3d,dg::DefaultLimiter(), dg::NEU, rk4eps);
 
-    dg::DDS ds ( dsFA, field, dg::not_normed, dg::centered), 
+    dg::geo::DS< ds ( dsFA, field, dg::not_normed, dg::centered), 
         dsNU ( dsNUFA, field, dg::not_normed, dg::centered);
 
 //     dg::DS<dg::DMatrix, dg::DVec> dsNEU( field, g3d, g3d.hz(), rk4eps, dg::DefaultLimiter(), dg::NEU);
diff --git a/inc/geometries/ribeiro_mpit.cu b/inc/geometries/ribeiro_mpit.cu
index f47fc04eb..758c5833a 100644
--- a/inc/geometries/ribeiro_mpit.cu
+++ b/inc/geometries/ribeiro_mpit.cu
@@ -7,11 +7,7 @@
 
 #include <mpi.h>
 
-#include "dg/backend/xspacelib.cuh"
-#include "dg/functors.h"
-
-#include "dg/backend/timer.cuh"
-#include "dg/backend/mpi_init.h"
+#include "dg/algorithm.h"
 #include "mpi_curvilinear.h"
 //#include "guenther.h"
 #include "solovev.h"
-- 
GitLab


From 1d43d56a4ed1d84608a640c05d271cd0156f7f32 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Thu, 2 Nov 2017 14:31:38 +0100
Subject: [PATCH 430/453] make guenther_ds_b compile again

---
 inc/geometries/geometries_doc.h |  14 +--
 inc/geometries/guenther.h       | 196 --------------------------------
 inc/geometries/guenther_ds_b.cu |  32 +++---
 inc/geometries/magnetic_field.h |  18 ++-
 4 files changed, 40 insertions(+), 220 deletions(-)

diff --git a/inc/geometries/geometries_doc.h b/inc/geometries/geometries_doc.h
index d889c3c02..5810db360 100644
--- a/inc/geometries/geometries_doc.h
+++ b/inc/geometries/geometries_doc.h
@@ -8,9 +8,9 @@
  * @defgroup grids 2. New geometric grids
  * @defgroup fluxfunctions 3. New functors based on the magnetic field geometry
 
- All functors in this section model two or three-dimensional functions, i.e. they all overload the operator() like aBinaryOperator
+ All functors in this section model two or three-dimensional functions, i.e. they all overload the operator() like \c aBinaryFunctor
  * @{
-      @defgroup geom 3.1 new flux functions and derivatives
+      @defgroup geom 3.1 New flux functions and derivatives
       @{
         @defgroup solovev The solovev magnetic field
         @defgroup taylor The Taylor state magnetic field
@@ -18,23 +18,23 @@
         @defgroup toroidal The Purely Toroidal magnetic field
         @defgroup circular The Circular magnetic field
       @}
-      @defgroup magnetic 3.2 magnetic field and associated functors
-      @defgroup profiles 3.3 miscellaneous functors based on flux functions
+      @defgroup magnetic 3.2 Magnetic field and associated functors
+      @defgroup profiles 3.3 Profile functors based on flux functions
  * @}
  * @defgroup fieldaligned 4. Fieldaligned derivatives
  * @defgroup misc_geo 5. Miscellaneous additions
  *
  * Objects that are used to define and integrate the magnetic field lines. 
- * All objects can be used in the evaluation() function.
+ * All objects can be used in the \c evaluation and \c pullback functions.
  * 
  */
 /*! @mainpage
  * This extension adds new features to the FELTOR core dg library. 
  *
  * - several grid generator classes are added, among them our new Hector class
- *   are added to the dg::geo namespace
+ *   are added to the \c dg::geo namespace
  * - a bunch of new functors implementing various magnetic field geometries
- *   and profiles are added to the dg::geo namespace
+ *   and profiles are added to the \c dg::geo namespace
  * - there are some miscellaneous additions like a flux surface average class
  * and one used to integrate the field lines for parallel derivatives all in the dg::geo namespace.
  * 
diff --git a/inc/geometries/guenther.h b/inc/geometries/guenther.h
index 253794a13..b21092a79 100644
--- a/inc/geometries/guenther.h
+++ b/inc/geometries/guenther.h
@@ -153,203 +153,7 @@ TokamakMagneticField createMagField( double R_0, double I_0)
 }
 ///@}
 
-/////////////////////////////////////////These should not be necessary!////////
 ///@cond
-struct InvB
-{
-    InvB( double R_0, double I_0 ):  R_0(R_0), I_0(I_0){}
-
-    double operator()(double R, double Z) const
-    {    
-        return sqrt(8.)*R/sqrt(8.*I_0*I_0+ M_PI*M_PI-M_PI*M_PI* cos(M_PI*(R-R_0))*cos(M_PI*Z))/R_0;
-    }
-    double operator()(double R, double Z, double phi) const
-    {  
-
-        return sqrt(8.)*R/sqrt(8.*I_0*I_0+ M_PI*M_PI-M_PI*M_PI* cos(M_PI*(R-R_0))*cos(M_PI*Z))/R_0;
-
-    }
-  private: 
-    double R_0,I_0;
-
-};
-struct B
-{
-    B( double R_0, double I_0):  R_0(R_0), I_0(I_0){}
-
-    double operator()(double R, double Z) const
-    {    
-        return sqrt(8.*I_0*I_0+ M_PI*M_PI-M_PI*M_PI* cos(M_PI*(R-R_0))*cos(M_PI*Z))*R_0/ (sqrt(8.)*R);
-    }
-    double operator()(double R, double Z, double phi) const
-    {    
-        return sqrt(8.*I_0*I_0+ M_PI*M_PI-M_PI*M_PI* cos(M_PI*(R-R_0))*cos(M_PI*Z))*R_0/ (sqrt(8.)*R);
-
-    }
-  private:
-    double R_0,I_0;
-
-};
-struct LnB
-{
-    LnB( double R_0, double I_0 ):  R_0(R_0), I_0(I_0) {}
-
-    double operator()(double R, double Z) const
-    {    
-        double invB = sqrt(8.)*R/sqrt(8.*I_0*I_0+ M_PI*M_PI-M_PI*M_PI* cos(M_PI*(R-R_0))*cos(M_PI*Z))/R_0;
-        return log(1./invB);    }
-    double operator()(double R, double Z, double phi) const
-    {    
-        double invB = sqrt(8.)*R/sqrt(8.*I_0*I_0+ M_PI*M_PI-M_PI*M_PI* cos(M_PI*(R-R_0))*cos(M_PI*Z))/R_0;
-
-        return log(1./invB);    }
-  private:
-    double R_0,I_0;
-
-};
-struct GradLnB
-{
-    GradLnB(double R_0, double I_0):  R_0(R_0), I_0(I_0) {} 
- 
-    double operator()( double R, double Z) const
-    {
-        double fac1 = sqrt(8.*I_0*I_0+ M_PI*M_PI-M_PI*M_PI* cos(M_PI*(R-R_0))*cos(M_PI*Z));
-        double z1 = cos(M_PI*0.5*(R-R_0))*(32.*I_0*I_0+5.*M_PI*M_PI)+
-        M_PI*M_PI* cos(M_PI*3.*(R-R_0)/2.)+
-        M_PI*R*sin(M_PI*3.*(R-R_0)/2.) ;
-        double z2 = cos(M_PI*0.5*(R-R_0)) + 
-        cos(M_PI*3*(R-R_0)/2) + 
-        M_PI*R*sin(M_PI*0.5*(R-R_0));
-        double nenner = fac1*fac1*fac1*2.*sqrt(2.)*R;
-        double divb = -M_PI*(z1*sin(M_PI*Z*0.5)-z2*M_PI*M_PI*sin(M_PI*Z*3./2.))/(nenner);
-       return -divb ;
-    }
-    /*
-    double operator()( double R, double Z, double phi) const
-    {
-        double fac1 = sqrt(8.*I_0*I_0+ M_PI*M_PI-M_PI*M_PI* cos(M_PI*(R-R_0))*cos(M_PI*Z));
-        double z1 = cos(M_PI*0.5*(R-R_0))*(32.*I_0*I_0+5.*M_PI*M_PI)+
-        M_PI*M_PI* cos(M_PI*3.*(R-R_0)/2.)+
-        M_PI*R*sin(M_PI*3.*(R-R_0)/2.) ;
-        double z2 = cos(M_PI*0.5*(R-R_0)) + 
-        cos(M_PI*3*(R-R_0)/2) + 
-        M_PI*R*sin(M_PI*0.5*(R-R_0));
-        double nenner = fac1*fac1*fac1*2.*sqrt(2.)*R;
-        double divb = -M_PI*(z1*sin(M_PI*Z*0.5)-z2*M_PI*M_PI*sin(M_PI*Z*3./2.))/(nenner);
-       return -divb ;
-    }
-    */
-    double operator()( double R, double Z, double phi)const{return operator()(R,Z);}
-    private:
-    double R_0,I_0;
-
-};
-struct Field
-{
-     Field( double R0, double I0):  R_0(R0), I_0(I0){}
-     Field( Parameters gp ):  R_0(gp.R_0), I_0(gp.I_0){}
-    void operator()( const std::vector<thrust::host_vector<double> >& y, std::vector<thrust::host_vector<double> >& yp) const
-    {
-        for( unsigned i=0; i<y[0].size(); i++)
-        {        
-
-            yp[2][i] = y[0][i]*sqrt(1.+ M_PI*M_PI*(1.- cos(M_PI*(y[0][i]-R_0))*cos(M_PI*y[1][i]))/8./I_0/I_0);            
-            yp[0][i] = -M_PI*y[0][i]*cos(M_PI*(y[0][i]-R_0)/2.)*sin(M_PI*y[1][i]/2.)/2./I_0;
-            yp[1][i] =  M_PI*y[0][i]*sin(M_PI*(y[0][i]-R_0)/2.)*cos(M_PI*y[1][i]/2.)/2./I_0 ;
-
-        }
-    }
-    void operator()( const thrust::host_vector<double> & y, thrust::host_vector<double> & yp) const
-    {
-            yp[2] = y[0]*sqrt(1.+ M_PI*M_PI*(1.-cos(M_PI*(y[0]-R_0))*cos(M_PI*y[1]))/8./I_0/I_0);
-            yp[0] = -M_PI*y[0]*cos(M_PI*(y[0]-R_0)/2.)*sin(M_PI*y[1]/2.)/2./I_0;
-            yp[1] =  M_PI*y[0]*sin(M_PI*(y[0]-R_0)/2.)*cos(M_PI*y[1]/2.)/2./I_0 ;
-    }
-    double operator()( double R, double Z) const
-    {
-
-        return sqrt(8.)*R/sqrt(8.*I_0*I_0+ M_PI*M_PI-M_PI*M_PI* cos(M_PI*(R-R_0))*cos(M_PI*Z))/R_0;
-    }
-    double operator()( double R, double Z, double phi) const
-    {
-
-        return sqrt(8.)*R/sqrt(8.*I_0*I_0+ M_PI*M_PI-M_PI*M_PI* cos(M_PI*(R-R_0))*cos(M_PI*Z))/R_0;
-        
-    }
-    double error( const dg::HVec& x0, const dg::HVec& x1)
-    {
-        return sqrt( (x0[0]-x1[0])*(x0[0]-x1[0]) +(x0[1]-x1[1])*(x0[1]-x1[1])+(x0[2]-x1[2])*(x0[2]-x1[2]));
-    }
-    bool monitor( const dg::HVec& end){ 
-        if ( std::isnan(end[0]) || std::isnan(end[1]) || std::isnan(end[2]) ) 
-        {
-            return false;
-        }
-        //if new integrated point outside domain
-        if ((1e-5 > end[0]  ) || (1e10 < end[0])  ||(-1e10  > end[1]  ) || (1e10 < end[1])||(-1e10 > end[2]  ) || (1e10 < end[2])  )
-        {
-            return false;
-        }
-        return true;
-    }
-    private:
-    double R_0, I_0;
-   
-};
-/**
- * @brief \f[ b^R\f]
- */
-struct FieldR
-{
-    FieldR( double R_0, double I_0):R_0(R_0), I_0(I_0){}
-    double operator()( double R, double Z, double phi) const
-    {
-        return -sqrt(2.)*M_PI*cos(M_PI*(R-R_0)/2.)*sin(M_PI*Z/2)/sqrt(8.*I_0*I_0+ M_PI*M_PI-M_PI*M_PI* cos(M_PI*(R-R_0))*cos(M_PI*Z));
-    }
-    private:
-    double R_0, I_0;
-};
-/**
- * @brief \f[ b^Z)\f]
- */
-struct FieldZ
-{
-    FieldZ( double R_0, double I_0):R_0(R_0), I_0(I_0){}
-    double operator()( double R, double Z, double phi) const
-    {
-        return sqrt(2.)*M_PI*sin(M_PI*(R-R_0)/2.)*cos(M_PI*Z/2)/sqrt(8.*I_0*I_0+ M_PI*M_PI-M_PI*M_PI* cos(M_PI*(R-R_0))*cos(M_PI*Z));
-
-    }
-    private:
-    double R_0, I_0;
-};
-/**
- * @brief \f[ b^\phi\f]
- */
-struct FieldP
-{
-    FieldP( double R_0, double I_0):R_0(R_0), I_0(I_0){}
-    double operator()( double R, double Z, double phi) const
-    {
-        return 2.*sqrt(2.)*I_0/R/sqrt(8.*I_0*I_0+ M_PI*M_PI-M_PI*M_PI* cos(M_PI*(R-R_0))*cos(M_PI*Z));
-    }
-    private:
-    double R_0, I_0;
-}; 
-/**
- * @brief \f[ b_t\f]
- */
-struct bt
-{
-    bt( double R_0, double I_0):R_0(R_0), I_0(I_0){}
-    double operator()( double R, double Z, double phi) const
-    {
-        double invB = 2.*sqrt(2.)*R/sqrt(8.*I_0*I_0+ M_PI*M_PI-M_PI*M_PI* cos(M_PI*(R-R_0))*cos(M_PI*Z))/R_0;
-        return invB*I_0*R_0/R;
-    }
-    private:
-    double R_0, I_0;
-}; 
 /////////////////////Test functions//////////////////
 /**
  * @brief \f[ f(R,Z,\phi)\f]
diff --git a/inc/geometries/guenther_ds_b.cu b/inc/geometries/guenther_ds_b.cu
index f060f6338..49d9bdc37 100644
--- a/inc/geometries/guenther_ds_b.cu
+++ b/inc/geometries/guenther_ds_b.cu
@@ -30,21 +30,21 @@ int main( )
     double Zmax=1.0*gp.a*gp.elongation;
     /////////////////////////////////////////////initialze fields /////////////////////
     
-    dg::geo::guenther::Field field(gp.R_0, gp.I_0);
-    dg::geo::guenther::InvB invb(gp.R_0, gp.I_0);
-    dg::geo::guenther::GradLnB gradlnB(gp.R_0, gp.I_0);
-    dg::geo::guenther::LnB lnB(gp.R_0, gp.I_0);
-    dg::geo::guenther::FieldR bR_(gp.R_0, gp.I_0);
-    dg::geo::guenther::FieldZ bZ_(gp.R_0, gp.I_0);
-    dg::geo::guenther::FieldP bPhi_(gp.R_0, gp.I_0);
+    dg::geo::TokamakMagneticField mag = dg::geo::createGuentherField(gp.R_0, gp.I_0);
+    dg::geo::InvB invb(mag);
+    dg::geo::GradLnB gradlnB(mag);
+    dg::geo::LnB lnB(mag);
+    dg::geo::FieldR bR_(mag);
+    dg::geo::FieldZ bZ_(mag);
+    dg::geo::FieldP bPhi_(mag);
+    dg::geo::Divb divb(mag);
+    dg::geo::Bmodule B(mag);
     dg::geo::guenther::FuncNeu funcNEU(gp.R_0,gp.I_0);
     dg::geo::guenther::FuncNeu2 funcNEU2(gp.R_0,gp.I_0);
     dg::geo::guenther::DeriNeu deriNEU(gp.R_0,gp.I_0);
     dg::geo::guenther::DeriNeu2 deriNEU2(gp.R_0,gp.I_0);
     dg::geo::guenther::DeriNeuT2 deriNEUT2(gp.R_0,gp.I_0);
     dg::geo::guenther::DeriNeuT deriNEUT(gp.R_0,gp.I_0);
-    dg::geo::guenther::Divb divb(gp.R_0,gp.I_0);
-    dg::geo::guenther::B Bfield(gp.R_0, gp.I_0);
     
     std::cout << "Type n, Nx, Ny, Nz\n";
     //std::cout << "Note, that function is resolved exactly in R,Z for n > 2\n";
@@ -66,7 +66,7 @@ int main( )
 
 
 
-        dg::CylindricalGrid3d<dg::DVec> g3d( Rmin,Rmax, Zmin,Zmax, z0, z1,  n,Nxn ,Nyn, Nzn,dg::DIR, dg::DIR, dg::PER);
+        dg::CylindricalGrid3d g3d( Rmin,Rmax, Zmin,Zmax, z0, z1,  n,Nxn ,Nyn, Nzn,dg::DIR, dg::DIR, dg::PER);
         dg::Grid2d g2d( Rmin,Rmax, Zmin,Zmax,  n, Nxn ,Nyn);
 
         std::cout << "NR = " << Nxn << std::endl;
@@ -82,12 +82,12 @@ int main( )
     const dg::DVec v3d = dg::create::inv_volume( g3d);
 
     std::cout << "computing dsDIR" << std::endl;
-    dg::geo::FieldAligned dsFA( field, g3d, dg::DefaultLimiter(), dg::DIR, rk4eps);
+    dg::geo::Fieldaligned<dg::aProductGeometry3d, dg::IDMatrix, dg::DVec>  dsFA( mag, g3d, dg::DIR, dg::DIR, dg::geo::FullLimiter(), rk4eps);
     std::cout << "computing dsNEU" << std::endl;
-    dg::geo::FieldAligned dsNUFA( field, g3d,dg::DefaultLimiter(), dg::NEU, rk4eps);
+    dg::geo::Fieldaligned<dg::aProductGeometry3d, dg::IDMatrix, dg::DVec> dsNUFA( mag, g3d,dg::NEU, dg::NEU, dg::geo::FullLimiter(), rk4eps);
 
-    dg::geo::DS< ds ( dsFA, field, dg::not_normed, dg::centered), 
-        dsNU ( dsNUFA, field, dg::not_normed, dg::centered);
+    dg::geo::DS<dg::aProductGeometry3d, dg::IDMatrix, dg::DMatrix, dg::DVec> ds ( dsFA, dg::not_normed, dg::centered), 
+        dsNU ( dsNUFA, dg::not_normed, dg::centered);
 
 //     dg::DS<dg::DMatrix, dg::DVec> dsNEU( field, g3d, g3d.hz(), rk4eps, dg::DefaultLimiter(), dg::NEU);
     
@@ -223,7 +223,7 @@ int main( )
 //     ellipticsym.symv(function,dsTds);
 //     dg::blas1::scal(dsTds,-1.0);
 // //     ds.centeredT(ones,divbT);
-    ds.forwardT( derivativef, dsTdsf);  //dsT(ds(f))
+    ds.forwardDiv( derivativef, dsTdsf);  //dsT(ds(f))
 //     ds.backwardT( derivativeb, dsTdsb); //dsT(ds(f))
 
 //     //centered
@@ -242,7 +242,7 @@ int main( )
         //dg::blas1::pointwiseDivide( dsTdsfb, inverseB, dsTdsfb);
 //     ds.centeredT( derivative2, dsTds2); //dsT(ds(f))
 //     dg::blas1::pointwiseDivide(ones,  inverseB, temp2); //B
-     ds.centeredT( ones, divbT);
+     ds.centeredDiv( ones, divbT);
 //     
 //     double normdsds =dg::blas2::dot(derivative2, w3d,derivative2);
 //     double normds1ds =dg::blas2::dot(derivativeones, w3d,derivative2);
diff --git a/inc/geometries/magnetic_field.h b/inc/geometries/magnetic_field.h
index a8b46a470..98e2f265b 100644
--- a/inc/geometries/magnetic_field.h
+++ b/inc/geometries/magnetic_field.h
@@ -276,10 +276,26 @@ struct GradLnB: public aCloneableBinaryFunctor<GradLnB>
     BR bR_;
     BZ bZ_;   
 };
+/**
+ * @brief \f[  \nabla \cdot \vec b \f]
+ *
+ *\f[  \nabla\cdot \vec b = -\nabla_\parallel \ln B \f]
+ *@sa \c GradLnB
+ */ 
+struct Divb: public aCloneableBinaryFunctor<GradLnB>
+{
+    Divb( const TokamakMagneticField& mag): m_gradLnB(mag) { } 
+    private:
+    double do_compute( double R, double Z) const
+    {
+        return -m_gradLnB(R,Z);
+    }
+    GradLnB m_gradLnB;
+};
 
 /**
  * @brief \f[ B_\varphi = R_0I/R^2\f]
-*/
+ */
 struct FieldP: public aCloneableBinaryFunctor<FieldP>
 {
     FieldP( const TokamakMagneticField& mag): mag_(mag){}
-- 
GitLab


From 8976bbf59747f391550821cf5ab6f31ccb374189 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Thu, 2 Nov 2017 16:38:09 +0100
Subject: [PATCH 431/453] all programs in geometries should compile

---
 inc/dg/Doxyfile                          |   1 +
 inc/dg/Makefile                          |  13 ++-
 inc/dg/algorithm.h                       |   7 +-
 inc/dg/average.h                         |   6 +-
 inc/dg/backend/mpi_matrix.h              |   1 -
 inc/dg/backend/mpi_matrix_blas.h         |   1 +
 inc/dg/backend/transpose.h               |   2 +-
 inc/dg/blas2.h                           |   2 +-
 inc/geometries/Makefile                  |  11 +--
 inc/geometries/ds.h                      |   4 +-
 inc/geometries/ds_geom_t.cu              |  33 ++++----
 inc/geometries/dz_mpit.cu                |  16 +---
 inc/geometries/dz_t.cu                   |   3 -
 inc/geometries/fieldaligned.h            |   8 +-
 inc/geometries/geometryX_elliptic_b.cu   |   9 +-
 inc/geometries/geometry_advection_b.cu   |   7 +-
 inc/geometries/geometry_elliptic_b.cu    |   6 +-
 inc/geometries/geometry_elliptic_mpib.cu |  41 +++++----
 inc/geometries/guenther_ds_b.cu          |  20 ++---
 inc/geometries/guenther_ds_mpib.cu       | 101 ++++++++++------------
 inc/geometries/mpi_fieldaligned.h        |   8 +-
 inc/geometries/ribeiroX.h                |   2 +
 inc/geometries/ribeiroX_t.cu             | 102 +++++++----------------
 inc/geometries/ribeiro_mpit.cu           |   6 +-
 inc/geometries/ribeiro_t.cu              |  22 ++---
 25 files changed, 172 insertions(+), 260 deletions(-)

diff --git a/inc/dg/Doxyfile b/inc/dg/Doxyfile
index 09576dbd7..1ac1c8e28 100644
--- a/inc/dg/Doxyfile
+++ b/inc/dg/Doxyfile
@@ -830,6 +830,7 @@ EXCLUDE                = ../dg/backend/creation.cuh \
                          ../dg/backend/sparseblockmat_gpu_kernels.cuh \
                          ../dg/backend/sparseblockmat_omp_kernels.h \
                          ../dg/backend/average.h \
+                         ../dg/average.h \
                          ../dg/backend/ell_interpolation.cuh \
 
 # The EXCLUDE_SYMLINKS tag can be used to select whether or not files or
diff --git a/inc/dg/Makefile b/inc/dg/Makefile
index d1597176f..3cf9adb57 100644
--- a/inc/dg/Makefile
+++ b/inc/dg/Makefile
@@ -13,6 +13,11 @@ CUFILES=$(wildcard *.cu)
 
 all: $(CPPFILES:%.cpp=%) $(CUFILES:%.cu=%)
 
+%_t: %_t.cu
+	$(CC) $(OPT) $(INCLUDE) -DDG_DEBUG $(CFLAGS) $< -o $@  -g
+
+%_b: %_b.cu 
+	$(CC) $(OPT) $(CFLAGS) -DDG_BENCHMARK $< -o $@ $(INCLUDE) -g 
 
 %_mpit: %_mpit.cu 
 	$(MPICC) $(INCLUDE) -DDG_DEBUG $(MPICFLAGS) $< -o $@ -g
@@ -20,14 +25,8 @@ all: $(CPPFILES:%.cpp=%) $(CUFILES:%.cu=%)
 %_mpib: %_mpib.cu
 	$(MPICC) $(OPT) $(MPICFLAGS) $< -o $@ $(INCLUDE) -g
 
-%_t: %_t.cu
-	$(CC) $(OPT) $(INCLUDE) -DDG_DEBUG $(CFLAGS) $< -o $@ 
-
-%_b: %_b.cu 
-	$(CC) $(OPT) $(CFLAGS) -DDG_BENCHMARK $< -o $@ $(INCLUDE) -g 
-
 bathRZ_t: bathRZ_t.cu 
-	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(GLFLAGS) $(INCLUDE) 
+	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(GLFLAGS) $(INCLUDE)  -g
 	
 .PHONY: clean doc
 
diff --git a/inc/dg/algorithm.h b/inc/dg/algorithm.h
index 84bde4976..09ed20564 100644
--- a/inc/dg/algorithm.h
+++ b/inc/dg/algorithm.h
@@ -7,7 +7,6 @@
  */
 #include "backend/timer.cuh"
 #include "backend/split_and_join.h"
-#include "backend/transpose.h"
 #include "backend/xspacelib.cuh"
 #include "backend/evaluationX.cuh"
 #include "backend/derivativesX.h"
@@ -15,6 +14,7 @@
 #include "backend/interpolationX.cuh"
 #include "backend/projectionX.h"
 #include "blas.h"
+#include "backend/transpose.h"
 #include "geometry/geometry.h"
 #include "helmholtz.h"
 #include "cg.h"
@@ -24,7 +24,10 @@
 #include "runge_kutta.h"
 #include "multigrid.h"
 #include "refined_elliptic.h"
-#ifdef MPI_VERSION
 #include "arakawa.h"
+#include "poisson.h"
+#include "backend/average.cuh"
+#ifdef MPI_VERSION
+#include "backend/average.h"
 #include "backend/mpi_init.h"
 #endif
diff --git a/inc/dg/average.h b/inc/dg/average.h
index 5f8d771d4..49a060bcd 100644
--- a/inc/dg/average.h
+++ b/inc/dg/average.h
@@ -5,8 +5,4 @@
 #ifdef MPI_VERSION
 #include "backend/average.h"
 #endif //MPI_VERSION
-
-/*!@file 
- *
- * @brief This file includes the appropriate headers for parallel derivatives
- */
+///@deprecated This header is deprecated in favour of algorithm.h
diff --git a/inc/dg/backend/mpi_matrix.h b/inc/dg/backend/mpi_matrix.h
index f11dbdcc2..5d607d254 100644
--- a/inc/dg/backend/mpi_matrix.h
+++ b/inc/dg/backend/mpi_matrix.h
@@ -2,7 +2,6 @@
 
 #include "mpi_vector.h"
 #include "memory.h"
-#include "mpi_matrix_blas.h"
 
 /*!@file
 
diff --git a/inc/dg/backend/mpi_matrix_blas.h b/inc/dg/backend/mpi_matrix_blas.h
index 517af6749..e54c5771e 100644
--- a/inc/dg/backend/mpi_matrix_blas.h
+++ b/inc/dg/backend/mpi_matrix_blas.h
@@ -1,6 +1,7 @@
 #pragma once
 
 #include "matrix_traits.h"
+#include "mpi_matrix.h"
 
 ///@cond
 namespace dg
diff --git a/inc/dg/backend/transpose.h b/inc/dg/backend/transpose.h
index c14396fa2..852c08bcf 100644
--- a/inc/dg/backend/transpose.h
+++ b/inc/dg/backend/transpose.h
@@ -3,7 +3,7 @@
 #include "cusp_matrix_blas.cuh"
 #include "matrix_traits.h"
 #ifdef MPI_VERSION
-#include "mpi_matrix.h"
+#include "mpi_matrix_blas.h"
 #endif //MPI_VERSION
 
 namespace dg
diff --git a/inc/dg/blas2.h b/inc/dg/blas2.h
index 28e55671e..df5bf8395 100644
--- a/inc/dg/blas2.h
+++ b/inc/dg/blas2.h
@@ -9,7 +9,7 @@
 #include "backend/sparseblockmat.cuh"
 #include "backend/selfmade_blas.cuh"
 #ifdef MPI_VERSION
-#include "backend/mpi_matrix.h"
+#include "backend/mpi_matrix_blas.h"
 #include "backend/mpi_precon_blas.h"
 #endif //MPI_VERSION
 #include "backend/std_matrix_blas.cuh"
diff --git a/inc/geometries/Makefile b/inc/geometries/Makefile
index 0a8af7645..f882d5737 100644
--- a/inc/geometries/Makefile
+++ b/inc/geometries/Makefile
@@ -12,7 +12,7 @@ CUFILES=$(wildcard *.cu)
 all: $(CUFILES:%.cu=%)
 
 ds_geom_t: ds_geom_t.cu solovev.h 
-	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(GLFLAGS) $(INCLUDE) $(LIBS) $(JSONLIB)  -DDG_BENCHMARK 
+	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(GLFLAGS) $(INCLUDE) $(LIBS) $(JSONLIB)  -DDG_BENCHMARK -g
 
 geometry_diag: geometry_diag.cu solovev.h 
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(LIBS) $(INCLUDE) $(JSONLIB) -g
@@ -20,14 +20,15 @@ geometry_diag: geometry_diag.cu solovev.h
 %_t: %_t.cu 
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -DDG_DEBUG -g
 
+%_b: %_b.cu 
+	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -g 
+
 %_mpit: %_mpit.cu 
-	$(MPICC) $(OPT) $(INCLUDE) $(MPICFLAGS)  $< -o $@ -g $(LIBS) $(JSONLIB) -DDG_DEBUG
+	$(MPICC) $(OPT) $(INCLUDE) $(MPICFLAGS)  $< -o $@ $(LIBS) $(JSONLIB) -DDG_DEBUG -g
 
 %_mpib: %_mpib.cu
-	$(MPICC) $(OPT) $(MPICFLAGS) $< -o $@ -g $(INCLUDE) $(LIBS) $(JSONLIB)
+	$(MPICC) $(OPT) $(MPICFLAGS) $< -o $@ -g $(INCLUDE) $(LIBS) $(JSONLIB) -g
 
-%_b: %_b.cu 
-	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -g 
 
 
 .PHONY: clean doc
diff --git a/inc/geometries/ds.h b/inc/geometries/ds.h
index 4b05d6d4d..28cadea84 100644
--- a/inc/geometries/ds.h
+++ b/inc/geometries/ds.h
@@ -54,7 +54,7 @@ struct DS
         dg::bc bcy = dg::NEU, 
         Limiter limit = FullLimiter(), 
         dg::norm no=dg::normed, dg::direction dir = dg::centered, 
-        double eps = 1e-5, unsigned multiplyX=5, unsigned multiplyY=5, bool dependsOnX = true, bool dependsOnY=true)
+        double eps = 1e-5, unsigned multiplyX=20, unsigned multiplyY=20, bool dependsOnX = true, bool dependsOnY=true)
     {
         dg::geo::BinaryVectorLvl0 bhat( (dg::geo::BHatR)(vec), (dg::geo::BHatZ)(vec), (dg::geo::BHatP)(vec));
         m_fa.construct( bhat, grid, bcx, bcy, limit, eps, multiplyX, multiplyY, dependsOnX, dependsOnY);
@@ -87,7 +87,7 @@ struct DS
         dg::bc bcy = dg::NEU, 
         Limiter limit = FullLimiter(), 
         dg::norm no=dg::normed, dg::direction dir = dg::centered, 
-        double eps = 1e-5, unsigned multiplyX=5, unsigned multiplyY=5, bool dependsOnX = true, bool dependsOnY=true)
+        double eps = 1e-5, unsigned multiplyX=20, unsigned multiplyY=20, bool dependsOnX = true, bool dependsOnY=true)
     {
         m_fa.construct( vec, grid, bcx, bcy, limit, eps, multiplyX, multiplyY, dependsOnX, dependsOnY);
         construct( m_fa, no, dir);
diff --git a/inc/geometries/ds_geom_t.cu b/inc/geometries/ds_geom_t.cu
index b4152d29f..1572756dc 100644
--- a/inc/geometries/ds_geom_t.cu
+++ b/inc/geometries/ds_geom_t.cu
@@ -10,12 +10,7 @@
 
 #include "file/nc_utilities.h"
 #include "draw/host_window.h"
-#include "dg/backend/timer.cuh"
-#include "dg/backend/xspacelib.cuh"
 #include "dg/algorithm.h"
-#include "dg/poisson.h"
-#include "dg/backend/functions.h"
-#include "dg/backend/interpolation.cuh"
 
 #include "solovev.h"
 #include "init.h"
@@ -114,8 +109,8 @@ int main( int argc, char* argv[])
     dg::geo::BHatR bhatR(c);
     dg::geo::BHatZ bhatZ(c);
     dg::geo::BHatP bhatP(c);
-    dg::DSFieldCylindrical field(dg::geo::BinaryVectorLvl0(bhatR, bhatZ, bhatP));
-    dg::Grid3d grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI,p.n, p.Nx, p.Ny,p.Nz);
+    dg::geo::BinaryVectorLvl0 vec(bhatR, bhatZ, bhatP);
+    dg::CylindricalGrid3d grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI,p.n, p.Nx, p.Ny,p.Nz);
     dg::HVec vecR = dg::evaluate( fieldR, grid);
     dg::HVec vecZ = dg::evaluate( fieldZ, grid);
     dg::HVec vecP = dg::evaluate( fieldP, grid);
@@ -135,16 +130,16 @@ int main( int argc, char* argv[])
     err = nc_put_var_double( ncid, vecID[1], vecZ.data());
     err = nc_put_var_double( ncid, vecID[2], vecP.data());
     nc_close(ncid);
-    std::cout << "-----(0) Check single field by integrating from 0 to 2pi (psi=0 surface)" << "\n";
-    thrust::host_vector<double>  in(3);
-    thrust::host_vector<double>  out(3);
-    in[0]=gp.R_0+gp.a*0.6; 
-    in[1]=0.0;
-    in[2]=0.0;
-    dg::integrateRK4( field, in, out,  2*M_PI, gp.rk4eps);
-    
-    std::cout <<"Rin =  "<< in[0] <<" Zin =  "<<in[1] <<" sin  = "<<in[2]<<"\n";
-    std::cout <<"Rout = "<< out[0]<<" Zout = "<<out[1]<<" sout = "<<out[2]<<"\n";
+    //std::cout << "-----(0) Check single field by integrating from 0 to 2pi (psi=0 surface)" << "\n";
+    //thrust::host_vector<double>  in(3);
+    //thrust::host_vector<double>  out(3);
+    //in[0]=gp.R_0+gp.a*0.6; 
+    //in[1]=0.0;
+    //in[2]=0.0;
+    //dg::integrateRK4( field, in, out,  2*M_PI, gp.rk4eps);
+    //
+    //std::cout <<"Rin =  "<< in[0] <<" Zin =  "<<in[1] <<" sin  = "<<in[2]<<"\n";
+    //std::cout <<"Rout = "<< out[0]<<" Zout = "<<out[1]<<" sout = "<<out[2]<<"\n";
 
 
     
@@ -176,8 +171,8 @@ int main( int argc, char* argv[])
                 std::cout << "Construct parallel  derivative\n";
                 dg::Timer t;
                 t.tic();
-                dg::FieldAligned<dg::aGeometry3d, dg::IDMatrix, dg::DVec > dsFA( field, g3d, gp.rk4eps, dg::geo::PsiLimiter(c.psip(), gp.psipmaxlim), g3d.bcx()); 
-                dg::DS<dg::aGeometry3d, dg::IDMatrix, dg::DMatrix, dg::DVec>  ds( dsFA, field, dg::normed, dg::centered); //choose bc of grid
+                dg::geo::Fieldaligned<dg::aProductGeometry3d, dg::IDMatrix, dg::DVec > dsFA( vec, g3d, dg::NEU, dg::NEU, dg::geo::PsiLimiter(c.psip(), gp.psipmaxlim), gp.rk4eps); 
+                dg::geo::DS<dg::aProductGeometry3d, dg::IDMatrix, dg::DMatrix, dg::DVec>  ds( dsFA, dg::normed, dg::centered); //choose bc of grid
                 t.toc();
                 std::cout << "-----> Creation of parallel Derivative took"<<t.diff()<<"s\n";
 
diff --git a/inc/geometries/dz_mpit.cu b/inc/geometries/dz_mpit.cu
index c89515c64..d65ce1ec8 100644
--- a/inc/geometries/dz_mpit.cu
+++ b/inc/geometries/dz_mpit.cu
@@ -1,18 +1,8 @@
 #include <iostream>
 
-#include <cusp/print.h>
-
 #include <mpi.h>
-#include "blas.h"
+#include "dg/algorithm.h"
 #include "ds.h"
-#include "functors.h"
-
-#include "backend/functions.h"
-#include "backend/timer.cuh"
-#include "backend/mpi_evaluation.h"
-#include "backend/mpi_derivatives.h"
-#include "backend/mpi_init.h"
-#include "geometry.h"
 
 double sine(double x, double y, double z){return sin(z);}
 double cosine(double x, double y, double z){return cos(z);}
@@ -44,9 +34,9 @@ int main(int argc, char **argv)
     const dg::MDVec w3d = dg::create::volume( g3d);
     dg::Timer t;
     t.tic();
-    dg::MDDS::FieldAligned dsFA( dg::DefaultField(), g3d, 1e-10, dg::DefaultLimiter(), dg::NEU);
+    dg::geo::BinaryVectorLvl0 vec( dg::geo::Constant(0), dg::geo::Constant(0), dg::geo::Constant(1));
 
-    dg::MDDS ds ( dsFA, dg::DefaultField(), dg::not_normed, dg::centered);
+    dg::geo::DS<dg::CartesianMPIGrid3d, dg::MIDMatrix, dg::MDMatrix, dg::MDVec> ds ( vec, g3d, dg::DIR, dg::DIR, dg::geo::FullLimiter(), dg::not_normed, dg::centered);
     t.toc();
     if(rank==0)std::cout << "TEST STRAIGHT FIELD LINES AND BOUNDARIES IN Z\n";
     if(rank==0)std::cout << "Creation of parallel Derivative took     "<<t.diff()<<"s\n";
diff --git a/inc/geometries/dz_t.cu b/inc/geometries/dz_t.cu
index db4691505..0a01cd10e 100644
--- a/inc/geometries/dz_t.cu
+++ b/inc/geometries/dz_t.cu
@@ -1,9 +1,6 @@
 #include <iostream>
 
-#include <cusp/print.h>
-
 #include "dg/algorithm.h"
-
 #include "ds.h"
 
 double sine(double x, double y, double z){return sin(z);}
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index 225226414..c358e6bcc 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -328,7 +328,7 @@ struct Fieldaligned
         dg::bc globalbcy = dg::NEU, 
         Limiter limit = FullLimiter(), 
         double eps = 1e-5,
-        unsigned multiplyX=5, unsigned multiplyY=5, 
+        unsigned multiplyX=20, unsigned multiplyY=20, 
         bool dependsOnX=true, bool dependsOnY=true, 
         double deltaPhi = -1)
     {
@@ -344,7 +344,7 @@ struct Fieldaligned
         dg::bc globalbcy = dg::NEU, 
         Limiter limit = FullLimiter(), 
         double eps = 1e-5,
-        unsigned multiplyX=5, unsigned multiplyY=5, 
+        unsigned multiplyX=20, unsigned multiplyY=20, 
         bool dependsOnX=true, bool dependsOnY=true, 
         double deltaPhi = -1)
     {
@@ -379,7 +379,7 @@ struct Fieldaligned
         dg::bc globalbcy = dg::NEU, 
         Limiter limit = FullLimiter(), 
         double eps = 1e-5, 
-        unsigned multiplyX=5, unsigned multiplyY=5, 
+        unsigned multiplyX=20, unsigned multiplyY=20, 
         bool dependsOnX=true, bool dependsOnY=true, 
         double deltaPhi = -1);
 
@@ -431,7 +431,7 @@ struct Fieldaligned
      */
     void set_boundaries( dg::bc bcz, const container& global, double scal_left, double scal_right)
     {
-        dg::split( global, m_temp);
+        dg::split( global, m_temp, m_g.get());
         dg::blas1::axpby( scal_left,  m_temp[0],      0, m_left);
         dg::blas1::axpby( scal_right, m_temp[m_Nz-1], 0, m_right);
         m_bcz = bcz;
diff --git a/inc/geometries/geometryX_elliptic_b.cu b/inc/geometries/geometryX_elliptic_b.cu
index d6cb51bfd..36fdde535 100644
--- a/inc/geometries/geometryX_elliptic_b.cu
+++ b/inc/geometries/geometryX_elliptic_b.cu
@@ -2,14 +2,7 @@
 
 #include "file/nc_utilities.h"
 
-#include "dg/geometry/refined_gridX.h"
-#include "dg/backend/timer.cuh"
-#include "dg/backend/grid.h"
-#include "dg/backend/gridX.h"
-#include "dg/backend/derivativesX.h"
-#include "dg/backend/evaluationX.cuh"
-#include "dg/cg.h"
-#include "dg/elliptic.h"
+#include "dg/algorithm.h"
 
 #include "solovev.h"
 #include "taylor.h"
diff --git a/inc/geometries/geometry_advection_b.cu b/inc/geometries/geometry_advection_b.cu
index e21c3a70f..33aad4e3c 100644
--- a/inc/geometries/geometry_advection_b.cu
+++ b/inc/geometries/geometry_advection_b.cu
@@ -1,9 +1,7 @@
 #include <iostream>
 #include <iomanip>
 
-#include "dg/arakawa.h"
-#include "dg/poisson.h"
-#include "dg/geometry/geometry.h"
+#include "dg/algorithm.h"
 
 #include "curvilinear.h"
 
@@ -13,7 +11,6 @@
 #include "solovev.h"
 #include "magnetic_field.h"
 #include "testfunctors.h"
-#include "dg/backend/timer.cuh"
 
 struct FuncDirPer2
 {
@@ -136,7 +133,7 @@ int main(int argc, char** argv)
     //dg::geo::RibeiroFluxGenerator ribeiro( c.get_psip(), psi_0, psi_1, gp.R_0, 0., 1);
     dg::geo::FluxGenerator ribeiro( c.get_psip(), c.get_ipol(), psi_0, psi_1, gp.R_0, 0., 1);
     //dg::geo::SimpleOrthogonal ribeiro( c.get_psip(), psi_0, psi_1, gp.R_0, 0., 1);
-    dg::CurvilinearGrid2d grid(ribeiro, n, Nx, Ny, dg::DIR); //2d
+    dg::geo::CurvilinearGrid2d grid(ribeiro, n, Nx, Ny, dg::DIR); //2d
     t.toc();
     std::cout << "Construction took "<<t.diff()<<"s"<<std::endl;
     grid.display();
diff --git a/inc/geometries/geometry_elliptic_b.cu b/inc/geometries/geometry_elliptic_b.cu
index a6e10fd01..c7c737b68 100644
--- a/inc/geometries/geometry_elliptic_b.cu
+++ b/inc/geometries/geometry_elliptic_b.cu
@@ -3,11 +3,7 @@
 
 #include "file/nc_utilities.h"
 
-#include "dg/backend/timer.cuh"
-#include "dg/backend/grid.h"
-#include "dg/geometry/geometry.h"
-#include "dg/elliptic.h"
-#include "dg/cg.h"
+#include "dg/algorithm.h"
 
 #include "solovev.h"
 #include "guenther.h"
diff --git a/inc/geometries/geometry_elliptic_mpib.cu b/inc/geometries/geometry_elliptic_mpib.cu
index 45cd6797e..44258c1ce 100644
--- a/inc/geometries/geometry_elliptic_mpib.cu
+++ b/inc/geometries/geometry_elliptic_mpib.cu
@@ -1,15 +1,12 @@
 #include <iostream>
+#include <memory>
 #include <mpi.h>
 
 #include <netcdf_par.h>
 
 #include "file/nc_utilities.h"
 
-#include "dg/backend/timer.cuh"
-#include "dg/backend/mpi_init.h"
-#include "dg/backend/grid.h"
-#include "dg/elliptic.h"
-#include "dg/cg.h"
+#include "dg/algorithm.h"
 
 #include "solovev.h"
 //#include "guenther.h"
@@ -52,8 +49,8 @@ int main(int argc, char**argv)
     t.tic();
     dg::geo::SimpleOrthogonal generator( c.get_psip(), psi_0, psi_1, gp.R_0, 0., 1);
     dg::geo::CurvilinearProductMPIGrid3d g3d( generator, n, Nx, Ny,Nz, dg::DIR, dg::PER, dg::PER, comm);
-    dg::geo::CurvilinearMPIGrid2d g2d = g3d.perp_grid();
-    dg::Elliptic<dg::geo::CurvilinearMPIGrid2d, dg::MDMatrix, dg::MDVec> pol( g2d, dg::not_normed, dg::forward);
+    std::unique_ptr<dg::aMPIGeometry2d> g2d(g3d.perp_grid());
+    dg::Elliptic<dg::aMPIGeometry2d, dg::MDMatrix, dg::MDVec> pol( *g2d, dg::not_normed, dg::forward);
     t.toc();
     if(rank==0)std::cout << "Construction took "<<t.diff()<<"s\n";
     ///////////////////////////////////////////////////////////////////////////
@@ -62,7 +59,7 @@ int main(int argc, char**argv)
     MPI_Info info = MPI_INFO_NULL;
     ncerr = nc_create_par( "testE_mpi.nc", NC_NETCDF4|NC_MPIIO|NC_CLOBBER, comm, info, &ncid); //MPI ON
     int dim2d[2];
-    ncerr = file::define_dimensions(  ncid, dim2d, g2d.global());
+    ncerr = file::define_dimensions(  ncid, dim2d, g2d->global());
     int coordsID[2], psiID, functionID, function2ID;
     ncerr = nc_def_var( ncid, "x_XYP", NC_DOUBLE, 2, dim2d, &coordsID[0]);
     ncerr = nc_def_var( ncid, "y_XYP", NC_DOUBLE, 2, dim2d, &coordsID[1]);
@@ -71,8 +68,8 @@ int main(int argc, char**argv)
     ncerr = nc_def_var( ncid, "ana_solution", NC_DOUBLE, 2, dim2d, &function2ID);
 
     int dims[2], periods[2],  coords[2];
-    MPI_Cart_get( g2d.communicator(), 2, dims, periods, coords);
-    size_t count[2] = {g2d.n()*g2d.Ny(), g2d.n()*g2d.Nx()};
+    MPI_Cart_get( g2d->communicator(), 2, dims, periods, coords);
+    size_t count[2] = {g2d->local().n()*g2d->local().Ny(), g2d->local().n()*g2d->local().Nx()};
     size_t start[2] = {coords[1]*count[0], coords[0]*count[1]};
 
     ncerr = nc_var_par_access( ncid, coordsID[0], NC_COLLECTIVE);
@@ -81,20 +78,20 @@ int main(int argc, char**argv)
     ncerr = nc_var_par_access( ncid, functionID, NC_COLLECTIVE);
     ncerr = nc_var_par_access( ncid, function2ID, NC_COLLECTIVE);
 
-    dg::HVec X( g2d.size()), Y(X); //P = dg::pullback( dg::coo3, g);
-    for( unsigned i=0; i<g2d.size(); i++)
+    dg::HVec X( g2d->local().size()), Y(X); //P = dg::pullback( dg::coo3, g);
+    for( unsigned i=0; i<g2d->local().size(); i++)
     {
-        X[i] = g2d.map()[0].data()[i];
-        Y[i] = g2d.map()[1].data()[i];
+        X[i] = g2d->map()[0].data()[i];
+        Y[i] = g2d->map()[1].data()[i];
     }
     ncerr = nc_put_vara_double( ncid, coordsID[0], start, count, X.data());
     ncerr = nc_put_vara_double( ncid, coordsID[1], start, count, Y.data());
     ///////////////////////////////////////////////////////////////////////////
-    dg::MDVec x =    dg::evaluate( dg::zero, g2d);
-    const dg::MDVec b =    dg::pullback( dg::geo::EllipticDirPerM(c, psi_0, psi_1, 4), g2d);
-    const dg::MDVec chi =  dg::pullback( dg::geo::Bmodule(c), g2d);
-    const dg::MDVec solution = dg::pullback( dg::geo::FuncDirPer(c, psi_0, psi_1, 4), g2d);
-    const dg::MDVec vol3d = dg::create::volume( g2d);
+    dg::MDVec x =    dg::evaluate( dg::zero, *g2d);
+    const dg::MDVec b =    dg::pullback( dg::geo::EllipticDirPerM(c, psi_0, psi_1, 4), *g2d);
+    const dg::MDVec chi =  dg::pullback( dg::geo::Bmodule(c), *g2d);
+    const dg::MDVec solution = dg::pullback( dg::geo::FuncDirPer(c, psi_0, psi_1, 4), *g2d);
+    const dg::MDVec vol3d = dg::create::volume( *g2d);
     pol.set_chi( chi);
     //compute error
     dg::MDVec error( solution);
@@ -111,14 +108,14 @@ int main(int argc, char**argv)
     const double norm = dg::blas2::dot( vol3d, solution);
     if(rank==0)std::cout << sqrt( err/norm) << "\t";
 
-    dg::SparseTensor<dg::MDVec> metric = g2d.metric();
+    dg::SparseTensor<dg::MDVec> metric = g2d->metric();
     dg::MDVec gyy = metric.value(1,1), gxx=metric.value(0,0), vol = dg::tensor::volume(metric).value();
     dg::blas1::transform( gxx, gxx, dg::SQRT<double>());
     dg::blas1::transform( gyy, gyy, dg::SQRT<double>());
     dg::blas1::pointwiseDot( gxx, vol, gxx);
     dg::blas1::pointwiseDot( gyy, vol, gyy);
-    dg::blas1::scal( gxx, g2d.hx());
-    dg::blas1::scal( gyy, g2d.hy());
+    dg::blas1::scal( gxx, g2d->hx());
+    dg::blas1::scal( gyy, g2d->hy());
     if(rank==0)std::cout << "(Max elements on first process)\t";
     if(rank==0)std::cout << *thrust::max_element( gxx.data().begin(), gxx.data().end()) << "\t";
     if(rank==0)std::cout << *thrust::max_element( gyy.data().begin(), gyy.data().end()) << "\t";
diff --git a/inc/geometries/guenther_ds_b.cu b/inc/geometries/guenther_ds_b.cu
index 49d9bdc37..e25b8c2cb 100644
--- a/inc/geometries/guenther_ds_b.cu
+++ b/inc/geometries/guenther_ds_b.cu
@@ -46,7 +46,7 @@ int main( )
     dg::geo::guenther::DeriNeuT2 deriNEUT2(gp.R_0,gp.I_0);
     dg::geo::guenther::DeriNeuT deriNEUT(gp.R_0,gp.I_0);
     
-    std::cout << "Type n, Nx, Ny, Nz\n";
+    //std::cout << "Type n, Nx, Ny, Nz\n";
     //std::cout << "Note, that function is resolved exactly in R,Z for n > 2\n";
     unsigned n=3, Nx=5, Ny=5, Nz=5;
     //std::cin >> n>> Nx>>Ny>>Nz;
@@ -82,9 +82,9 @@ int main( )
     const dg::DVec v3d = dg::create::inv_volume( g3d);
 
     std::cout << "computing dsDIR" << std::endl;
-    dg::geo::Fieldaligned<dg::aProductGeometry3d, dg::IDMatrix, dg::DVec>  dsFA( mag, g3d, dg::DIR, dg::DIR, dg::geo::FullLimiter(), rk4eps);
+    dg::geo::Fieldaligned<dg::aProductGeometry3d, dg::IDMatrix, dg::DVec>  dsFA( mag, g3d, dg::DIR, dg::DIR, dg::geo::FullLimiter(), rk4eps, 50, 50);
     std::cout << "computing dsNEU" << std::endl;
-    dg::geo::Fieldaligned<dg::aProductGeometry3d, dg::IDMatrix, dg::DVec> dsNUFA( mag, g3d,dg::NEU, dg::NEU, dg::geo::FullLimiter(), rk4eps);
+    dg::geo::Fieldaligned<dg::aProductGeometry3d, dg::IDMatrix, dg::DVec> dsNUFA( mag, g3d,dg::NEU, dg::NEU, dg::geo::FullLimiter(), rk4eps, 50, 50);
 
     dg::geo::DS<dg::aProductGeometry3d, dg::IDMatrix, dg::DMatrix, dg::DVec> ds ( dsFA, dg::not_normed, dg::centered), 
         dsNU ( dsNUFA, dg::not_normed, dg::centered);
@@ -236,19 +236,15 @@ int main( )
 //     dg::blas1::axpby(0.5,dsTdsbd,0.5,dsTdsfd,dsTdsfbd); 
     ds.symv(function,dsTdsfb);
     dg::blas1::pointwiseDot(v3d,dsTdsfb,dsTdsfb);
-        //ds( function, temp);
-        //dg::blas1::pointwiseDot( temp, inverseB, temp);
-        //ds(temp, dsTdsfb);
-        //dg::blas1::pointwiseDivide( dsTdsfb, inverseB, dsTdsfb);
 //     ds.centeredT( derivative2, dsTds2); //dsT(ds(f))
 //     dg::blas1::pointwiseDivide(ones,  inverseB, temp2); //B
-     ds.centeredDiv( ones, divbT);
+    ds.centeredDiv( ones, divbT);
 //     
 //     double normdsds =dg::blas2::dot(derivative2, w3d,derivative2);
 //     double normds1ds =dg::blas2::dot(derivativeones, w3d,derivative2);
 //     double normdivBT =dg::blas2::dot(divBT, w3d,divBT);
-     double normdivbT =dg::blas2::dot(divbT, w3d,divbT);
-     double normdivb =dg::blas2::dot(divbsol, w3d,divbsol); 
+    double normdivbT =dg::blas2::dot(divbT, w3d,divbT);
+    double normdivb =dg::blas2::dot(divbsol, w3d,divbsol); 
 //     double normdsTf = dg::blas2::dot(derivativeT2, w3d, function2);
 //     double normdsT_1 = dg::blas2::dot(derivativeT2, w3d, ones);
 //     double normdsT1 = dg::blas2::dot(derivativeTones, w3d, function2);
@@ -394,8 +390,8 @@ int main( )
     
     
     double eps =1e-8;   
-    dg::Invert< dg::DVec> invert( dg::evaluate(dg::zero,g3d), w3d.size(), eps );  
-    std::cout << "MAX # iterations = " << w3d.size() << std::endl;
+    dg::Invert< dg::DVec> invert( dg::evaluate(dg::zero,g3d), g3d.size(), eps );  
+    std::cout << "MAX # iterations = " << g3d.size() << std::endl;
 // 
 //    const dg::DVec rhs = dg::evaluate( solovev::DeriNeuT2( gp.R_0, gp.I_0), g3d);
 // // 
diff --git a/inc/geometries/guenther_ds_mpib.cu b/inc/geometries/guenther_ds_mpib.cu
index e89c91633..12c1a8c7a 100644
--- a/inc/geometries/guenther_ds_mpib.cu
+++ b/inc/geometries/guenther_ds_mpib.cu
@@ -4,23 +4,13 @@
 #include <cusp/print.h>
 #include <cusp/csr_matrix.h>
 
-#include "dg/backend/xspacelib.cuh"
-#include "dg/backend/evaluation.cuh"
-#include "dg/backend/timer.cuh"
-#include "dg/blas.h"
-#include "dg/ds.h"
-#include "dg/backend/functions.h"
-#include "dg/functors.h"
-#include "dg/elliptic.h"
-#include "dg/cg.h"
-#include "dg/backend/interpolation.cuh"
-#include "dg/backend/typedefs.cuh"
+#include "dg/algorithm.h"
+#include "ds.h"
 // #include "draw/host_window.h"
 #include "guenther.h"
 #include "magnetic_field.h"
 #include "testfunctors.h"
 
-using namespace dg::geo::guenther;
 
 int main( int argc, char* argv[])
 {
@@ -46,7 +36,7 @@ int main( int argc, char* argv[])
     Json::Value js;
     std::ifstream is("guenther_params.js");
     reader.parse(is,js,false);
-    GeomParameters gp(js);
+    dg::geo::guenther::Parameters gp(js);
 //     gp.display( std::cout);
 
     //////////////////////////////////////////////////////////////////////////
@@ -57,21 +47,21 @@ int main( int argc, char* argv[])
     double Zmax=1.0*gp.a*gp.elongation;
     /////////////////////////////////////////////initialze fields /////////////////////
     
-    Field field(gp.R_0, gp.I_0);
-    InvB invb(gp.R_0, gp.I_0);
-    GradLnB gradlnB(gp.R_0, gp.I_0);
-    LnB lnB(gp.R_0, gp.I_0);
-    FieldR bR_(gp.R_0, gp.I_0);
-    FieldZ bZ_(gp.R_0, gp.I_0);
-    FieldP bPhi_(gp.R_0, gp.I_0);
-    FuncNeu funcNEU(gp.R_0,gp.I_0);
-    FuncNeu2 funcNEU2(gp.R_0,gp.I_0);
-    DeriNeu deriNEU(gp.R_0,gp.I_0);
-    DeriNeu2 deriNEU2(gp.R_0,gp.I_0);
-    DeriNeuT2 deriNEUT2(gp.R_0,gp.I_0);
-    DeriNeuT deriNEUT(gp.R_0,gp.I_0);
-    Divb divb(gp.R_0,gp.I_0);
-    B Bfield(gp.R_0, gp.I_0);
+    dg::geo::TokamakMagneticField mag = dg::geo::createGuentherField(gp.R_0, gp.I_0);
+    dg::geo::InvB invb(mag);
+    dg::geo::GradLnB gradlnB(mag);
+    dg::geo::LnB lnB(mag);
+    dg::geo::FieldR bR_(mag);
+    dg::geo::FieldZ bZ_(mag);
+    dg::geo::FieldP bPhi_(mag);
+    dg::geo::Divb divb(mag);
+    dg::geo::Bmodule B(mag);
+    dg::geo::guenther::FuncNeu funcNEU(gp.R_0,gp.I_0);
+    dg::geo::guenther::FuncNeu2 funcNEU2(gp.R_0,gp.I_0);
+    dg::geo::guenther::DeriNeu deriNEU(gp.R_0,gp.I_0);
+    dg::geo::guenther::DeriNeu2 deriNEU2(gp.R_0,gp.I_0);
+    dg::geo::guenther::DeriNeuT2 deriNEUT2(gp.R_0,gp.I_0);
+    dg::geo::guenther::DeriNeuT deriNEUT(gp.R_0,gp.I_0);
     
     //std::cout << "Type n, Nx, Ny, Nz\n";
     //std::cout << "Note, that function is resolved exactly in R,Z for n > 2\n";
@@ -93,7 +83,7 @@ int main( int argc, char* argv[])
 
 
 
-        dg::CylindricalMPIGrid3d<dg::MDVec> g3d( Rmin,Rmax, Zmin,Zmax, z0, z1,  n,Nxn ,Nyn, Nzn,dg::DIR, dg::DIR, dg::PER, comm);
+        dg::CylindricalMPIGrid3d g3d( Rmin,Rmax, Zmin,Zmax, z0, z1,  n,Nxn ,Nyn, Nzn,dg::DIR, dg::DIR, dg::PER, comm);
         dg::MPIGrid2d g2d( Rmin,Rmax, Zmin,Zmax,  n, Nxn ,Nyn, dg::DIR, dg::DIR, comm);
 
         if(rank==0)std::cout << "NR = " << Nxn << std::endl;
@@ -109,13 +99,12 @@ int main( int argc, char* argv[])
     const dg::MDVec v3d = dg::create::inv_volume( g3d);
 
     if(rank==0)std::cout << "computing dsDIR" << std::endl;
-    dg::MDDS::FieldAligned dsFA( field, g3d, rk4eps, dg::DefaultLimiter(), dg::DIR);
+    dg::geo::Fieldaligned<dg::aProductMPIGeometry3d, dg::MIDMatrix, dg::MDVec>  dsFA( mag, g3d, dg::DIR, dg::DIR, dg::geo::FullLimiter(), rk4eps, 50, 50);
     if(rank==0)std::cout << "computing dsNEU" << std::endl;
-    dg::MDDS::FieldAligned dsNUFA( field, g3d, rk4eps, dg::DefaultLimiter(), dg::NEU);
+    dg::geo::Fieldaligned<dg::aProductMPIGeometry3d, dg::MIDMatrix, dg::MDVec> dsNUFA( mag, g3d,dg::NEU, dg::NEU, dg::geo::FullLimiter(), rk4eps, 50, 50);
 
-
-    dg::MDDS ds ( dsFA, field, dg::not_normed, dg::centered), 
-         dsNU ( dsNUFA, field, dg::not_normed, dg::centered);
+    dg::geo::DS<dg::aProductMPIGeometry3d, dg::MIDMatrix, dg::MDMatrix, dg::MDVec> ds ( dsFA, dg::not_normed, dg::centered), 
+        dsNU ( dsNUFA, dg::not_normed, dg::centered);
 
 //     dg::DS<dg::DMatrix, dg::MDVec> dsNEU( field, g3d, g3d.hz(), rk4eps, dg::DefaultLimiter(), dg::NEU);
     
@@ -224,15 +213,15 @@ int main( int argc, char* argv[])
 
 //     
 //     
-//     ds.centeredT(function, derivativeT); //ds(f)
+//     ds.centeredDiv(function, derivativeT); //ds(f)
 // 
 //     //divB
 //     dg::blas1::pointwiseDivide(ones,  inverseB, temp2); //B
-//     ds.centeredT(temp2, divBT); // dsT B
+//     ds.centeredDiv(temp2, divBT); // dsT B
 // 
 //     
-//     ds.centeredT( function2, derivativeT2); //ds(f)
-//     ds.centeredT( ones, derivativeTones); //ds(f)
+//     ds.centeredDiv( function2, derivativeT2); //ds(f)
+//     ds.centeredDiv( ones, derivativeTones); //ds(f)
     //B ds f/B
 //     dg::blas1::pointwiseDot( inverseB, function, temp);
 //     ds( temp, derivativeTds);
@@ -246,13 +235,13 @@ int main( int argc, char* argv[])
     
     //     dg::blas1::pointwiseDivide( derivativeTds, inverseB, derivativeTds);
 //     
-//     ds.centeredT( derivative, dsTds); //dsT(ds(f))
+//     ds.centeredDiv( derivative, dsTds); //dsT(ds(f))
 //     
 //     //overwrite with sym from adjoint dg
 //     ellipticsym.symv(function,dsTds);
 //     dg::blas1::scal(dsTds,-1.0);
-// //     ds.centeredT(ones,divbT);
-    ds.forwardT( derivativef, dsTdsf);  //dsT(ds(f))
+// //     ds.centeredDiv(ones,divbT);
+    ds.forwardDiv( derivativef, dsTdsf);  //dsT(ds(f))
 //     ds.backwardT( derivativeb, dsTdsb); //dsT(ds(f))
 
 //     //centered
@@ -265,15 +254,15 @@ int main( int argc, char* argv[])
 //     dg::blas1::axpby(0.5,dsTdsbd,0.5,dsTdsfd,dsTdsfbd); 
     ds.symv(function,dsTdsfb);
     dg::blas1::pointwiseDot(v3d,dsTdsfb,dsTdsfb);
-//     ds.centeredT( derivative2, dsTds2); //dsT(ds(f))
+//     ds.centeredDiv( derivative2, dsTds2); //dsT(ds(f))
 //     dg::blas1::pointwiseDivide(ones,  inverseB, temp2); //B
-//     ds.centeredT( ones, divbT);
+    ds.centeredDiv( ones, divbT);
 //     
 //     double normdsds =dg::blas2::dot(derivative2, w3d,derivative2);
 //     double normds1ds =dg::blas2::dot(derivativeones, w3d,derivative2);
 //     double normdivBT =dg::blas2::dot(divBT, w3d,divBT);
-//     double normdivbT =dg::blas2::dot(divbT, w3d,divbT);
-//     double normdivb =dg::blas2::dot(divbsol, w3d,divbsol); 
+    double normdivbT =dg::blas2::dot(divbT, w3d,divbT);
+    double normdivb =dg::blas2::dot(divbsol, w3d,divbsol); 
 //     double normdsTf = dg::blas2::dot(derivativeT2, w3d, function2);
 //     double normdsT_1 = dg::blas2::dot(derivativeT2, w3d, ones);
 //     double normdsT1 = dg::blas2::dot(derivativeTones, w3d, function2);
@@ -303,14 +292,14 @@ int main( int argc, char* argv[])
 //     errRZPhi =dg::blas2::dot( w3d, derivativeRZPhi);    
 //     std::cout << "Relative Difference in DS is "<< sqrt( errRZPhi/norm )<<"\n"; 
 //     
-//     std::cout << "--------------------testing dsT" << std::endl;
-//     std::cout << "|| divbsol ||  "<<sqrt( normdivb)<<"\n";
-//     std::cout << "|| divbT  ||   "<<sqrt( normdivbT)<<"\n";
-//     dg::blas1::axpby( 1., divbsol, -1., divbT);
-//     normdivbT =dg::blas2::dot(divbT, w3d,divbT);
-//     std::cout << "Relative Difference in DST is   "<<sqrt( normdivbT)<<"\n";
-//     std::cout << "-------------------- " << std::endl;
-//     std::cout << "|| divB || "<<sqrt( normdivBT)<<"\n";
+     if(rank==0)std::cout << "--------------------testing dsT" << std::endl;
+     if(rank==0)std::cout << "|| divbsol ||  "<<sqrt( normdivb)<<"\n";
+     if(rank==0)std::cout << "|| divbT  ||   "<<sqrt( normdivbT)<<"\n";
+     dg::blas1::axpby( 1., divbsol, -1., divbT);
+     normdivbT =dg::blas2::dot(divbT, w3d,divbT);
+     if(rank==0)std::cout << "Relative Difference in DST is   "<<sqrt( normdivbT)<<"\n";
+     if(rank==0)std::cout << "-------------------- " << std::endl;
+     //std::cout << "|| divB || "<<sqrt( normdivBT)<<"\n";
 // 
 //     
 //     std::cout << "-------------------- " << std::endl;
@@ -352,8 +341,10 @@ int main( int argc, char* argv[])
     
     if(rank==0)std::cout << "--------------------testing dsTdsfb " << std::endl;
     if(rank==0)std::cout << "|| SolutionT ||      "<<sqrt( normdsTds)<<"\n";
+    double remainder =dg::blas1::dot( w3d,dsTdsfb);
     double errdsTdsfb =dg::blas2::dot( w3d,dsTdsfb);
     if(rank==0)std::cout << "|| DerivativeTds ||  "<<sqrt( errdsTdsfb)<<"\n";
+    if(rank==0)std::cout << "   Integral          "<<remainder<<"\n";
     dg::blas1::axpby( 1., solutiondsTds, -1., dsTdsfb);
     errdsTdsfb =dg::blas2::dot( w3d, dsTdsfb);
     if(rank==0)std::cout << "Relative Difference in DST is "<< sqrt( errdsTdsfb/normdsTds )<<"\n";
@@ -417,8 +408,8 @@ int main( int argc, char* argv[])
     
     
     double eps =1e-8;   
-    dg::Invert< dg::MDVec> invert( dg::evaluate(dg::zero,g3d), w3d.size(), eps );  
-    if(rank==0)std::cout << "MAX # iterations = " << w3d.size() << std::endl;
+    dg::Invert< dg::MDVec> invert( dg::evaluate(dg::zero,g3d), g3d.size(), eps );  
+    if(rank==0)std::cout << "MAX # iterations = " << g3d.size() << std::endl;
 // 
 //    const dg::MDVec rhs = dg::evaluate( solovev::DeriNeuT2( gp.R_0, gp.I_0), g3d);
 // // 
diff --git a/inc/geometries/mpi_fieldaligned.h b/inc/geometries/mpi_fieldaligned.h
index 683357b47..f1fa42098 100644
--- a/inc/geometries/mpi_fieldaligned.h
+++ b/inc/geometries/mpi_fieldaligned.h
@@ -68,7 +68,7 @@ struct Fieldaligned< ProductMPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY
         dg::bc globalbcy = dg::NEU, 
         Limiter limit = FullLimiter(), 
         double eps = 1e-5,
-        unsigned multiplyX=5, unsigned multiplyY=5, 
+        unsigned multiplyX=20, unsigned multiplyY=20, 
         bool dependsOnX=true, bool dependsOnY=true, 
         double deltaPhi = -1)
     {
@@ -82,7 +82,7 @@ struct Fieldaligned< ProductMPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY
         dg::bc globalbcy = dg::NEU, 
         Limiter limit = FullLimiter(), 
         double eps = 1e-5,
-        unsigned multiplyX=5, unsigned multiplyY=5, 
+        unsigned multiplyX=20, unsigned multiplyY=20, 
         bool dependsOnX=true, bool dependsOnY=true, 
         double deltaPhi = -1)
     {
@@ -95,7 +95,7 @@ struct Fieldaligned< ProductMPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY
         dg::bc globalbcy = dg::NEU, 
         Limiter limit = FullLimiter(), 
         double eps = 1e-5, 
-        unsigned multiplyX=5, unsigned multiplyY=5, 
+        unsigned multiplyX=20, unsigned multiplyY=20, 
         bool dependsOnX=true, bool dependsOnY=true, 
         double deltaPhi = -1);
 
@@ -105,7 +105,7 @@ struct Fieldaligned< ProductMPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY
     void set_boundaries( dg::bc bcz, double left, double right)
     {
         m_bcz = bcz; 
-        const dg::MPIGrid2d g2d( 0., 1., 0., 1., m_g.get().global().n(), m_g.get().global().Nx(), m_g.get().global().Ny(), m_g.get().perp_communicator() );
+        const dg::MPIGrid2d g2d( 0., 1., 0., 1., m_g.get().global().n(), m_g.get().global().Nx(), m_g.get().global().Ny(), m_g.get().get_perp_comm() );
         m_left  = dg::evaluate( dg::CONSTANT(left), g2d);
         m_right = dg::evaluate( dg::CONSTANT(right),g2d);
     }
diff --git a/inc/geometries/ribeiroX.h b/inc/geometries/ribeiroX.h
index c3239091a..385f96660 100644
--- a/inc/geometries/ribeiroX.h
+++ b/inc/geometries/ribeiroX.h
@@ -251,6 +251,8 @@ struct RibeiroX : public aGeneratorX2d
         zeta1_= -fx/(1.-fx)*zeta0_;
         x0_=x0, y0_=y0, psi0_=psi_0;
     }
+
+    virtual RibeiroX* clone() const{return new RibeiroX(*this);}
     private:
     bool isConformal()const{return false;}
     bool do_isOrthogonal()const{return false;}
diff --git a/inc/geometries/ribeiroX_t.cu b/inc/geometries/ribeiroX_t.cu
index 88544e1b3..88bf40080 100644
--- a/inc/geometries/ribeiroX_t.cu
+++ b/inc/geometries/ribeiroX_t.cu
@@ -5,23 +5,19 @@
 #include <sstream>
 #include <cmath>
 
-#include "dg/backend/xspacelib.cuh"
-#include "dg/functors.h"
+#include "dg/algorithm.h"
 #include "file/nc_utilities.h"
 
-#include "dg/backend/timer.cuh"
 #include "solovev.h"
 #include "taylor.h"
 //#include "guenther.h"
 #include "curvilinearX.h"
 #include "ribeiroX.h"
-#include "dg/ds.h"
 #include "init.h"
+#include "ds.h"
 
-//typedef dg::FieldAligned< solovev::ConformalXGrid3d<dg::DVec> , dg::IDMatrix, dg::DVec> DFA;
 double sine( double x) {return sin(x);}
 double cosine( double x) {return cos(x);}
-typedef dg::FieldAligned< dg::geo::CurvilinearGridX3d<dg::DVec> , dg::IDMatrix, dg::DVec> DFA;
 
 thrust::host_vector<double> periodify( const thrust::host_vector<double>& in, const dg::GridX3d& g)
 {
@@ -102,12 +98,12 @@ int main( int argc, char* argv[])
     std::cout << "Psi min "<<psip.f()(gp.R_0, 0)<<"\n";
     double R_X = gp.R_0-1.1*gp.triangularity*gp.a;
     double Z_X = -1.1*gp.elongation*gp.a;
-    dg::geo::findXpoint( psipR, psipZ, psipRR, psipRZ, psipZZ, R_X, Z_X);
+    dg::geo::findXpoint( psip, R_X, Z_X);
 
     double R0 = gp.R_0, Z0 = 0;
     dg::geo::RibeiroX generator(psip, psi_0, fx_0, R_X,Z_X, R0, Z0);
-    dg::geo::CurvilinearGridX3d g3d(generator, psi_0, fx_0, fy_0, n, Nx, Ny,Nz, dg::DIR, dg::NEU);
-    dg::geo::CurvilinearGridX2d g2d = g3d.perp_grid();
+    dg::geo::CurvilinearProductGridX3d g3d(generator, fx_0, fy_0, n, Nx, Ny,Nz, dg::DIR, dg::NEU);
+    dg::geo::CurvilinearGridX2d g2d(generator, fx_0, fy_0, n, Nx, Ny, dg::DIR, dg::NEU);
     t.toc();
     dg::GridX3d g3d_periodic(g3d.x0(), g3d.x1(), g3d.y0(), g3d.y1(), g3d.z0(), g3d.z1(), g3d.fx(), g3d.fy(), g3d.n(), g3d.Nx(), g3d.Ny(), 2); 
     std::cout << "Construction took "<<t.diff()<<"s"<<std::endl;
@@ -136,15 +132,15 @@ int main( int argc, char* argv[])
     err = nc_def_var( ncid, "volume", NC_DOUBLE, 3, dim3d, &volID);
     err = nc_def_var( ncid, "divB", NC_DOUBLE, 3, dim3d, &divBID);
 
-    thrust::host_vector<double> psi_p = dg::pullback( psip, g2d);
+    thrust::host_vector<double> psi_p = dg::pullback( psip.f(), g2d);
     g2d.display();
     err = nc_put_var_double( ncid, onesID, periodify(psi_p, g3d_periodic).data());
     //err = nc_put_var_double( ncid, onesID, periodify(g2d.g(), g3d_periodic).data());
     dg::HVec X( g2d.size()), Y(X); //P = dg::pullback( dg::coo3, g);
     for( unsigned i=0; i<g2d.size(); i++)
     {
-        X[i] = g2d.r()[i];
-        Y[i] = g2d.z()[i];
+        X[i] = g2d.map()[0][i];
+        Y[i] = g2d.map()[1][i];
     }
 
     dg::DVec ones = dg::evaluate( dg::one, g2d);
@@ -153,23 +149,19 @@ int main( int argc, char* argv[])
 
     err = nc_put_var_double( ncid, coordsID[0], periodify(X, g3d_periodic).data());
     err = nc_put_var_double( ncid, coordsID[1], periodify(Y, g3d_periodic).data());
-    //err = nc_put_var_double( ncid, coordsID[0], X.data());
-    //err = nc_put_var_double( ncid, coordsID[1], Y.data());
-    //err = nc_put_var_double( ncid, coord1D[0], g3d.rx0().data());
-    //err = nc_put_var_double( ncid, coord1D[1], g3d.zx0().data());
-    //err = nc_put_var_double( ncid, coord1D[2], g3d.rx1().data());
-    //err = nc_put_var_double( ncid, coord1D[3], g3d.zx1().data());
-    //err = nc_put_var_double( ncid, coord1D[4], periodify(g3d.f_x(), g3d_periodic).data());
-    //err = nc_put_var_double( ncid, coord1D[4], g3d.f_x().data());
-    //err = nc_put_var_double( ncid, coordsID[2], g.z().data());
 
-    dg::blas1::pointwiseDivide( g2d.g_yy(), g2d.g_xx(), temp0);
+    dg::SparseTensor<dg::DVec> metric = g2d.metric();
+    dg::DVec g_xx = metric.value(0,0), g_xy = metric.value(0,1), g_yy=metric.value(1,1);
+    dg::SparseElement<dg::DVec> vol_ = dg::tensor::volume(metric);
+    dg::DVec vol = vol_.value();
+
+    dg::blas1::pointwiseDivide( g_yy, g_xx, temp0);
     dg::blas1::axpby( 1., ones, -1., temp0, temp0);
     dg::blas1::transfer( temp0, X);
     err = nc_put_var_double( ncid, defID, periodify(X, g3d_periodic).data());
     //err = nc_put_var_double( ncid, defID, X.data());
-    dg::blas1::transfer( g2d.vol(), X);
-    dg::blas1::transfer( g2d.g_yy(),Y);
+    dg::blas1::transfer( vol, X);
+    dg::blas1::transfer( g_yy,Y);
     dg::blas1::pointwiseDot( Y, X, X);
     err = nc_put_var_double( ncid, volID, periodify(X, g3d_periodic).data());
     //err = nc_put_var_double( ncid, volID, X.data());
@@ -177,42 +169,42 @@ int main( int argc, char* argv[])
     std::cout << "Construction successful!\n";
 
     //compute error in volume element (in conformal grid g^xx is the volume element)
-    dg::blas1::pointwiseDot( g2d.g_xx(), g2d.g_yy(), temp0);
-    dg::blas1::pointwiseDot( g2d.g_xy(), g2d.g_xy(), temp1);
+    dg::blas1::pointwiseDot( g_xx, g_yy, temp0);
+    dg::blas1::pointwiseDot( g_xy, g_xy, temp1);
     dg::blas1::axpby( 1., temp0, -1., temp1, temp0);
-    dg::blas1::transfer( g2d.g_xx(),  temp1);
+    dg::blas1::transfer( g_xx,  temp1);
     dg::blas1::pointwiseDot( temp1, temp1, temp1);
     dg::blas1::axpby( 1., temp1, -1., temp0, temp0);
     double error = sqrt( dg::blas2::dot( temp0, w2d, temp0)/dg::blas2::dot( temp1, w2d, temp1));
     std::cout<< "Rel Error in Determinant is "<<error<<"\n";
 
     //compute error in determinant vs volume form
-    dg::blas1::pointwiseDot( g2d.g_xx(), g2d.g_yy(), temp0);
-    dg::blas1::pointwiseDot( g2d.g_xy(), g2d.g_xy(), temp1);
+    dg::blas1::pointwiseDot( g_xx, g_yy, temp0);
+    dg::blas1::pointwiseDot( g_xy, g_xy, temp1);
     dg::blas1::axpby( 1., temp0, -1., temp1, temp0);
     dg::blas1::transform( temp0, temp0, dg::SQRT<double>());
     dg::blas1::pointwiseDivide( ones, temp0, temp0);
     dg::blas1::transfer( temp0, X);
     err = nc_put_var_double( ncid, volID, periodify(X, g3d_periodic).data());
-    dg::blas1::axpby( 1., temp0, -1., g2d.vol(), temp0);
-    error = sqrt(dg::blas2::dot( temp0, w2d, temp0)/dg::blas2::dot( g2d.vol(), w2d, g2d.vol()));
+    dg::blas1::axpby( 1., temp0, -1., vol, temp0);
+    error = sqrt(dg::blas2::dot( temp0, w2d, temp0)/dg::blas2::dot( vol, w2d, vol));
     std::cout << "Rel Consistency  of volume is "<<error<<"\n";
 
     //compare g^xx to volume form
-    dg::blas1::transfer( g2d.g_xx(), temp0);
+    dg::blas1::transfer( g_xx, temp0);
     dg::blas1::pointwiseDivide( ones, temp0, temp0);
-    dg::blas1::axpby( 1., temp0, -1., g2d.vol(), temp0);
-    error=sqrt(dg::blas2::dot( temp0, w2d, temp0))/sqrt( dg::blas2::dot(g2d.vol(), w2d, g2d.vol()));
+    dg::blas1::axpby( 1., temp0, -1., vol, temp0);
+    error=sqrt(dg::blas2::dot( temp0, w2d, temp0))/sqrt( dg::blas2::dot(vol, w2d, vol));
     std::cout << "Rel Error of volume form is "<<error<<"\n";
 
     std::cout << "TEST VOLUME IS:\n";
     dg::CartesianGrid2d g2dC( gp.R_0 -1.2*gp.a, gp.R_0 + 1.2*gp.a, -2.0*gp.a*gp.elongation, 1.2*gp.a*gp.elongation, 1, 5e3, 1e4, dg::PER, dg::PER);
     gp.psipmax = 0., gp.psipmin = psi_0;
-    Iris<solovev::Psip> iris( psip, gp.psipmin, gp.psipmax);
+    dg::geo::Iris iris( psip.f(), gp.psipmin, gp.psipmax);
     dg::HVec vec  = dg::evaluate( iris, g2dC);
-    dg::DVec cutter = dg::pullback( iris, g2d), vol( cutter);
-    dg::blas1::pointwiseDot(cutter, w2d, vol);
-    double volume = dg::blas1::dot( g2d.vol(), vol);
+    dg::DVec cutter = dg::pullback( iris, g2d), cut_vol( cutter);
+    dg::blas1::pointwiseDot(cutter, w2d, cut_vol);
+    double volume = dg::blas1::dot( vol, cut_vol);
     dg::HVec g2d_weights = dg::create::volume( g2dC);
     double volumeRZP = dg::blas1::dot( vec, g2d_weights);
     std::cout << "volumeXYP is "<< volume<<std::endl;
@@ -220,40 +212,6 @@ int main( int argc, char* argv[])
     std::cout << "relative difference in volume is "<<fabs(volumeRZP - volume)/volume<<std::endl;
     std::cout << "Note that the error might also come from the volume in RZP!\n";
 
-   // ///////////////////////////TEST 3d grid//////////////////////////////////////
-   // std::cout << "Start DS test!"<<std::endl;
-   // const dg::DVec vol3d = dg::create::volume( g3d);
-   // //DFA fieldaligned(OrthogonalXField( gp, g2d, g2d.g()), g3d, gp.rk4eps, dg::NoLimiter(), dg::NEU); 
-   // DFA fieldaligned( CurvilinearField( gp, g2d.x(), g2d.f_x()), g3d, gp.rk4eps, dg::NoLimiter(), dg::NEU); 
-
-   // //dg::DS<DFA, dg::Composite<dg::DMatrix>, dg::DVec> ds( fieldaligned, OrthogonalXField(gp, g2d, g2d.g()), dg::normed, dg::centered, false);
-   // dg::DS<DFA, dg::Composite<dg::DMatrix>, dg::DVec> ds( fieldaligned, CurvilinearField(gp, g2d.x(), g2d.f_x()), dg::normed, dg::centered, false);
-   // dg::DVec B = dg::pullback( solovev::InvB(gp), g3d), divB(B);
-   // dg::DVec lnB = dg::pullback( solovev::LnB(gp), g3d), gradB(B);
-   // const dg::DVec gradLnB = dg::pullback( solovev::GradLnB(gp), g3d);
-   // dg::blas1::pointwiseDivide( ones, B, B);
-
-   // ds.centeredT( B, divB);
-   // std::cout << "Divergence of B is "<<sqrt( dg::blas2::dot( divB, vol3d, divB))<<"\n";
-   // ds.centered( lnB, gradB);
-   // dg::blas1::axpby( 1., gradB, -1., gradLnB, gradB);
-   // //test if topological shift was correct!!
-   // X = gradB;
-   // dg::blas1::pointwiseDot(cutter, gradB, gradB);
-   // double norm = sqrt( dg::blas2::dot( gradLnB, vol3d, gradLnB) );
-   // std::cout << "rel. error of lnB is    "<<sqrt( dg::blas2::dot( gradB, vol3d, gradB))/norm<<" (doesn't fullfill boundary conditions so it was cut at separatrix)\n";
-
-   // const dg::DVec function = dg::pullback(solovev::FuncNeu(gp), g3d);
-   // dg::DVec temp(function);
-   // const dg::DVec derivative = dg::pullback(solovev::DeriNeu(gp), g3d);
-   // ds( function, temp);
-   // dg::blas1::axpby( 1., temp, -1., derivative, temp);
-   // norm = sqrt( dg::blas2::dot( derivative, vol3d, derivative) );
-   // std::cout << "rel. error of DS  is    "<<sqrt( dg::blas2::dot( temp, vol3d, temp))/norm<<"\n";
-   // err = nc_put_var_double( ncid, divBID, periodify(X, g3d_periodic).data());
-   // //err = nc_put_var_double( ncid, divBID, X.data());
     err = nc_close( ncid);
-
-
     return 0;
 }
diff --git a/inc/geometries/ribeiro_mpit.cu b/inc/geometries/ribeiro_mpit.cu
index 758c5833a..46d11204a 100644
--- a/inc/geometries/ribeiro_mpit.cu
+++ b/inc/geometries/ribeiro_mpit.cu
@@ -81,7 +81,7 @@ int main( int argc, char* argv[])
 
     int dims[2], periods[2],  coords[2];
     MPI_Cart_get( g2d.get().communicator(), 2, dims, periods, coords);
-    size_t count[2] = {g2d.get().n()*g2d.get().Ny(), g2d.get().n()*g2d.get().Nx()};
+    size_t count[2] = {g2d.get().local().n()*g2d.get().local().Ny(), g2d.get().local().n()*g2d.get().local().Nx()};
     size_t start[2] = {coords[1]*count[0], coords[0]*count[1]};
     err = nc_var_par_access( ncid, coordsID[0], NC_COLLECTIVE);
     err = nc_var_par_access( ncid, coordsID[1], NC_COLLECTIVE);
@@ -92,8 +92,8 @@ int main( int argc, char* argv[])
     dg::MHVec psi_p = dg::pullback( psip.f(), g2d.get());
     //g.display();
     err = nc_put_vara_double( ncid, onesID, start, count, psi_p.data().data());
-    dg::HVec X( g2d.get().size()), Y(X); //P = dg::pullback( dg::coo3, g);
-    for( unsigned i=0; i<g2d.get().size(); i++)
+    dg::HVec X( g2d.get().local().size()), Y(X); //P = dg::pullback( dg::coo3, g);
+    for( unsigned i=0; i<g2d.get().local().size(); i++)
     {
         X[i] = g2d.get().map()[0].data()[i];
         Y[i] = g2d.get().map()[0].data()[i];
diff --git a/inc/geometries/ribeiro_t.cu b/inc/geometries/ribeiro_t.cu
index 79935d55c..1b659752f 100644
--- a/inc/geometries/ribeiro_t.cu
+++ b/inc/geometries/ribeiro_t.cu
@@ -70,8 +70,8 @@ int main( int argc, char* argv[])
     t.tic();
     dg::geo::Ribeiro ribeiro( psip, psi_0, psi_1, gp.R_0, 0., 1);
     dg::geo::CurvilinearProductGrid3d g3d(ribeiro, n, Nx, Ny,Nz, dg::DIR);
-    dg::geo::CurvilinearGrid2d g2d = g3d.perp_grid();
-    dg::Grid2d g2d_periodic(g2d.x0(), g2d.x1(), g2d.y0(), g2d.y1(), g2d.n(), g2d.Nx(), g2d.Ny()+1); 
+    std::unique_ptr<dg::aGeometry2d> g2d( g3d.perp_grid());
+    dg::Grid2d g2d_periodic(g2d->x0(), g2d->x1(), g2d->y0(), g2d->y1(), g2d->n(), g2d->Nx(), g2d->Ny()+1); 
     t.toc();
     std::cout << "Construction took "<<t.diff()<<"s"<<std::endl;
     int ncid;
@@ -89,30 +89,30 @@ int main( int argc, char* argv[])
     err = nc_def_var( ncid, "volume", NC_DOUBLE, 2, dim3d, &volID);
     err = nc_def_var( ncid, "divB", NC_DOUBLE, 2, dim3d, &divBID);
 
-    thrust::host_vector<double> psi_p = dg::pullback( psip.f(), g2d);
+    thrust::host_vector<double> psi_p = dg::pullback( psip.f(), *g2d);
     //g.display();
     err = nc_put_var_double( ncid, onesID, periodify(psi_p, g2d_periodic).data());
-    dg::HVec X( g2d.size()), Y(X); //P = dg::pullback( dg::coo3, g);
-    for( unsigned i=0; i<g2d.size(); i++)
+    dg::HVec X( g2d->size()), Y(X); //P = dg::pullback( dg::coo3, g);
+    for( unsigned i=0; i<g2d->size(); i++)
     {
-        X[i] = g2d.map()[0][i];
-        Y[i] = g2d.map()[1][i];
+        X[i] = g2d->map()[0][i];
+        Y[i] = g2d->map()[1][i];
     }
 
-    dg::HVec temp0( g2d.size()), temp1(temp0);
-    dg::HVec w2d = dg::create::weights( g2d);
+    dg::HVec temp0( g2d->size()), temp1(temp0);
+    dg::HVec w2d = dg::create::weights( *g2d);
 
     err = nc_put_var_double( ncid, coordsID[0], periodify(X, g2d_periodic).data());
     err = nc_put_var_double( ncid, coordsID[1], periodify(Y, g2d_periodic).data());
 
-    dg::SparseTensor<dg::HVec> metric = g2d.metric();
+    dg::SparseTensor<dg::HVec> metric = g2d->metric();
     dg::HVec g_xx = metric.value(0,0), g_xy = metric.value(0,1), g_yy=metric.value(1,1);
     dg::SparseElement<dg::HVec> vol_ = dg::tensor::volume(metric);
     dg::HVec vol = vol_.value();
     //err = nc_put_var_double( ncid, coordsID[2], g.z().data());
     //compute and write deformation into netcdf
     dg::blas1::pointwiseDivide( g_xy, g_xx, temp0);
-    const dg::HVec ones = dg::evaluate( dg::one, g2d);
+    const dg::HVec ones = dg::evaluate( dg::one, *g2d);
     X=g_yy;
     err = nc_put_var_double( ncid, defID, periodify(X, g2d_periodic).data());
     //compute and write ribeiroratio into netcdf
-- 
GitLab


From 9a1ef4c1b7fc7d5ceef71ee7cf576858626fb846 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Thu, 2 Nov 2017 16:42:32 +0100
Subject: [PATCH 432/453] pay attention to _mpi programs Nx() and Ny() need to
 be local parameters

---
 src/toefl/toefl_mpi.cu | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/toefl/toefl_mpi.cu b/src/toefl/toefl_mpi.cu
index 17ce61da6..ee0ddea43 100644
--- a/src/toefl/toefl_mpi.cu
+++ b/src/toefl/toefl_mpi.cu
@@ -120,7 +120,7 @@ int main( int argc, char* argv[])
       MPI_Cart_get( comm, 2, dims, periods, coords);
       dg::MDVec transfer( dg::evaluate( dg::zero, grid));
     ///////////////////////////////////first output/////////////////////////
-    size_t count[3] = {1, grid_out.n()*grid_out.Ny(), grid_out.n()*grid_out.Nx()};
+    size_t count[3] = {1, grid_out.n()*grid_out.local().Ny(), grid_out.n()*grid_out.local().Nx()};
     size_t start[3] = {0, coords[1]*count[1], coords[0]*count[2]};
     size_t Ecount[] = {1};
     size_t Estart[] = {0};
-- 
GitLab


From 65a24c1cffe253402cd3508ceff6bb38900cc3b6 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Thu, 2 Nov 2017 18:01:57 +0100
Subject: [PATCH 433/453] make diag programs work with the new interface

---
 diag/Makefile              |  6 ++---
 diag/feltorSHdiag.cpp      |  5 ----
 diag/feltorSHdiag.cu       |  8 ++----
 diag/feltorSHvmaxdiag.cu   |  3 +--
 diag/feltordiag.cu         | 51 +++++++++++++++--------------------
 diag/filamentdiag.cu       | 34 ++++++++----------------
 diag/impRdiag.cu           |  8 +-----
 diag/ncdiag.cpp            | 54 ++++++++++++++++----------------------
 diag/toeflEPdiag.cu        |  2 +-
 diag/toeflRdiag.cu         |  2 +-
 inc/dg/backend/average.cuh | 49 +++++++++++-----------------------
 inc/geometries/average.h   |  2 +-
 12 files changed, 80 insertions(+), 144 deletions(-)

diff --git a/diag/Makefile b/diag/Makefile
index b6a36d92b..621ffb012 100644
--- a/diag/Makefile
+++ b/diag/Makefile
@@ -8,7 +8,7 @@ include ../config/devices/devices.mk
 INCLUDE+= -I../inc/# include files libs
 INCLUDE+= -I../src/# include files from source code
 
-TARGETS =  compare histdiag ncdiag filamentdiag fftwdiag crosscoherencdiag feltorSesoldiag feltorShwdiag feltorSHdiag feltordiag vmaxnc feltorSHvmaxdiag toeflRdiag impRdiag
+TARGETS =  compare histdiag ncdiag filamentdiag fftwdiag crosscoherencdiag feltorSesoldiag feltorShwdiag feltorSHdiag feltordiag vmaxnc feltorSHvmaxdiag toeflRdiag impRdiag feltorShwmerger feltorShwradstat feltorShwstat growthrate toeflEPdiag
 
 all: $(TARGETS)
 
@@ -32,9 +32,9 @@ feltorSHvmaxdiag: feltorSHvmaxdiag.cu
 vmaxnc: vmaxnc.cu
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) 
 fftwdiag: fftwdiag.cpp
-	$(CC) $(OPT) $(CFLAGS) -std=c++0x $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -I$(HOME)/include/spectral -lfftw3  -DTL_DEBUG -g
+	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -I$(HOME)/include/spectral -lfftw3  -DTL_DEBUG -g
 growthrate: growthrate.cpp
-	$(CC) $(OPT) $(CFLAGS) -std=c++0x $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -I$(HOME)/include/spectral -lfftw3  -DTL_DEBUG -g
+	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -I$(HOME)/include/spectral -lfftw3  -DTL_DEBUG -g
 histdiag: histdiag.cpp
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) -DDG_DEBUG -g
 compare: compare.cpp
diff --git a/diag/feltorSHdiag.cpp b/diag/feltorSHdiag.cpp
index 8a4980255..f9371a724 100644
--- a/diag/feltorSHdiag.cpp
+++ b/diag/feltorSHdiag.cpp
@@ -6,11 +6,6 @@
 #include <sstream>
 
 #include "dg/algorithm.h"
-#include "dg/poisson.h"
-
-#include "dg/backend/interpolation.cuh"
-#include "dg/backend/xspacelib.cuh"
-#include "dg/functors.h"
 
 #include "file/nc_utilities.h"
 #include "feltorSH/parameters.h"
diff --git a/diag/feltorSHdiag.cu b/diag/feltorSHdiag.cu
index 187edf193..6bfb21e80 100644
--- a/diag/feltorSHdiag.cu
+++ b/diag/feltorSHdiag.cu
@@ -6,11 +6,7 @@
 #include <sstream>
 
 #include "dg/algorithm.h"
-#include "dg/poisson.h"
-//#include "dg/weiss.h" //MW: not commited?
-#include "dg/backend/interpolation.cuh"
-#include "dg/backend/xspacelib.cuh"
-#include "dg/functors.h"
+#include "geometries/geometries.h"
 
 
 #include "file/nc_utilities.h"
@@ -25,7 +21,7 @@ struct Heaviside2d
     Heaviside2d( double sigma):sigma2_(sigma*sigma/4.), x_(0), y_(0){}
 //     Heaviside2d( double sigma):sigma2_(sigma*sigma), x_(0), y_(0){}
     void set_origin( double x0, double y0){ x_=x0, y_=y0;}
-    double operator()(double x, double y)
+    double operator()(double x, double y)const
     {
         double r2 = (x-x_)*(x-x_)+(y-y_)*(y-y_);
         if( r2 >= sigma2_)
diff --git a/diag/feltorSHvmaxdiag.cu b/diag/feltorSHvmaxdiag.cu
index c9d40d89e..2d4347987 100644
--- a/diag/feltorSHvmaxdiag.cu
+++ b/diag/feltorSHvmaxdiag.cu
@@ -7,7 +7,6 @@
 #include <math.h>
 
 #include "dg/algorithm.h"
-#include "dg/backend/xspacelib.cuh"
 #include "feltorSH/parameters.h"
 #include "file/nc_utilities.h"
 
@@ -61,7 +60,7 @@ int main( int argc, char* argv[])
         
         //Find integer time value of t=17 gamma
         double dpe=neamp+teamp+neamp*teamp;
-        double gammath =20./sqrt((2+p.amp)*p.amp*p.mcv/(p.sigma*(1.)));
+        //double gammath =20./sqrt((2+p.amp)*p.amp*p.mcv/(p.sigma*(1.)));
         double gammathreal = sqrt((2+neamp)*neamp*p.mcv/p.sigma);
         double gammathrealdpe = sqrt(dpe*p.mcv/p.sigma);
 //         if (p.sigma==10.)
diff --git a/diag/feltordiag.cu b/diag/feltordiag.cu
index 617fe85df..a967c2d67 100644
--- a/diag/feltordiag.cu
+++ b/diag/feltordiag.cu
@@ -5,15 +5,7 @@
 #include <string>
 
 #include "dg/algorithm.h"
-#include "dg/poisson.h"
-
-#include "dg/backend/interpolation.cuh"
-#include "dg/backend/xspacelib.cuh"
-#include "dg/backend/average.cuh"
-#include "dg/functors.h"
-
 #include "file/nc_utilities.h"
-
 #include "geometries/geometries.h"
 #include "feltor/parameters.h"
 
@@ -53,7 +45,7 @@ int main( int argc, char* argv[])
     reader.parse( input, js, false);
     const eule::Parameters p(js);
     reader.parse( geom, gs, false);
-    const dg::geo::solovev::GeomParameters gp(gs);
+    const dg::geo::solovev::Parameters gp(gs);
     p.display();
     gp.display();
     ///////////////////////////////////////////////////////////////////////////
@@ -64,7 +56,7 @@ int main( int argc, char* argv[])
     double Zmax=p.boxscaleZp*gp.a*gp.elongation;
     //Grids
 
-    dg::CylindricalGrid3d<dg::DVec> g3d_out( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n_out, p.Nx_out, p.Ny_out, p.Nz_out, p.bc, p.bc, dg::PER); 
+    dg::CylindricalGrid3d g3d_out( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n_out, p.Nx_out, p.Ny_out, p.Nz_out, p.bc, p.bc, dg::PER); 
     dg::Grid2d  g2d_out( Rmin,Rmax, Zmin,Zmax, p.n_out, p.Nx_out, p.Ny_out,  p.bc, p.bc); 
     //1d grid
 
@@ -77,7 +69,7 @@ int main( int argc, char* argv[])
     double psipmin = (double)thrust::reduce( psipog2d.begin(), psipog2d.end(), 0.0,thrust::minimum<double>()  );
     double psipmax = (double)thrust::reduce( psipog2d.begin(), psipog2d.end(), psipmin,thrust::maximum<double>()  );
 //     double psipmax = 0.0;
-    dg::geo::PsiPupil<dg::geo::solovev::Psip> psipupil(psip,psipmax);
+    dg::geo::PsiPupil psipupil(psip,psipmax);
     dg::HVec psipupilog2d   = dg::evaluate( psipupil, g2d_out);    
     dg::HVec psipupilog3d   = dg::evaluate( psipupil, g3d_out);    
 
@@ -138,28 +130,28 @@ int main( int argc, char* argv[])
 //     double energy_0 =0.,U_i_0=0.,U_e_0=0.,U_phi_0=0.,U_pare_0=0.,U_pari_0=0.,mass_0=0.;
 //     os << "#Time(1) mass(2) Ue(3) Ui(4) Uphi(5) Upare(6) Upari(7) Utot(8) EDiff(9)\n";
     std::cout << "Compute safety factor   "<< "\n";
-    dg::geo::solovev::MagneticField c(gp);
-    dg::geo::Alpha<dg::geo::solovev::MagneticField> alpha(c); 
+    dg::geo::TokamakMagneticField c = dg::geo::createSolovevField(gp);
+    dg::geo::Alpha alpha(c); 
     dg::DVec alphaog2d   = dg::evaluate( alpha, g2d_out);      
     dg::DVec abs = dg::evaluate( dg::cooX1d, g1d_out);
-    dg::geo::SafetyFactor<dg::geo::solovev::MagneticField, dg::DVec> qprofile(g2d_out, gp, alphaog2d );
+    dg::geo::SafetyFactor<dg::DVec> qprofile(g2d_out, c, alphaog2d );
     dg::DVec sf = dg::evaluate(qprofile, g1d_out);
 
 
     //perp laplacian for computation of vorticity
 
     dg::DVec vor3d    = dg::evaluate( dg::zero, g3d_out);
-    dg::Elliptic<dg::CylindricalGrid3d<dg::DVec>, dg::DMatrix, dg::DVec> laplacian(g3d_out,dg::DIR, dg::DIR, dg::normed, dg::centered); 
+    dg::Elliptic<dg::CylindricalGrid3d, dg::DMatrix, dg::DVec> laplacian(g3d_out,dg::DIR, dg::DIR, dg::normed, dg::centered); 
     dg::IDMatrix fsaonrzmatrix,fsaonrzphimatrix;     
     fsaonrzmatrix    =  dg::create::interpolation(psipupilog2d ,g1d_out);    
     fsaonrzphimatrix =  dg::create::interpolation(psipupilog3d ,g1d_out);    
     
     //Vectors and Matrices for Diffusion coefficient
 
-    const dg::DVec curvR = dg::evaluate( dg::geo::CurvatureNablaBR<dg::geo::solovev::MagneticField>(c, gp.R_0), g3d_out);
-    const dg::DVec curvZ = dg::evaluate( dg::geo::CurvatureNablaBZ<dg::geo::solovev::MagneticField>(c, gp.R_0), g3d_out);
-    dg::Poisson<dg::CylindricalGrid3d<dg::DVec>,dg::DMatrix, dg::DVec> poisson(g3d_out,  dg::DIR, dg::DIR,  g3d_out.bcx(), g3d_out.bcy());
-    const dg::DVec binv = dg::evaluate( dg::geo::Field<dg::geo::solovev::MagneticField>(c, gp.R_0) , g3d_out) ;
+    const dg::DVec curvR = dg::evaluate( dg::geo::CurvatureNablaBR(c), g3d_out);
+    const dg::DVec curvZ = dg::evaluate( dg::geo::CurvatureNablaBZ(c), g3d_out);
+    dg::Poisson<dg::CylindricalGrid3d,dg::DMatrix, dg::DVec> poisson(g3d_out,  dg::DIR, dg::DIR,  g3d_out.bcx(), g3d_out.bcy());
+    const dg::DVec binv = dg::evaluate( dg::geo::InvB(c) , g3d_out) ;
     dg::DVec temp1 = dg::evaluate(dg::zero , g3d_out) ;
     dg::DVec temp2 = dg::evaluate(dg::zero , g3d_out) ;
     dg::DVec temp3 = dg::evaluate(dg::zero , g3d_out) ;
@@ -179,7 +171,6 @@ int main( int argc, char* argv[])
     std::vector<dg::HVec> fields3d_h(5,dg::evaluate(dg::zero,g3d_out));
     std::vector<dg::DVec> fields3d(5,dg::evaluate(dg::zero,g3d_out));
     std::vector<dg::DVec> fields2d(5,dg::evaluate(dg::zero,g3d_out));
-    dg::ToroidalAverage<dg::DVec> toravg(g3d_out);
     unsigned outlim = 0.; int timeID;
     size_t steps;
     err = nc_open( argv[1], NC_NOWRITE, &ncid); //open 3d file
@@ -222,7 +213,7 @@ int main( int argc, char* argv[])
             fields3d[j] = fields3d_h[j];
     
             //get 2d data and sum up for avg
-            toravg(fields3d[j],data2davg);
+            dg::toroidal_average(fields3d[j],data2davg,g3d_out);
 
             //get 2d data of MidPlane
             unsigned kmp = (g3d_out.Nz()/2);
@@ -235,7 +226,7 @@ int main( int argc, char* argv[])
 
 
             //computa fsa of quantities
-            dg::geo::FluxSurfaceAverage<dg::geo::solovev::MagneticField, dg::DVec> fsadata(g2d_out,c, data2davg );
+            dg::geo::FluxSurfaceAverage<dg::DVec> fsadata(g2d_out,c, data2davg );
             dg::DVec data1dfsa = dg::evaluate(fsadata,g1d_out);
             dg::blas1::transfer(data1dfsa,transfer1d);
             err1d = nc_put_vara_double( ncid1d, dataIDs1d[j], start1d, count1d,  transfer1d.data());
@@ -249,11 +240,11 @@ int main( int argc, char* argv[])
         }
         //----------------Start vorticity computation
         dg::blas2::gemv( laplacian,fields3d[4],vor3d);
-        toravg(vor3d,vor2davg);
+        dg::toroidal_average(vor3d,vor2davg,g3d_out);
         dg::blas1::transfer(vor2davg,transfer2d);     
 
         err2d = nc_put_vara_double( ncid2d, dataIDs2d[10],   start2d, count2d, transfer2d.data());
-        dg::geo::FluxSurfaceAverage<dg::geo::solovev::MagneticField, dg::DVec> fsavor(g2d_out,c, vor2davg );
+        dg::geo::FluxSurfaceAverage<dg::DVec> fsavor(g2d_out,c, vor2davg );
         dg::DVec vor1dfsa = dg::evaluate(fsavor,g1d_out);
         dg::blas1::transfer(vor1dfsa,transfer1d);
         err1d = nc_put_vara_double( ncid1d, dataIDs1d[6], start1d, count1d,  transfer1d.data()); 
@@ -286,21 +277,21 @@ int main( int argc, char* argv[])
         dg::blas1::transform(temp1, temp1, dg::SQRT<double>());  // sqrt(psipR^2 +   psipZ^2)
         dg::blas1::pointwiseDivide( Depsip3d, temp1, Depsip3d); //Depsip3d = N_e*(1/B*[phi,psi_p]_RZ - K(psi_p) + 0.5*nu_e*U_e^2*K(psi_p))
       
-        toravg(Depsip3d,Depsip2davg);
+        dg::toroidal_average(Depsip3d,Depsip2davg,g3d_out);
 
-        dg::geo::FluxSurfaceAverage<dg::geo::solovev::MagneticField, dg::DVec> fsaDepsip(g2d_out,c, Depsip2davg );
+        dg::geo::FluxSurfaceAverage<dg::DVec> fsaDepsip(g2d_out,c, Depsip2davg );
         dg::DVec  Depsip1Dfsa = dg::evaluate(fsaDepsip,g1d_out);
         //compute delta f on midplane : d Depsip2d = Depsip - <Depsip>       
         dg::blas2::gemv(fsaonrzphimatrix, Depsip1Dfsa , Depsip3dfluc ); //fsa on RZ grid
         dg::blas1::axpby(1.0,Depsip3d,-1.0, Depsip3dfluc, Depsip3dfluc); 
         //Same procedure for fluc
-        toravg(Depsip3dfluc,Depsip2dflucavg);
+        dg::toroidal_average(Depsip3dfluc,Depsip2dflucavg,g3d_out);
         //fluctuation
 //         transfer2d = Depsip2dflucavg;
         //toroidal avg
         transfer2d = Depsip2davg;
         err2d = nc_put_vara_double( ncid2d, dataIDs2d[11],   start2d, count2d, transfer2d.data());
-        dg::geo::FluxSurfaceAverage<dg::geo::solovev::MagneticField, dg::DVec> fsaDepsipfluc(g2d_out,c, Depsip2dflucavg );
+        dg::geo::FluxSurfaceAverage< dg::DVec> fsaDepsipfluc(g2d_out,c, Depsip2dflucavg );
         dg::DVec  Depsip1Dflucfsa = dg::evaluate(fsaDepsipfluc,g1d_out);
         transfer1d =Depsip1Dflucfsa;
         err1d = nc_put_vara_double( ncid1d, dataIDs1d[7], start1d, count1d,   transfer1d.data()); 
@@ -311,10 +302,10 @@ int main( int argc, char* argv[])
         dg::blas1::transform(temp3, temp1, dg::LN<double>()); // lnN
         poisson.variationRHS(temp1,temp2); // (nabla_perp N)^2
         dg::blas1::transform(temp2, Lperpinv3d, dg::SQRT<double>()); // |(nabla_perp N)|
-        toravg(Lperpinv3d,Lperpinv2davg);
+        dg::toroidal_average(Lperpinv3d,Lperpinv2davg, g3d_out);
         transfer2d = Lperpinv2davg;
         err2d = nc_put_vara_double( ncid2d, dataIDs2d[12],   start2d, count2d, transfer2d.data());
-        dg::geo::FluxSurfaceAverage<dg::geo::solovev::MagneticField, dg::DVec> fsaLperpinv(g2d_out,c, Lperpinv2davg );
+        dg::geo::FluxSurfaceAverage< dg::DVec> fsaLperpinv(g2d_out,c, Lperpinv2davg );
         dg::DVec  Lperpinv1Dfsa = dg::evaluate(fsaLperpinv,g1d_out);
         transfer1d =Lperpinv1Dfsa;
         err1d = nc_put_vara_double( ncid1d, dataIDs1d[8], start1d, count1d,   transfer1d.data()); 
diff --git a/diag/filamentdiag.cu b/diag/filamentdiag.cu
index 759710546..93ddfacb8 100644
--- a/diag/filamentdiag.cu
+++ b/diag/filamentdiag.cu
@@ -5,18 +5,10 @@
 #include <string>
 
 #include "dg/algorithm.h"
-#include "dg/poisson.h"
-
-#include "dg/backend/interpolation.cuh"
-#include "dg/backend/xspacelib.cuh"
-#include "dg/backend/average.cuh"
-#include "dg/functors.h"
-
+#include "geometries/geometries.h"
 #include "file/nc_utilities.h"
 
-#include "dg/geometry.h"
 #include "feltor/parameters.h"
-#include "geometries/geometries.h"
 
 int main( int argc, char* argv[])
 {
@@ -48,7 +40,7 @@ int main( int argc, char* argv[])
     reader.parse( input, js, false);
     const eule::Parameters p(js);
     reader.parse( geom, gs, false);
-    const dg::geo::solovev::GeomParameters gp(gs);
+    const dg::geo::solovev::Parameters gp(gs);
     p.display();
     gp.display();
     /////////////////////////////create Grids and weights//////////////////////////////////
@@ -58,7 +50,7 @@ int main( int argc, char* argv[])
     double Rmax=gp.R_0+p.boxscaleRp*gp.a; 
     double Zmax=p.boxscaleZp*gp.a*gp.elongation;
     //Grids
-    dg::CylindricalGrid3d<dg::DVec> g3d_out( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n_out, p.Nx_out, p.Ny_out, p.Nz_out, dg::NEU, dg::NEU, dg::PER);  
+    dg::CylindricalGrid3d g3d_out( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n_out, p.Nx_out, p.Ny_out, p.Nz_out, dg::NEU, dg::NEU, dg::PER);  
     dg::CartesianGrid2d  g2d_out( Rmin,Rmax, Zmin,Zmax, p.n_out, p.Nx_out, p.Ny_out, dg::NEU, dg::NEU);
     //--------generate nc file and define dimensions----------------
     int ncidout; 
@@ -93,22 +85,18 @@ int main( int argc, char* argv[])
     double time=0.;
 
     ///////////////////////Define helper variables for computations////////
-    dg::geo::solovev::MagneticField c(gp);
-    dg::DDS dsN( typename dg::DDS::FieldAligned(
-                dg::geo::Field<dg::geo::solovev::MagneticField>(c, gp.R_0), 
-                g3d_out, gp.rk4eps, 
-                dg::geo::PsiLimiter<dg::geo::solovev::Psip>(c.psip, gp.psipmaxlim), 
-                dg::NEU,(2*M_PI)/((double)p.Nz)), 
-            dg::geo::Field<dg::geo::solovev::MagneticField>(c, gp.R_0), dg::normed, dg::centered );
+    dg::geo::TokamakMagneticField c= dg::geo::createSolovevField(gp);
+    dg::geo::DS<dg::aProductGeometry3d,dg::IDMatrix, dg::DMatrix, dg::DVec> dsN( 
+            c, g3d_out, dg::NEU, dg::NEU, dg::geo::FullLimiter(), dg::normed, dg::centered, gp.rk4eps);
     dg::DVec vor3d    = dg::evaluate( dg::zero, g3d_out);
     dg::HVec transfer = dg::evaluate( dg::zero, g3d_out);
-    dg::Elliptic<dg::CylindricalGrid3d<dg::DVec>, dg::DMatrix, dg::DVec> laplacian(g3d_out,dg::DIR, dg::DIR, dg::normed, dg::centered); 
+    dg::Elliptic<dg::CylindricalGrid3d, dg::DMatrix, dg::DVec> laplacian(g3d_out,dg::DIR, dg::DIR, dg::normed, dg::centered); 
     const dg::DVec w3d = dg::create::weights( g3d_out);   
     const dg::DVec w2d = dg::create::weights( g2d_out);   
-    const dg::DVec curvR = dg::evaluate( dg::geo::CurvatureNablaBR<dg::geo::solovev::MagneticField>(c, gp.R_0), g3d_out);
-    const dg::DVec curvZ = dg::evaluate( dg::geo::CurvatureNablaBZ<dg::geo::solovev::MagneticField>(c, gp.R_0), g3d_out);
-    dg::Poisson<dg::CylindricalGrid3d<dg::DVec>, dg::DMatrix, dg::DVec> poisson(g3d_out,  dg::DIR, dg::DIR,  g3d_out.bcx(), g3d_out.bcy());
-    const dg::DVec binv = dg::evaluate( dg::geo::Field<dg::geo::solovev::MagneticField>(c, gp.R_0) , g3d_out) ;
+    const dg::DVec curvR = dg::evaluate( dg::geo::CurvatureNablaBR(c), g3d_out);
+    const dg::DVec curvZ = dg::evaluate( dg::geo::CurvatureNablaBZ(c), g3d_out);
+    dg::Poisson<dg::CylindricalGrid3d, dg::DMatrix, dg::DVec> poisson(g3d_out,  dg::DIR, dg::DIR,  g3d_out.bcx(), g3d_out.bcy());
+    const dg::DVec binv = dg::evaluate( dg::geo::InvB(c) , g3d_out) ;
     const dg::DVec one3d    =  dg::evaluate(dg::one,g3d_out);
     const dg::DVec one2d    =  dg::evaluate(dg::one,g2d_out);
     dg::DVec temp1 = dg::evaluate(dg::zero , g3d_out) ;
diff --git a/diag/impRdiag.cu b/diag/impRdiag.cu
index 78fe8136a..cef4d5234 100644
--- a/diag/impRdiag.cu
+++ b/diag/impRdiag.cu
@@ -6,21 +6,15 @@
 #include <sstream>
 
 #include "dg/algorithm.h"
-#include "dg/backend/xspacelib.cuh"
 #include "file/nc_utilities.h"
 #include "impurities/parameters.h"
 
-#include "dg/backend/timer.cuh"
-//dg::Timer t;
-//t.tic();
-//t.toc();
-
 struct Heaviside2d
 { Heaviside2d( double sigma):sigma2_(sigma*sigma), x_(0), y_(0) {}
   void set_origin( double x0, double y0)
   { x_ = x0, y_ = y0;
   }
-  double operator()(double x, double y)
+  double operator()(double x, double y)const
   { double r2 = (x-x_)*(x-x_)+(y-y_)*(y-y_);
     if( r2 >= sigma2_)
       return 0.;
diff --git a/diag/ncdiag.cpp b/diag/ncdiag.cpp
index 8432bd741..928f99cb0 100644
--- a/diag/ncdiag.cpp
+++ b/diag/ncdiag.cpp
@@ -5,17 +5,8 @@
 #include <string>
 
 #include "dg/algorithm.h"
-#include "dg/poisson.h"
-
-#include "dg/backend/interpolation.cuh"
-#include "dg/backend/xspacelib.cuh"
-#include "dg/backend/average.cuh"
-#include "dg/functors.h"
-#include "dg/geometry.h"
-
-#include "file/nc_utilities.h"
-
 #include "geometries/geometries.h"
+#include "file/nc_utilities.h"
 
 #include "feltor/parameters.h"
 
@@ -55,7 +46,7 @@ int main( int argc, char* argv[])
     reader.parse( input, js, false);
     const eule::Parameters p(js);
     reader.parse( geom, gs, false);
-    const dg::geo::solovev::GeomParameters gp(gs);
+    const dg::geo::solovev::Parameters gp(gs);
     p.display();
     gp.display();
     ///////////////////////////////////////////////////////////////////////////
@@ -71,7 +62,7 @@ int main( int argc, char* argv[])
 //     double Zmax=p.boxscaleRp*gp.a*gp.elongation;
 
     //Grids
-    dg::CylindricalGrid3d<dg::HVec> g3d_out( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n_out, p.Nx_out, p.Ny_out, p.Nz_out, dg::NEU, dg::NEU, dg::PER);  
+    dg::CylindricalGrid3d g3d_out( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n_out, p.Nx_out, p.Ny_out, p.Nz_out, dg::NEU, dg::NEU, dg::PER);  
     dg::Grid2d  g2d_out( Rmin,Rmax, Zmin,Zmax,p.n_out, p.Nx_out, p.Ny_out, dg::NEU, dg::NEU);
     //1d grid
     dg::geo::solovev::Psip psip(gp);
@@ -81,7 +72,7 @@ int main( int argc, char* argv[])
     double psipmin = (double)thrust::reduce( psipog2d.begin(), psipog2d.end(), 0.0,thrust::minimum<double>()  );
     double psipmax = (double)thrust::reduce( psipog2d.begin(), psipog2d.end(),psipmin,thrust::maximum<double>()  );
 //     double psipmax = 0.0;
-    dg::geo::PsiPupil<dg::geo::solovev::Psip> psipupil(psip,psipmax);
+    dg::geo::PsiPupil psipupil(psip,psipmax);
     dg::HVec psipupilog2d   = dg::evaluate( psipupil, g2d_out);    
     dg::HVec psipupilog3d   = dg::evaluate( psipupil, g3d_out);    
 
@@ -153,17 +144,17 @@ int main( int argc, char* argv[])
     //perp laplacian for computation of vorticity
 
     dg::HVec vor3d    = dg::evaluate( dg::zero, g3d_out);
-    dg::Elliptic<dg::CylindricalGrid3d<dg::HVec>, dg::HMatrix, dg::HVec> laplacian(g3d_out,dg::DIR, dg::DIR, dg::normed, dg::centered); 
+    dg::Elliptic<dg::CylindricalGrid3d, dg::HMatrix, dg::HVec> laplacian(g3d_out,dg::DIR, dg::DIR, dg::normed, dg::centered); 
     dg::IHMatrix fsaonrzmatrix,fsaonrzphimatrix;     
     fsaonrzmatrix    =  dg::create::interpolation(psipupilog2d ,g1d_out);    
     fsaonrzphimatrix =  dg::create::interpolation(psipupilog3d ,g1d_out);    
     
     //Vectors and Matrices for Diffusion coefficient
-    dg::geo::solovev::MagneticField c(gp);
-    const dg::HVec curvR = dg::evaluate( dg::geo::CurvatureNablaBR<dg::geo::solovev::MagneticField>(c, gp.R_0), g3d_out);
-    const dg::HVec curvZ = dg::evaluate( dg::geo::CurvatureNablaBZ<dg::geo::solovev::MagneticField>(c, gp.R_0), g3d_out);
-    dg::Poisson<dg::CylindricalGrid3d<dg::HVec>, dg::HMatrix, dg::HVec> poisson(g3d_out,  dg::DIR, dg::DIR,  g3d_out.bcx(), g3d_out.bcy());
-    const dg::HVec binv = dg::evaluate(dg::geo::Field<dg::geo::solovev::MagneticField>(c, gp.R_0) , g3d_out) ;
+    dg::geo::TokamakMagneticField c = dg::geo::createSolovevField(gp);
+    const dg::HVec curvR = dg::evaluate( dg::geo::CurvatureNablaBR(c), g3d_out);
+    const dg::HVec curvZ = dg::evaluate( dg::geo::CurvatureNablaBZ(c), g3d_out);
+    dg::Poisson<dg::CylindricalGrid3d, dg::HMatrix, dg::HVec> poisson(g3d_out,  dg::DIR, dg::DIR,  g3d_out.bcx(), g3d_out.bcy());
+    const dg::HVec binv = dg::evaluate(dg::geo::InvB(c) , g3d_out) ;
     dg::HVec Deperp3d =  dg::evaluate(dg::zero , g3d_out) ; 
     dg::HVec temp1 = dg::evaluate(dg::zero , g3d_out) ;
     dg::HVec temp2 = dg::evaluate(dg::zero , g3d_out) ;
@@ -183,7 +174,6 @@ int main( int argc, char* argv[])
 #endif
     std::vector<dg::HVec> fields3d(5,dg::evaluate(dg::zero,g3d_out));
     std::vector<dg::HVec> fields2d(5,dg::evaluate(dg::zero,g3d_out));
-    dg::ToroidalAverage<dg::HVec> toravg(g3d_out);
     unsigned outlim = p.maxout;
     std::cout << "number of outputs ? : " << std::endl;
     std::cin>>outlim;
@@ -224,7 +214,7 @@ int main( int argc, char* argv[])
             err = nc_close(ncid);  //close 3d file
     
             //get 2d data and sum up for avg
-            toravg(fields3d[j],data2davg);
+            dg::toroidal_average(fields3d[j],data2davg, g3d_out);
 
             //get 2d data of MidPlane
             unsigned kmp = (g3d_out.Nz()/2);
@@ -238,7 +228,7 @@ int main( int argc, char* argv[])
 
 
             //computa fsa of quantities
-            dg::geo::FluxSurfaceAverage<dg::geo::solovev::MagneticField, dg::HVec> fsadata(g2d_out,c, data2davg );
+            dg::geo::FluxSurfaceAverage<dg::HVec> fsadata(g2d_out,c, data2davg );
             dg::HVec data1dfsa = dg::evaluate(fsadata,g1d_out);
             err1d = nc_put_vara_double( ncid1d, dataIDs1d[j], start1d, count1d,  data1dfsa.data());
             
@@ -251,9 +241,9 @@ int main( int argc, char* argv[])
         }
         //----------------Start vorticity computation
         dg::blas2::gemv( laplacian,fields3d[4],vor3d);
-        toravg(vor3d,vor2davg);
+        dg::toroidal_average(vor3d,vor2davg, g3d_out);
         err2d = nc_put_vara_double( ncid2d, dataIDs2d[10],   start2d, count2d, vor2davg.data());
-        dg::geo::FluxSurfaceAverage<dg::geo::solovev::MagneticField, dg::HVec> fsavor(g2d_out,c, vor2davg );
+        dg::geo::FluxSurfaceAverage<dg::HVec> fsavor(g2d_out,c, vor2davg );
         dg::HVec vor1dfsa = dg::evaluate(fsavor,g1d_out);
         err1d = nc_put_vara_double( ncid1d, dataIDs1d[6], start1d, count1d,  vor1dfsa.data()); 
         //----------------Stop vorticity computation
@@ -274,10 +264,10 @@ int main( int argc, char* argv[])
         dg::blas1::axpby(  0.5*p.mu[0], temp1,1.0,  Deperp3d);  //D_perp,e = 1/B*[phi,N_e]_RZ - 1*K(N_e) + 0.5*nu_e*U_e^2*K(N_e)
         dg::blas1::pointwiseDot( Deperp3d, temp3, Deperp3d); //D_perp,e = N_e*(1/B*[phi,N_e]_RZ - 0.5*K(N_e))
 
-        toravg(Deperp3d,Deperp2davg);
+        dg::toroidal_average(Deperp3d,Deperp2davg, g3d_out);
 
         err2d = nc_put_vara_double( ncid2d, dataIDs2d[11],   start2d, count2d, Deperp2davg.data());
-        dg::geo::FluxSurfaceAverage<dg::geo::solovev::MagneticField, dg::HVec> fsaDeperp(g2d_out,c, Deperp2davg );
+        dg::geo::FluxSurfaceAverage<dg::HVec> fsaDeperp(g2d_out,c, Deperp2davg );
         dg::HVec  Deperp1Dfsa = dg::evaluate(fsaDeperp,g1d_out);
         err1d = nc_put_vara_double( ncid1d, dataIDs1d[7], start1d, count1d,   Deperp1Dfsa.data()); 
         //--------------- Stop Perpendicular ELECTRONDENSITYFLUX computation
@@ -306,20 +296,20 @@ int main( int argc, char* argv[])
         dg::blas1::transform(temp1, temp1, dg::SQRT<double>());  // sqrt(psipR^2 +   psipZ^2)
         dg::blas1::pointwiseDivide( Depsip3d, temp1, Depsip3d); //Depsip3d = N_e*(1/B*[phi,psi_p]_RZ - K(psi_p) + 0.5*nu_e*U_e^2*K(psi_p))
       
-        toravg(Depsip3d,Depsip2davg);
+        dg::toroidal_average(Depsip3d,Depsip2davg,g3d_out);
 
-        dg::geo::FluxSurfaceAverage<dg::geo::solovev::MagneticField, dg::HVec> fsaDepsip(g2d_out,c, Depsip2davg );
+        dg::geo::FluxSurfaceAverage<dg::HVec> fsaDepsip(g2d_out,c, Depsip2davg );
         dg::HVec  Depsip1Dfsa = dg::evaluate(fsaDepsip,g1d_out);
         //compute delta f on midplane : d Depsip2d = Depsip - <Depsip>       
         dg::blas2::gemv(fsaonrzphimatrix, Depsip1Dfsa , Depsip3dfluc ); //fsa on RZ grid
         dg::blas1::axpby(1.0,Depsip3d,-1.0, Depsip3dfluc, Depsip3dfluc); 
         //Same procedure for fluc
-        toravg(Depsip3dfluc,Depsip2dflucavg);
+        dg::toroidal_average(Depsip3dfluc,Depsip2dflucavg, g3d_out);
         //fluctuation
 //         err2d = nc_put_vara_double( ncid2d, dataIDs2d[12],   start2d, count2d, Depsip2dflucavg.data());
         //toroidal avg
         err2d = nc_put_vara_double( ncid2d, dataIDs2d[12],   start2d, count2d, Depsip2davg.data());
-        dg::geo::FluxSurfaceAverage<dg::geo::solovev::MagneticField, dg::HVec> fsaDepsipfluc(g2d_out,c, Depsip2dflucavg );
+        dg::geo::FluxSurfaceAverage<dg::HVec> fsaDepsipfluc(g2d_out,c, Depsip2dflucavg );
         dg::HVec  Depsip1Dflucfsa = dg::evaluate(fsaDepsipfluc,g1d_out);
         err1d = nc_put_vara_double( ncid1d, dataIDs1d[8], start1d, count1d,   Depsip1Dflucfsa.data()); 
         std::cout << "Depsip =" << dg::blas2::dot(psipupilog3d,w3d, Depsip3dfluc) << std::endl;
@@ -329,9 +319,9 @@ int main( int argc, char* argv[])
         dg::blas1::transform(temp3, temp1, dg::LN<double>());
         poisson.variationRHS(temp1,temp2);
         dg::blas1::transform(temp2, Lperpinv3d, dg::SQRT<double>());
-        toravg(Lperpinv3d,Lperpinv2davg);
+        dg::toroidal_average(Lperpinv3d,Lperpinv2davg,g3d_out);
         err2d = nc_put_vara_double( ncid2d, dataIDs2d[13],   start2d, count2d, Lperpinv2davg.data());
-        dg::geo::FluxSurfaceAverage<dg::geo::solovev::MagneticField, dg::HVec> fsaLperpinv(g2d_out,c, Lperpinv2davg );
+        dg::geo::FluxSurfaceAverage<dg::HVec> fsaLperpinv(g2d_out,c, Lperpinv2davg );
         dg::HVec  Lperpinv1Dfsa = dg::evaluate(fsaLperpinv,g1d_out);
         err1d = nc_put_vara_double( ncid1d, dataIDs1d[9], start1d, count1d,   Lperpinv1Dfsa .data()); 
         std::cout << "Lperpinv=" <<dg::blas2::dot(psipupilog3d,w3d, Lperpinv3d) << std::endl;
diff --git a/diag/toeflEPdiag.cu b/diag/toeflEPdiag.cu
index 516ab3c6a..41cc136b5 100644
--- a/diag/toeflEPdiag.cu
+++ b/diag/toeflEPdiag.cu
@@ -24,7 +24,7 @@ struct Heaviside2d
     Heaviside2d( double sigma):sigma2_(sigma*sigma), x_(0), y_(0){}
 //     Heaviside2d( double sigma):sigma2_(sigma*sigma), x_(0), y_(0){}
     void set_origin( double x0, double y0){ x_=x0, y_=y0;}
-    double operator()(double x, double y)
+    double operator()(double x, double y) const
     {
         double r2 = (x-x_)*(x-x_)+(y-y_)*(y-y_);
         if( r2 >= sigma2_)
diff --git a/diag/toeflRdiag.cu b/diag/toeflRdiag.cu
index ef06c1035..7aa1c0643 100644
--- a/diag/toeflRdiag.cu
+++ b/diag/toeflRdiag.cu
@@ -24,7 +24,7 @@ struct Heaviside2d
     Heaviside2d( double sigma):sigma2_(sigma*sigma), x_(0), y_(0){}
 //     Heaviside2d( double sigma):sigma2_(sigma*sigma), x_(0), y_(0){}
     void set_origin( double x0, double y0){ x_=x0, y_=y0;}
-    double operator()(double x, double y)
+    double operator()(double x, double y)const
     {
         double r2 = (x-x_)*(x-x_)+(y-y_)*(y-y_);
         if( r2 >= sigma2_)
diff --git a/inc/dg/backend/average.cuh b/inc/dg/backend/average.cuh
index 59f2d68d7..137d15942 100644
--- a/inc/dg/backend/average.cuh
+++ b/inc/dg/backend/average.cuh
@@ -4,6 +4,7 @@
 #include "xspacelib.cuh"
 #include "memory.h"
 #include "../blas1.h"
+#include "split_and_join.h"
 
 /*! @file 
   @brief contains classes for poloidal and toroidal average computations.
@@ -76,41 +77,23 @@ struct PoloidalAverage
 };
 
 /**
- * @brief Class for phi average computations
+ * @brief Compute the average in the 3rd direction
  *
+ * @tparam container must be either \c dg::HVec or \c dg::DVec
+ * @param src 3d Source vector 
+ * @param res contains the 2d result on output (may not equal src, gets resized properly)
+ * @param t Contains the grid sizes 
  * @ingroup utilities
- * @tparam container Vector class to be used
  */
-template< class container = thrust::host_vector<double> >
-struct ToroidalAverage
+template<class container>
+void toroidal_average( const container& src, container& res, const aTopology3d& t)
 {
-    /**
-     * @brief Construct from grid object
-     *
-     * @param g3d 3d Grid
-     */
-    ToroidalAverage(const dg::aTopology3d& g3d):
-        g3d_(g3d),
-        sizeg2d_(g3d_.get().size()/g3d_.get().Nz())
-    {        
-    }
-    /**
-     * @brief Compute the average in phi-direction
-     *
-     * @param src 3d Source vector 
-     * @param res contains the 2d result on output (may not equal src)
-     */
-    void operator()(const container& src, container& res)
-    {
-        for( unsigned k=0; k<g3d_.get().Nz(); k++)
-        {
-            container data2d(src.begin() + k*sizeg2d_,src.begin() + (k+1)*sizeg2d_);
-            dg::blas1::axpby(1.0,data2d,1.0,res); 
-        }
-        dg::blas1::scal(res,1./g3d_.get().Nz()); //scale avg
-    }
-    private:
-    Handle<dg::aTopology3d> g3d_;
-    unsigned sizeg2d_;
-};
+    std::vector<container> split_src;
+    dg::split( src, split_src, t);
+    res = split_src[0];
+    for( unsigned k=1; k<t.Nz(); k++)
+        dg::blas1::axpby(1.0,split_src[k],1.0,res); 
+    dg::blas1::scal(res,1./(double)t.Nz()); //scale avg
+}
+
 }//namespace dg
diff --git a/inc/geometries/average.h b/inc/geometries/average.h
index 214cbd08e..15838cde6 100644
--- a/inc/geometries/average.h
+++ b/inc/geometries/average.h
@@ -147,7 +147,7 @@ struct FluxSurfaceAverage
 
 where \f$ \alpha\f$ is the dg::geo::Alpha functor.
  * @copydoc hide_container
- * @ingroup misc
+ * @ingroup misc_geo
  *
  */
 template <class container = thrust::host_vector<double> >
-- 
GitLab


From d2e90b3b06e461bc289ead03e12dc7bb5653246e Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 2 Nov 2017 23:24:46 +0100
Subject: [PATCH 434/453] adapted reco2d, lamb_dipole and impurities to new
 interface

---
 inc/dg/backend/mpi_collective.h            | 15 ++++----
 inc/geometries/ds.h                        |  6 ++--
 inc/geometries/fieldaligned.h              |  6 ++--
 inc/geometries/mpi_fieldaligned.h          |  6 ++--
 src/README                                 |  7 ----
 src/README.adoc                            | 10 ++++++
 src/impurities/toeflI.cuh                  | 11 +++---
 src/impurities/toefl_hpc.cu                |  8 ++---
 src/impurities/toefl_mpi.cu                | 22 ++++++------
 src/lamb_dipole/shu.cuh                    | 12 +++----
 src/lamb_dipole/shu_hpc.cu                 | 11 ------
 src/lamb_dipole/shu_t.cu                   |  9 ++---
 src/reco2D/Makefile                        |  6 ++--
 src/reco2D/README                          |  4 ---
 src/reco2D/parameters.h                    |  4 +--
 src/reco2D/{asela.cu => reconnection.cu}   | 28 +++++++--------
 src/reco2D/{asela.cuh => reconnection.cuh} | 40 +++++++---------------
 17 files changed, 83 insertions(+), 122 deletions(-)
 delete mode 100644 src/README
 create mode 100644 src/README.adoc
 delete mode 100644 src/reco2D/README
 rename src/reco2D/{asela.cu => reconnection.cu} (91%)
 rename src/reco2D/{asela.cuh => reconnection.cuh} (87%)

diff --git a/inc/dg/backend/mpi_collective.h b/inc/dg/backend/mpi_collective.h
index 4547d0df8..3853ef923 100644
--- a/inc/dg/backend/mpi_collective.h
+++ b/inc/dg/backend/mpi_collective.h
@@ -332,18 +332,19 @@ struct SurjectiveComm : public aCommunicator<Vector>
         //now gather the localGatherMap from the buffer to the store to get the final gather map 
         Vector localGatherMap_d;
         dg::blas1::transfer( localGatherMap, localGatherMap_d);
-        Index gatherMap = bijectiveComm_.global_gather( localGatherMap_d);
-        dg::blas1::transfer(gatherMap, gatherMap_);
+        Vector gatherMap_V = bijectiveComm_.global_gather( localGatherMap_d);
+        Index gatherMap_I;
+        dg::blas1::transfer(gatherMap_V, gatherMap_I);
+        gatherMap_ = gatherMap_I;
         store_size_ = gatherMap_.size();
         store_.data().resize( store_size_);
         keys_.data().resize( store_size_);
 
         //now prepare a reduction map and a scatter map
-        thrust::host_vector<int> sortMap(gatherMap);
-        thrust::sequence( sortMap.begin(), sortMap.end());
-        thrust::stable_sort_by_key( gatherMap.begin(), gatherMap.end(), sortMap.begin());//note: this also sorts the gatherMap
-        dg::blas1::transfer( sortMap, sortMap_);
-        dg::blas1::transfer( gatherMap, sortedGatherMap_);
+        sortMap_ = gatherMap_I;
+        thrust::sequence( sortMap_.begin(), sortMap_.end());
+        thrust::stable_sort_by_key( gatherMap_I.begin(), gatherMap_I.end(), sortMap_.begin());//note: this also sorts the gatherMap
+        dg::blas1::transfer( gatherMap_I, sortedGatherMap_);
         //now we can repeat/invert the sort by a gather/scatter operation with sortMap as map 
     }
     unsigned buffer_size_, store_size_;
diff --git a/inc/geometries/ds.h b/inc/geometries/ds.h
index 28cadea84..65f2cd5c9 100644
--- a/inc/geometries/ds.h
+++ b/inc/geometries/ds.h
@@ -46,7 +46,7 @@ struct DS
     
     /**
      * @brief Create the magnetic unit vector field and construct
-     * @copydetails DS(const dg::geo::BinaryVectorLvl0&,const ProductGeometry&,unsigned,unsigned,bool,bool,double,dg::norm,dg::direction)
+     * @copydetails DS(const dg::geo::BinaryVectorLvl0&,const ProductGeometry&,dg::bc,dg::bc,Limiter,dg::norm,dg::direction,double,unsigned,unsigned,bool,bool)
      */
     template <class Limiter>
     DS(const dg::geo::TokamakMagneticField& vec, const ProductGeometry& grid, 
@@ -54,7 +54,7 @@ struct DS
         dg::bc bcy = dg::NEU, 
         Limiter limit = FullLimiter(), 
         dg::norm no=dg::normed, dg::direction dir = dg::centered, 
-        double eps = 1e-5, unsigned multiplyX=20, unsigned multiplyY=20, bool dependsOnX = true, bool dependsOnY=true)
+        double eps = 1e-5, unsigned multiplyX=10, unsigned multiplyY=10, bool dependsOnX = true, bool dependsOnY=true)
     {
         dg::geo::BinaryVectorLvl0 bhat( (dg::geo::BHatR)(vec), (dg::geo::BHatZ)(vec), (dg::geo::BHatP)(vec));
         m_fa.construct( bhat, grid, bcx, bcy, limit, eps, multiplyX, multiplyY, dependsOnX, dependsOnY);
@@ -87,7 +87,7 @@ struct DS
         dg::bc bcy = dg::NEU, 
         Limiter limit = FullLimiter(), 
         dg::norm no=dg::normed, dg::direction dir = dg::centered, 
-        double eps = 1e-5, unsigned multiplyX=20, unsigned multiplyY=20, bool dependsOnX = true, bool dependsOnY=true)
+        double eps = 1e-5, unsigned multiplyX=10, unsigned multiplyY=10, bool dependsOnX = true, bool dependsOnY=true)
     {
         m_fa.construct( vec, grid, bcx, bcy, limit, eps, multiplyX, multiplyY, dependsOnX, dependsOnY);
         construct( m_fa, no, dir);
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index c358e6bcc..36c751ab8 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -328,7 +328,7 @@ struct Fieldaligned
         dg::bc globalbcy = dg::NEU, 
         Limiter limit = FullLimiter(), 
         double eps = 1e-5,
-        unsigned multiplyX=20, unsigned multiplyY=20, 
+        unsigned multiplyX=10, unsigned multiplyY=10, 
         bool dependsOnX=true, bool dependsOnY=true, 
         double deltaPhi = -1)
     {
@@ -344,7 +344,7 @@ struct Fieldaligned
         dg::bc globalbcy = dg::NEU, 
         Limiter limit = FullLimiter(), 
         double eps = 1e-5,
-        unsigned multiplyX=20, unsigned multiplyY=20, 
+        unsigned multiplyX=10, unsigned multiplyY=10, 
         bool dependsOnX=true, bool dependsOnY=true, 
         double deltaPhi = -1)
     {
@@ -379,7 +379,7 @@ struct Fieldaligned
         dg::bc globalbcy = dg::NEU, 
         Limiter limit = FullLimiter(), 
         double eps = 1e-5, 
-        unsigned multiplyX=20, unsigned multiplyY=20, 
+        unsigned multiplyX=10, unsigned multiplyY=10, 
         bool dependsOnX=true, bool dependsOnY=true, 
         double deltaPhi = -1);
 
diff --git a/inc/geometries/mpi_fieldaligned.h b/inc/geometries/mpi_fieldaligned.h
index f1fa42098..293906109 100644
--- a/inc/geometries/mpi_fieldaligned.h
+++ b/inc/geometries/mpi_fieldaligned.h
@@ -68,7 +68,7 @@ struct Fieldaligned< ProductMPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY
         dg::bc globalbcy = dg::NEU, 
         Limiter limit = FullLimiter(), 
         double eps = 1e-5,
-        unsigned multiplyX=20, unsigned multiplyY=20, 
+        unsigned multiplyX=10, unsigned multiplyY=10, 
         bool dependsOnX=true, bool dependsOnY=true, 
         double deltaPhi = -1)
     {
@@ -82,7 +82,7 @@ struct Fieldaligned< ProductMPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY
         dg::bc globalbcy = dg::NEU, 
         Limiter limit = FullLimiter(), 
         double eps = 1e-5,
-        unsigned multiplyX=20, unsigned multiplyY=20, 
+        unsigned multiplyX=10, unsigned multiplyY=10, 
         bool dependsOnX=true, bool dependsOnY=true, 
         double deltaPhi = -1)
     {
@@ -95,7 +95,7 @@ struct Fieldaligned< ProductMPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY
         dg::bc globalbcy = dg::NEU, 
         Limiter limit = FullLimiter(), 
         double eps = 1e-5, 
-        unsigned multiplyX=20, unsigned multiplyY=20, 
+        unsigned multiplyX=10, unsigned multiplyY=10, 
         bool dependsOnX=true, bool dependsOnY=true, 
         double deltaPhi = -1);
 
diff --git a/src/README b/src/README
deleted file mode 100644
index 9e589c501..000000000
--- a/src/README
+++ /dev/null
@@ -1,7 +0,0 @@
-The src folder contains specific projects that are based on the numerical algorithms
-in the inc folder
-
-In order to check if an update introduces an unwanted bug, do the following steps:
-- execute the project with default/verification input for the official branch
-- execute the same project with same input on the new branch
-- use the feltor/diag/compare program to compare the outputs
diff --git a/src/README.adoc b/src/README.adoc
new file mode 100644
index 000000000..14a39362d
--- /dev/null
+++ b/src/README.adoc
@@ -0,0 +1,10 @@
+== The source folder
+
+The src folder contains specific projects that are based on the numerical algorithms
+in the inc folder
+
+In order to check if an update introduces an unwanted bug, do the following steps:
+
+* execute the project with default/verification input for the official branch
+* execute the same project with same input on the new branch
+* use the feltor/diag/compare program to compare the outputs
diff --git a/src/impurities/toeflI.cuh b/src/impurities/toeflI.cuh
index 7df388931..9e5884f71 100644
--- a/src/impurities/toeflI.cuh
+++ b/src/impurities/toeflI.cuh
@@ -8,10 +8,6 @@
 #endif
 
 
-// TODO es wäre besser, wenn ToeflI auch einen Zeitschritt berechnen würde
-// dann wäre die Rückgabe der Felder (Potential vs. Masse vs. exp( y)) konsistenter
-// (nur das Objekt weiß welches Feld zu welchem Zeitschritt gehört)
-
 namespace dg
 {
 template<class Geometry, class Matrix, class container>
@@ -23,7 +19,7 @@ struct Diffusion
         LaplacianM_perp ( g, g.bcx(), g.bcy(), dg::normed, dg::centered)
     {
     }
-    void operator()( std::vector<container>& x, std::vector<container>& y)
+    void operator()( const std::vector<container>& x, std::vector<container>& y)
     {
         /* x[0] := N_e - 1
            x[1] := N_i - 1
@@ -43,6 +39,7 @@ struct Diffusion
     }
     dg::Elliptic<Geometry, Matrix, container>& laplacianM() {return LaplacianM_perp;}
     const container& weights(){return LaplacianM_perp.weights();}
+    const container& inv_weights(){return LaplacianM_perp.inv_weights();}
     const container& precond(){return LaplacianM_perp.precond();}
   private:
     const imp::Parameters p;
@@ -85,7 +82,7 @@ struct ToeflI
      * @param y input vector
      * @param yp the rhs yp = f(y)
      */
-    void operator()( std::vector<container>& y, std::vector<container>& yp);
+    void operator()( const std::vector<container>& y, std::vector<container>& yp);
 
     /**
      * @brief Return the mass of the last field in operator() in a global computation
@@ -199,7 +196,7 @@ const container& ToeflI<G, Matrix, container>::polarization( const std::vector<c
 }
 
 template< class G, class M, class container>
-void ToeflI< G, M, container>::operator()(std::vector<container>& y, std::vector<container>& yp)
+void ToeflI< G, M, container>::operator()(const std::vector<container>& y, std::vector<container>& yp)
 {
     //y[0] = N_e - 1
     //y[1] = N_i - 1
diff --git a/src/impurities/toefl_hpc.cu b/src/impurities/toefl_hpc.cu
index d76f7aad9..5da770992 100644
--- a/src/impurities/toefl_hpc.cu
+++ b/src/impurities/toefl_hpc.cu
@@ -110,10 +110,10 @@ int main( int argc, char* argv[])
     int ncid;
     err = nc_create( argv[2],NC_NETCDF4|NC_CLOBBER, &ncid);
     err = nc_put_att_text( ncid, NC_GLOBAL, "inputfile", input.size(), input.data());
-    const int version[3] = {FELTOR_MAJOR_VERSION, FELTOR_MINOR_VERSION, FELTOR_SUBMINOR_VERSION};
-    err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_major_version",    NC_INT, 1, &version[0]);
-    err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_minor_version",    NC_INT, 1, &version[1]);
-    err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_subminor_version", NC_INT, 1, &version[2]);
+    //const int version[3] = {FELTOR_MAJOR_VERSION, FELTOR_MINOR_VERSION, FELTOR_SUBMINOR_VERSION}; REMOVED (MW)
+    //err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_major_version",    NC_INT, 1, &version[0]);
+    //err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_minor_version",    NC_INT, 1, &version[1]);
+    //err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_subminor_version", NC_INT, 1, &version[2]);
     int dim_ids[3], tvarID;
     err = file::define_dimensions( ncid, dim_ids, &tvarID, grid_out);
     //field IDs
diff --git a/src/impurities/toefl_mpi.cu b/src/impurities/toefl_mpi.cu
index a7c451da0..7f541acd6 100644
--- a/src/impurities/toefl_mpi.cu
+++ b/src/impurities/toefl_mpi.cu
@@ -50,18 +50,18 @@ int main( int argc, char* argv[])
     MPI_Comm comm;
     MPI_Cart_create( MPI_COMM_WORLD, 2, np, periods, true, &comm);
     ////////////////////////Parameter initialisation//////////////////////////
-    std::string input;
+    Json::Reader reader;
+    Json::Value js;
     if( argc != 3)
     {   if(rank==0)std::cerr << "ERROR: Wrong number of arguments!\nUsage: "<< argv[0]<<" [inputfile] [outputfile]\n";
         return -1;
     }
     else
-    {   input = file::read_file( argv[1]);
+    {   
+        std::ifstream is(argv[1]);
+        reader.parse( is, js, false); //read input without comments
     }
-    Json::Reader reader;
-    Json::Value js;
-    reader.parse( input, js, false); //read input without comments
-    input = js.toStyledString(); //save input without comments, which is important if netcdf file is later read by another parser
+    std::string input = js.toStyledString(); //save input without comments, which is important if netcdf file is later read by another parser
     const imp::Parameters p( js);
     if(rank==0)p.display( std::cout);
     ////////////////////////////////set up computations///////////////////////////
@@ -141,10 +141,10 @@ int main( int argc, char* argv[])
     MPI_Info info = MPI_INFO_NULL;
     err = nc_create_par( argv[2], NC_NETCDF4|NC_MPIIO|NC_CLOBBER, comm, info, &ncid); //MPI ON
     err = nc_put_att_text( ncid, NC_GLOBAL, "inputfile", input.size(), input.data());
-    const int version[3] = {FELTOR_MAJOR_VERSION, FELTOR_MINOR_VERSION, FELTOR_SUBMINOR_VERSION};
-    err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_major_version",    NC_INT, 1, &version[0]);
-    err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_minor_version",    NC_INT, 1, &version[1]);
-    err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_subminor_version", NC_INT, 1, &version[2]);
+    //const int version[3] = {FELTOR_MAJOR_VERSION, FELTOR_MINOR_VERSION, FELTOR_SUBMINOR_VERSION}; REMOVED (MW)
+    //err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_major_version",    NC_INT, 1, &version[0]);
+    //err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_minor_version",    NC_INT, 1, &version[1]);
+    //err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_subminor_version", NC_INT, 1, &version[2]);
     int dim_ids[3], tvarID;
     err = file::define_dimensions( ncid, dim_ids, &tvarID, grid_out.global());
     //field IDs
@@ -175,7 +175,7 @@ int main( int argc, char* argv[])
     if(rank==0)std::cout << "First output ... \n";
     int dims[2],  coords[2];
     MPI_Cart_get( comm, 2, dims, periods, coords);
-    size_t count[3] = {1, grid_out.n()*grid_out.Ny(), grid_out.n()*grid_out.Nx()};
+    size_t count[3] = {1, grid_out.n()*grid_out.local().Ny(), grid_out.n()*grid_out.local().Nx()};
     size_t start[3] = {0, coords[1]*count[1], coords[0]*count[2]};
     dg::MDVec transfer( dg::evaluate(dg::zero, grid));
     dg::DVec transferD( dg::evaluate(dg::zero, grid_out.local()));
diff --git a/src/lamb_dipole/shu.cuh b/src/lamb_dipole/shu.cuh
index cb9a81c2b..076006667 100644
--- a/src/lamb_dipole/shu.cuh
+++ b/src/lamb_dipole/shu.cuh
@@ -4,10 +4,7 @@
 #include <exception>
 #include <cusp/ell_matrix.h>
 
-#include "dg/blas.h"
-#include "dg/arakawa.h"
-#include "dg/elliptic.h"
-#include "dg/cg.h"
+#include "dg/algorithm.h"
 
 namespace dg
 {
@@ -25,6 +22,7 @@ struct Diffusion
         dg::blas1::scal( y, -nu_);
     }
     const container& weights(){return w2d;}
+    const container& inv_weights(){return v2d;}
     const container& precond(){return v2d;}
   private:
     double nu_;
@@ -49,7 +47,7 @@ struct Shu
      * @return psi is the potential
      */
     const container& potential( ) {return psi;}
-    void operator()( Vector& y, Vector& yp);
+    void operator()( const Vector& y, Vector& yp);
   private:
     //typedef typename VectorTraits< Vector>::value_type value_type;
     container psi, w2d, v2d;
@@ -69,9 +67,9 @@ Shu< Matrix, container>::Shu( const Grid2d& g, double eps):
 }
 
 template< class Matrix, class container>
-void Shu<Matrix, container>::operator()( Vector& y, Vector& yp)
+void Shu<Matrix, container>::operator()( const Vector& y, Vector& yp)
 {
-    invert( laplaceM, psi, y, w2d, v2d);
+    invert( laplaceM, psi, y);
     arakawa_( y, psi, yp); //A(y,psi)-> yp
 }
 
diff --git a/src/lamb_dipole/shu_hpc.cu b/src/lamb_dipole/shu_hpc.cu
index 234f875e7..c165e86b5 100644
--- a/src/lamb_dipole/shu_hpc.cu
+++ b/src/lamb_dipole/shu_hpc.cu
@@ -1,18 +1,7 @@
 #include <iostream>
 #include <iomanip>
 
-#include "dg/backend/timer.cuh"
-#include "dg/functors.h"
-#include "dg/backend/evaluation.cuh"
-#include "dg/runge_kutta.h"
-#include "dg/multistep.h"
-#include "dg/backend/xspacelib.cuh"
-#include "dg/backend/typedefs.cuh"
-
-#include "dg/exceptions.h"
-
 #include "file/nc_utilities.h"
-
 #include "shu.cuh"
 #include "parameters.h"
 
diff --git a/src/lamb_dipole/shu_t.cu b/src/lamb_dipole/shu_t.cu
index b6868d418..23e8464fa 100644
--- a/src/lamb_dipole/shu_t.cu
+++ b/src/lamb_dipole/shu_t.cu
@@ -6,12 +6,7 @@
 
 #include "draw/host_window.h"
 
-#include "dg/functors.h"
-#include "dg/backend/evaluation.cuh"
-#include "dg/backend/xspacelib.cuh"
-#include "dg/runge_kutta.h"
-#include "dg/multistep.h"
-#include "dg/helmholtz.h"
+#include "dg/algorithm.h"
 
 #include "shu.cuh"
 
@@ -75,7 +70,7 @@ int main()
         cout << "Total vorticity is: "<<blas2::dot( stencil, w2d, y0) << "\n";
         cout << "Total enstrophy is: "<<blas2::dot( w2d, y0)<<"\n";
         //compute the color scale
-        dg::blas2::mv( equidistant, y0, visual );
+        dg::blas2::symv( equidistant, y0, visual );
         colors.scale() =  (float)thrust::reduce( visual.begin(), visual.end(), -1., dg::AbsMax<float>() );
         std::cout << "Color scale " << colors.scale() <<"\n";
         //draw and swap buffers
diff --git a/src/reco2D/Makefile b/src/reco2D/Makefile
index 4e63d7858..054fc4408 100644
--- a/src/reco2D/Makefile
+++ b/src/reco2D/Makefile
@@ -8,9 +8,9 @@ include ../../config/devices/devices.mk
 INCLUDE+= -I../         # other src libraries
 INCLUDE+= -I../../inc   # other project libraries
 
-all: asela
+all: reconnection
 
-asela: asela.cu asela.cuh 
+reconnection: reconnection.cu reconnection.cuh 
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(GLFLAGS) $(JSONLIB) -DDG_BENCHMARK 
 
 .PHONY: clean
@@ -19,4 +19,4 @@ doc:
 	doxygen Doxyfile
 	
 clean:
-	rm -f asela 
+	rm -f reconnection 
diff --git a/src/reco2D/README b/src/reco2D/README
deleted file mode 100644
index 158d449b2..000000000
--- a/src/reco2D/README
+++ /dev/null
@@ -1,4 +0,0 @@
-project description: 2d electromagnetic turbulence 
-
-cp input/default.json input.json
-./asela
diff --git a/src/reco2D/parameters.h b/src/reco2D/parameters.h
index 08d1ffd1d..031e1bb53 100644
--- a/src/reco2D/parameters.h
+++ b/src/reco2D/parameters.h
@@ -2,7 +2,7 @@
 #include "dg/enums.h"
 #include "json/json.h"
 
-namespace eule
+namespace reco
 {
 /**
  * @brief Provide a mapping between input file and named parameters
@@ -77,7 +77,7 @@ struct Parameters
             <<"Number of outputs:       "<<maxout<<std::endl; //the endl is for the implicit flush 
     }
 };
-} //namespace eule
+} //namespace reco
 
 
     
diff --git a/src/reco2D/asela.cu b/src/reco2D/reconnection.cu
similarity index 91%
rename from src/reco2D/asela.cu
rename to src/reco2D/reconnection.cu
index 5b6b9948c..f2f35b0e6 100644
--- a/src/reco2D/asela.cu
+++ b/src/reco2D/reconnection.cu
@@ -6,11 +6,7 @@
 #include "draw/host_window.h"
 //#include "draw/device_window.cuh"
 
-#include "asela.cuh"
-
-#include "dg/runge_kutta.h"
-#include "dg/multistep.h"
-#include "dg/backend/timer.cuh"
+#include "reconnection.cuh"
 #include "parameters.h"
 
 /*
@@ -44,7 +40,7 @@ int main( int argc, char* argv[])
         std::cerr << "ERROR: Too many arguments!\nUsage: "<< argv[0]<<" [filename]\n";
         return -1;
     }
-    const eule::Parameters p( js);
+    const reco::Parameters p( js);
     p.display( std::cout);
     /////////glfw initialisation ////////////////////////////////////////////
     std::stringstream title;
@@ -57,8 +53,8 @@ int main( int argc, char* argv[])
 
     dg::Grid2d grid( -p.lxhalf, p.lxhalf, -p.lyhalf, p.lyhalf , p.n, p.Nx, p.Ny, p.bc_x, p.bc_y);
     //create RHS 
-    eule::Asela< dg::CartesianGrid2d, dg::DMatrix, dg::DVec > asela( grid, p); 
-    eule::Diffusion<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> diffusion( grid, p.nu, 1., 1. );
+    reco::Explicit< dg::CartesianGrid2d, dg::DMatrix, dg::DVec > reconnection( grid, p); 
+    reco::Diffusion<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> diffusion( grid, p.nu, 1., 1. );
     //create initial vector
     std::vector<dg::DVec> y0(4, dg::evaluate( dg::one, grid)), y1(y0); // n_e' = gaussian
     y0[2] = y0[3] = dg::evaluate( aparallel, grid);
@@ -79,9 +75,9 @@ int main( int argc, char* argv[])
     //create timer
     dg::Timer t;
     double time = 0;
-    //ab.init( asela, y0, p.dt);
-    ab.init( asela, diffusion, y0, p.dt);
-    //ab( asela, y0, y1, p.dt);
+    //ab.init( reconnection, y0, p.dt);
+    ab.init( reconnection, diffusion, y0, p.dt);
+    //ab( reconnection, y0, y1, p.dt);
     //y0.swap( y1); 
     std::cout << "Begin computation \n";
     std::cout << std::scientific << std::setprecision( 2);
@@ -113,7 +109,7 @@ int main( int argc, char* argv[])
         render.renderQuad( visual, grid.n()*grid.Nx(), grid.n()*grid.Ny(), colors);
 
 
-        dvisual = asela.potential()[0];
+        dvisual = reconnection.potential()[0];
         dg::blas1::transfer(dvisual, hvisual);
         dg::blas2::gemv( equi, hvisual, visual);
         //compute the color scale
@@ -124,7 +120,7 @@ int main( int argc, char* argv[])
 
 
         //transform phi
-        dg::blas2::gemv(laplaceM, asela.potential()[0], dvisual);
+        dg::blas2::gemv(laplaceM, reconnection.potential()[0], dvisual);
         dg::blas1::transfer(dvisual, hvisual);
         dg::blas2::gemv( equi, hvisual, visual);
         //compute the color scale
@@ -135,7 +131,7 @@ int main( int argc, char* argv[])
 
 
         //transform Aparallel
-        dvisual = asela.aparallel();
+        dvisual = reconnection.aparallel();
         dg::blas1::transfer(dvisual, hvisual);
         dg::blas2::gemv( equi, hvisual, visual);
         //compute the color scale
@@ -146,7 +142,7 @@ int main( int argc, char* argv[])
 
 
         //transform Aparallel
-        dg::blas2::gemv( laplaceM, asela.aparallel(), dvisual);
+        dg::blas2::gemv( laplaceM, reconnection.aparallel(), dvisual);
         dg::blas1::transfer(dvisual, hvisual);
         dg::blas2::gemv( equi, hvisual, visual);
         //compute the color scale
@@ -170,7 +166,7 @@ int main( int argc, char* argv[])
         for( unsigned i=0; i<p.itstp; i++)
         {
             step++;
-            try{ ab( asela, diffusion, y0);}
+            try{ ab( reconnection, diffusion, y0);}
             catch( dg::Fail& fail) { 
                 std::cerr << "CG failed to converge to "<<fail.epsilon()<<"\n";
                 std::cerr << "Does Simulation respect CFL condition?\n";
diff --git a/src/reco2D/asela.cuh b/src/reco2D/reconnection.cuh
similarity index 87%
rename from src/reco2D/asela.cuh
rename to src/reco2D/reconnection.cuh
index c34ebf957..fdbe2a647 100644
--- a/src/reco2D/asela.cuh
+++ b/src/reco2D/reconnection.cuh
@@ -1,19 +1,9 @@
 #pragma once
 
-
-#include "parameters.h"
-
-#include "dg/backend/xspacelib.cuh"
-#include "dg/cg.h"
 #include "dg/algorithm.h"
+#include "parameters.h"
 
-#ifdef DG_BENCHMARK
-#include "dg/backend/timer.cuh"
-#endif
-
-
-
-namespace eule
+namespace reco
 {
 
 template<class Geometry, class Matrix, class container>
@@ -38,6 +28,7 @@ struct Diffusion
 
     }
     const container& weights(){return w2d_;}
+    const container& inv_weights(){return v2d_;}
     const container& precond(){return v2d_;}
 
   private:
@@ -48,52 +39,49 @@ struct Diffusion
 };
 
 template< class Geometry, class Matrix, class container=thrust::device_vector<double> >
-struct Asela
+struct Explicit
 {
-    Asela( const Geometry& g, Parameters p);
+    Explicit( const Geometry& g, Parameters p);
 
     const std::vector<container>& potential( ) const { return phi;}
     const container& aparallel( ) const { return apar;}
-    void operator()( std::vector<container>& y, std::vector<container>& yp);
+    void operator()( const std::vector<container>& y, std::vector<container>& yp);
 
   private:
     const container w2d, v2d, one;
     container rho, omega, apar;
 
-    std::vector<container> phi, arakAN, arakAU, u;
-    std::vector<container> expy;
+    std::vector<container> phi, expy, arakAN, arakAU, u;
 
     //matrices and solvers
 
     dg::ArakawaX<Geometry, Matrix, container> arakawa; 
     dg::Invert<container> invert_pol, invert_maxwell; 
     dg::Helmholtz<Geometry, Matrix, container> maxwell;
-
     dg::Elliptic< Geometry, Matrix, container > pol, laplaceM; 
 
     Parameters p;
-
 };
 
 template< class G, class M, class container>
-Asela< G, M, container>::Asela( const G& grid, Parameters p ): 
+Explicit< G, M, container>::Explicit( const G& grid, Parameters p ): 
     w2d( dg::create::weights(grid)),
     v2d( dg::create::inv_weights(grid)),
     one( dg::evaluate( dg::one, grid)),
     rho( dg::evaluate( dg::zero, grid)),
     omega(rho), apar(rho),
     phi( 2, rho), expy( phi), arakAN( phi), arakAU( phi), u(phi), 
-    laplaceM ( grid, dg::normed, dg::centered),
     arakawa( grid), 
-    maxwell( grid),
     invert_pol( rho, rho.size(), p.eps_pol),
     invert_maxwell( rho, rho.size(), p.eps_maxwell),
+    maxwell( grid),
     pol(     grid), 
+    laplaceM ( grid, dg::normed, dg::centered),
     p(p)
 { }
 
 template<class G, class M, class container>
-void Asela< G, M, container>::operator()( std::vector<container>& y, std::vector<container>& yp)
+void Explicit< G, M, container>::operator()( const std::vector<container>& y, std::vector<container>& yp) 
 {
     assert( y.size() == 4);
     assert( y.size() == yp.size());
@@ -104,7 +92,7 @@ void Asela< G, M, container>::operator()( std::vector<container>& y, std::vector
     dg::blas1::axpby( p.dhat[1]*p.dhat[1], expy[1], 0., omega);
     pol.set_chi(omega);
     dg::blas1::axpby( -p.dhat[0], expy[0], p.dhat[1], expy[1], rho);
-    invert_pol( pol, phi[0], rho, w2d, v2d);
+    invert_pol( pol, phi[0], rho);
     //compute phi[1]
     arakawa.variation( phi[0], phi[1]);
     dg::blas1::axpby( 1., phi[0], -0.5*p.dhat[1], phi[1]);////////////////////
@@ -137,7 +125,5 @@ void Asela< G, M, container>::operator()( std::vector<container>& y, std::vector
 
 }
 
-
-
-}//namespace eule
+}//namespace reco
 
-- 
GitLab


From 9d2913ff3ccca844ac627029ea0e0932e4eecba3 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Thu, 2 Nov 2017 23:39:41 +0100
Subject: [PATCH 435/453] updated hasegawa folder and added local to
 feltorShw_mpi.cu

---
 src/feltorShw/feltor_mpi.cu |  2 +-
 src/hasegawa/hw.cu          | 10 ++--
 src/hasegawa/hw.cuh         | 96 ++++++++++++++++++-------------------
 src/hasegawa/mima.cu        |  8 ++--
 src/hasegawa/mima.cuh       | 55 ++++++++++-----------
 5 files changed, 80 insertions(+), 91 deletions(-)

diff --git a/src/feltorShw/feltor_mpi.cu b/src/feltorShw/feltor_mpi.cu
index 79de6e29b..449a6f0e4 100644
--- a/src/feltorShw/feltor_mpi.cu
+++ b/src/feltorShw/feltor_mpi.cu
@@ -277,7 +277,7 @@ int main( int argc, char* argv[])
     if(rank==0) std::cout << "First output ... \n";
     int dims[2],  coords[2];
     MPI_Cart_get( comm, 2, dims, periods, coords);
-    size_t count[3] = {1, grid_out.n()*grid_out.Ny(), grid_out.n()*grid_out.Nx()};  
+    size_t count[3] = {1, grid_out.n()*grid_out.local().Ny(), grid_out.n()*grid_out.local().Nx()};  
     size_t start[3] = {0, coords[1]*count[1],          coords[0]*count[2]}; 
     dg::MDVec transfer( dg::evaluate(dg::zero, grid));
     dg::DVec transferD( dg::evaluate(dg::zero, grid_out.local()));
diff --git a/src/hasegawa/hw.cu b/src/hasegawa/hw.cu
index 3db8620c7..fa92739ad 100644
--- a/src/hasegawa/hw.cu
+++ b/src/hasegawa/hw.cu
@@ -48,10 +48,10 @@ int main( int argc, char* argv[])
     GLFWwindow* w = draw::glfwInitAndCreateWindow( js["width"].asDouble(), js["height"].asDouble(), "");
     draw::RenderHostData render(js["rows"].asDouble(), js["cols"].asDouble());
     /////////////////////////////////////////////////////////////////////////
-    dg::Grid2d grid( 0, p.lx, 0, p.ly, p.n, p.Nx, p.Ny, p.bc_x, p.bc_y);
+    dg::CartesianGrid2d grid( 0, p.lx, 0, p.ly, p.n, p.Nx, p.Ny, p.bc_x, p.bc_y);
     //create RHS 
     bool mhw = (p.equations == "modified");
-    dg::HW<dg::DMatrix, dg::DVec > test( grid, p.kappa, p.tau, p.nu, p.eps_pol, mhw); 
+    hw::HW<dg::DMatrix, dg::DVec > test( grid, p.kappa, p.tau, p.nu, p.eps_pol, mhw); 
     dg::DVec one( grid.size(), 1.);
     //create initial vector
     dg::Gaussian gaussian( p.posX*grid.lx(), p.posY*grid.ly(), p.sigma, p.sigma, p.amp); //gaussian width is in absolute values
@@ -69,7 +69,7 @@ int main( int argc, char* argv[])
     //dg::AB< k, std::vector<dg::DVec> > ab( y0);
     //dg::TVB< std::vector<dg::DVec> > ab( y0);
     dg::Karniadakis<std::vector<dg::DVec> > ab( y0, y0[0].size(), p.eps_time);
-    dg::Diffusion<dg::DMatrix, dg::DVec> diffusion( grid, p.nu);
+    hw::Diffusion<dg::DMatrix, dg::DVec> diffusion( grid, p.nu);
 
     dg::DVec dvisual( grid.size(), 0.);
     dg::HVec hvisual( grid.size(), 0.), visual(hvisual);
@@ -81,8 +81,8 @@ int main( int argc, char* argv[])
     ab.init( test, diffusion, y0, p.dt);
     //ab( test, y0, y1, p.dt);
     //y0.swap( y1); 
-    double E0 = test.energy(), energy0 = E0, E1 = 0, diff = 0;
-    double Ezf0 = test.zonal_flow_energy(), energyzf0 = Ezf0, Ezf1 = 0, diffzf = 0;
+    double E0 = test.energy(), E1 = 0, diff = 0; //energy0 = E0;
+    double Ezf0 = test.zonal_flow_energy(), Ezf1 = 0, diffzf = 0; //energyzf0 = Ezf0;
     std::cout << "Begin computation \n";
     std::cout << std::scientific << std::setprecision( 2);
     unsigned step = 0;
diff --git a/src/hasegawa/hw.cuh b/src/hasegawa/hw.cuh
index 6f490053e..098f5d0fd 100644
--- a/src/hasegawa/hw.cuh
+++ b/src/hasegawa/hw.cuh
@@ -3,21 +3,16 @@
 #include <exception>
 
 #include "dg/algorithm.h"
-#include "dg/average.h"
-
-#ifdef DG_BENCHMARK
-#include "dg/backend/timer.cuh"
-#endif
 
 ///@note This is an old copy of the toefl project and shouldn't be taken as a basis for a new project
 
-namespace dg
+namespace hw
 {
 
 template< class Matrix, class container>
 struct Diffusion
 {
-    Diffusion( const dg::Grid2d& g, double nu): nu_(nu),
+    Diffusion( const dg::CartesianGrid2d& g, double nu): nu_(nu),
         w2d(dg::create::weights( g)), v2d( dg::create::inv_weights(g)), temp( g.size()), LaplacianM( g, dg::normed, dg::centered) {
         }
     void operator()( const std::vector<container>& x, std::vector<container>& y)
@@ -33,12 +28,13 @@ struct Diffusion
         }
     }
     const container& weights(){return w2d;}
+    const container& inv_weights(){return v2d;}
     const container& precond(){return v2d;}
   private:
     double nu_;
     const container w2d, v2d;
     container temp;
-    Elliptic<dg::CartesianGrid2d, Matrix, container> LaplacianM;
+    dg::Elliptic<dg::CartesianGrid2d, Matrix, container> LaplacianM;
 };
 
 
@@ -61,7 +57,7 @@ struct HW
      * @param eps_gamma stopping criterion for Gamma operator
      * @param global local or global computation
      */
-    HW( const Grid2d& g, double , double , double , double , bool);
+    HW( const dg::CartesianGrid2d& g, double , double , double , double , bool);
 
     /**
      * @brief Returns phi and psi that belong to the last y in operator()
@@ -78,7 +74,7 @@ struct HW
      * @param y input vector
      * @param yp the rhs yp = f(y)
      */
-    void operator()( std::vector<container>& y, std::vector<container>& yp);
+    void operator()( const std::vector<container>& y, std::vector<container>& yp);
 
     /**
      * @brief Return the mass of the last field in operator() in a global computation
@@ -121,10 +117,10 @@ struct HW
     std::vector<container> lapy, laplapy;
 
     //matrices and solvers
-    ArakawaX< dg::CartesianGrid2d, Matrix, container> arakawa; 
-    CG<container > pcg;
-    PoloidalAverage<container, thrust::device_vector<int> > average;
-    Elliptic<dg::CartesianGrid2d, Matrix, container> A, laplaceM;
+    dg::ArakawaX< dg::CartesianGrid2d, Matrix, container> arakawa; 
+    dg::CG<container > pcg;
+    dg::PoloidalAverage<container, thrust::device_vector<int> > average;
+    dg::Elliptic<dg::CartesianGrid2d, Matrix, container> A, laplaceM;
 
     const container w2d, v2d, one;
     const double alpha;
@@ -139,14 +135,14 @@ struct HW
 };
 
 template< class Matrix, class container>
-HW<Matrix, container>::HW( const Grid2d& grid, double alpha, double g, double nu, double eps_pol, bool mhw ): 
+HW<Matrix, container>::HW( const dg::CartesianGrid2d& grid, double alpha, double g, double nu, double eps_pol, bool mhw ): 
     chi( grid.size(), 0.), omega(chi), phi( chi), phi_old( chi), dyphi( chi),
     lapphiM(chi), lapy( 2, chi),  laplapy( lapy),
-    A( grid, not_normed, centered), laplaceM( grid, normed, centered),
+    A( grid, dg::not_normed, dg::centered), laplaceM( grid, dg::normed, dg::centered),
     arakawa( grid), 
     pcg( omega, omega.size()), 
     average( grid),
-    w2d( create::weights(grid)), v2d( create::inv_weights(grid)), one( dg::evaluate(dg::one, grid)),
+    w2d( dg::create::weights(grid)), v2d( dg::create::inv_weights(grid)), one( dg::evaluate(dg::one, grid)),
     alpha( alpha), g(g), nu( nu), eps_pol(eps_pol), mhw( mhw)
 {
 
@@ -157,17 +153,17 @@ template<class M, class container>
 const container& HW<M, container>::polarisation( const std::vector<container>& y)
 {
     //extrapolate phi and gamma_n
-    blas1::axpby( 2., phi, -1.,  phi_old);
+    dg::blas1::axpby( 2., phi, -1.,  phi_old);
     phi.swap( phi_old);
 #ifdef DG_BENCHMARK
-    Timer t; 
+    dg::Timer t; 
     t.tic();
 #endif
-    blas1::axpby( 1., y[1], -1., y[0], lapphiM); //n_i - n_e = omega
-    blas2::symv( w2d, lapphiM, omega); 
+    dg::blas1::axpby( 1., y[1], -1., y[0], lapphiM); //n_i - n_e = omega
+    dg::blas2::symv( w2d, lapphiM, omega); 
     unsigned number = pcg( A, phi, omega, v2d, eps_pol);
     if( number == pcg.get_max())
-        throw Fail( eps_pol);
+        throw dg::Fail( eps_pol);
 #ifdef DG_BENCHMARK
     std::cout << "# of pcg iterations for phi \t"<< number <<"\t";
     t.toc();
@@ -179,7 +175,7 @@ const container& HW<M, container>::polarisation( const std::vector<container>& y
 }
 
 template< class M, class container>
-void HW< M, container>::operator()( std::vector<container>& y, std::vector<container>& yp)
+void HW< M, container>::operator()( const std::vector<container>& y, std::vector<container>& yp)
 {
     assert( y.size() == 2);
     assert( y.size() == yp.size());
@@ -192,56 +188,56 @@ void HW< M, container>::operator()( std::vector<container>& y, std::vector<conta
 
 
     //compute derivatives
-    blas2::gemv( arakawa.dy(), phi, dyphi);
+    dg::blas2::gemv( arakawa.dy(), phi, dyphi);
     //gradient terms
-    blas1::axpby( g, dyphi, 1., yp[0]);
-    blas1::pointwiseDot( dyphi, lapphiM, omega);
-    //blas1::axpby( -2.*0.05, omega, 1., yp[0]);
-    blas1::axpby( g, dyphi, 1., yp[1]);
-    blas1::axpby( -2.*0.05, omega, 1., yp[1]);
+    dg::blas1::axpby( g, dyphi, 1., yp[0]);
+    dg::blas1::pointwiseDot( dyphi, lapphiM, omega);
+    //dg::blas1::axpby( -2.*0.05, omega, 1., yp[0]);
+    dg::blas1::axpby( g, dyphi, 1., yp[1]);
+    dg::blas1::axpby( -2.*0.05, omega, 1., yp[1]);
     //hw term
-    blas1::axpby( 1., phi, 0., omega);
-    blas1::axpby( -1., y[0], 1., omega);
+    dg::blas1::axpby( 1., phi, 0., omega);
+    dg::blas1::axpby( -1., y[0], 1., omega);
     if( mhw) 
     {
         average( phi, chi);
-        blas1::axpby( -1., chi, 1., omega );
+        dg::blas1::axpby( -1., chi, 1., omega );
         average( y[0], chi);
-        blas1::axpby( 1., chi, 1., omega);
+        dg::blas1::axpby( 1., chi, 1., omega);
     }
-    blas1::axpby( alpha, omega, 1., yp[0]);
+    dg::blas1::axpby( alpha, omega, 1., yp[0]);
 
     //add laplacians
     for( unsigned i=0; i<y.size(); i++)
     {
-        blas2::gemv( laplaceM, y[i], lapy[i]);
-        blas2::gemv( laplaceM, lapy[i], laplapy[i]);
-       // blas1::axpby( -nu, laplapy[i], 1., yp[i]); //rescale 
+        dg::blas2::gemv( laplaceM, y[i], lapy[i]);
+        dg::blas2::gemv( laplaceM, lapy[i], laplapy[i]);
+       // dg::blas1::axpby( -nu, laplapy[i], 1., yp[i]); //rescale 
     }
 
-    double ue = 0.5*blas2::dot( y[0], w2d, y[0]);
-    double ui = 0.5*blas2::dot( y[1], w2d, y[1]);
-    double ei = 0.5*blas2::dot( phi, w2d, lapphiM);
+    double ue = 0.5*dg::blas2::dot( y[0], w2d, y[0]);
+    double ui = 0.5*dg::blas2::dot( y[1], w2d, y[1]);
+    double ei = 0.5*dg::blas2::dot( phi, w2d, lapphiM);
     energy_ = ue + ui + ei;
-    flux_ = g*blas2::dot( y[0], w2d, dyphi);
-    jot_ = -alpha*blas2::dot( omega, w2d, omega);
-    ediff_ = nu * blas2::dot( phi, w2d, laplapy[0]) -nu*blas2::dot( y[0], w2d, laplapy[0]) - nu * blas2::dot( phi, w2d, laplapy[1]);
+    flux_ = g*dg::blas2::dot( y[0], w2d, dyphi);
+    jot_ = -alpha*dg::blas2::dot( omega, w2d, omega);
+    ediff_ = nu * dg::blas2::dot( phi, w2d, laplapy[0]) -nu*dg::blas2::dot( y[0], w2d, laplapy[0]) - nu * dg::blas2::dot( phi, w2d, laplapy[1]);
 
      
     
     average( phi, chi);
     average( lapphiM, omega);
-    uzf_ = 0.5*blas2::dot( chi, w2d, omega);
-    blas1::pointwiseDot( dyphi, lapphiM, omega);
-    blas2::symv( arakawa.dx(), omega, lapy[0]);
+    uzf_ = 0.5*dg::blas2::dot( chi, w2d, omega);
+    dg::blas1::pointwiseDot( dyphi, lapphiM, omega);
+    dg::blas2::symv( arakawa.dx(), omega, lapy[0]);
     average( lapy[0], omega);
-    capitalR_ = blas2::dot( chi, w2d, omega);
+    capitalR_ = dg::blas2::dot( chi, w2d, omega);
     average( laplapy[0], omega);
-    diff_ = nu * blas2::dot( phi, w2d, omega);
+    diff_ = nu * dg::blas2::dot( phi, w2d, omega);
     average( laplapy[1], omega);
-    diff_ += -nu * blas2::dot( phi, w2d, omega);
+    diff_ += -nu * dg::blas2::dot( phi, w2d, omega);
 
 }
 
 
-}//namespace dg
+}//namespace hw
diff --git a/src/hasegawa/mima.cu b/src/hasegawa/mima.cu
index 9a928ee50..707171a73 100644
--- a/src/hasegawa/mima.cu
+++ b/src/hasegawa/mima.cu
@@ -7,8 +7,6 @@
 //#include "draw/device_window.cuh"
 
 #include "mima.cuh"
-#include "dg/multistep.h"
-#include "dg/backend/timer.cuh"
 #include "../toefl/parameters.h"
 
 /*
@@ -47,10 +45,10 @@ int main( int argc, char* argv[])
     GLFWwindow* w = draw::glfwInitAndCreateWindow( js["width"].asDouble(), js["height"].asDouble(), "");
     draw::RenderHostData render(js["rows"].asDouble(), js["cols"].asDouble());
     /////////////////////////////////////////////////////////////////////////
-    dg::Grid2d grid( 0, p.lx, 0, p.ly, p.n, p.Nx, p.Ny, p.bc_x, p.bc_y);
+    dg::CartesianGrid2d grid( 0, p.lx, 0, p.ly, p.n, p.Nx, p.Ny, p.bc_x, p.bc_y);
     //create RHS 
     bool mhw = ( p.equations == "modified");
-    dg::Mima< dg::DMatrix, dg::DVec > mima( grid, p.kappa, p.eps_pol, mhw); 
+    mima::Mima< dg::DMatrix, dg::DVec > mima( grid, p.kappa, p.eps_pol, mhw); 
     dg::DVec one( grid.size(), 1.);
     //create initial vector
     dg::Gaussian gaussian( p.posX*grid.lx(), p.posY*grid.ly(), p.sigma, p.sigma, p.amp); //gaussian width is in absolute values
@@ -69,7 +67,7 @@ int main( int argc, char* argv[])
         dg::blas1::axpby( -meanMass, one, 1., y0);
     }
     dg::Karniadakis<dg::DVec > ab( y0, y0.size(), p.eps_time);
-    dg::Diffusion<dg::DMatrix,dg::DVec> diffusion( grid, p.nu);
+    mima::Diffusion<dg::DMatrix,dg::DVec> diffusion( grid, p.nu);
 
     dg::DVec dvisual( grid.size(), 0.);
     dg::HVec hvisual( grid.size(), 0.), visual(hvisual);
diff --git a/src/hasegawa/mima.cuh b/src/hasegawa/mima.cuh
index 0c05f9f77..a9176acd2 100644
--- a/src/hasegawa/mima.cuh
+++ b/src/hasegawa/mima.cuh
@@ -2,22 +2,16 @@
 
 #include <exception>
 
-#include "dg/backend/xspacelib.cuh"
 #include "dg/algorithm.h"
 
-#ifdef DG_BENCHMARK
-#include "dg/backend/timer.cuh"
-#endif
+///@note This is an old copy of the toefl project and shouldn't be taken as a basis for a new project
 
-
-//@note This is an old copy of the toefl project and shouldn't be taken as a basis for a new project
-
-namespace dg
+namespace mima
 {
 template< class Matrix, class container>
 struct Diffusion
 {
-    Diffusion( const dg::Grid2d& g, double nu): nu_(nu),
+    Diffusion( const dg::CartesianGrid2d& g, double nu): nu_(nu),
         w2d(dg::create::weights( g)), v2d( dg::create::inv_weights(g)), temp( g.size()), LaplacianM( g, dg::normed, dg::centered) {
         }
     void operator()( const container& x, container& y)
@@ -29,12 +23,13 @@ struct Diffusion
         dg::blas1::scal( y, -nu_);
     }
     const container& weights(){return w2d;}
+    const container& inv_weights(){return v2d;}
     const container& precond(){return v2d;}
   private:
     double nu_;
     const container w2d, v2d;
     container temp;
-    Elliptic<dg::CartesianGrid2d, Matrix, container> LaplacianM;
+    dg::Elliptic<dg::CartesianGrid2d, Matrix, container> LaplacianM;
 };
 
 
@@ -57,7 +52,7 @@ struct Mima
      * @param eps_gamma stopping criterion for Gamma operator
      * @param global local or global computation
      */
-    Mima( const Grid2d& g, double kappa, double eps, bool global);
+    Mima( const dg::CartesianGrid2d& g, double kappa, double eps, bool global);
 
     /**
      * @brief Returns phi and psi that belong to the last y in operator()
@@ -84,24 +79,24 @@ struct Mima
     container dxxphi, dxyphi;
 
     //matrices and solvers
-    Elliptic<dg::CartesianGrid2d, Matrix, container> laplaceM;
-    ArakawaX<dg::CartesianGrid2d, Matrix, container> arakawa; 
+    dg::Elliptic<dg::CartesianGrid2d, Matrix, container> laplaceM;
+    dg::ArakawaX<dg::CartesianGrid2d, Matrix, container> arakawa; 
     const container w2d, v2d;
-    Invert<container> invert;
-    Helmholtz<dg::CartesianGrid2d, Matrix, container> helmholtz;
+    dg::Invert<container> invert;
+    dg::Helmholtz<dg::CartesianGrid2d, Matrix, container> helmholtz;
 
 
 
 };
 
 template< class M, class container>
-Mima< M, container>::Mima( const Grid2d& grid, double kappa, double eps, bool global ): 
+Mima< M, container>::Mima( const dg::CartesianGrid2d& grid, double kappa, double eps, bool global ): 
     kappa( kappa), global(global),
     phi( grid.size(), 0.), dxphi( phi), dyphi( phi), omega(phi),
     dxxphi( phi), dxyphi(phi),
     arakawa( grid), 
-    w2d( create::weights(grid)), v2d( create::inv_weights(grid)),
-    laplaceM( grid, normed, dg::centered),
+    w2d( dg::create::weights(grid)), v2d( dg::create::inv_weights(grid)),
+    laplaceM( grid, dg::normed, dg::centered),
     helmholtz( grid, -1),
     invert( phi, grid.size(), eps)
 {
@@ -116,22 +111,22 @@ void Mima< M, container>::operator()( const container& y, container& yp)
 
     arakawa( phi, omega, yp);
     //compute derivatives
-    blas2::gemv( arakawa.dx(), phi, dxphi);
-    blas2::gemv( arakawa.dy(), phi, dyphi);
-    blas2::gemv( arakawa.dx(), dxphi, dxxphi);
-    blas2::gemv( arakawa.dy(), dxphi, dxyphi);
+    dg::blas2::gemv( arakawa.dx(), phi, dxphi);
+    dg::blas2::gemv( arakawa.dy(), phi, dyphi);
+    dg::blas2::gemv( arakawa.dx(), dxphi, dxxphi);
+    dg::blas2::gemv( arakawa.dy(), dxphi, dxyphi);
     //gradient terms
-    blas1::axpby( -1, dyphi, 1., yp);
+    dg::blas1::axpby( -1, dyphi, 1., yp);
 
-    blas1::pointwiseDot( dyphi, omega, omega);
-    blas1::axpby( -2*kappa, omega, 1., yp);
+    dg::blas1::pointwiseDot( dyphi, omega, omega);
+    dg::blas1::axpby( -2*kappa, omega, 1., yp);
 
     if( global)
     {
-        blas1::pointwiseDot( dxphi, dxyphi, omega);
-        blas1::axpby( -kappa, omega, 1., yp);
-        blas1::pointwiseDot( dyphi, dxxphi, omega);
-        blas1::axpby( +kappa, omega, 1., yp);
+        dg::blas1::pointwiseDot( dxphi, dxyphi, omega);
+        dg::blas1::axpby( -kappa, omega, 1., yp);
+        dg::blas1::pointwiseDot( dyphi, dxxphi, omega);
+        dg::blas1::axpby( +kappa, omega, 1., yp);
     }
     //dg::blas1::scal(yp, -1.);
 
@@ -139,5 +134,5 @@ void Mima< M, container>::operator()( const container& y, container& yp)
 }
 
 
-}//namespace dg
+}//namespace mima
 
-- 
GitLab


From 8de931cee6e1c2f3389ba2529dca160bf4e19667 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Fri, 3 Nov 2017 00:25:27 +0100
Subject: [PATCH 436/453] adapt ep and Markus projects

---
 inc/dg/backend/cusp_matrix_blas.cuh |  1 +
 src/ep/toeflR.cu                    |  6 +--
 src/ep/toeflR.cuh                   | 69 +++++++++++++----------------
 src/ep/toefl_mpi.cu                 | 10 ++---
 src/feltorSH/feltor.cuh             |  9 ++--
 src/feltorSH/feltor_hpc.cu          |  8 ++--
 src/feltorSH/feltor_mpi.cu          | 10 ++---
 src/feltorSHp/feltor.cuh            |  9 ++--
 src/feltorSHp/feltor_hpc.cu         |  8 ++--
 src/feltorSHp/feltor_mpi.cu         | 10 ++---
 src/feltorSesol/feltor.cuh          |  7 +--
 src/feltorSesol/feltor_mpi.cu       |  2 +-
 src/polar/polar_mpi.cu              | 10 +----
 13 files changed, 73 insertions(+), 86 deletions(-)

diff --git a/inc/dg/backend/cusp_matrix_blas.cuh b/inc/dg/backend/cusp_matrix_blas.cuh
index 48849e22e..50c076645 100644
--- a/inc/dg/backend/cusp_matrix_blas.cuh
+++ b/inc/dg/backend/cusp_matrix_blas.cuh
@@ -6,6 +6,7 @@
 #endif //DG_DEBUG
 
 #include <typeinfo>
+#include <limits.h>
 #include <cusp/multiply.h>
 #include <cusp/convert.h>
 #include <cusp/array1d.h>
diff --git a/src/ep/toeflR.cu b/src/ep/toeflR.cu
index 3aa961ce7..c181ddd2a 100644
--- a/src/ep/toeflR.cu
+++ b/src/ep/toeflR.cu
@@ -49,10 +49,10 @@ int main( int argc, char* argv[])
     draw::RenderHostData render(js["rows"].asDouble(), js["cols"].asDouble());
     /////////////////////////////////////////////////////////////////////////
 
-    dg::Grid2d grid( 0, p.lx, 0, p.ly, p.n, p.Nx, p.Ny, p.bc_x, p.bc_y);
+    dg::CartesianGrid2d grid( 0, p.lx, 0, p.ly, p.n, p.Nx, p.Ny, p.bc_x, p.bc_y);
     //create RHS 
-    dg::ToeflR<dg::CartesianGrid2d, dg::DMatrix, dg::DVec > test( grid, p); 
-    dg::Diffusion<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> diffusion( grid, p.nu);
+    ep::ToeflR<dg::CartesianGrid2d, dg::DMatrix, dg::DVec > test( grid, p); 
+    ep::Diffusion<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> diffusion( grid, p.nu);
     //create initial vector
     dg::Gaussian g( p.posX*p.lx, p.posY*p.ly, p.sigma, p.sigma, p.amp); //gaussian width is in absolute values
     dg::DVec gauss = dg::evaluate( g, grid);
diff --git a/src/ep/toeflR.cuh b/src/ep/toeflR.cuh
index 18615700d..e1eff047c 100644
--- a/src/ep/toeflR.cuh
+++ b/src/ep/toeflR.cuh
@@ -3,15 +3,9 @@
 #include <exception>
 
 #include "dg/algorithm.h"
-#include "dg/backend/typedefs.cuh"
 #include "parameters.h"
 
-#ifdef DG_BENCHMARK
-#include "dg/backend/timer.cuh"
-#endif
-
-
-namespace dg
+namespace ep
 {
 
 template<class Geometry, class Matrix, class container>
@@ -22,7 +16,7 @@ struct Diffusion
         temp( dg::evaluate(dg::zero, g)), 
         LaplacianM_perp( g, dg::normed, dg::centered){
     }
-    void operator()( std::vector<container>& x, std::vector<container>& y)
+    void operator()( const std::vector<container>& x, std::vector<container>& y)
     {
         /* x[0] := N_e - 1
          * x[2] := N_i - 1 
@@ -38,6 +32,7 @@ struct Diffusion
     }
     dg::Elliptic<Geometry, Matrix, container>& laplacianM() {return LaplacianM_perp;}
     const container& weights(){return LaplacianM_perp.weights();}
+    const container& inv_weights(){return LaplacianM_perp.inv_weights();}
     const container& precond(){return LaplacianM_perp.precond();}
 
   private:
@@ -83,7 +78,7 @@ struct ToeflR
      * @param y input vector
      * @param yp the rhs yp = f(y)
      */
-    void operator()( std::vector<container>& y, std::vector<container>& yp);
+    void operator()( const std::vector<container>& y, std::vector<container>& yp);
 
     /**
      * @brief Return the mass of the last field in operator() in a global computation
@@ -122,14 +117,14 @@ struct ToeflR
     container chi, omega;
     const container binv; //magnetic field
 
+    container gamma_n, potential_;
     std::vector<container> psi, dypsi, ype;
     std::vector<container> dyy, lny, lapy;
-    container gamma_n, potential_;
 
     //matrices and solvers
-    Elliptic<Geometry, Matrix, container> pol, laplaceM; //contains normalized laplacian
-    Helmholtz<Geometry,  Matrix, container> gamma1;
-    ArakawaX< Geometry, Matrix, container> arakawa; 
+    dg::Elliptic<Geometry, Matrix, container> pol, laplaceM; //contains normalized laplacian
+    dg::Helmholtz<Geometry,  Matrix, container> gamma1;
+    dg::ArakawaX< Geometry, Matrix, container> arakawa; 
 
     dg::Invert<container> invert_pol, invert_invgamma;
 
@@ -147,16 +142,16 @@ struct ToeflR
 template< class Geometry, class M, class container>
 ToeflR< Geometry, M, container>::ToeflR( const Geometry& grid, const Parameters& p ): 
     chi( evaluate( dg::zero, grid)), omega(chi),
-    binv( evaluate( LinearX( p.kappa, 1.-p.kappa*p.posX*p.lx), grid)), 
+    binv( evaluate( dg::LinearX( p.kappa, 1.-p.kappa*p.posX*p.lx), grid)), 
     gamma_n(chi), potential_(chi), psi( 2, chi), dypsi( psi), ype(psi),
     dyy(2,chi), lny( dyy), lapy(dyy), 
-    pol(     grid, not_normed, dg::centered), 
-    laplaceM( grid, normed, centered),
+    pol(     grid, dg::not_normed, dg::centered), 
+    laplaceM( grid, dg::normed, dg::centered),
     gamma1(  grid, 0., dg::centered),
     arakawa( grid), 
     invert_pol(      omega, omega.size(), p.eps_pol),
     invert_invgamma( omega, omega.size(), p.eps_gamma),
-    w2d( create::volume(grid)), v2d( create::inv_volume(grid)), one( dg::evaluate(dg::one, grid)),
+    w2d( dg::create::volume(grid)), v2d( dg::create::inv_volume(grid)), one( dg::evaluate(dg::one, grid)),
     eps_pol(p.eps_pol), eps_gamma( p.eps_gamma), kappa(p.kappa), nu(p.nu), debye_( p.debye)
 {
     tau[0] = p.tau[0], tau[1] = p.tau[1];
@@ -193,8 +188,8 @@ const container& ToeflR<G, M, container>::polarisation( const std::vector<contai
         dg::blas1::transfer( y[i], omega);
         dg::blas1::plus( omega, 1.); 
         dg::blas1::scal( omega, z[i]*mu[i]);
-        blas1::pointwiseDot( binv, omega, omega); 
-        blas1::pointwiseDot( binv, omega, omega); //\omega *= binv^2
+        dg::blas1::pointwiseDot( binv, omega, omega); 
+        dg::blas1::pointwiseDot( binv, omega, omega); //\omega *= binv^2
         dg::blas1::axpby( 1., omega, 1, chi);
     }
     pol.set_chi( chi);
@@ -206,7 +201,7 @@ const container& ToeflR<G, M, container>::polarisation( const std::vector<contai
         unsigned number = invert_invgamma( gamma1, gamma_n, y[i]);
         if(  number == invert_invgamma.get_max())
             throw dg::Fail( eps_gamma);
-        blas1::axpby( z[i], gamma_n, 1., omega); 
+        dg::blas1::axpby( z[i], gamma_n, 1., omega); 
     }
     unsigned number = invert_pol( pol, potential_, omega);
     if(  number == invert_pol.get_max())
@@ -215,7 +210,7 @@ const container& ToeflR<G, M, container>::polarisation( const std::vector<contai
 }
 
 template< class G, class M, class container>
-void ToeflR<G, M, container>::operator()( std::vector<container>& y, std::vector<container>& yp)
+void ToeflR<G, M, container>::operator()( const std::vector<container>& y, std::vector<container>& yp)
 {
     //y[0] = N_e - 1
     //y[1] = N_i - 1 || y[1] = Omega
@@ -233,20 +228,20 @@ void ToeflR<G, M, container>::operator()( std::vector<container>& y, std::vector
         dg::blas2::symv( laplaceM, y[i], lapy[i]);
     }
 
-        mass_ = blas2::dot( one, w2d, y[0] ) + blas2::dot( one, w2d, y[1]); //take real ion density which is electron density!!
-        diff_ = nu*( blas2::dot( one, w2d, lapy[0]) + blas2::dot( one, w2d, lapy[1]));
-        double Ue = z[0]*tau[0]*blas2::dot( lny[0], w2d, ype[0]);
-        double Up = z[1]*tau[1]*blas2::dot( lny[1], w2d, ype[1]);
-        double Uphi = 0.5*blas2::dot( ype[0], w2d, omega) + 0.5*blas2::dot( ype[1], w2d, omega); 
+        mass_ = dg::blas2::dot( one, w2d, y[0] ) + dg::blas2::dot( one, w2d, y[1]); //take real ion density which is electron density!!
+        diff_ = nu*( dg::blas2::dot( one, w2d, lapy[0]) + dg::blas2::dot( one, w2d, lapy[1]));
+        double Ue = z[0]*tau[0]*dg::blas2::dot( lny[0], w2d, ype[0]);
+        double Up = z[1]*tau[1]*dg::blas2::dot( lny[1], w2d, ype[1]);
+        double Uphi = 0.5*dg::blas2::dot( ype[0], w2d, omega) + 0.5*dg::blas2::dot( ype[1], w2d, omega); 
         arakawa.variation(potential_, omega); 
         double UE = debye_*dg::blas2::dot( one, w2d, omega);
         energy_ = Ue + Up + Uphi + UE;
         //std::cout << "Ue "<<Ue<< "Up "<<Up<< "Uphi "<<Uphi<< "UE "<<UE<<"\n";
 
-        double Ge = - tau[0]*(blas2::dot( one, w2d, lapy[0]) + blas2::dot( lapy[0], w2d, lny[0])); // minus because of laplace
-        double Gp = - tau[1]*(blas2::dot( one, w2d, lapy[1]) + blas2::dot( lapy[1], w2d, lny[1])); // minus because of laplace
-        double Gpsie = -blas2::dot( psi[0], w2d, lapy[0]);
-        double Gpsip = -blas2::dot( psi[1], w2d, lapy[1]);
+        double Ge = - tau[0]*(dg::blas2::dot( one, w2d, lapy[0]) + dg::blas2::dot( lapy[0], w2d, lny[0])); // minus because of laplace
+        double Gp = - tau[1]*(dg::blas2::dot( one, w2d, lapy[1]) + dg::blas2::dot( lapy[1], w2d, lny[1])); // minus because of laplace
+        double Gpsie = -dg::blas2::dot( psi[0], w2d, lapy[0]);
+        double Gpsip = -dg::blas2::dot( psi[1], w2d, lapy[1]);
         //std::cout << "ge "<<Ge<<" gp "<<Gp<<" gpsie "<<Gpsie<<" gpsip "<<Gpsip<<"\n";
         ediff_ = nu*( z[0]*Ge + z[1]*Gp + z[0]*Gpsie + z[1]*Gpsip);
     }
@@ -255,15 +250,15 @@ void ToeflR<G, M, container>::operator()( std::vector<container>& y, std::vector
     for( unsigned i=0; i<y.size(); i++)
     {
         arakawa( y[i], psi[i], yp[i]);
-        blas1::pointwiseDot( binv, yp[i], yp[i]);
-        blas2::gemv( arakawa.dy(), y[i], dyy[i]);
-        blas2::gemv( arakawa.dy(), psi[i], dypsi[i]);
-        blas1::pointwiseDot( dypsi[i], ype[i], dypsi[i]);
-        blas1::axpby( kappa, dypsi[i], 1., yp[i]);
-        blas1::axpby( tau[i]*kappa, dyy[i], 1., yp[i]);
+        dg::blas1::pointwiseDot( binv, yp[i], yp[i]);
+        dg::blas2::gemv( arakawa.dy(), y[i], dyy[i]);
+        dg::blas2::gemv( arakawa.dy(), psi[i], dypsi[i]);
+        dg::blas1::pointwiseDot( dypsi[i], ype[i], dypsi[i]);
+        dg::blas1::axpby( kappa, dypsi[i], 1., yp[i]);
+        dg::blas1::axpby( tau[i]*kappa, dyy[i], 1., yp[i]);
     }
     return;
 }
 
-}//namespace dg
+}//namespace ep
 
diff --git a/src/ep/toefl_mpi.cu b/src/ep/toefl_mpi.cu
index 510778661..bf32942b0 100644
--- a/src/ep/toefl_mpi.cu
+++ b/src/ep/toefl_mpi.cu
@@ -74,8 +74,8 @@ int main( int argc, char* argv[])
     dg::MPIGrid2d grid( 0, p.lx, 0, p.ly, p.n, p.Nx, p.Ny, p.bc_x, p.bc_y, comm);
     dg::MPIGrid2d grid_out( 0., p.lx, 0.,p.ly, p.n_out, p.Nx_out, p.Ny_out, p.bc_x, p.bc_y, comm);  
     //create RHS 
-    dg::ToeflR< dg::CartesianMPIGrid2d, dg::MDMatrix, dg::MDVec > test( grid, p); 
-    dg::Diffusion<dg::CartesianMPIGrid2d, dg::MDMatrix, dg::MDVec> diffusion( grid, p.nu);
+    ep::ToeflR< dg::CartesianMPIGrid2d, dg::MDMatrix, dg::MDVec > test( grid, p); 
+    ep::Diffusion<dg::CartesianMPIGrid2d, dg::MDMatrix, dg::MDVec> diffusion( grid, p.nu);
     //create initial vector
     dg::Gaussian g( p.posX*p.lx, p.posY*p.ly, p.sigma, p.sigma, p.amp); 
     dg::MDVec gauss = dg::evaluate( g, grid);
@@ -101,10 +101,6 @@ int main( int argc, char* argv[])
     MPI_Info info = MPI_INFO_NULL;
     err = nc_create_par( argv[2],NC_NETCDF4|NC_MPIIO|NC_CLOBBER,comm,info, &ncid);
     err = nc_put_att_text( ncid, NC_GLOBAL, "inputfile", input.size(), input.data());
-    const int version[3] = {FELTOR_MAJOR_VERSION, FELTOR_MINOR_VERSION, FELTOR_SUBMINOR_VERSION}; //write maybe to json file!?
-    err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_major_version", NC_INT, 1, &version[0]);
-    err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_minor_version", NC_INT, 1, &version[1]);
-    err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_subminor_version", NC_INT, 1, &version[2]);
     int dim_ids[3], tvarID;
     err = file::define_dimensions( ncid, dim_ids, &tvarID, grid_out.global());
     //field IDs
@@ -134,7 +130,7 @@ int main( int argc, char* argv[])
     ///////////////////////////////////first output/////////////////////////
     int dims[2],  coords[2];
     MPI_Cart_get( comm, 2, dims, periods, coords);
-    size_t count[3] = {1, grid_out.n()*grid_out.Ny(), grid_out.n()*grid_out.Nx()};
+    size_t count[3] = {1, grid_out.n()*grid_out.local().Ny(), grid_out.n()*grid_out.local().Nx()};
     size_t start[3] = {0, coords[1]*count[1], coords[0]*count[2]};
     size_t Estart[] = {0};
     size_t Ecount[] = {1};
diff --git a/src/feltorSH/feltor.cuh b/src/feltorSH/feltor.cuh
index c62506365..2828e8f7f 100644
--- a/src/feltorSH/feltor.cuh
+++ b/src/feltorSH/feltor.cuh
@@ -32,7 +32,7 @@ struct Rolkar
         LaplacianM_perp ( g,g.bcx(),g.bcy(), dg::normed, dg::centered)
     {
     }
-    void operator()( std::vector<container>& x, std::vector<container>& y)
+    void operator()( const std::vector<container>& x, std::vector<container>& y)
     {
         /* x[0] := N_e - (bgamp+profamp)
            x[1] := N_i - (bgamp+profamp)
@@ -53,6 +53,7 @@ struct Rolkar
     }
     dg::Elliptic<Geometry, Matrix, container>& laplacianM() {return LaplacianM_perp;}
     const container& weights(){return LaplacianM_perp.weights();}
+    const container& inv_weights(){return LaplacianM_perp.inv_weights();}
     const container& precond(){return LaplacianM_perp.precond();}
   private:
     const eule::Parameters p;
@@ -82,7 +83,7 @@ struct Feltor
     void initializene( const container& y, const container& helper, container& target);
     void initializepi( const container& y, const container& helper, container& target);
 
-    void operator()( std::vector<container>& y, std::vector<container>& yp);
+    void operator()( const std::vector<container>& y, std::vector<container>& yp);
 
     double mass( ) {return mass_;}
     double mass_diffusion( ) {return diff_;}
@@ -393,7 +394,7 @@ void Feltor<G, Matrix, container>::initializepi( const container& src, const con
 }
 
 template<class G, class Matrix, class container>
-void Feltor<G, Matrix, container>::operator()( std::vector<container>& y, std::vector<container>& yp)
+void Feltor<G, Matrix, container>::operator()( const std::vector<container>& y, std::vector<container>& yp)
 {
  /* y[0] := N_e - (p.bgprofamp + p.nprofileamp)
        y[1] := N_i - (p.bgprofamp + p.nprofileamp)
@@ -408,7 +409,7 @@ void Feltor<G, Matrix, container>::operator()( std::vector<container>& y, std::v
     if (p.iso == 1)    {
         dg::blas1::scal( y[2], 0.0);
         dg::blas1::scal( y[3], 0.0); 
-    }
+    } //it's not safe to change the input since this is hold by the time integrator, if you really need to do this you could copy y into your own variable and set that to zero (MW)
     
     for(unsigned i=0; i<4; i++)
     {
diff --git a/src/feltorSH/feltor_hpc.cu b/src/feltorSH/feltor_hpc.cu
index 9596b7363..efc0be0ea 100644
--- a/src/feltorSH/feltor_hpc.cu
+++ b/src/feltorSH/feltor_hpc.cu
@@ -105,10 +105,10 @@ int main( int argc, char* argv[])
     int ncid;
     err = nc_create( argv[2],NC_NETCDF4|NC_CLOBBER, &ncid);
     err = nc_put_att_text( ncid, NC_GLOBAL, "inputfile", input.size(), input.data());
-    const int version[3] = {FELTOR_MAJOR_VERSION, FELTOR_MINOR_VERSION, FELTOR_SUBMINOR_VERSION};
-    err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_major_version", NC_INT, 1, &version[0]);
-    err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_minor_version", NC_INT, 1, &version[1]);
-    err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_subminor_version", NC_INT, 1, &version[2]);
+    //const int version[3] = {FELTOR_MAJOR_VERSION, FELTOR_MINOR_VERSION, FELTOR_SUBMINOR_VERSION}; REMOVED (MW)
+    //err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_major_version", NC_INT, 1, &version[0]);
+    //err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_minor_version", NC_INT, 1, &version[1]);
+    //err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_subminor_version", NC_INT, 1, &version[2]);
     int dim_ids[3], tvarID;
     err = file::define_dimensions( ncid, dim_ids, &tvarID, grid_out);
     err = nc_enddef( ncid);
diff --git a/src/feltorSH/feltor_mpi.cu b/src/feltorSH/feltor_mpi.cu
index 00bcc5b9f..d0d243e3e 100644
--- a/src/feltorSH/feltor_mpi.cu
+++ b/src/feltorSH/feltor_mpi.cu
@@ -137,10 +137,10 @@ int main( int argc, char* argv[])
 //     err = nc_create( argv[2],NC_NETCDF4|NC_CLOBBER, &ncid);//MPI OFF
     err = nc_create_par( argv[2], NC_NETCDF4|NC_MPIIO|NC_CLOBBER, comm, info, &ncid); //MPI ON
     err = nc_put_att_text( ncid, NC_GLOBAL, "inputfile", input.size(), input.data());
-    const int version[3] = {FELTOR_MAJOR_VERSION, FELTOR_MINOR_VERSION, FELTOR_SUBMINOR_VERSION};
-    err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_major_version", NC_INT, 1, &version[0]);
-    err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_minor_version", NC_INT, 1, &version[1]);
-    err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_subminor_version", NC_INT, 1, &version[2]);
+    //const int version[3] = {FELTOR_MAJOR_VERSION, FELTOR_MINOR_VERSION, FELTOR_SUBMINOR_VERSION};
+    //err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_major_version", NC_INT, 1, &version[0]);
+    //err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_minor_version", NC_INT, 1, &version[1]);
+    //err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_subminor_version", NC_INT, 1, &version[2]);
     int dim_ids[3], tvarID;
     dg::Grid2d global_grid_out ( 0., p.lx, 0.,p.ly, p.n_out, p.Nx_out, p.Ny_out, p.bc_x, p.bc_y);  
     err = file::define_dimensions( ncid, dim_ids, &tvarID, global_grid_out);
@@ -182,7 +182,7 @@ int main( int argc, char* argv[])
     if(rank==0) std::cout << "First output ... \n";
     int dims[2],  coords[2];
     MPI_Cart_get( comm, 2, dims, periods, coords);
-    size_t count[3] = {1, grid_out.n()*grid_out.Ny(), grid_out.n()*grid_out.Nx()};
+    size_t count[3] = {1, grid_out.n()*grid_out.local().Ny(), grid_out.n()*grid_out.local().Nx()};
     size_t start[3] = {0, coords[1]*count[1],          coords[0]*count[2]}; 
     dg::MDVec transfer( dg::evaluate(dg::zero, grid));
     dg::DVec transferD( dg::evaluate(dg::zero, grid_out.local()));
diff --git a/src/feltorSHp/feltor.cuh b/src/feltorSHp/feltor.cuh
index f35b26d2a..06509f434 100644
--- a/src/feltorSHp/feltor.cuh
+++ b/src/feltorSHp/feltor.cuh
@@ -38,7 +38,7 @@ struct Rolkar
         LaplacianM_perp ( g,g.bcx(),g.bcy(), dg::normed, dg::centered)
     {
     }
-    void operator()( std::vector<container>& x, std::vector<container>& y)
+    void operator()( const std::vector<container>& x, std::vector<container>& y)
     {
         /* x[0] := n_e - (bgamp+profamp)
            x[1] := N_i - (bgamp+profamp)
@@ -58,6 +58,7 @@ struct Rolkar
     }
     dg::Elliptic<Geometry, Matrix, container>& laplacianM() {return LaplacianM_perp;}
     const container& weights(){return LaplacianM_perp.weights();}
+    const container& inv_weights(){return LaplacianM_perp.inv_weights();}
     const container& precond(){return LaplacianM_perp.precond();}
   private:
     const eule::Parameters p;
@@ -104,7 +105,7 @@ struct Feltor
      */
     void initializepi( const container& y, const container& helper, container& target);
 
-    void operator()( std::vector<container>& y, std::vector<container>& yp);
+    void operator()( const std::vector<container>& y, std::vector<container>& yp);
 
     double mass( ) {return mass_;}
     double mass_diffusion( ) {return diff_;}
@@ -413,7 +414,7 @@ void Feltor<G, Matrix, container>::initializepi( const container& src, const con
 }
 
 template<class G, class Matrix, class container>
-void Feltor<G, Matrix, container>::operator()( std::vector<container>& y, std::vector<container>& yp)
+void Feltor<G, Matrix, container>::operator()( const std::vector<container>& y, std::vector<container>& yp)
 {
    /* y[0] := N_e - (p.bgprofamp + p.nprofileamp)
        y[1] := N_i - (p.bgprofamp + p.nprofileamp)
@@ -428,7 +429,7 @@ void Feltor<G, Matrix, container>::operator()( std::vector<container>& y, std::v
     if (p.iso == 1)    {
         dg::blas1::axpby((p.bgprofamp + p.nprofileamp),y[0],0., y[2],y[2] );
         dg::blas1::axpby((p.bgprofamp + p.nprofileamp),y[1],0., y[3],y[3] ); 
-    }
+    } //see same place in feltorSH  (MW)
     
     for(unsigned i=0; i<2; i++)
     {
diff --git a/src/feltorSHp/feltor_hpc.cu b/src/feltorSHp/feltor_hpc.cu
index 3a6e7c166..0c5b60ab3 100644
--- a/src/feltorSHp/feltor_hpc.cu
+++ b/src/feltorSHp/feltor_hpc.cu
@@ -100,10 +100,10 @@ int main( int argc, char* argv[])
     int ncid;
     err = nc_create( argv[2],NC_NETCDF4|NC_CLOBBER, &ncid);
     err = nc_put_att_text( ncid, NC_GLOBAL, "inputfile", input.size(), input.data());
-    const int version[3] = {FELTOR_MAJOR_VERSION, FELTOR_MINOR_VERSION, FELTOR_SUBMINOR_VERSION};
-    err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_major_version", NC_INT, 1, &version[0]);
-    err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_minor_version", NC_INT, 1, &version[1]);
-    err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_subminor_version", NC_INT, 1, &version[2]);
+    //const int version[3] = {FELTOR_MAJOR_VERSION, FELTOR_MINOR_VERSION, FELTOR_SUBMINOR_VERSION}; REMOVED (MW)
+    //err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_major_version", NC_INT, 1, &version[0]);
+    //err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_minor_version", NC_INT, 1, &version[1]);
+    //err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_subminor_version", NC_INT, 1, &version[2]);
     int dim_ids[3], tvarID;
     err = file::define_dimensions( ncid, dim_ids, &tvarID, grid_out);
     err = nc_enddef( ncid);
diff --git a/src/feltorSHp/feltor_mpi.cu b/src/feltorSHp/feltor_mpi.cu
index 1a145e73c..65d216bec 100644
--- a/src/feltorSHp/feltor_mpi.cu
+++ b/src/feltorSHp/feltor_mpi.cu
@@ -130,10 +130,10 @@ int main( int argc, char* argv[])
 //     err = nc_create( argv[2],NC_NETCDF4|NC_CLOBBER, &ncid);//MPI OFF
     err = nc_create_par( argv[2], NC_NETCDF4|NC_MPIIO|NC_CLOBBER, comm, info, &ncid); //MPI ON
     err = nc_put_att_text( ncid, NC_GLOBAL, "inputfile", input.size(), input.data());
-    const int version[3] = {FELTOR_MAJOR_VERSION, FELTOR_MINOR_VERSION, FELTOR_SUBMINOR_VERSION};
-    err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_major_version", NC_INT, 1, &version[0]);
-    err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_minor_version", NC_INT, 1, &version[1]);
-    err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_subminor_version", NC_INT, 1, &version[2]);
+    //const int version[3] = {FELTOR_MAJOR_VERSION, FELTOR_MINOR_VERSION, FELTOR_SUBMINOR_VERSION};
+    //err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_major_version", NC_INT, 1, &version[0]);
+    //err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_minor_version", NC_INT, 1, &version[1]);
+    //err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_subminor_version", NC_INT, 1, &version[2]);
     int dim_ids[3], tvarID;
     dg::Grid2d global_grid_out ( 0., p.lx, 0.,p.ly, p.n_out, p.Nx_out, p.Ny_out, p.bc_x, p.bc_y);  
     err = file::define_dimensions( ncid, dim_ids, &tvarID, global_grid_out);
@@ -175,7 +175,7 @@ int main( int argc, char* argv[])
     if(rank==0) std::cout << "First output ... \n";
     int dims[2],  coords[2];
     MPI_Cart_get( comm, 2, dims, periods, coords);
-    size_t count[3] = {1, grid_out.n()*grid_out.Ny(), grid_out.n()*grid_out.Nx()};
+    size_t count[3] = {1, grid_out.n()*grid_out.local().Ny(), grid_out.n()*grid_out.local().Nx()};
     size_t start[3] = {0, coords[1]*count[1],          coords[0]*count[2]}; 
     dg::MDVec transfer( dg::evaluate(dg::zero, grid));
     dg::DVec transferD( dg::evaluate(dg::zero, grid_out.local()));
diff --git a/src/feltorSesol/feltor.cuh b/src/feltorSesol/feltor.cuh
index 3677d7d70..62983d18c 100644
--- a/src/feltorSesol/feltor.cuh
+++ b/src/feltorSesol/feltor.cuh
@@ -35,7 +35,7 @@ struct Rolkar
         LaplacianM_perp_phi ( g,p.bc_x_phi,g.bcy(), dg::normed, dg::centered)
     {
     }
-    void operator()( std::vector<container>& x, std::vector<container>& y)
+    void operator()( const std::vector<container>& x, std::vector<container>& y)
     {
         /* x[0] := N_e - (bgamp+profamp)
            x[1] := N_i - (bgamp+profamp)
@@ -51,6 +51,7 @@ struct Rolkar
     }
     dg::Elliptic<Geometry, Matrix, container>& laplacianM() {return LaplacianM_perp_phi;}
     const container& weights(){return LaplacianM_perp.weights();}
+    const container& inv_weights(){return LaplacianM_perp.inv_weights();}
     const container& precond(){return LaplacianM_perp.precond();}
   private:
     const eule::Parameters p;
@@ -80,7 +81,7 @@ struct Feltor
     const std::vector<container>& potential( ) const { return phi;}
     void initializene( const container& y, container& target);
 
-    void operator()( std::vector<container>& y, std::vector<container>& yp);
+    void operator()( const std::vector<container>& y, std::vector<container>& yp);
 
     double mass( ) {return mass_;}
     double mass_diffusion( ) {return diff_;}
@@ -195,7 +196,7 @@ void Feltor<G, Matrix, container>::initializene( const container& src, container
 }
 
 template<class G, class Matrix, class container>
-void Feltor<G, Matrix, container>::operator()( std::vector<container>& y, std::vector<container>& yp)
+void Feltor<G, Matrix, container>::operator()( const std::vector<container>& y, std::vector<container>& yp)
 {
     /* y[0] := N_e - (p.bgprofamp + p.nprofileamp)
        y[1] := N_i - (p.bgprofamp + p.nprofileamp)
diff --git a/src/feltorSesol/feltor_mpi.cu b/src/feltorSesol/feltor_mpi.cu
index c21b1410a..5b60c6669 100644
--- a/src/feltorSesol/feltor_mpi.cu
+++ b/src/feltorSesol/feltor_mpi.cu
@@ -227,7 +227,7 @@ int main( int argc, char* argv[])
     if(rank==0) std::cout << "First output ... \n";
     int dims[2],  coords[2];
     MPI_Cart_get( comm, 2, dims, periods, coords);
-    size_t count[3] = {1, grid_out.n()*grid_out.Ny(), grid_out.n()*grid_out.Nx()};  
+    size_t count[3] = {1, grid_out.n()*grid_out.local().Ny(), grid_out.n()*grid_out.local().Nx()};  
     size_t start[3] = {0, coords[1]*count[1],          coords[0]*count[2]}; 
     dg::MDVec transfer( dg::evaluate(dg::zero, grid));
     dg::DVec transferD( dg::evaluate(dg::zero, grid_out.local()));
diff --git a/src/polar/polar_mpi.cu b/src/polar/polar_mpi.cu
index 4908d368b..31045e11f 100644
--- a/src/polar/polar_mpi.cu
+++ b/src/polar/polar_mpi.cu
@@ -6,15 +6,7 @@
 #include <thrust/remove.h>
 #include <thrust/host_vector.h>
 
-#include "dg/backend/timer.cuh"
-#include "dg/backend/evaluation.cuh"
-#include "dg/backend/xspacelib.cuh"
-#include "dg/runge_kutta.h"
-#include "dg/multistep.h"
-#include "dg/helmholtz.h"
-#include "dg/backend/typedefs.cuh"
-#include "dg/functors.h"
-
+#include "dg/algorithm.h"
 #include "geometries/geometries.h"
 
 #include "ns.h"
-- 
GitLab


From f9d4add0314df5cf11481b86b4656d93ed9deb01 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Fri, 3 Nov 2017 14:25:52 +0100
Subject: [PATCH 437/453] debugged asela

---
 src/asela/asela.cu     |  30 +++-----
 src/asela/asela.cuh    | 171 +++++++++++++++--------------------------
 src/asela/asela_hpc.cu |  23 +++---
 src/asela/asela_mpi.cu |  22 +++---
 src/asela/parameters.h |   2 +-
 5 files changed, 95 insertions(+), 153 deletions(-)

diff --git a/src/asela/asela.cu b/src/asela/asela.cu
index 59e3cf4e4..b048f9a15 100644
--- a/src/asela/asela.cu
+++ b/src/asela/asela.cu
@@ -7,11 +7,7 @@
 
 #include "draw/host_window.h"
 //#include "draw/device_window.cuh"
-#include "dg/backend/xspacelib.cuh"
-#include "dg/backend/sparseblockmat.cuh"
-#include "dg/backend/timer.cuh"
-#include "dg/backend/average.cuh"
-#include "dg/backend/typedefs.cuh"
+#include "dg/algorithm.h"
 #include "geometries/geometries.h"
 
 #include "asela.cuh"
@@ -23,7 +19,6 @@
    - integrates the ToeflR - functor and 
    - directly visualizes results on the screen using parameters in window_params.txt
 */
-typedef dg::FieldAligned< dg::CylindricalGrid3d, dg::IDMatrix, dg::DVec> DFA;
 
 int main( int argc, char* argv[])
 {
@@ -70,14 +65,14 @@ int main( int argc, char* argv[])
 
     dg::CylindricalGrid3d grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n, p.Nx, p.Ny, p.Nz, p.bc, p.bc, dg::PER);      //create RHS 
     std::cout << "Constructing Asela...\n";
-    asela::Asela<dg::CylindricalGrid3d, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec> asela( grid, p,gp); //initialize before rolkar!
+    asela::Asela<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec> asela( grid, p,gp); //initialize before rolkar!
     std::cout << "Constructing Implicit...\n";
-    asela::Implicit<dg::CylindricalGrid3d, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec> rolkar(  grid, p, gp, asela.ds(), asela.dsDIR());
+    asela::Implicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec> rolkar(  grid, p, gp, asela.ds(), asela.dsDIR());
     std::cout << "Done!\n";
 
    /////////////////////The initial field///////////////////////////////////////////
     //background profile
-    dg::geo::Nprofile<Psip> prof(p.bgprofamp, p.nprofileamp, gp, Psip(gp)); //initial background profile
+    dg::geo::Nprofile prof(p.bgprofamp, p.nprofileamp, gp, dg::geo::solovev::Psip(gp)); //initial background profile
     std::vector<dg::DVec> y0(4, dg::evaluate( prof, grid)), y1(y0); 
     //perturbation 
     dg::GaussianZ gaussianZ( 0., p.sigma_z*M_PI, 1); //modulation along fieldline
@@ -96,14 +91,14 @@ int main( int argc, char* argv[])
     }
     if( p.mode == 3)
     {
-        dg::geo::ZonalFlow<Psip> init0(p.amp, p.k_psi, gp, Psip(gp));
+        dg::geo::ZonalFlow init0(p.amp, p.k_psi, gp, dg::geo::solovev::Psip(gp));
         y1[1] = asela.ds().fieldaligned().evaluate( init0, gaussianZ, (unsigned)p.Nz/2, 1); 
     }
     dg::blas1::axpby( 1., y1[1], 1., y0[1]); //sum up background and perturbation
     dg::blas1::plus(y0[1], -1); //initialize ni-1
     if( p.mode == 2 || p.mode == 3)
     {
-        dg::DVec damping = dg::evaluate( dg::geo::GaussianProfXDamping<Psip>(Psip(gp), gp), grid);
+        dg::DVec damping = dg::evaluate( dg::geo::GaussianProfXDamping(dg::geo::solovev::Psip(gp), gp), grid);
         dg::blas1::pointwiseDot(damping,y0[1], y0[1]); //damp with gaussprofdamp
     }
     std::cout << "intiialize ne" << std::endl;
@@ -124,7 +119,6 @@ int main( int argc, char* argv[])
     dg::HVec hvisual( grid.size(), 0.), visual(hvisual),avisual(hvisual);
     dg::IHMatrix equi = dg::create::backscatter( grid);
     draw::ColorMapRedBlueExtMinMax colors(-1.0, 1.0);
-    dg::ToroidalAverage<dg::HVec> toravg(grid);
 
     //create timer
     dg::Timer t;
@@ -151,7 +145,7 @@ int main( int argc, char* argv[])
             render.renderQuad( part, grid.n()*grid.Nx(), grid.n()*grid.Ny(), colors);
         }
         dg::blas1::axpby(0.0,avisual,0.0,avisual);
-        toravg(visual,avisual);
+        dg::toroidal_average(visual,avisual,grid);
         render.renderQuad( avisual, grid.n()*grid.Nx(), grid.n()*grid.Ny(), colors);
         //draw ions
         dg::blas1::transfer( y0[1], hvisual);
@@ -167,7 +161,7 @@ int main( int argc, char* argv[])
             render.renderQuad( part, grid.n()*grid.Nx(), grid.n()*grid.Ny(), colors);
         }
         dg::blas1::axpby(0.0,avisual,0.0,avisual);
-        toravg(visual,avisual);
+        dg::toroidal_average(visual,avisual,grid);
         render.renderQuad( avisual, grid.n()*grid.Nx(), grid.n()*grid.Ny(), colors);
         //draw Potential
         dg::blas1::transfer( asela.potential()[0], hvisual);
@@ -187,7 +181,7 @@ int main( int argc, char* argv[])
             render.renderQuad( part, grid.n()*grid.Nx(), grid.n()*grid.Ny(), colors);
         }
         dg::blas1::axpby(0.0,avisual,0.0,avisual);
-        toravg(visual,avisual);
+        dg::toroidal_average(visual,avisual,grid);
         render.renderQuad( avisual, grid.n()*grid.Nx(), grid.n()*grid.Ny(), colors);
 
         //draw U_e
@@ -203,7 +197,7 @@ int main( int argc, char* argv[])
             render.renderQuad( part, grid.n()*grid.Nx(), grid.n()*grid.Ny(), colors);
         }
         dg::blas1::axpby(0.0,avisual,0.0,avisual);
-        toravg(visual,avisual);
+        dg::toroidal_average(visual,avisual,grid);
         render.renderQuad( avisual, grid.n()*grid.Nx(), grid.n()*grid.Ny(), colors);
         //draw U_i
         dg::blas1::transfer( asela.uparallel()[1], hvisual);
@@ -218,7 +212,7 @@ int main( int argc, char* argv[])
             render.renderQuad( part, grid.n()*grid.Nx(), grid.n()*grid.Ny(), colors);
         }
         dg::blas1::axpby(0.0,avisual,0.0,avisual);
-        toravg(visual,avisual);
+        dg::toroidal_average(visual,avisual,grid);
         render.renderQuad( avisual, grid.n()*grid.Nx(), grid.n()*grid.Ny(), colors);
         //draw a parallel
         dg::blas1::transfer(asela.aparallel(), hvisual);
@@ -233,7 +227,7 @@ int main( int argc, char* argv[])
             render.renderQuad( part, grid.n()*grid.Nx(), grid.n()*grid.Ny(), colors);
         }
         dg::blas1::axpby(0.0,avisual,0.0,avisual);
-        toravg(visual,avisual);
+        dg::toroidal_average(visual,avisual,grid);
         render.renderQuad( avisual, grid.n()*grid.Nx(), grid.n()*grid.Ny(), colors);
  
         
diff --git a/src/asela/asela.cuh b/src/asela/asela.cuh
index bd8c51bd1..73c457237 100644
--- a/src/asela/asela.cuh
+++ b/src/asela/asela.cuh
@@ -1,15 +1,9 @@
 #pragma once
 
 #include "dg/algorithm.h"
-#include "dg/poisson.h"
 #include "parameters.h"
-
 #include "geometries/geometries.h"
 
-#ifdef DG_BENCHMARK
-#include "dg/backend/timer.cuh"
-#endif //DG_BENCHMARK
-
 // #define APAR
 namespace asela
 {
@@ -30,7 +24,7 @@ namespace asela
  * @tparam container The Vector class 
  * @tparam container The container class
  */
-template<class Geometry, class DS, class Matrix, class container>
+template<class Geometry, class IMatrix, class Matrix, class container>
 struct Implicit
 {
         /**
@@ -41,7 +35,7 @@ struct Implicit
      * @param p the physics parameters
      * @param gp the geometry parameters
      */
-    Implicit( const Geometry& g, eule::Parameters p, dg::geo::solovev::Parameters gp, DS& dsN, DS& dsDIR):
+    Implicit( const Geometry& g, asela::Parameters p, dg::geo::solovev::Parameters gp, dg::geo::DS<Geometry, IMatrix, Matrix, container>& dsN, dg::geo::DS<Geometry, IMatrix, Matrix,  container>& dsDIR):
         p(p),
         gp(gp),
         LaplacianM_perpN  ( g, g.bcx(), g.bcy(), dg::normed, dg::centered),
@@ -51,7 +45,7 @@ struct Implicit
     {
         using dg::geo::solovev::Psip;
         dg::blas1::transfer( dg::evaluate( dg::zero, g), temp);
-        dg::blas1::transfer( dg::pullback( dg::geo::GaussianDamping<Psip>(Psip(gp), gp.psipmaxcut, gp.alpha), g), dampgauss_);
+        dg::blas1::transfer( dg::pullback( dg::geo::GaussianDamping(Psip(gp), gp.psipmaxcut, gp.alpha), g), dampgauss_);
     }
         /**
      * @brief Return implicit terms
@@ -59,7 +53,7 @@ struct Implicit
      * @param x input vector (x[0] := N_e -1, x[1] := N_i-1, x[2] := w_e, x[3] = w_i)
      * @param y output vector
      */
-    void operator()( std::vector<container>& x, std::vector<container>& y)
+    void operator()( const std::vector<container>& x, std::vector<container>& y)
     {
         /* x[0] := N_e - 1
            x[1] := N_i - 1
@@ -100,30 +94,16 @@ struct Implicit
      */
     dg::Elliptic<Geometry, Matrix, container>& laplacianM() {return LaplacianM_perpDIR;}
 
-    /**
-     * @brief Model function for Inversion
-     *
-     * @return weights for the inversion function in
-     */
     const container& weights(){return LaplacianM_perpDIR.weights();}
-    /**
-     * @brief Model function for Inversion
-     *
-     * @return preconditioner for the inversion function in
-     */
+    const container& inv_weights(){return LaplacianM_perpDIR.inv_weights();}
     const container& precond(){return LaplacianM_perpDIR.precond();}
-    /**
-     * @brief Damping used in the diffusion equations
-     *
-     * @return Vector containing damping 
-     */
   private:
-    const eule::Parameters p;
+    const asela::Parameters p;
     const dg::geo::solovev::Parameters gp;
     container temp;
     container dampgauss_;
     dg::Elliptic<Geometry, Matrix, container> LaplacianM_perpN,LaplacianM_perpDIR;
-    DS& dsN_,dsDIR_;
+    dg::geo::DS<Geometry, IMatrix, Matrix, container> dsN_,dsDIR_;
 
 };
 
@@ -134,7 +114,7 @@ struct Implicit
  * @tparam container main container to hold the vectors
  * @tparam container class of the weights
  */
-template< class Geometry, class DS, class Matrix, class container >
+template< class Geometry, class IMatrix, class Matrix, class container >
 struct Asela
 {
     /**
@@ -145,15 +125,10 @@ struct Asela
      * @param p the physics parameters
      * @param gp the geometry parameters
      */
-    Asela( const Geometry& g, eule::Parameters p, dg::geo::solovev::Parameters gp);
+    Asela( const Geometry& g, asela::Parameters p, dg::geo::solovev::Parameters gp);
+    dg::geo::DS<Geometry, IMatrix, Matrix, container>& ds(){return dsN_;}
+    dg::geo::DS<Geometry, IMatrix, Matrix, container>& dsDIR(){return dsDIR_;}
     /**
-     * @brief Return a ds class for evaluation purposes
-     *
-     * @return 
-     */
-    DS& ds(){return dsN_;}
-    DS& dsDIR(){return dsDIR_;}
-        /**
      * @brief Returns phi and psi that belong to the last solve of the polarization equation
      *
      * In a multistep scheme this corresponds to the point HEAD-1
@@ -187,7 +162,7 @@ struct Asela
      * @param y y[0] := N_e - 1, y[1] := N_i - 1, y[2] := w_e, y[3] := w_i
      * @param yp Result
      */
-    void operator()( std::vector<container>& y, std::vector<container>& yp);
+    void operator()( const std::vector<container>& y, std::vector<container>& yp);
     
     /**
      * @brief \f[ M := \int_V (n_e-1) dV \f]
@@ -245,13 +220,13 @@ struct Asela
     double fieldalignment() { return aligned_;}
     
   private:
-    void vecdotnablaN(const container& x, const container& y, container& z, container& target);
-    void vecdotnablaDIR(const container& x, const container& y, container& z, container& target);
+    void vecdotnablaN(const container& x, const container& y, const container& z, container& target);
+    void vecdotnablaDIR(const container& x, const container& y, const container& z, container& target);
     //extrapolates and solves for phi[1], then adds square velocity ( omega)
     container& compute_psi( container& potential);
     container& polarisation( const std::vector<container>& y); //solves polarisation equation
     container& induct(const std::vector<container>& y);//solves induction equation
-    double add_parallel_dynamics( std::vector<container>& y, std::vector<container>& yp);
+    double add_parallel_dynamics( const std::vector<container>& y, std::vector<container>& yp);
 
     container chi, omega,lambda;//1d container
 
@@ -265,13 +240,13 @@ struct Asela
     std::vector<container> dsy, curvy,curvkappay;  //4d container
 
     //matrices and solvers
-    DS dsDIR_,dsN_;
+    dg::geo::DS<Geometry, IMatrix, Matrix, container> dsDIR_,dsN_;
     dg::Poisson< Geometry, Matrix, container > poissonN,poissonDIR; 
     dg::Elliptic<  Geometry, Matrix, container  > pol,lapperpN,lapperpDIR; //note the host vector
     dg::Helmholtz< Geometry, Matrix, container  > maxwell, invgammaDIR, invgammaN;
     dg::Invert<container> invert_maxwell, invert_pol, invert_invgammaN,invert_invgammaNW,invert_invgammaA, invert_invgammaPhi;
 
-    const eule::Parameters p;
+    const asela::Parameters p;
     const dg::geo::solovev::Parameters gp;
 
     double mass_, energy_, diff_, ediff_, aligned_;
@@ -280,35 +255,10 @@ struct Asela
 };
 ///@}
 
-template<class Grid, class DS, class Matrix, class container>
-Asela<Grid, DS, Matrix, container>::Asela( const Grid& g, Parameters p, dg::geo::solovev::Parameters gp): 
-    dsDIR_( typename DS::FieldAligned( 
-                dg::geo::Field<dg::geo::solovev::MagneticField>(
-                    dg::geo::solovev::MagneticField(gp), gp.R_0
-                    ), 
-                g, gp.rk4eps, 
-                dg::geo::PsiLimiter<dg::geo::solovev::Psip>(
-                    dg::geo::solovev::Psip(gp), gp.psipmaxlim
-                    ), 
-                dg::DIR, (2*M_PI)/((double)p.Nz)
-                ), 
-            dg::geo::Field<dg::geo::solovev::MagneticField>(
-                dg::geo::solovev::MagneticField(gp), gp.R_0
-                ), 
-            dg::normed, dg::forward ),
-    dsN_( typename DS::FieldAligned(
-                dg::geo::Field<dg::geo::solovev::MagneticField>(
-                    dg::geo::solovev::MagneticField(gp), gp.R_0), 
-                g, gp.rk4eps, 
-                dg::geo::PsiLimiter<dg::geo::solovev::Psip>(
-                    dg::geo::solovev::Psip(gp), gp.psipmaxlim
-                    ), 
-                g.bcx(), (2*M_PI)/((double)p.Nz)
-                ), 
-          dg::geo::Field<dg::geo::solovev::MagneticField>(
-              dg::geo::solovev::MagneticField(gp), gp.R_0
-              ), 
-          dg::normed, dg::forward ),
+template<class Grid, class IMatrix, class Matrix, class container>
+Asela<Grid, IMatrix, Matrix, container>::Asela( const Grid& g, Parameters p, dg::geo::solovev::Parameters gp): 
+    dsDIR_( dg::geo::createSolovevField(gp), g, dg::DIR, dg::DIR, dg::geo::PsiLimiter( dg::geo::solovev::Psip(gp), gp.psipmaxlim), dg::normed, dg::forward, gp.rk4eps),
+    dsN_( dg::geo::createSolovevField(gp), g, dg::NEU, dg::NEU, dg::geo::PsiLimiter( dg::geo::solovev::Psip(gp), gp.psipmaxlim), dg::normed, dg::forward, gp.rk4eps),
     //////////the poisson operators ////////////////////////////////////////
     poissonN(g, g.bcx(), g.bcy(), dg::DIR, dg::DIR), //first N/U then phi BCC
     poissonDIR(g, dg::DIR, dg::DIR, dg::DIR, dg::DIR), //first N/U then phi BCC
@@ -341,18 +291,17 @@ Asela<Grid, DS, Matrix, container>::Asela( const Grid& g, Parameters p, dg::geo:
     invert_invgammaA.construct(   omega, p.Nx*p.Ny*p.Nz*p.n*p.n, p.eps_gamma); 
     invert_invgammaPhi.construct( omega, p.Nx*p.Ny*p.Nz*p.n*p.n, p.eps_gamma); 
     //////////////////////////////init fields /////////////////////
-    using namespace dg::geo::solovev;
-    MagneticField mf(gp);
-    dg::blas1::transfer(  dg::pullback(dg::geo::Field<MagneticField>(mf, gp.R_0),                     g), binv);
-    dg::blas1::transfer(  dg::pullback(dg::geo::GradLnB<MagneticField>(mf, gp.R_0),                   g), dslnB);
-    dg::blas1::transfer(  dg::pullback(dg::geo::TanhSource<Psip>(mf.psip, gp.psipmin, gp.alpha),      g), source);
-    dg::blas1::transfer(  dg::pullback(dg::geo::GaussianDamping<Psip>(mf.psip, gp.psipmax, gp.alpha), g), damping);
+    dg::geo::TokamakMagneticField mf = dg::geo::createSolovevField(gp);
+    dg::blas1::transfer(  dg::pullback(dg::geo::InvB(mf),            g), binv);
+    dg::blas1::transfer(  dg::pullback(dg::geo::GradLnB(mf),         g), dslnB);
+    dg::blas1::transfer(  dg::pullback(dg::geo::TanhSource(mf.psip(), gp.psipmin, gp.alpha),      g), source);
+    dg::blas1::transfer(  dg::pullback(dg::geo::GaussianDamping(mf.psip(), gp.psipmax, gp.alpha), g), damping);
     ////////////////////////////transform curvature components////////
-    dg::geo::pushForwardPerp(dg::geo::CurvatureNablaBR<MagneticField>(mf, gp.R_0), dg::geo::CurvatureNablaBZ<MagneticField>(mf, gp.R_0), curvX, curvY, g);
-    dg::blas1::transfer(  dg::pullback(dg::geo::DivCurvatureKappa<MagneticField>(mf, gp.R_0), g), divCurvKappa);
+    dg::pushForwardPerp(dg::geo::CurvatureNablaBR(mf), dg::geo::CurvatureNablaBZ(mf), curvX, curvY, g);
+    dg::blas1::transfer(  dg::pullback(dg::geo::DivCurvatureKappa(mf), g), divCurvKappa);
     if (p.curvmode==1) 
     {
-        dg::geo::pushForwardPerp(dg::geo::CurvatureKappaR(), dg::geo::CurvatureKappaZ<MagneticField>(mf, gp.R_0), curvKappaX, curvKappaY, g);
+        dg::pushForwardPerp(dg::geo::CurvatureKappaR(), dg::geo::CurvatureKappaZ(mf), curvKappaX, curvKappaY, g);
         dg::blas1::axpby( 1.,curvX,1.,curvKappaX,curvX);
         dg::blas1::axpby( 1.,curvY,1.,curvKappaY,curvY);
     }
@@ -365,7 +314,7 @@ Asela<Grid, DS, Matrix, container>::Asela( const Grid& g, Parameters p, dg::geo:
         dg::blas1::scal(divCurvKappa,0.);
     }
     ///////////////////init densities//////////////////////////////
-    dg::blas1::transfer( dg::pullback(dg::geo::Nprofile<Psip>(p.bgprofamp, p.nprofileamp, gp, mf.psip),g), profne);
+    dg::blas1::transfer( dg::pullback(dg::geo::Nprofile(p.bgprofamp, p.nprofileamp, gp, mf.psip()),g), profne);
     dg::blas1::transfer(  profne ,profNi);
     dg::blas1::plus( profNi, -1); 
     initializene(profNi, profne); //ne = Gamma N_i (needs Invert object)
@@ -379,8 +328,8 @@ Asela<Grid, DS, Matrix, container>::Asela( const Grid& g, Parameters p, dg::geo:
 
 
 //computes and modifies expy!!
-template<class Geometry, class DS, class Matrix, class container>
-container& Asela<Geometry, DS, Matrix, container>::polarisation( const std::vector<container>& y)
+template<class Geometry, class IMatrix, class Matrix, class container>
+container& Asela<Geometry, IMatrix, Matrix, container>::polarisation( const std::vector<container>& y)
 {
     dg::blas1::axpby( p.mu[1], y[1], 0, chi);        //chi =  \mu_i (n_i-1) 
     dg::blas1::plus( chi, p.mu[1]);
@@ -397,8 +346,8 @@ container& Asela<Geometry, DS, Matrix, container>::polarisation( const std::vect
     return phi[0];
 }
 
-template<class Geometry, class DS, class Matrix, class container>
-container& Asela<Geometry, DS, Matrix, container>::induct(const std::vector<container>& y)
+template<class Geometry, class IMatrix, class Matrix, class container>
+container& Asela<Geometry, IMatrix, Matrix, container>::induct(const std::vector<container>& y)
 {
     if (p.flrmode == 0)
     {
@@ -432,8 +381,8 @@ container& Asela<Geometry, DS, Matrix, container>::induct(const std::vector<cont
     }
     return apar[0];
 }
-template<class Geometry, class DS, class Matrix, class container>
-container& Asela<Geometry, DS, Matrix,container>::compute_psi( container& potential)
+template<class Geometry, class IMatrix, class Matrix, class container>
+container& Asela<Geometry, IMatrix, Matrix,container>::compute_psi( container& potential)
 {
     invert_invgammaPhi(invgammaDIR,chi,potential);                    //chi  Gamma phi
     poissonN.variationRHS(potential, omega);
@@ -443,14 +392,14 @@ container& Asela<Geometry, DS, Matrix,container>::compute_psi( container& potent
     return phi[1];    
 }
 
-template<class Geometry, class DS, class Matrix, class container>
-void Asela<Geometry, DS, Matrix, container>::initializene( const container& src, container& target)
+template<class Geometry, class IMatrix, class Matrix, class container>
+void Asela<Geometry, IMatrix, Matrix, container>::initializene( const container& src, container& target)
 { 
     invert_invgammaN(invgammaN,target,src); //=ne-1 = Gamma (ni-1)    
 }
 
-template<class G, class DS, class M, class V>
-double Asela<G, DS, M, V>::add_parallel_dynamics( std::vector<V>& y, std::vector<V>& yp)
+template<class G, class IMatrix, class M, class V>
+double Asela<G, IMatrix, M, V>::add_parallel_dynamics(const  std::vector<V>& y, std::vector<V>& yp)
 {
     double z[2]     = {-1.0,1.0};
     double Dpar[4]  = {0.0, 0.0,0.0,0.0};
@@ -523,10 +472,14 @@ double Asela<G, DS, M, V>::add_parallel_dynamics( std::vector<V>& y, std::vector
         if (p.pardiss==1)
         {
             dsN_.forward( y[i], omega); 
-            dsN_.forwardTD(omega,lambda);
-            dg::blas1::axpby( 0.5*nu_parallel[i], lambda, 0., lambda,lambda);  //lambda = 0.5 nu_parallel ds^2_f N
+            dg::blas1::pointwiseDot( omega, binv, omega);
+            dsN_.backwardDiv(omega,lambda);
+            dg::blas1::pointwiseDivide( lambda, binv, lambda);
+            dg::blas1::scal( lambda, 0.5*nu_parallel[i]);  //lambda = 0.5 nu_parallel ds^2_f N
             dsN_.backward( y[i], omega); 
-            dsN_.backwardTD(omega,chi);
+            dg::blas1::pointwiseDot( omega, binv, omega);
+            dsN_.forwardDiv(omega,chi);
+            dg::blas1::pointwiseDivide( chi, binv, chi);
             dg::blas1::axpby( 0.5*nu_parallel[i],chi, 1., lambda,lambda);    //lambda = 0.5 nu_parallel ds^2_f N + 0.5 nu_parallel ds^2_b N
             dg::blas1::axpby( 1., lambda, 1., yp[i]);  //add to yp //dtN += 0.5 nu_parallel ds^2_f N + 0.5 nu_parallel ds^2_b N
         }           
@@ -557,10 +510,14 @@ double Asela<G, DS, M, V>::add_parallel_dynamics( std::vector<V>& y, std::vector
         if (p.pardiss==1)
         {
             dsDIR_.forward( u[i], omega); 
-            dsDIR_.forwardTD(omega,lambda);
-            dg::blas1::axpby( 0.5*nu_parallel[i+2], lambda, 0., lambda,lambda); //lambda = 0.5 nu_parallel ds^2_f U
+            dg::blas1::pointwiseDot( omega, binv, omega);
+            dsDIR_.backwardDiv(omega,lambda);
+            dg::blas1::pointwiseDivide( lambda, binv, lambda);
+            dg::blas1::scal( lambda, 0.5*nu_parallel[i+2]); //lambda = 0.5 nu_parallel ds^2_f U
             dsDIR_.backward( u[i], omega); 
-            dsDIR_.backwardTD(omega,chi);
+            dg::blas1::pointwiseDot( omega, binv, omega);
+            dsDIR_.forwardDiv(omega,chi);
+            dg::blas1::pointwiseDivide( chi, binv, chi);
             dg::blas1::axpby( 0.5*nu_parallel[i+2], chi, 1., lambda,lambda);  //lambda = 0.5 nu_parallel ds^2_f U + 0.5 nu_parallel ds^2_b U
             dg::blas1::axpby( 1., lambda, 1., yp[i+2]); //0.5 nu_parallel ds^2_f U + 0.5 nu_parallel ds^2_b U
         }   
@@ -578,8 +535,8 @@ double Asela<G, DS, M, V>::add_parallel_dynamics( std::vector<V>& y, std::vector
 }
 
 // #endif
-template<class Geometry, class DS, class Matrix, class container>
-void Asela<Geometry, DS, Matrix, container>::operator()( std::vector<container>& y, std::vector<container>& yp)
+template<class Geometry, class IMatrix, class Matrix, class container>
+void Asela<Geometry, IMatrix, Matrix, container>::operator()( const std::vector<container>& y, std::vector<container>& yp)
 {   
     /*  y[0] := N_e - 1
         y[1] := N_i - 1
@@ -773,26 +730,24 @@ void Asela<Geometry, DS, Matrix, container>::operator()( std::vector<container>&
 }
 
 //Computes curvature operator
-template<class Geometry, class DS, class Matrix, class container>
-void Asela<Geometry, DS, Matrix, container>::vecdotnablaN(const container& vecX, const container& vecY, container& src, container& target)
+template<class Geometry, class IMatrix, class Matrix, class container>
+void Asela<Geometry, IMatrix, Matrix, container>::vecdotnablaN(const container& vecX, const container& vecY, const container& src, container& target)
 {
     container temp1(src);
     dg::blas2::gemv( poissonN.dxlhs(), src, target); //d_R src
-    dg::blas1::pointwiseDot( vecX, target, target); // C^R d_R src
     dg::blas2::gemv( poissonN.dylhs(), src, temp1);  //d_Z src
-    dg::blas1::pointwiseDot( 1., vecY, temp1, 1., target);   // C^Z d_Z src + C^R d_R src
+    dg::blas1::pointwiseDot( 1., vecX, target, 1., vecY, temp1, 0., target);   // C^Z d_Z src + C^R d_R src
 }
 
-template<class Geometry, class DS, class Matrix, class container>
-void Asela<Geometry, DS, Matrix, container>::vecdotnablaDIR(const container& vecX, const container& vecY,  container& src, container& target)
+template<class Geometry, class IMatrix, class Matrix, class container>
+void Asela<Geometry, IMatrix, Matrix, container>::vecdotnablaDIR(const container& vecX, const container& vecY,  const container& src, container& target)
 {
     container temp1(src);
     dg::blas2::gemv( poissonDIR.dxrhs(), src, target); //d_R src
-    dg::blas1::pointwiseDot( vecX, target, target); // C^R d_R src
     dg::blas2::gemv( poissonDIR.dyrhs(), src, temp1);  //d_Z src
-    dg::blas1::pointwiseDot( 1., vecY, temp1, 1., target);// C^Z d_Z src + C^R d_R src
+    dg::blas1::pointwiseDot( 1., vecX, target, 1., vecY, temp1, 0., target);   // C^Z d_Z src + C^R d_R src
 }
 
 ///@endcond
 
-} //namespace eule
+} //namespace asela
diff --git a/src/asela/asela_hpc.cu b/src/asela/asela_hpc.cu
index 31f06ca7c..fe6e8a1a2 100644
--- a/src/asela/asela_hpc.cu
+++ b/src/asela/asela_hpc.cu
@@ -5,9 +5,6 @@
 #include <cmath>
 
 #include "dg/algorithm.h"
-#include "dg/backend/timer.cuh"
-#include "dg/backend/xspacelib.cuh"
-#include "dg/backend/interpolation.cuh"
 
 #include "file/nc_utilities.h"
 
@@ -21,8 +18,6 @@
 
 */
 
-typedef dg::FieldAligned< dg::CylindricalGrid3d, dg::IDMatrix, dg::DVec> DFA;
-using namespace dg::geo::solovev;
 int main( int argc, char* argv[])
 {
     ////////////////////////Parameter initialisation//////////////////////////
@@ -58,14 +53,14 @@ int main( int argc, char* argv[])
      
     //create RHS 
     std::cout << "Constructing Asela...\n";
-    asela::Asela<dg::CylindricalGrid3d, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec> asela( grid, p, gp); //initialize before rolkar!
+    asela::Asela<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec> asela( grid, p, gp); //initialize before rolkar!
     std::cout << "Constructing Implicit...\n";
-    asela::Implicit< dg::CylindricalGrid3d, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec > rolkar( grid, p, gp, asela.ds(), asela.dsDIR());
+    asela::Implicit< dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec > rolkar( grid, p, gp, asela.ds(), asela.dsDIR());
     std::cout << "Done!\n";
 
     /////////////////////The initial field//////////////////////////////////////////
     //background profile
-    dg::geo::Nprofile<Psip> prof(p.bgprofamp, p.nprofileamp, gp, Psip(gp)); //initial background profile
+    dg::geo::Nprofile prof(p.bgprofamp, p.nprofileamp, gp, dg::geo::solovev::Psip(gp)); //initial background profile
     std::vector<dg::DVec> y0(4, dg::evaluate( prof, grid)), y1(y0); 
     //perturbation 
     dg::GaussianZ gaussianZ( 0., p.sigma_z*M_PI, 1); //modulation along fieldline
@@ -84,14 +79,14 @@ int main( int argc, char* argv[])
     }
     if( p.mode == 3)
     {
-        dg::geo::ZonalFlow<Psip> init0(p.amp, p.k_psi, gp, Psip(gp));
+        dg::geo::ZonalFlow init0(p.amp, p.k_psi, gp, dg::geo::solovev::Psip(gp));
         y1[1] = asela.ds().fieldaligned().evaluate( init0, gaussianZ, (unsigned)p.Nz/2, 1); 
     }
     dg::blas1::axpby( 1., y1[1], 1., y0[1]); //sum up background and perturbation
     dg::blas1::plus(y0[1], -1); //initialize ni-1
     if( p.mode == 2 || p.mode == 3)
     {
-        dg::DVec damping = dg::evaluate( dg::geo::GaussianProfXDamping<Psip>(Psip(gp), gp), grid);
+        dg::DVec damping = dg::evaluate( dg::geo::GaussianProfXDamping(dg::geo::solovev::Psip(gp), gp), grid);
         dg::blas1::pointwiseDot(damping, y0[1], y0[1]); //damp with gaussprofdamp
     }
     std::cout << "intiialize ne" << std::endl;
@@ -111,10 +106,10 @@ int main( int argc, char* argv[])
     int dim_ids[4], tvarID;
     {
         err = file::define_dimensions( ncid, dim_ids, &tvarID, grid_out);
-        MagneticField c(gp);
-        dg::geo::FieldR<MagneticField> fieldR(c, gp.R_0);
-        dg::geo::FieldZ<MagneticField> fieldZ(c, gp.R_0);
-        dg::geo::FieldP<MagneticField> fieldP(c, gp.R_0);
+        dg::geo::TokamakMagneticField c = dg::geo::createSolovevField(gp);
+        dg::geo::FieldR fieldR(c);
+        dg::geo::FieldZ fieldZ(c);
+        dg::geo::FieldP fieldP(c);
         dg::HVec vecR = dg::evaluate( fieldR, grid_out);
         dg::HVec vecZ = dg::evaluate( fieldZ, grid_out);
         dg::HVec vecP = dg::evaluate( fieldP, grid_out);
diff --git a/src/asela/asela_mpi.cu b/src/asela/asela_mpi.cu
index 579f447ca..e441cb558 100644
--- a/src/asela/asela_mpi.cu
+++ b/src/asela/asela_mpi.cu
@@ -24,8 +24,6 @@
         output dimensions must be divisible by the mpi process numbers
 */
 
-typedef dg::MPI_FieldAligned< dg::CylindricalMPIGrid3d, dg::IDMatrix,dg::BijectiveComm< dg::iDVec, dg::DVec >, dg::DVec> DFA;
-using namespace dg::geo::solovev;
 int main( int argc, char* argv[])
 {
     ////////////////////////////////setup MPI///////////////////////////////
@@ -90,14 +88,14 @@ int main( int argc, char* argv[])
      
     //create RHS 
     if(rank==0)std::cout << "Constructing Asela...\n";
-    asela::Asela<dg::CylindricalMPIGrid3d, dg::DS<DFA, dg::MDMatrix, dg::MDVec>, dg::MDMatrix, dg::MDVec> asela( grid, p, gp); //initialize before rolkar!
+    asela::Asela<dg::CylindricalMPIGrid3d, dg::MIDMatrix, dg::MDMatrix, dg::MDVec> asela( grid, p, gp); //initialize before rolkar!
     if(rank==0)std::cout << "Constructing Implicit...\n";
-    asela::Implicit< dg::CylindricalMPIGrid3d, dg::DS<DFA, dg::MDMatrix, dg::MDVec>, dg::MDMatrix, dg::MDVec > rolkar( grid, p, gp, asela.ds(), asela.dsDIR());
+    asela::Implicit< dg::CylindricalMPIGrid3d, dg::MIDMatrix, dg::MDMatrix, dg::MDVec > rolkar( grid, p, gp, asela.ds(), asela.dsDIR());
     if(rank==0)std::cout << "Done!\n";
 
     /////////////////////The initial field/////////////////////////////////////////
     //background profile
-    dg::geo::Nprofile<Psip> prof(p.bgprofamp, p.nprofileamp, gp, Psip(gp)); //initial background profile
+    dg::geo::Nprofile prof(p.bgprofamp, p.nprofileamp, gp, dg::geo::solovev::Psip(gp)); //initial background profile
     std::vector<dg::MDVec> y0(4, dg::evaluate( prof, grid)), y1(y0); 
     //perturbation 
     dg::GaussianZ gaussianZ( 0., p.sigma_z*M_PI, 1); //modulation along fieldline
@@ -116,14 +114,14 @@ int main( int argc, char* argv[])
     }
     if( p.mode == 3)
     {
-        dg::geo::ZonalFlow<Psip> init0(p.amp, p.k_psi, gp, Psip(gp));
+        dg::geo::ZonalFlow init0(p.amp, p.k_psi, gp, dg::geo::solovev::Psip(gp));
         y1[1] = asela.ds().fieldaligned().evaluate( init0, gaussianZ, (unsigned)p.Nz/2, 1); 
     }
     dg::blas1::axpby( 1., y1[1], 1., y0[1]); //sum up background and perturbation
     dg::blas1::plus(y0[1], -1); //initialize ni-1
     if( p.mode == 2 || p.mode == 3)
     {
-        dg::MDVec damping = dg::evaluate( dg::geo::GaussianProfXDamping<Psip>(Psip(gp), gp), grid);
+        dg::MDVec damping = dg::evaluate( dg::geo::GaussianProfXDamping(dg::geo::solovev::Psip(gp), gp), grid);
         dg::blas1::pointwiseDot(damping, y0[1], y0[1]); //damp with gaussprofdamp
     }
     std::cout << "intiialize ne" << std::endl;
@@ -147,10 +145,10 @@ int main( int argc, char* argv[])
     {
         err = file::define_dimensions( ncid, dimids, &tvarID, grid_out.global());
 
-        MagneticField c(gp);
-        dg::geo::FieldR<MagneticField> fieldR(c, gp.R_0);
-        dg::geo::FieldZ<MagneticField> fieldZ(c, gp.R_0);
-        dg::geo::FieldP<MagneticField> fieldP(c, gp.R_0);
+        dg::geo::TokamakMagneticField c = dg::geo::createSolovevField(gp);
+        dg::geo::FieldR fieldR(c);
+        dg::geo::FieldZ fieldZ(c);
+        dg::geo::FieldP fieldP(c);
         dg::HVec vecR = dg::evaluate( fieldR, grid_out.global());
         dg::HVec vecZ = dg::evaluate( fieldZ, grid_out.global());
         dg::HVec vecP = dg::evaluate( fieldP, grid_out.global());
@@ -216,7 +214,7 @@ int main( int argc, char* argv[])
     if(rank==0)std::cout << "First output ... \n";
     int dims[3],  coords[3];
     MPI_Cart_get( comm, 3, dims, periods, coords);
-    size_t count[4] = {1, grid_out.Nz(), grid_out.n()*(grid_out.Ny()), grid_out.n()*(grid_out.Nx())};
+    size_t count[4] = {1, grid_out.local().Nz(), grid_out.n()*(grid_out.local().Ny()), grid_out.n()*(grid_out.local().Nx())};
     size_t start[4] = {0, coords[2]*count[1], coords[1]*count[2], coords[0]*count[3]};
     dg::MDVec transfer( dg::evaluate(dg::zero, grid));
     dg::DVec transferD( dg::evaluate(dg::zero, grid_out.local()));
diff --git a/src/asela/parameters.h b/src/asela/parameters.h
index 29ef5c926..5f5856d9f 100644
--- a/src/asela/parameters.h
+++ b/src/asela/parameters.h
@@ -2,7 +2,7 @@
 #include "dg/enums.h"
 #include "json/json.h"
 
-namespace eule{
+namespace asela{
 /**
  * @brief Provide a mapping between input file and named parameters
  */
-- 
GitLab


From 157a77fc75704517ce2d2e438a21752706cdbdd8 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Fri, 3 Nov 2017 14:52:36 +0100
Subject: [PATCH 438/453] feltor project also compiles

---
 src/asela/asela.cuh      |  15 ++----
 src/feltor/feltor.cu     |  24 ++++-----
 src/feltor/feltor.cuh    | 114 +++++++++++++++++++--------------------
 src/feltor/feltor_hpc.cu |  25 +++------
 src/feltor/feltor_mpi.cu |  29 ++++------
 src/feltor/parameters.h  |   4 +-
 6 files changed, 88 insertions(+), 123 deletions(-)

diff --git a/src/asela/asela.cuh b/src/asela/asela.cuh
index 73c457237..6bced49c4 100644
--- a/src/asela/asela.cuh
+++ b/src/asela/asela.cuh
@@ -299,20 +299,15 @@ Asela<Grid, IMatrix, Matrix, container>::Asela( const Grid& g, Parameters p, dg:
     ////////////////////////////transform curvature components////////
     dg::pushForwardPerp(dg::geo::CurvatureNablaBR(mf), dg::geo::CurvatureNablaBZ(mf), curvX, curvY, g);
     dg::blas1::transfer(  dg::pullback(dg::geo::DivCurvatureKappa(mf), g), divCurvKappa);
-    if (p.curvmode==1) 
-    {
-        dg::pushForwardPerp(dg::geo::CurvatureKappaR(), dg::geo::CurvatureKappaZ(mf), curvKappaX, curvKappaY, g);
-        dg::blas1::axpby( 1.,curvX,1.,curvKappaX,curvX);
-        dg::blas1::axpby( 1.,curvY,1.,curvKappaY,curvY);
-    }
+    dg::pushForwardPerp(dg::geo::CurvatureKappaR(), dg::geo::CurvatureKappaZ(mf), curvKappaX, curvKappaY, g);
     if (p.curvmode==0) 
     {
-        dg::blas1::transfer(  tempX, curvKappaX);
-        dg::blas1::transfer(  tempY, curvKappaY);
-        dg::blas1::axpby( 1.,curvX,1.,curvKappaX,curvX);
-        dg::blas1::axpby( 1.,curvY,1.,curvKappaY,curvY);
+        dg::blas1::transfer(  curvX, curvKappaX);
+        dg::blas1::transfer(  curvY, curvKappaY);
         dg::blas1::scal(divCurvKappa,0.);
     }
+    dg::blas1::axpby( 1.,curvX,1.,curvKappaX,curvX);
+    dg::blas1::axpby( 1.,curvY,1.,curvKappaY,curvY);
     ///////////////////init densities//////////////////////////////
     dg::blas1::transfer( dg::pullback(dg::geo::Nprofile(p.bgprofamp, p.nprofileamp, gp, mf.psip()),g), profne);
     dg::blas1::transfer(  profne ,profNi);
diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index d18f79842..45b0d00ee 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -7,11 +7,6 @@
 
 #include "draw/host_window.h"
 //#include "draw/device_window.cuh"
-#include "dg/backend/xspacelib.cuh"
-#include "dg/backend/sparseblockmat.cuh"
-#include "dg/backend/timer.cuh"
-#include "dg/backend/average.cuh"
-#include "dg/backend/typedefs.cuh"
 
 #include "feltor.cuh"
 
@@ -67,14 +62,14 @@ int main( int argc, char* argv[])
 
     //create RHS 
     std::cout << "Constructing Explicit...\n";
-    feltor::Explicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec> feltor( grid, mag, p, gp); //initialize before rolkar!
+    feltor::Explicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec> feltor( grid, p, gp); //initialize before rolkar!
     std::cout << "Constructing Implicit...\n";
     feltor::Implicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec> rolkar( grid, p, gp, feltor.ds(), feltor.dsDIR());
     std::cout << "Done!\n";
 
     /////////////////////The initial field///////////////////////////////////////////
     //background profile
-    dg::geo::Nprofile<Psip> prof(p.bgprofamp, p.nprofileamp, gp, Psip(gp)); //initial background profile
+    dg::geo::Nprofile prof(p.bgprofamp, p.nprofileamp, gp, dg::geo::solovev::Psip(gp)); //initial background profile
     std::vector<dg::DVec> y0(4, dg::evaluate( prof, grid)), y1(y0); 
     //perturbation 
     dg::GaussianZ gaussianZ( 0., p.sigma_z*M_PI, 1); //modulation along fieldline
@@ -93,14 +88,14 @@ int main( int argc, char* argv[])
     }
     if( p.mode == 3)
     {
-        dg::geo::ZonalFlow<Psip> init0(p.amp, p.k_psi, gp, Psip(gp));
+        dg::geo::ZonalFlow init0(p.amp, p.k_psi, gp, dg::geo::solovev::Psip(gp));
         y1[1] = feltor.ds().fieldaligned().evaluate( init0, gaussianZ, (unsigned)p.Nz/2, 1); 
     }
     dg::blas1::axpby( 1., y1[1], 1., y0[1]); //sum up background and perturbation
     dg::blas1::plus(y0[1], -1); //initialize ni-1
     if( p.mode == 2 || p.mode == 3)
     {
-        dg::DVec damping = dg::evaluate( dg::geo::GaussianProfXDamping<Psip>(Psip(gp), gp), grid);
+        dg::DVec damping = dg::evaluate( dg::geo::GaussianProfXDamping(dg::geo::solovev::Psip(gp), gp), grid);
         dg::blas1::pointwiseDot(damping,y0[1], y0[1]); //damp with gaussprofdamp
     }
     std::cout << "intiialize ne" << std::endl;
@@ -121,7 +116,6 @@ int main( int argc, char* argv[])
     dg::HVec hvisual( grid.size(), 0.), visual(hvisual),avisual(hvisual);
     dg::IHMatrix equi = dg::create::backscatter( grid);
     draw::ColorMapRedBlueExtMinMax colors(-1.0, 1.0);
-    dg::ToroidalAverage<dg::HVec> toravg(grid);
     //create timer
     dg::Timer t;
     double time = 0;
@@ -159,7 +153,7 @@ int main( int argc, char* argv[])
             render.renderQuad( part, grid.n()*grid.Nx(), grid.n()*grid.Ny(), colors);
         }
         dg::blas1::axpby(0.0,avisual,0.0,avisual);
-        toravg(visual,avisual);
+        dg::toroidal_average(visual,avisual,grid);
         render.renderQuad( avisual, grid.n()*grid.Nx(), grid.n()*grid.Ny(), colors);
         //draw ions
         //thrust::transform( y1[1].begin(), y1[1].end(), dvisual.begin(), dg::PLUS<double>(-0.));//ne-1
@@ -180,7 +174,7 @@ int main( int argc, char* argv[])
             render.renderQuad( part, grid.n()*grid.Nx(), grid.n()*grid.Ny(), colors);
         }
         dg::blas1::axpby(0.0,avisual,0.0,avisual);
-        toravg(visual,avisual);
+        dg::toroidal_average(visual,avisual,grid);
         render.renderQuad( avisual, grid.n()*grid.Nx(), grid.n()*grid.Ny(), colors);
         
         //draw potential
@@ -202,7 +196,7 @@ int main( int argc, char* argv[])
             render.renderQuad( part, grid.n()*grid.Nx(), grid.n()*grid.Ny(), colors);
         }
         dg::blas1::axpby(0.0,avisual,0.0,avisual);
-        toravg(visual,avisual);
+        dg::toroidal_average(visual,avisual,grid);
         render.renderQuad( avisual, grid.n()*grid.Nx(), grid.n()*grid.Ny(), colors);
         
         //draw U_e
@@ -220,7 +214,7 @@ int main( int argc, char* argv[])
             render.renderQuad( part, grid.n()*grid.Nx(), grid.n()*grid.Ny(), colors);
         }
         dg::blas1::axpby(0.0,avisual,0.0,avisual);
-        toravg(visual,avisual);
+        dg::toroidal_average(visual,avisual,grid);
         render.renderQuad( avisual, grid.n()*grid.Nx(), grid.n()*grid.Ny(), colors);      
         
         //draw U_i
@@ -238,7 +232,7 @@ int main( int argc, char* argv[])
             render.renderQuad( part, grid.n()*grid.Nx(), grid.n()*grid.Ny(), colors);
         }
         dg::blas1::axpby(0.0,avisual,0.0,avisual);
-        toravg(visual,avisual);
+        dg::toroidal_average(visual,avisual,grid);
         render.renderQuad( avisual, grid.n()*grid.Nx(), grid.n()*grid.Ny(), colors);
         
         title << std::fixed; 
diff --git a/src/feltor/feltor.cuh b/src/feltor/feltor.cuh
index a0fe0350a..c14638662 100644
--- a/src/feltor/feltor.cuh
+++ b/src/feltor/feltor.cuh
@@ -22,7 +22,7 @@ template<class Geometry, class IMatrix, class Matrix, class container>
 struct Implicit
 {
 
-    Implicit( const Geometry& g, feltor::Parameters p, dg::geo::solovev::Parameters gp, DS& dsN, DS& dsDIR):
+    Implicit( const Geometry& g, feltor::Parameters p, dg::geo::solovev::Parameters gp, dg::geo::DS<Geometry, IMatrix, Matrix, container>& dsN, dg::geo::DS<Geometry, IMatrix, Matrix,  container>& dsDIR):
         p(p),
         gp(gp),
         LaplacianM_perpN  ( g, g.bcx(), g.bcy(), dg::normed, dg::centered),
@@ -84,17 +84,17 @@ struct Implicit
     container temp;
     container dampgauss_;
     dg::Elliptic<Geometry, Matrix, container> LaplacianM_perpN,LaplacianM_perpDIR;
-    DS& dsN_,dsDIR_;
+    dg::geo::DS<Geometry, IMatrix, Matrix, container> dsN_,dsDIR_;
 };
 
 template< class Geometry, class IMatrix, class Matrix, class container >
 struct Explicit
 {
-    Explicit( const Geometry& g, const dg::geo::TokamakMagneticField& mag, feltor::Parameters p, dg::geo::solovev::Parameters gp);
+    Explicit( const Geometry& g, feltor::Parameters p, dg::geo::solovev::Parameters gp);
 
 
-    dg::DS<Geometry, IMatrix, Matrix, container>& ds(){return dsN_;}
-    dg::DS<Geometry, IMatrix, Matrix, container>& dsDIR(){return dsDIR_;}
+    dg::geo::DS<Geometry, IMatrix, Matrix, container>& ds(){return dsN_;}
+    dg::geo::DS<Geometry, IMatrix, Matrix, container>& dsDIR(){return dsDIR_;}
 
     /**
      * @brief Returns phi and psi that belong to the last solve of the polarization equation
@@ -167,8 +167,8 @@ struct Explicit
     double fieldalignment() { return aligned_;}
 
   private:
-    void vecdotnablaN(const container& x, const container& y, container& z, container& target);
-    void vecdotnablaDIR(const container& x, const container& y, container& z, container& target);
+    void vecdotnablaN(const container& x, const container& y, const container& z, container& target);
+    void vecdotnablaDIR(const container& x, const container& y, const container& z, container& target);
     //extrapolates and solves for phi[1], then adds square velocity ( omega)
     container& compute_psi( const container& potential);
     container& polarisation( const std::vector<container>& y); //solves polarisation equation
@@ -188,7 +188,7 @@ struct Explicit
     std::vector<container> dsy, curvy,curvkappay; 
 
     //matrices and solvers
-    dg::DS<Geometry, IMatrix, Matrix, container> dsDIR_, dsN_;
+    dg::geo::DS<Geometry, IMatrix, Matrix, container> dsDIR_, dsN_;
     dg::Poisson<    Geometry, Matrix, container> poissonN,poissonDIR; 
     dg::Elliptic<   Geometry, Matrix, container> pol,lapperpN,lapperpDIR;
     dg::Helmholtz< Geometry, Matrix, container > invgammaDIR, invgammaN;
@@ -205,19 +205,9 @@ struct Explicit
 
 ///@cond
 template<class Grid, class IMatrix, class Matrix, class container>
-Explicit<Grid, IMatrix, Matrix, container>::Explicit( const Grid& g, const TokamakMagneticField& mag, feltor::Parameters p, dg::geo::solovev::Parameters gp): 
-    dsDIR_( dg::FieldAligned<Grid, IMatrix, container>( mag, g, gp.rk4eps, 
-                dg::geo::PsiLimiter(mag.psip(), gp.psipmaxlim), 
-                dg::DIR, (2*M_PI)/((double)p.Nz)
-                ), 
-            dg::geo::InvB(mag), 
-            dg::normed, dg::forward ),
-    dsN_( dg::FieldAligned<Grid, IMatrix, container>( mag, g, gp.rk4eps, 
-                dg::geo::PsiLimiter(mag.psip(), gp.psipmaxlim), 
-                g.bcx(), (2*M_PI)/((double)p.Nz)
-                ), 
-            dg::geo::InvB(mag), 
-            dg::normed, dg::forward ),
+Explicit<Grid, IMatrix, Matrix, container>::Explicit( const Grid& g, feltor::Parameters p, dg::geo::solovev::Parameters gp): 
+    dsDIR_( dg::geo::createSolovevField(gp), g, dg::DIR, dg::DIR, dg::geo::PsiLimiter( dg::geo::solovev::Psip(gp), gp.psipmaxlim), dg::normed, dg::forward, gp.rk4eps),
+    dsN_( dg::geo::createSolovevField(gp), g, dg::NEU, dg::NEU, dg::geo::PsiLimiter( dg::geo::solovev::Psip(gp), gp.psipmaxlim), dg::normed, dg::forward, gp.rk4eps),
     //////////the poisson operators ////////////////////////////////////////
     poissonN(  g, g.bcx(), g.bcy(), dg::DIR, dg::DIR), //first N/U then phi BCC
     poissonDIR(g, dg::DIR, dg::DIR, dg::DIR, dg::DIR), //first N/U then phi BCC
@@ -243,29 +233,25 @@ Explicit<Grid, IMatrix, Matrix, container>::Explicit( const Grid& g, const Tokam
     invert_invgammaN.construct(   omega, p.Nx*p.Ny*p.Nz*p.n*p.n, p.eps_gamma); 
     invert_invgammaPhi.construct( omega, p.Nx*p.Ny*p.Nz*p.n*p.n, p.eps_gamma); 
     //////////////////////////////init fields /////////////////////
-    dg::blas1::transfer(  dg::pullback(dg::geo::InvB(mag),                                         g), binv);
-    dg::blas1::transfer(  dg::pullback(dg::geo::GradLnB(mag),                                      g), gradlnB);
-    dg::blas1::transfer(  dg::pullback(dg::geo::TanhSource(mf.psip, gp.psipmin, gp.alpha),         g), source);
-    dg::blas1::transfer(  dg::pullback(dg::geo::GaussianDamping(mag.psip(), gp.psipmax, gp.alpha), g), damping);
+    dg::geo::TokamakMagneticField mf = dg::geo::createSolovevField(gp);
+    dg::blas1::transfer(  dg::pullback(dg::geo::InvB(mf),      g), binv);
+    dg::blas1::transfer(  dg::pullback(dg::geo::GradLnB(mf),   g), gradlnB);
+    dg::blas1::transfer(  dg::pullback(dg::geo::TanhSource(mf.psip(), gp.psipmin, gp.alpha),         g), source);
+    dg::blas1::transfer(  dg::pullback(dg::geo::GaussianDamping(mf.psip(), gp.psipmax, gp.alpha), g), damping);
     ////////////////////////////transform curvature components////////
-    dg::geo::pushForwardPerp(dg::geo::CurvatureNablaBR(mag), dg::geo::CurvatureNablaBZ(mag), curvX, curvY, g);
-    dg::blas1::transfer(  dg::pullback(dg::geo::DivCurvatureKappa(mag), g), divCurvKappa);
-    if (p.curvmode==1) 
-    {
-        dg::geo::pushForwardPerp(dg::geo::CurvatureKappaR(), dg::geo::CurvatureKappaZ(mag), curvKappaX, curvKappaY, g);
-        dg::blas1::axpby( 1.,curvX,1.,curvKappaX,curvX);
-        dg::blas1::axpby( 1.,curvY,1.,curvKappaY,curvY);
-    }
+    dg::pushForwardPerp(dg::geo::CurvatureNablaBR(mf), dg::geo::CurvatureNablaBZ(mf), curvX, curvY, g);
+    dg::blas1::transfer(  dg::pullback(dg::geo::DivCurvatureKappa(mf), g), divCurvKappa);
+    dg::pushForwardPerp(dg::geo::CurvatureKappaR(), dg::geo::CurvatureKappaZ(mf), curvKappaX, curvKappaY, g);
     if (p.curvmode==0) 
     {
-        dg::blas1::transfer(  tempX, curvKappaX);
-        dg::blas1::transfer(  tempY, curvKappaY);
-        dg::blas1::axpby( 1.,curvX,1.,curvKappaX,curvX);
-        dg::blas1::axpby( 1.,curvY,1.,curvKappaY,curvY);
+        dg::blas1::transfer(  curvX, curvKappaX);
+        dg::blas1::transfer(  curvY, curvKappaY);
         dg::blas1::scal(divCurvKappa,0.);
     }
+    dg::blas1::axpby( 1.,curvX,1.,curvKappaX,curvX);
+    dg::blas1::axpby( 1.,curvY,1.,curvKappaY,curvY);
     ///////////////////init densities//////////////////////////////
-    dg::blas1::transfer( dg::pullback(dg::geo::Nprofile(p.bgprofamp, p.nprofileamp, gp, mag.psip()),g), profne);
+    dg::blas1::transfer( dg::pullback(dg::geo::Nprofile(p.bgprofamp, p.nprofileamp, gp, mf.psip()),g), profne);
     dg::blas1::transfer(  profne ,profNi);
     dg::blas1::plus( profNi, -1); 
     initializene(profNi, profne); //ne = Gamma N_i (needs Invert object)
@@ -276,8 +262,8 @@ Explicit<Grid, IMatrix, Matrix, container>::Explicit( const Grid& g, const Tokam
     dg::blas1::transfer( dg::create::inv_volume(g), v3d);
 }
 
-template<class Geometry, class DS, class Matrix, class container>
-container& Explicit<Geometry, DS, Matrix, container>::polarisation( const std::vector<container>& y)
+template<class Geometry, class IMatrix, class Matrix, class container>
+container& Explicit<Geometry, IMatrix, Matrix, container>::polarisation( const std::vector<container>& y)
 {
     dg::blas1::axpby( p.mu[1], y[1], 0, chi);       //chi =  \mu_i (n_i-1) 
     dg::blas1::plus( chi, p.mu[1]);
@@ -293,8 +279,8 @@ container& Explicit<Geometry, DS, Matrix, container>::polarisation( const std::v
     return phi[0];
 }
 
-template< class Geometry, class DS, class Matrix, class container>
-container& Explicit<Geometry, DS, Matrix,container>::compute_psi( const container& potential)
+template< class Geometry, class IMatrix, class Matrix, class container>
+container& Explicit<Geometry, IMatrix, Matrix,container>::compute_psi( const container& potential)
 {
     invert_invgammaPhi(invgammaDIR,chi,potential);                    //chi  Gamma phi
     poissonN.variationRHS(potential, omega);
@@ -304,15 +290,15 @@ container& Explicit<Geometry, DS, Matrix,container>::compute_psi( const containe
     return phi[1];    
 }
 
-template<class Geometry, class DS, class Matrix, class container>
-void Explicit<Geometry, DS, Matrix, container>::initializene( const container& src, container& target)
+template<class Geometry, class IMatrix, class Matrix, class container>
+void Explicit<Geometry, IMatrix, Matrix, container>::initializene( const container& src, container& target)
 { 
     invert_invgammaN(invgammaN,target,src); //=ne-1 = Gamma (ni-1)    
 }
 
 
-template<class G, class DS, class M, class V>
-double Explicit<G, DS, M, V>::add_parallel_dynamics( const std::vector<V>& y, std::vector<V>& yp)
+template<class G, class IMatrix, class M, class V>
+double Explicit<G, IMatrix, M, V>::add_parallel_dynamics( const std::vector<V>& y, std::vector<V>& yp)
 {
     double z[2]     = {-1.0,1.0};
     double Dpar[4]  = {0.0, 0.0,0.0,0.0};
@@ -373,10 +359,14 @@ double Explicit<G, DS, M, V>::add_parallel_dynamics( const std::vector<V>& y, st
         if (p.pardiss==1)
         {
             dsN_.forward( y[i], omega); 
-            dsN_.forwardTD(omega,lambda);
-            dg::blas1::axpby( 0.5*nu_parallel[i], lambda, 0., lambda,lambda);  //lambda = 0.5 nu_parallel ds^2_f N
+            dg::blas1::pointwiseDot( omega, binv, omega);
+            dsN_.backwardDiv(omega,lambda);
+            dg::blas1::pointwiseDivide( lambda, binv, lambda);
+            dg::blas1::scal( lambda, 0.5*nu_parallel[i]);  //lambda = 0.5 nu_parallel ds^2_f N
             dsN_.backward( y[i], omega); 
-            dsN_.backwardTD(omega,chi);
+            dg::blas1::pointwiseDot( omega, binv, omega);
+            dsN_.forwardDiv(omega,chi);
+            dg::blas1::pointwiseDivide( chi, binv, chi);
             dg::blas1::axpby( 0.5*nu_parallel[i],chi, 1., lambda,lambda);    //lambda = 0.5 nu_parallel ds^2_f N + 0.5 nu_parallel ds^2_b N
             dg::blas1::axpby( 1., lambda, 1., yp[i]);  //add to yp //dtN += 0.5 nu_parallel ds^2_f N + 0.5 nu_parallel ds^2_b N
         }           
@@ -408,10 +398,14 @@ double Explicit<G, DS, M, V>::add_parallel_dynamics( const std::vector<V>& y, st
         if (p.pardiss==1)
         {
             dsDIR_.forward( y[i+2], omega); 
-            dsDIR_.forwardTD(omega,lambda);
-            dg::blas1::axpby( 0.5*nu_parallel[i+2], lambda, 0., lambda,lambda); //lambda = 0.5 nu_parallel ds^2_f U
+            dg::blas1::pointwiseDot( omega, binv, omega);
+            dsDIR_.backwardDiv(omega,lambda);
+            dg::blas1::pointwiseDivide( lambda, binv, lambda);
+            dg::blas1::scal( lambda, 0.5*nu_parallel[i+2]); //lambda = 0.5 nu_parallel ds^2_f U
             dsDIR_.backward( y[i+2], omega); 
-            dsDIR_.backwardTD(omega,chi);
+            dg::blas1::pointwiseDot( omega, binv, omega);
+            dsDIR_.forwardDiv(omega,chi);
+            dg::blas1::pointwiseDivide( chi, binv, chi);
             dg::blas1::axpby( 0.5*nu_parallel[i+2], chi, 1., lambda,lambda);  //lambda = 0.5 nu_parallel ds^2_f U + 0.5 nu_parallel ds^2_b U
             dg::blas1::axpby( 1., lambda, 1., yp[i+2]); //0.5 nu_parallel ds^2_f U + 0.5 nu_parallel ds^2_b U
         }   
@@ -430,8 +424,8 @@ double Explicit<G, DS, M, V>::add_parallel_dynamics( const std::vector<V>& y, st
 }
 
 
-template<class Geometry, class DS, class Matrix, class container>
-void Explicit<Geometry, DS, Matrix, container>::operator()( const std::vector<container>& y, std::vector<container>& yp)
+template<class Geometry, class IMatrix, class Matrix, class container>
+void Explicit<Geometry, IMatrix, Matrix, container>::operator()( const std::vector<container>& y, std::vector<container>& yp)
 {
     /* y[0] := N_e - 1
        y[1] := N_i - 1
@@ -602,22 +596,22 @@ void Explicit<Geometry, DS, Matrix, container>::operator()( const std::vector<co
 
 
 //Computes curvature operator
-template<class Geometry, class DS, class Matrix, class container>
-void Explicit<Geometry, DS, Matrix, container>::vecdotnablaN(const container& vecX, const container& vecY, container& src, container& target)
+template<class Geometry, class IMatrix, class Matrix, class container>
+void Explicit<Geometry, IMatrix, Matrix, container>::vecdotnablaN(const container& vecX, const container& vecY, const container& src, container& target)
 {
     container temp1(src);
     dg::blas2::gemv( poissonN.dxlhs(), src, target); //d_R src
     dg::blas2::gemv( poissonN.dylhs(), src, temp1);  //d_Z src
-    dg::blas1::pointwiseDot( 1., vecY, temp1, 1., vecX, target, 0, target);   // C^Z d_Z src + C^R d_R src
+    dg::blas1::pointwiseDot( 1., vecX, target, 1., vecY, temp1, 0., target);   // C^Z d_Z src + C^R d_R src
 }
 
-template<class Geometry, class DS, class Matrix, class container>
-void Explicit<Geometry, DS, Matrix, container>::vecdotnablaDIR(const container& vecX, const container& vecY,  container& src, container& target)
+template<class Geometry, class IMatrix, class Matrix, class container>
+void Explicit<Geometry, IMatrix, Matrix, container>::vecdotnablaDIR(const container& vecX, const container& vecY,  const container& src, container& target)
 {
     container temp1(src);
     dg::blas2::gemv( poissonDIR.dxrhs(), src, target); //d_R src
     dg::blas2::gemv( poissonDIR.dyrhs(), src, temp1);  //d_Z src
-    dg::blas1::pointwiseDot( 1., vecY, temp1, 1., vecX, target, 0, target);   // C^Z d_Z src + C^R d_R src
+    dg::blas1::pointwiseDot( 1., vecX, target, 1., vecY, temp1, 0., target);   // C^Z d_Z src + C^R d_R src
 }
 
 ///@endcond
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index ce6e6140d..6c3175e02 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -4,16 +4,7 @@
 #include <sstream>
 #include <cmath>
 
-
-
-#include "dg/algorithm.h"
-#include "dg/backend/timer.cuh"
-#include "dg/backend/xspacelib.cuh"
-#include "dg/backend/interpolation.cuh"
-
-
 #include "file/nc_utilities.h"
-
 #include "feltor.cuh"
 
 /*
@@ -24,8 +15,6 @@
 
 */
 
-typedef dg::FieldAligned< dg::CylindricalGrid3d, dg::IDMatrix, dg::DVec> DFA;
-using namespace dg::geo::solovev;
 int main( int argc, char* argv[])
 {
     ////////////////////////Parameter initialisation//////////////////////////
@@ -67,7 +56,7 @@ int main( int argc, char* argv[])
 
     /////////////////////The initial field//////////////////////////////////////////
     //background profile
-    dg::geo::Nprofile<Psip> prof(p.bgprofamp, p.nprofileamp, gp, Psip(gp)); //initial background profile
+    dg::geo::Nprofile prof(p.bgprofamp, p.nprofileamp, gp, dg::geo::solovev::Psip(gp)); //initial background profile
     std::vector<dg::DVec> y0(4, dg::evaluate( prof, grid)), y1(y0); 
     //perturbation 
     dg::GaussianZ gaussianZ( 0., p.sigma_z*M_PI, 1); //modulation along fieldline
@@ -86,14 +75,14 @@ int main( int argc, char* argv[])
     }
     if( p.mode == 3)
     {
-        dg::geo::ZonalFlow<Psip> init0(p.amp, p.k_psi, gp, Psip(gp));
+        dg::geo::ZonalFlow init0(p.amp, p.k_psi, gp, dg::geo::solovev::Psip(gp));
         y1[1] = feltor.ds().fieldaligned().evaluate( init0, gaussianZ, (unsigned)p.Nz/2, 1); 
     }
     dg::blas1::axpby( 1., y1[1], 1., y0[1]); //sum up background and perturbation
     dg::blas1::plus(y0[1], -1); //initialize ni-1
     if( p.mode == 2 || p.mode == 3)
     {
-        dg::DVec damping = dg::evaluate( dg::geo::GaussianProfXDamping<Psip>(Psip(gp), gp), grid);
+        dg::DVec damping = dg::evaluate( dg::geo::GaussianProfXDamping(dg::geo::solovev::Psip(gp), gp), grid);
         dg::blas1::pointwiseDot(damping, y0[1], y0[1]); //damp with gaussprofdamp
     }
     std::cout << "intiialize ne" << std::endl;
@@ -113,10 +102,10 @@ int main( int argc, char* argv[])
     int dim_ids[4], tvarID;
     {
         err = file::define_dimensions( ncid, dim_ids, &tvarID, grid_out);
-        MagneticField c(gp);
-        dg::geo::FieldR<MagneticField> fieldR(c, gp.R_0);
-        dg::geo::FieldZ<MagneticField> fieldZ(c, gp.R_0);
-        dg::geo::FieldP<MagneticField> fieldP(c, gp.R_0);
+        dg::geo::TokamakMagneticField c=dg::geo::createSolovevField(gp);
+        dg::geo::FieldR fieldR(c);
+        dg::geo::FieldZ fieldZ(c);
+        dg::geo::FieldP fieldP(c);
 
         dg::HVec vecR = dg::evaluate( fieldR, grid_out);
         dg::HVec vecZ = dg::evaluate( fieldZ, grid_out);
diff --git a/src/feltor/feltor_mpi.cu b/src/feltor/feltor_mpi.cu
index 91917f840..fd32ce0ec 100644
--- a/src/feltor/feltor_mpi.cu
+++ b/src/feltor/feltor_mpi.cu
@@ -6,11 +6,6 @@
 
 #include <mpi.h> //activate mpi
 
-#include "dg/algorithm.h"
-#include "dg/backend/timer.cuh"
-#include "dg/backend/xspacelib.cuh"
-#include "dg/backend/interpolation.cuh"
-
 #include "netcdf_par.h" //exclude if par netcdf=OFF
 #include "file/nc_utilities.h"
 
@@ -24,8 +19,6 @@
         output dimensions must be divisible by the mpi process numbers
 */
 
-typedef dg::MPI_FieldAligned< dg::CylindricalMPIGrid3d, dg::IDMatrix,dg::BijectiveComm< dg::iDVec, dg::DVec >, dg::DVec> DFA;
-using namespace dg::geo::solovev;
 int main( int argc, char* argv[])
 {
     ////////////////////////////////setup MPI///////////////////////////////
@@ -73,7 +66,7 @@ int main( int argc, char* argv[])
         reader.parse(ks,gs,false);
     }
     const feltor::Parameters p( js);
-    const dg::geo::solovev::GeomParameters gp(gs);
+    const dg::geo::solovev::Parameters gp(gs);
     if(rank==0)p.display( std::cout);
     if(rank==0)gp.display( std::cout);
     std::string input = js.toStyledString(), geom = gs.toStyledString();
@@ -89,14 +82,14 @@ int main( int argc, char* argv[])
      
     //create RHS 
     if(rank==0)std::cout << "Constructing Explicit...\n";
-    feltor::Explicit<dg::CylindricalMPIGrid3d, dg::IDMatrix, dg::MDMatrix, dg::MDVec> feltor( grid, p, gp); //initialize before rolkar!
+    feltor::Explicit<dg::CylindricalMPIGrid3d, dg::MIDMatrix, dg::MDMatrix, dg::MDVec> feltor( grid, p, gp); //initialize before rolkar!
     if(rank==0)std::cout << "Constructing Implicit...\n";
-    feltor::Implicit< dg::CylindricalMPIGrid3d, dg::IDMatrix, dg::MDMatrix, dg::MDVec > rolkar( grid, p, gp, feltor.ds(), feltor.dsDIR());
+    feltor::Implicit< dg::CylindricalMPIGrid3d, dg::MIDMatrix, dg::MDMatrix, dg::MDVec > rolkar( grid, p, gp, feltor.ds(), feltor.dsDIR());
     if(rank==0)std::cout << "Done!\n";
 
     /////////////////////The initial field/////////////////////////////////////////
     //background profile
-    dg::geo::Nprofile<Psip> prof(p.bgprofamp, p.nprofileamp, gp, Psip(gp)); //initial background profile
+    dg::geo::Nprofile prof(p.bgprofamp, p.nprofileamp, gp, dg::geo::solovev::Psip(gp)); //initial background profile
     std::vector<dg::MDVec> y0(4, dg::evaluate( prof, grid)), y1(y0); 
     //perturbation 
     dg::GaussianZ gaussianZ( 0., p.sigma_z*M_PI, 1); //modulation along fieldline
@@ -115,14 +108,14 @@ int main( int argc, char* argv[])
     }
     if( p.mode == 3)
     {
-        dg::geo::ZonalFlow<Psip> init0(p.amp, p.k_psi, gp, Psip(gp));
+        dg::geo::ZonalFlow init0(p.amp, p.k_psi, gp, dg::geo::solovev::Psip(gp));
         y1[1] = feltor.ds().fieldaligned().evaluate( init0, gaussianZ, (unsigned)p.Nz/2, 1); 
     }
     dg::blas1::axpby( 1., y1[1], 1., y0[1]); //sum up background and perturbation
     dg::blas1::plus(y0[1], -1); //initialize ni-1
     if( p.mode == 2 || p.mode == 3)
     {
-        dg::MDVec damping = dg::evaluate( dg::geo::GaussianProfXDamping<Psip>(Psip(gp), gp), grid);
+        dg::MDVec damping = dg::evaluate( dg::geo::GaussianProfXDamping(dg::geo::solovev::Psip(gp), gp), grid);
         dg::blas1::pointwiseDot(damping, y0[1], y0[1]); //damp with gaussprofdamp
     }
     std::cout << "intiialize ne" << std::endl;
@@ -144,11 +137,11 @@ int main( int argc, char* argv[])
     err = nc_put_att_text( ncid, NC_GLOBAL, "geomfile",  geom.size(), geom.data());
     int dimids[4], tvarID;
     {
-        MagneticField c(gp);
+        dg::geo::TokamakMagneticField c = dg::geo::createSolovevField(gp);
         err = file::define_dimensions( ncid, dimids, &tvarID, grid_out.global());
-        dg::geo::FieldR<MagneticField> fieldR(c, gp.R_0);
-        dg::geo::FieldZ<MagneticField> fieldZ(c, gp.R_0);
-        dg::geo::FieldP<MagneticField> fieldP(c, gp.R_0);
+        dg::geo::FieldR fieldR(c);
+        dg::geo::FieldZ fieldZ(c);
+        dg::geo::FieldP fieldP(c);
         dg::HVec vecR = dg::evaluate( fieldR, grid_out.global());
         dg::HVec vecZ = dg::evaluate( fieldZ, grid_out.global());
         dg::HVec vecP = dg::evaluate( fieldP, grid_out.global());
@@ -214,7 +207,7 @@ int main( int argc, char* argv[])
     if(rank==0)std::cout << "First output ... \n";
     int dims[3],  coords[3];
     MPI_Cart_get( comm, 3, dims, periods, coords);
-    size_t count[4] = {1, grid_out.Nz(), grid_out.n()*(grid_out.Ny()), grid_out.n()*(grid_out.Nx())};
+    size_t count[4] = {1, grid_out.local().Nz(), grid_out.n()*(grid_out.local().Ny()), grid_out.n()*(grid_out.local().Nx())};
     size_t start[4] = {0, coords[2]*count[1], coords[1]*count[2], coords[0]*count[3]};
     dg::MDVec transfer( dg::evaluate(dg::zero, grid));
     dg::DVec transferD( dg::evaluate(dg::zero, grid_out.local()));
diff --git a/src/feltor/parameters.h b/src/feltor/parameters.h
index d509fc8aa..1c6a3e460 100644
--- a/src/feltor/parameters.h
+++ b/src/feltor/parameters.h
@@ -2,7 +2,7 @@
 #include "dg/enums.h"
 #include "json/json.h"
 
-namespace eule{
+namespace feltor{
 /**
  * @brief Provide a mapping between input file and named parameters
  */
@@ -168,7 +168,7 @@ struct Parameters
     }
 };
 
-}//namespace eule
+}//namespace feltor
 
 
     
-- 
GitLab


From 6ccec66210b6a9d655db62ebf458d3d4badbd281 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Fri, 3 Nov 2017 22:48:13 +0100
Subject: [PATCH 439/453] removed if statement of p.iso == 1 in feltorSH and
 feltorSHp

---
 src/feltorSH/feltor.cuh  |  9 ---------
 src/feltorSHp/feltor.cuh | 10 ----------
 2 files changed, 19 deletions(-)

diff --git a/src/feltorSH/feltor.cuh b/src/feltorSH/feltor.cuh
index 2828e8f7f..e33d746b9 100644
--- a/src/feltorSH/feltor.cuh
+++ b/src/feltorSH/feltor.cuh
@@ -1,12 +1,7 @@
 #pragma once
 
 #include "dg/algorithm.h"
-#include "dg/poisson.h"
 #include "parameters.h"
-#include <cusp/multiply.h>
-#ifdef DG_BENCHMARK
-#include "dg/backend/timer.cuh"
-#endif //DG_BENCHMARK
 /*!@file
 
   Contains the solvers 
@@ -406,10 +401,6 @@ void Feltor<G, Matrix, container>::operator()( const std::vector<container>& y,
     assert( y.size() == 4);
     assert( y.size() == yp.size());
     //transform compute n and logn and energies
-    if (p.iso == 1)    {
-        dg::blas1::scal( y[2], 0.0);
-        dg::blas1::scal( y[3], 0.0); 
-    } //it's not safe to change the input since this is hold by the time integrator, if you really need to do this you could copy y into your own variable and set that to zero (MW)
     
     for(unsigned i=0; i<4; i++)
     {
diff --git a/src/feltorSHp/feltor.cuh b/src/feltorSHp/feltor.cuh
index 06509f434..862b5eef8 100644
--- a/src/feltorSHp/feltor.cuh
+++ b/src/feltorSHp/feltor.cuh
@@ -1,12 +1,7 @@
 #pragma once
 
 #include "dg/algorithm.h"
-#include "dg/poisson.h"
 #include "parameters.h"
-#include <cusp/multiply.h>
-#ifdef DG_BENCHMARK
-#include "dg/backend/timer.cuh"
-#endif //DG_BENCHMARK
 /*!@file
 
   Contains the solvers 
@@ -426,11 +421,6 @@ void Feltor<G, Matrix, container>::operator()( const std::vector<container>& y,
     assert( y.size() == 4);
     assert( y.size() == yp.size());
     
-    if (p.iso == 1)    {
-        dg::blas1::axpby((p.bgprofamp + p.nprofileamp),y[0],0., y[2],y[2] );
-        dg::blas1::axpby((p.bgprofamp + p.nprofileamp),y[1],0., y[3],y[3] ); 
-    } //see same place in feltorSH  (MW)
-    
     for(unsigned i=0; i<2; i++)
     {
         dg::blas1::transform( y[i], n[i], dg::PLUS<>(+(p.bgprofamp + p.nprofileamp))); //N = N_tilde +(p.bgprofamp + p.nprofileamp)
-- 
GitLab


From e57a6b2a3b924266b763c2fe34ad1bba354536dd Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Fri, 3 Nov 2017 23:34:45 +0100
Subject: [PATCH 440/453] cleanup backend README

---
 inc/dg/backend/README | 59 -------------------------------------------
 inc/dg/dg_doc.h       | 19 ++++++++------
 2 files changed, 11 insertions(+), 67 deletions(-)

diff --git a/inc/dg/backend/README b/inc/dg/backend/README
index 50f666e84..c36460d76 100644
--- a/inc/dg/backend/README
+++ b/inc/dg/backend/README
@@ -1,61 +1,2 @@
 contains engines for the container free numerical algorithms and test programs as
 well as additional utility functions
-
-Performance hints
-===========================================
-FIRST: (Small things to do)
-1. Implement transposition of mpi interpolation for dz.h (explanation further below)
-   - transpose interpolation matrices locally
-   - first apply collective.gather()
-   - then do a gemv on the buffer 
-
-2. Implement an mpi version of create::dz and the application of it in mpi_matrix.h (if needed in general_elliptic.h)
-
-THIRD: (road to MPI + X, where X is all we already have, i.e. OpenMP, MIC, or GPU)
-The general idea is to separate global communication from local parallelization and thus 
-readily reuse the existing, optimized library for the local part
-
-2. implement a general sparse mpi matrix format (unifying the existing mpi_matrix and the mpi-interpolation )
-The idea is that the mpi matrix takes care of the communication and then defers the actual computation 
-to an existing local matrix format.
-
-    There are two approaches to distribute a global sparse matrix to nodes: row distributed (RD) or 
-    column distributed (CD). In the first each node gets the same rows as the vector it holds, in the
-    second it gets the same columns as the vector it holds.
-    IDEA how to then implement gemv(input, output) algorithm:
-    RD: 1. create a local send buffer and locally gather values from input vector (c) into a send buffer (am Besten nach PID geordnet, note that a given value can be sent to several processes -> that's why it's a gather)
-        2. globally scatter these values into recv buffer (b)  (The first two points are a global gather)
-        3. then apply the local matrix to that buffer and store result in output vector (a)
-
-    CD: 1. apply the local matrix on the input vector and store result in a local send buffer,  (a)
-        2. globally scatter the values in this buffer to a recv buffer (b) (every value in the result belongs to exactly one line/process)
-        3. then permute and reduce the recv buffer on double indices and store result in output vector (c) 
-    
-        Transposition is easy: if a RD matrix is transposed you get a CD matrix, also transpose the Collective object (swap scatter and gather maps)
-
-    Note that there are three types of indices that you need to consider: 
-    1) the global vector index is the index of an element if there was only one vector that lay contiguously in memory. 
-    2) the local vector index is the index of the local chunk a process has. 
-    3) the buffer index is the index into the communication buffer. 
-    
-    The matrix class needs three private data members for the three steps: 
-    a) The local matrix, (using buffer indices, the type should probably be a template parameter)
-    b) A Collective object which holds a communication pattern to take care of the global scatter operations,  and the communication between device and host e.g.
-    c) A local index map, to map the local buffer indices to local row/col indices
-    
-    IDEA: how to create such a matrix:
-    If you can, manually create (a), (b), (c) and construct the matrix. 
-    The current mpi matrix is a row distributed matrix and the communication 
-    is done by manual MPI_SendRecv calls. This has to (maybe even should) be replaced by a Collective object, which
-    internally uses MPI_Alltoallv (If need be, the MPI_Alltoallv might be replaced by MPI_Neighbor_alltoallv in MPI 3)
-
-
-    If you cannot or you don't want to:
-    1. create local sparse matrices holding global row/col indices and distribute row-wise or column-wise
-    ( there needs to be a map or function that can map global indices to local ones and the corresponding PID)
-    2. unique_copy global row (CD)/ col (RD) indices into buffer (no mistake here :-))
-    3. map global indices in this buffer to local indices and PIDs
-    4. in the local matrix replace row( CD)/ col(RD) indices by the corresponding buffer indices (find & replace) -> (a)
-         (this might be slow) 
-    5. with the PIDs construct Collective (b), then globally scatter the indices -> (c)
-
diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index 76b8559c5..44b38f726 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -190,26 +190,28 @@
 
 In Feltor each mpi process gets an equally sized chunk of a vector.
 The corresponding structure in FELTOR is the \c dg::MPI_Vector, which is 
-nothing but a wrapper around any container type object and a MPI_Comm. 
+nothing but a wrapper around any container type object and a \c MPI_Comm. 
 With this the \c dg::blas1 functions can readily implemented by just redirecting to the
 implementation for the container type. The only functions that need
-communication are the \c dg::blas1::dot functions (MPI_Allreduce).
+communication are the \c dg::blas1::dot functions (\c MPI_Allreduce).
 
 @section mpi_matrix Row and column distributed matrices
 
 Contrary to a vector
-a matrix can be distributed in two ways, row-wise and column wise. 
-The structure \c dg::MPIDistMat is a wrapper around a LocalMatrix type object 
-and an instance of \c dg::aCommunicator
-In a row-distributed matrix each process gets the complete 
+a matrix can be distributed among processes in two ways: 
+\a row-distributed and \a column-distributed. 
+In a row-distributed matrix each process gets the 
 rows of the matrix that correspond to the indices in the 
 vector it holds. 
-In a column-distributed matrix each process gets the complete 
-columns of the matrix corresponding to the indices in the 
+In a column-distributed matrix each process gets the
+columns of the matrix that correspond to the indices in the 
 vector it holds. 
 When we implement a matrix-vector multiplication the order 
 of communication and computation depends on the distribution 
 of the matrix.
+First, we define the structure \c dg::MPIDistMat as a simple a wrapper around a 
+LocalMatrix type object 
+and an instance of a \c dg::aCommunicator.
 \subsection row Row distributed
 For the row-distributed matrix each process first has to gather 
 all elements of the input vector it needs to be able to compute the elements of the output. In general this requires MPI communication.
@@ -246,5 +248,6 @@ It turns out that a row-distributed matrix can be transposed
 by transposition of the local matrices and the gather matrix (s.a. \c dg::transpose).
 The result is then a column distributed matrix.
 The transpose of a column distributed matrix is a row-distributed matrix and vice-versa.
+You can create an MPI row-distributed matrix if you know the global column indices by our \c dg::convert function.
 @}
 */
-- 
GitLab


From 0169bdb37473652217e9abc5f18cd2c86a795149 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Fri, 3 Nov 2017 23:48:20 +0100
Subject: [PATCH 441/453] moved hiddenn scripts to backend

---
 .find_large_objects.sh => inc/dg/backend/.find_large_objects.sh | 2 +-
 .rm_big_objects.sh => inc/dg/backend/.rm_big_objects.sh         | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename .find_large_objects.sh => inc/dg/backend/.find_large_objects.sh (74%)
 rename .rm_big_objects.sh => inc/dg/backend/.rm_big_objects.sh (100%)

diff --git a/.find_large_objects.sh b/inc/dg/backend/.find_large_objects.sh
similarity index 74%
rename from .find_large_objects.sh
rename to inc/dg/backend/.find_large_objects.sh
index 7a8a7bf8a..bc1276708 100755
--- a/.find_large_objects.sh
+++ b/inc/dg/backend/.find_large_objects.sh
@@ -3,7 +3,7 @@
 #This scripts finds large objects that might accidentally be added to the repository and stores them in largeobjects.txt
 
 git rev-list --objects --all | sort -k 2 > allfileshas.txt
-git gc && git verify-pack -v .git/objects/pack/pack-*.idx | egrep "^\w+ blob\W+[0-9]+ [0-9]+ [0-9]+$" | sort -k 3 -n -r > bigobjects.txt
+git gc && git verify-pack -v ../../../.git/objects/pack/pack-*.idx | egrep "^\w+ blob\W+[0-9]+ [0-9]+ [0-9]+$" | sort -k 3 -n -r > bigobjects.txt
 for SHA in `cut -f 1 -d\  < bigobjects.txt`; do
     echo $(grep $SHA bigobjects.txt) $(grep $SHA allfileshas.txt) | awk '{print $1,$3,$7}' >> bigtosmall.txt
     done;
diff --git a/.rm_big_objects.sh b/inc/dg/backend/.rm_big_objects.sh
similarity index 100%
rename from .rm_big_objects.sh
rename to inc/dg/backend/.rm_big_objects.sh
-- 
GitLab


From 42d64177377bb9c9751c8baeb0816407b964d4e5 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 6 Nov 2017 11:46:44 +0100
Subject: [PATCH 442/453] changed DSFieldCylindrical

---
 inc/geometries/fieldaligned.h | 25 +++++++++++++++----------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index 36c751ab8..d3cf6191e 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -44,12 +44,16 @@ namespace detail{
 
 struct DSFieldCylindrical
 {
-    DSFieldCylindrical( const dg::geo::BinaryVectorLvl0& v):v_(v) { }
+    DSFieldCylindrical( const dg::geo::BinaryVectorLvl0& v, Grid2d boundary):v_(v), m_b(boundary) { }
     void operator()( const dg::HVec& y, dg::HVec& yp) const {
-        double vz = v_.z()(y[0], y[1]);
-        yp[0] = v_.x()(y[0], y[1])/vz; 
-        yp[1] = v_.y()(y[0], y[1])/vz;
+        double R = y[0], Z = y[1];
+        //m_b.shift_topologic( y[0], y[1], R, Z); //shift R,Z onto domain
+        double vz = v_.z()(R, Z);
+        yp[0] = v_.x()(R, Z)/vz; 
+        yp[1] = v_.y()(R, Z)/vz;
         yp[2] = 1./vz;
+        if( !m_b.contains( yp[0], yp[1])) 
+            yp[0] = yp[1] = 0;
     }
 
     double error( const dg::HVec& x0, const dg::HVec& x1)const {
@@ -60,16 +64,17 @@ struct DSFieldCylindrical
         {
             return false;
         }
-        //if new integrated point outside domain
-        if ((1e-5 > end[0]  ) || (1e10 < end[0])  ||(-1e10  > end[1]  ) || (1e10 < end[1])||(-1e10 > end[2]  ) || (1e10 < end[2])  )
-        {
-            return false;
-        }
+        ////if new integrated point outside domain
+        //if ((1e-5 > end[0]  ) || (1e10 < end[0])  ||(-1e10  > end[1]  ) || (1e10 < end[1])||(-1e10 > end[2]  ) || (1e10 < end[2])  )
+        //{
+        //    return false;
+        //}
         return true;
     }
     
     private:
     dg::geo::BinaryVectorLvl0 v_;
+    dg::Grid2d m_b;
 };
 
 struct DSField
@@ -272,7 +277,7 @@ void integrate_all_fieldlines2d( const dg::geo::BinaryVectorLvl0& vec, const dg:
     //construct field on high polynomial grid, then integrate it
     dg::geo::detail::DSField field( vec, grid_field);
     //field in case of cartesian grid
-    dg::geo::detail::DSFieldCylindrical cyl_field(vec);
+    dg::geo::detail::DSFieldCylindrical cyl_field(vec, (dg::Grid2d)grid_field);
     unsigned size = grid_evaluate.size();
     for( unsigned i=0; i<size; i++)
     {
-- 
GitLab


From 6826938099109a61e43a87b50d1290434dbb8833 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 6 Nov 2017 12:01:36 +0100
Subject: [PATCH 443/453] removed monitor in DSField

---
 inc/geometries/fieldaligned.h | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index d3cf6191e..f16a51782 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -47,7 +47,7 @@ struct DSFieldCylindrical
     DSFieldCylindrical( const dg::geo::BinaryVectorLvl0& v, Grid2d boundary):v_(v), m_b(boundary) { }
     void operator()( const dg::HVec& y, dg::HVec& yp) const {
         double R = y[0], Z = y[1];
-        //m_b.shift_topologic( y[0], y[1], R, Z); //shift R,Z onto domain
+        m_b.shift_topologic( y[0], y[1], R, Z); //shift R,Z onto domain
         double vz = v_.z()(R, Z);
         yp[0] = v_.x()(R, Z)/vz; 
         yp[1] = v_.y()(R, Z)/vz;
@@ -119,11 +119,11 @@ struct DSField
         {
             return false;
         }
-        //if new integrated point far outside domain
-        if ((end[0] < g_.get().x0()-1e4 ) || (g_.get().x1()+1e4 < end[0])  ||(end[1] < g_.get().y0()-1e4 ) || (g_.get().y1()+1e4 < end[1])||(-1e10 > end[2]  ) || (1e10 < end[2])  )
-        {
-            return false;
-        }
+        ////if new integrated point far outside domain (How could it end up there??)
+        //if ((end[0] < g_.get().x0()-1e4 ) || (g_.get().x1()+1e4 < end[0])  ||(end[1] < g_.get().y0()-1e4 ) || (g_.get().y1()+1e4 < end[1])||(-1e10 > end[2]  ) || (1e10 < end[2])  )
+        //{
+        //    return false;
+        //}
         return true;
     }
     private:
@@ -260,6 +260,7 @@ void boxintegrator( const Field& field, const Topology& grid,
         //now assume the rest is purely toroidal
         double deltaS = coords1[2];
         thrust::host_vector<double> temp=coords0;
+        //compute the vector value on the boundary point
         field(coords1, temp); //we are just interested in temp[2]
         coords1[2] = deltaS + (deltaPhi-phi1)*temp[2]; // ds + dphi*f[2]
     }
-- 
GitLab


From b7710174bbbc5ddbea0c458bced0f344f264ce81 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 6 Nov 2017 16:29:05 +0100
Subject: [PATCH 444/453] made additional parameter in ds to choose whether to
 interpolate or not

---
 inc/dg/runge_kutta.h              |  2 +-
 inc/geometries/ds.h               | 14 ++++++-----
 inc/geometries/ds_curv_mpit.cu    |  2 +-
 inc/geometries/ds_curv_t.cu       |  2 +-
 inc/geometries/ds_mpit.cu         |  2 +-
 inc/geometries/ds_t.cu            |  4 +--
 inc/geometries/fieldaligned.h     | 42 ++++++++++++++++++-------------
 inc/geometries/mpi_fieldaligned.h | 27 ++++++++++++--------
 src/toefl/Makefile                |  2 +-
 9 files changed, 55 insertions(+), 42 deletions(-)

diff --git a/inc/dg/runge_kutta.h b/inc/dg/runge_kutta.h
index 6cddc2c2c..28b9f6be4 100644
--- a/inc/dg/runge_kutta.h
+++ b/inc/dg/runge_kutta.h
@@ -599,7 +599,7 @@ int integrateRK(RHS& rhs, const container& begin, container& end, double T_max,
     }
     if( error > eps_abs )
     {
-        std::cerr << "ATTENTION: Runge Kutta failed to converge. Error is "<<error<<std::endl;
+        std::cerr << "ATTENTION: Runge Kutta failed to converge. Error is "<<error<<" with "<<NT<<" steps"<<std::endl;
         return -2;
     }
     return NT;
diff --git a/inc/geometries/ds.h b/inc/geometries/ds.h
index 65f2cd5c9..38400079e 100644
--- a/inc/geometries/ds.h
+++ b/inc/geometries/ds.h
@@ -54,10 +54,10 @@ struct DS
         dg::bc bcy = dg::NEU, 
         Limiter limit = FullLimiter(), 
         dg::norm no=dg::normed, dg::direction dir = dg::centered, 
-        double eps = 1e-5, unsigned multiplyX=10, unsigned multiplyY=10, bool dependsOnX = true, bool dependsOnY=true)
+        double eps = 1e-5, unsigned multiplyX=10, unsigned multiplyY=10, bool dependsOnX = true, bool dependsOnY=true, bool integrateAll=true)
     {
         dg::geo::BinaryVectorLvl0 bhat( (dg::geo::BHatR)(vec), (dg::geo::BHatZ)(vec), (dg::geo::BHatP)(vec));
-        m_fa.construct( bhat, grid, bcx, bcy, limit, eps, multiplyX, multiplyY, dependsOnX, dependsOnY);
+        m_fa.construct( bhat, grid, bcx, bcy, limit, eps, multiplyX, multiplyY, dependsOnX, dependsOnY,integrateAll);
         construct( m_fa, no, dir);
     }
     /**
@@ -77,8 +77,10 @@ struct DS
      * @param eps Desired accuracy of the fieldline integrator
      * @param multiplyX defines the resolution in X of the fine grid relative to grid
      * @param multiplyY defines the resolution in Y of the fine grid relative to grid
-     * @param dependsOnX indicates, whether the given vector field vec depends on the first coordinate
-     * @param dependsOnY indicates, whether the given vector field vec depends on the second coordinate
+     * @param dependsOnX indicates, whether the given vector field vec depends on the first coordinate (jump terms are added in symv)
+     * @param dependsOnY indicates, whether the given vector field vec depends on the second coordinate (jump terms are added in symv)
+     * @param integrateAll indicates, that all fieldlines of the fine grid should be integrated instead of interpolating it from the coarse grid. 
+     *  Should be true if the streamlines of the vector field cross the domain boudary. 
      * @sa \c Fieldaligned
      */
     template<class Limiter>
@@ -87,9 +89,9 @@ struct DS
         dg::bc bcy = dg::NEU, 
         Limiter limit = FullLimiter(), 
         dg::norm no=dg::normed, dg::direction dir = dg::centered, 
-        double eps = 1e-5, unsigned multiplyX=10, unsigned multiplyY=10, bool dependsOnX = true, bool dependsOnY=true)
+        double eps = 1e-5, unsigned multiplyX=10, unsigned multiplyY=10, bool dependsOnX = true, bool dependsOnY=true, bool integrateAll=true)
     {
-        m_fa.construct( vec, grid, bcx, bcy, limit, eps, multiplyX, multiplyY, dependsOnX, dependsOnY);
+        m_fa.construct( vec, grid, bcx, bcy, limit, eps, multiplyX, multiplyY, dependsOnX, dependsOnY, integrateAll);
         construct( m_fa, no, dir);
     }
     ///@copydoc construct
diff --git a/inc/geometries/ds_curv_mpit.cu b/inc/geometries/ds_curv_mpit.cu
index 7b60f863c..24a00b49e 100644
--- a/inc/geometries/ds_curv_mpit.cu
+++ b/inc/geometries/ds_curv_mpit.cu
@@ -47,7 +47,7 @@ int main(int argc, char * argv[])
     if(rank==0)std::cout << "Constructing Grid...\n";
     dg::geo::CurvilinearProductMPIGrid3d g3d(flux, n, Nx, Ny,Nz, dg::DIR, dg::PER, dg::PER, comm);
     if(rank==0)std::cout << "Constructing Fieldlines...\n";
-    dg::geo::DS<dg::aProductMPIGeometry3d, dg::MIHMatrix, dg::MHMatrix, dg::MHVec> ds( mag, g3d, dg::NEU, dg::NEU, dg::geo::FullLimiter(), dg::normed, dg::centered, 1e-8, mx, my, false, true);
+    dg::geo::DS<dg::aProductMPIGeometry3d, dg::MIHMatrix, dg::MHMatrix, dg::MHVec> ds( mag, g3d, dg::NEU, dg::NEU, dg::geo::FullLimiter(), dg::normed, dg::centered, 1e-8, mx, my, false, true,false);
     
     t.toc();
     if(rank==0)std::cout << "Construction took "<<t.diff()<<"s\n";
diff --git a/inc/geometries/ds_curv_t.cu b/inc/geometries/ds_curv_t.cu
index 3fd068846..f6e73dfa1 100644
--- a/inc/geometries/ds_curv_t.cu
+++ b/inc/geometries/ds_curv_t.cu
@@ -44,7 +44,7 @@ int main(int argc, char * argv[])
     dg::geo::CurvilinearProductGrid3d g3d(flux, n, Nx, Ny,Nz, dg::DIR);
     //dg::geo::Fieldaligned<dg::aGeometry3d, dg::IHMatrix, dg::HVec> fieldaligned( bhat, g3d, 1, 4, gp.rk4eps, dg::NoLimiter() ); 
     std::cout << "Constructing Fieldlines..."<<std::endl;
-    dg::geo::DS<dg::aProductGeometry3d, dg::IHMatrix, dg::HMatrix, dg::HVec> ds( mag, g3d, dg::NEU, dg::NEU, dg::geo::FullLimiter(), dg::normed, dg::centered, 1e-8, mx, my, false, true);
+    dg::geo::DS<dg::aProductGeometry3d, dg::IHMatrix, dg::HMatrix, dg::HVec> ds( mag, g3d, dg::NEU, dg::NEU, dg::geo::FullLimiter(), dg::normed, dg::centered, 1e-8, mx, my, false, true, false);
     
     t.toc();
     std::cout << "Construction took "<<t.diff()<<"s\n";
diff --git a/inc/geometries/ds_mpit.cu b/inc/geometries/ds_mpit.cu
index 88be045f4..fbd48b56e 100644
--- a/inc/geometries/ds_mpit.cu
+++ b/inc/geometries/ds_mpit.cu
@@ -36,7 +36,7 @@ int main(int argc, char* argv[])
     if(rank==0)std::cout << "Create parallel Derivative!\n";
     dg::geo::TokamakMagneticField mag = dg::geo::createCircularField( R_0, I_0);
     dg::geo::BinaryVectorLvl0 bhat( (dg::geo::BHatR)(mag), (dg::geo::BHatZ)(mag), (dg::geo::BHatP)(mag));
-    dg::geo::Fieldaligned<dg::aProductMPIGeometry3d,dg::MIDMatrix,dg::MDVec>  dsFA( bhat, g3d, dg::NEU, dg::NEU, dg::geo::NoLimiter(), 1e-6, 5,5,true,true);
+    dg::geo::Fieldaligned<dg::aProductMPIGeometry3d,dg::MIDMatrix,dg::MDVec>  dsFA( bhat, g3d, dg::NEU, dg::NEU, dg::geo::NoLimiter(), 1e-8);
     dg::geo::DS<dg::aProductMPIGeometry3d, dg::MIDMatrix, dg::MDMatrix, dg::MDVec> ds( dsFA, dg::not_normed, dg::centered);
     if(rank==0)std::cout << "Ready!\n";
 
diff --git a/inc/geometries/ds_t.cu b/inc/geometries/ds_t.cu
index 875241726..d50351152 100644
--- a/inc/geometries/ds_t.cu
+++ b/inc/geometries/ds_t.cu
@@ -30,7 +30,7 @@ int main(int argc, char * argv[])
     unsigned n, Nx, Ny, Nz;
     std::cin >> n>> Nx>>Ny>>Nz;
     std::cout << "You typed "<<n<<" "<<Nx<<" "<<Ny<<" "<<Nz<<std::endl;
-    std::cout << "Type mx (5) and my (5)\n";
+    std::cout << "Type mx (10) and my (10)\n";
     unsigned mx, my;
     std::cin >> mx>> my;
     std::cout << "You typed "<<mx<<" "<<my<<std::endl;
@@ -42,7 +42,7 @@ int main(int argc, char * argv[])
     const dg::geo::TokamakMagneticField mag = dg::geo::createCircularField( R_0, I_0);
     const dg::geo::BinaryVectorLvl0 bhat( (dg::geo::BHatR)(mag), (dg::geo::BHatZ)(mag), (dg::geo::BHatP)(mag));
     //create Fieldaligned object and construct DS from it
-    const dg::geo::Fieldaligned<dg::aProductGeometry3d,dg::IDMatrix,dg::DVec>  dsFA( bhat, g3d, dg::NEU, dg::NEU, dg::geo::NoLimiter(), 1e-6, mx, my, true,true);
+    const dg::geo::Fieldaligned<dg::aProductGeometry3d,dg::IDMatrix,dg::DVec>  dsFA( bhat, g3d, dg::NEU, dg::NEU, dg::geo::NoLimiter(), 1e-8, mx, my, true,true,true);
     dg::geo::DS<dg::aProductGeometry3d, dg::IDMatrix, dg::DMatrix, dg::DVec> ds( dsFA, dg::not_normed, dg::centered);
     ///##########################################################///
     //apply to function 
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index f16a51782..742ca62ae 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -52,8 +52,6 @@ struct DSFieldCylindrical
         yp[0] = v_.x()(R, Z)/vz; 
         yp[1] = v_.y()(R, Z)/vz;
         yp[2] = 1./vz;
-        if( !m_b.contains( yp[0], yp[1])) 
-            yp[0] = yp[1] = 0;
     }
 
     double error( const dg::HVec& x0, const dg::HVec& x1)const {
@@ -134,6 +132,7 @@ struct DSField
 
 void clip_to_boundary( double& x, double& y, const dg::aTopology2d& grid)
 {
+    //there is no shift_topologic because if interpolating the results
     if (!(x > grid.x0())) { x=grid.x0();}
     if (!(x < grid.x1())) { x=grid.x1();}
     if (!(y > grid.y0())) { y=grid.y0();}
@@ -216,7 +215,7 @@ struct BoxIntegrator
 };
 
 /**
- * @brief Integrate one field line in a given box, Result is guaranteed to lie inside the box modulo periodic boundary conditions
+ * @brief Integrate one field line in a given box
  *
  * @tparam Field Must be usable in the integrateRK function
  * @tparam Topology must provide 2d contains and shift_topologic function
@@ -335,11 +334,11 @@ struct Fieldaligned
         Limiter limit = FullLimiter(), 
         double eps = 1e-5,
         unsigned multiplyX=10, unsigned multiplyY=10, 
-        bool dependsOnX=true, bool dependsOnY=true, 
+        bool dependsOnX=true, bool dependsOnY=true, bool integrateAll = true,
         double deltaPhi = -1)
     {
         dg::geo::BinaryVectorLvl0 bhat( (dg::geo::BHatR)(vec), (dg::geo::BHatZ)(vec), (dg::geo::BHatP)(vec));
-        construct( bhat, grid, globalbcx, globalbcy, limit, eps, multiplyX, multiplyY, dependsOnX, dependsOnY, deltaPhi);
+        construct( bhat, grid, globalbcx, globalbcy, limit, eps, multiplyX, multiplyY, dependsOnX, dependsOnY, integrateAll, deltaPhi);
     }
 
     ///@copydoc construct()
@@ -351,10 +350,10 @@ struct Fieldaligned
         Limiter limit = FullLimiter(), 
         double eps = 1e-5,
         unsigned multiplyX=10, unsigned multiplyY=10, 
-        bool dependsOnX=true, bool dependsOnY=true, 
+        bool dependsOnX=true, bool dependsOnY=true, bool integrateAll = true,
         double deltaPhi = -1)
     {
-        construct( vec, grid, globalbcx, globalbcy, limit, eps, multiplyX, multiplyY, dependsOnX, dependsOnY, deltaPhi);
+        construct( vec, grid, globalbcx, globalbcy, limit, eps, multiplyX, multiplyY, dependsOnX, dependsOnY, integrateAll, deltaPhi);
     }
     /**
     * @brief Construct from a field and a grid
@@ -367,15 +366,17 @@ struct Fieldaligned
     * @param globalbcx Defines the interpolation behaviour when a fieldline intersects the boundary box in the perpendicular direction
     * @param globalbcy Defines the interpolation behaviour when a fieldline intersects the boundary box in the perpendicular direction
     * @param limit Instance of the limiter class (Default is a limiter everywhere, 
-        note that if grid.bcz() is periodic it doesn't matter if there is a limiter or not)
+        note that if \c grid.bcz() is periodic it doesn't matter if there is a limiter or not)
     * @param eps Desired accuracy of the fieldline integrator
     * @param multiplyX defines the resolution in X of the fine grid relative to grid
     * @param multiplyY defines the resolution in Y of the fine grid relative to grid
-    * @param dependsOnX indicates, whether the given vector field vec depends on the first coordinate
-    * @param dependsOnY indicates, whether the given vector field vec depends on the second coordinate
+    * @param dependsOnX indicates, whether the given vector field vec depends on the first coordinate 
+    * @param dependsOnY indicates, whether the given vector field vec depends on the second coordinate 
+    * @param integrateAll indicates, that all fieldlines of the fine grid should be integrated instead of interpolating it from the coarse grid. 
+    *  Should be true if the streamlines of the vector field cross the domain boudary. 
     * @param deltaPhi Is either <0 (then it's ignored), or may differ from grid.hz() if grid.Nz() == 1
     * @note If there is a limiter, the boundary condition on the first/last plane is set 
-        by the grid.bcz() variable and can be changed by the set_boundaries function. 
+        by the \c grid.bcz() variable and can be changed by the set_boundaries function. 
         If there is no limiter, the boundary condition is periodic.
     */
     template <class Limiter>
@@ -386,7 +387,7 @@ struct Fieldaligned
         Limiter limit = FullLimiter(), 
         double eps = 1e-5, 
         unsigned multiplyX=10, unsigned multiplyY=10, 
-        bool dependsOnX=true, bool dependsOnY=true, 
+        bool dependsOnX=true, bool dependsOnY=true, bool integrateAll = true,
         double deltaPhi = -1);
 
     bool dependsOnX()const{return m_dependsOnX;}
@@ -527,7 +528,7 @@ template <class Limiter>
 void Fieldaligned<Geometry, IMatrix, container>::construct(
     const dg::geo::BinaryVectorLvl0& vec, const Geometry& grid, 
     dg::bc globalbcx, dg::bc globalbcy, Limiter limit, double eps, 
-    unsigned mx, unsigned my, bool bx, bool by, double deltaPhi)
+    unsigned mx, unsigned my, bool bx, bool by, bool integrateAll, double deltaPhi)
 {
     m_dependsOnX=bx, m_dependsOnY=by;
     m_Nz=grid.Nz(), m_bcz=grid.bcz(); 
@@ -554,12 +555,21 @@ void Fieldaligned<Geometry, IMatrix, container>::construct(
 #endif
     dg::Handle<dg::aGeometry2d> grid_magnetic = grid_coarse;//INTEGRATE HIGH ORDER GRID
     grid_magnetic.get().set( 7, grid_magnetic.get().Nx(), grid_magnetic.get().Ny());
+    dg::Grid2d grid_fine( grid_coarse.get() );//FINE GRID
+    grid_fine.multiplyCellNumbers((double)mx, (double)my);
 #ifdef DG_BENCHMARK
     t.toc();
      std::cout << "High order grid gen   took: "<<t.diff()<<"\n";
     t.tic();
 #endif
-    detail::integrate_all_fieldlines2d( vec, grid_magnetic.get(), grid_coarse.get(), yp_coarse, ym_coarse, deltaPhi, eps);
+    if(integrateAll)
+        detail::integrate_all_fieldlines2d( vec, grid_magnetic.get(), grid_fine, yp, ym, deltaPhi, eps);
+    else
+    {
+        detail::integrate_all_fieldlines2d( vec, grid_magnetic.get(), grid_coarse.get(), yp_coarse, ym_coarse, deltaPhi, eps);
+        dg::IHMatrix interpolate = dg::create::interpolation( grid_fine, grid_coarse.get());  //INTERPOLATE TO FINE GRID
+        dg::geo::detail::interpolate_and_clip( interpolate, grid_fine, grid_fine, yp_coarse, ym_coarse, yp, ym);
+    }
 #ifdef DG_BENCHMARK
     t.toc();
     std::cout << "Fieldline integration took: "<<t.diff()<<"\n";
@@ -567,10 +577,6 @@ void Fieldaligned<Geometry, IMatrix, container>::construct(
     //%%%%%%%%%%%%%%%%%%Create interpolation and projection%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     t.tic();
 #endif
-    dg::Grid2d grid_fine( grid_coarse.get() );//FINE GRID
-    grid_fine.multiplyCellNumbers((double)mx, (double)my);
-    dg::IHMatrix interpolate = dg::create::interpolation( grid_fine, grid_coarse.get());  //INTERPOLATE TO FINE GRID
-    dg::geo::detail::interpolate_and_clip( interpolate, grid_fine, grid_fine, yp_coarse, ym_coarse, yp, ym);
     dg::IHMatrix plusFine  = dg::create::interpolation( yp[0], yp[1], grid_coarse.get(), globalbcx, globalbcy), plus, plusT;
     dg::IHMatrix minusFine = dg::create::interpolation( ym[0], ym[1], grid_coarse.get(), globalbcx, globalbcy), minus, minusT;
     dg::IHMatrix projection = dg::create::projection( grid_coarse.get(), grid_fine);
diff --git a/inc/geometries/mpi_fieldaligned.h b/inc/geometries/mpi_fieldaligned.h
index 293906109..e72df40c8 100644
--- a/inc/geometries/mpi_fieldaligned.h
+++ b/inc/geometries/mpi_fieldaligned.h
@@ -69,11 +69,11 @@ struct Fieldaligned< ProductMPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY
         Limiter limit = FullLimiter(), 
         double eps = 1e-5,
         unsigned multiplyX=10, unsigned multiplyY=10, 
-        bool dependsOnX=true, bool dependsOnY=true, 
+        bool dependsOnX=true, bool dependsOnY=true, bool integrateAll = true,
         double deltaPhi = -1)
     {
         dg::geo::BinaryVectorLvl0 bhat( (dg::geo::BHatR)(vec), (dg::geo::BHatZ)(vec), (dg::geo::BHatP)(vec));
-        construct( bhat, grid, globalbcx, globalbcy, limit, eps, multiplyX, multiplyY, dependsOnX, dependsOnY, deltaPhi);
+        construct( bhat, grid, globalbcx, globalbcy, limit, eps, multiplyX, multiplyY, dependsOnX, dependsOnY, integrateAll, deltaPhi);
     }
     template <class Limiter>
     Fieldaligned(const dg::geo::BinaryVectorLvl0& vec, 
@@ -83,10 +83,10 @@ struct Fieldaligned< ProductMPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY
         Limiter limit = FullLimiter(), 
         double eps = 1e-5,
         unsigned multiplyX=10, unsigned multiplyY=10, 
-        bool dependsOnX=true, bool dependsOnY=true, 
+        bool dependsOnX=true, bool dependsOnY=true, bool integrateAll = true,
         double deltaPhi = -1)
     {
-        construct( vec, grid, globalbcx, globalbcy, limit, eps, multiplyX, multiplyY, dependsOnX, dependsOnY, deltaPhi);
+        construct( vec, grid, globalbcx, globalbcy, limit, eps, multiplyX, multiplyY, dependsOnX, dependsOnY, integrateAll, deltaPhi);
     }
     template <class Limiter>
     void construct(const dg::geo::BinaryVectorLvl0& vec, 
@@ -96,7 +96,7 @@ struct Fieldaligned< ProductMPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY
         Limiter limit = FullLimiter(), 
         double eps = 1e-5, 
         unsigned multiplyX=10, unsigned multiplyY=10, 
-        bool dependsOnX=true, bool dependsOnY=true, 
+        bool dependsOnX=true, bool dependsOnY=true, bool integrateAll = true,
         double deltaPhi = -1);
 
     bool dependsOnX()const{return m_dependsOnX;}
@@ -162,7 +162,7 @@ template <class Limiter>
 void Fieldaligned<MPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vector<LocalContainer> >::construct(
     const dg::geo::BinaryVectorLvl0& vec, const MPIGeometry& grid, 
     dg::bc globalbcx, dg::bc globalbcy, Limiter limit, double eps, 
-    unsigned mx, unsigned my, bool bx, bool by, double deltaPhi)
+    unsigned mx, unsigned my, bool bx, bool by, bool integrateAll, double deltaPhi)
 {
     m_dependsOnX=bx, m_dependsOnY=by;
     m_Nz=grid.local().Nz(), m_bcz=grid.bcz(); 
@@ -195,12 +195,21 @@ void Fieldaligned<MPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vec
     dg::Handle<dg::aMPIGeometry2d> grid_magnetic = grid_coarse;//INTEGRATE HIGH ORDER GRID
     grid_magnetic.get().set( 7, grid_magnetic.get().Nx(), grid_magnetic.get().Ny());
     dg::Handle<dg::aGeometry2d> global_grid_magnetic = grid_magnetic.get().global_geometry();
+    dg::MPIGrid2d grid_fine( grid_coarse.get() );//FINE GRID
+    grid_fine.multiplyCellNumbers((double)mx, (double)my);
 #ifdef DG_BENCHMARK
     t.toc();
     if(rank==0) std::cout << "High order grid gen   took: "<<t.diff()<<"\n";
     t.tic();
 #endif
-    detail::integrate_all_fieldlines2d( vec, global_grid_magnetic.get(), grid_coarse.get().local(), yp_coarse, ym_coarse, deltaPhi, eps);
+    if(integrateAll)
+        detail::integrate_all_fieldlines2d( vec, global_grid_magnetic.get(), grid_fine.local(), yp, ym, deltaPhi, eps);
+    else
+    {
+        detail::integrate_all_fieldlines2d( vec, global_grid_magnetic.get(), grid_coarse.get().local(), yp_coarse, ym_coarse, deltaPhi, eps);
+        dg::IHMatrix interpolate = dg::create::interpolation( grid_fine.local(), grid_coarse.get().local());  //INTERPOLATE TO FINE GRID
+        dg::geo::detail::interpolate_and_clip( interpolate, grid_fine.local(), grid_fine.global(), yp_coarse, ym_coarse, yp, ym);
+    }
 #ifdef DG_BENCHMARK
     t.toc();
     if(rank==0) std::cout << "Fieldline integration took: "<<t.diff()<<"\n";
@@ -208,10 +217,6 @@ void Fieldaligned<MPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vec
     //%%%%%%%%%%%%%%%%%%Create interpolation and projection%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     t.tic();
 #endif
-    dg::MPIGrid2d grid_fine( grid_coarse.get() );//FINE GRID
-    grid_fine.multiplyCellNumbers((double)mx, (double)my);
-    dg::IHMatrix interpolate = dg::create::interpolation( grid_fine.local(), grid_coarse.get().local());  //INTERPOLATE TO FINE GRID
-    dg::geo::detail::interpolate_and_clip( interpolate, grid_fine.local(), grid_fine.global(), yp_coarse, ym_coarse, yp, ym);
     dg::IHMatrix plusFine  = dg::create::interpolation( yp[0], yp[1], grid_coarse.get().global(), globalbcx, globalbcy), plus;
     dg::IHMatrix minusFine = dg::create::interpolation( ym[0], ym[1], grid_coarse.get().global(), globalbcx, globalbcy), minus;
     dg::IHMatrix projection = dg::create::projection( grid_coarse.get().local(), grid_fine.local());
diff --git a/src/toefl/Makefile b/src/toefl/Makefile
index febec8d4d..762d57f64 100644
--- a/src/toefl/Makefile
+++ b/src/toefl/Makefile
@@ -17,7 +17,7 @@ toefl_hpc: toefl_hpc.cu toeflR.cuh
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -DDG_BENCHMARK  -g
 
 toefl_mpi: toefl_mpi.cu toeflR.cuh 
-	$(MPICC) $(OPT) $(MPICFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) #-DDG_BENCHMARK
+	$(MPICC) $(OPT) $(MPICFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -DDG_BENCHMARK -g
 
 doc: 
 	mkdir -p doc; \
-- 
GitLab


From 9d19bbe2dc2b74af95853ba71724155ae542e5a0 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 6 Nov 2017 16:40:46 +0100
Subject: [PATCH 445/453] asela2d compiles

---
 src/asela2D/asela.cu     | 20 ++++++++------------
 src/asela2D/asela_hpc.cu | 29 ++++++++++++-----------------
 src/asela2D/asela_mpi.cu | 29 ++++++++++++-----------------
 3 files changed, 32 insertions(+), 46 deletions(-)

diff --git a/src/asela2D/asela.cu b/src/asela2D/asela.cu
index 50b1a9e73..1aec35f7a 100644
--- a/src/asela2D/asela.cu
+++ b/src/asela2D/asela.cu
@@ -7,8 +7,7 @@
 
 #include "draw/host_window.h"
 //#include "draw/device_window.cuh"
-#include "dg/backend/xspacelib.cuh"
-#include "dg/backend/timer.cuh"
+#include "dg/algorithm.h"
 #include "geometries/geometries.h"
 
 #include "asela/asela.cuh"
@@ -20,8 +19,6 @@
    - directly visualizes results on the screen using parameters in window_params.txt
 */
 
-typedef dg::FieldAligned< dg::CylindricalGrid3d, dg::IDMatrix, dg::DVec> DFA;
-using namespace dg::geo::solovev;
 int main( int argc, char* argv[])
 {
     ////////////////////////Parameter initialisation//////////////////////////
@@ -46,8 +43,8 @@ int main( int argc, char* argv[])
         std::cerr << "ERROR: Too many arguments!\nUsage: "<< argv[0]<<" [inputfile] [geomfile] \n";
         return -1;
     }
-    const eule::Parameters p( js);
-    const dg::geo::solovev::GeomParameters gp(gs);
+    const asela::Parameters p( js);
+    const dg::geo::solovev::Parameters gp(gs);
     p.display( std::cout);
     gp.display( std::cout);
     /////////glfw initialisation ////////////////////////////////////////////
@@ -65,15 +62,14 @@ int main( int argc, char* argv[])
     //Make grid
     dg::CylindricalGrid3d grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n, p.Nx, p.Ny, 1, p.bc, p.bc, dg::PER);      //create RHS 
     std::cout << "Constructing Asela...\n";
-    eule::Asela<dg::CylindricalGrid3d, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec > asela( grid, p, gp); //initialize before rolkar!
-
+    asela::Asela<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec> asela( grid, p,gp); //initialize before rolkar!
     std::cout << "Constructing Rolkar...\n";
-    eule::Rolkar<dg::CylindricalGrid3d, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec> rolkar( grid, p, gp, asela.ds(), asela.dsDIR());
+    asela::Implicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec> rolkar(  grid, p, gp, asela.ds(), asela.dsDIR());
     std::cout << "Done!\n";
 
     /////////////////////The initial field///////////////////////////////////////////
     //background profile
-    dg::geo::Nprofile<Psip> prof(p.bgprofamp, p.nprofileamp, gp, Psip(gp)); //initial background profile
+    dg::geo::Nprofile prof(p.bgprofamp, p.nprofileamp, gp, dg::geo::solovev::Psip(gp)); //initial background profile
     std::vector<dg::DVec> y0(4, dg::evaluate( prof, grid)), y1(y0); 
     
     //initial perturbation
@@ -89,14 +85,14 @@ int main( int argc, char* argv[])
     }
     if (p.mode == 3) 
     { 
-        dg::geo::ZonalFlow<Psip> init0(p.amp, p.k_psi, gp, Psip(gp));
+        dg::geo::ZonalFlow init0(p.amp, p.k_psi, gp, dg::geo::solovev::Psip(gp));
         y1[1] = dg::evaluate( init0, grid);
     }
 
     
     dg::blas1::axpby( 1., y1[1], 1., y0[1]); //initialize ni
     dg::blas1::transform(y0[1], y0[1], dg::PLUS<>(-1)); //initialize ni-1
-    dg::DVec damping = dg::evaluate( dg::geo::GaussianProfXDamping<Psip>(Psip(gp), gp), grid);
+    dg::DVec damping = dg::evaluate( dg::geo::GaussianProfXDamping(dg::geo::solovev::Psip(gp), gp), grid);
     dg::blas1::pointwiseDot(damping,y0[1], y0[1]); //damp with gaussprofdamp
     std::cout << "intiialize ne" << std::endl;
     if( p.initcond == 0) asela.initializene( y0[1], y0[0]);
diff --git a/src/asela2D/asela_hpc.cu b/src/asela2D/asela_hpc.cu
index 3f560cd14..5f3661fc6 100644
--- a/src/asela2D/asela_hpc.cu
+++ b/src/asela2D/asela_hpc.cu
@@ -7,11 +7,8 @@
 
 
 
-#include "dg/backend/xspacelib.cuh"
-#include "dg/backend/timer.cuh"
-#include "dg/backend/interpolation.cuh"
 #include "file/nc_utilities.h"
-#include "geometries/solovev.h"
+#include "dg/algorithm.h"
 
 #include "asela/asela.cuh"
 #include "asela/parameters.h"
@@ -23,8 +20,6 @@
         density fields are the real densities in XSPACE ( not logarithmic values)
 */
 
-typedef dg::FieldAligned< dg::CylindricalGrid3d, dg::IDMatrix, dg::DVec> DFA;
-using namespace dg::geo::solovev;
 int main( int argc, char* argv[])
 {
     ////////////////////////Parameter initialisation//////////////////////////
@@ -42,8 +37,8 @@ int main( int argc, char* argv[])
         reader.parse(is,js,false);
         reader.parse(ks,gs,false);
     }
-    const eule::Parameters p( js);
-    const dg::geo::solovev::GeomParameters gp(gs);
+    const asela::Parameters p( js);
+    const dg::geo::solovev::Parameters gp(gs);
     p.display( std::cout);
     gp.display( std::cout);
     std::string input = js.toStyledString(), geom = gs.toStyledString();
@@ -58,14 +53,14 @@ int main( int argc, char* argv[])
     dg::CylindricalGrid3d grid_out( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n_out, p.Nx_out, p.Ny_out,1,p.bc, p.bc, dg::PER);  
     //create RHS 
     std::cout << "Constructing Asela...\n";
-    eule::Asela<dg::CylindricalGrid3d, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec > asela( grid, p, gp); //initialize before rolkar!
+    asela::Asela<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec> asela( grid, p,gp); //initialize before rolkar!
     std::cout << "Constructing Rolkar...\n";
-    eule::Rolkar<dg::CylindricalGrid3d, dg::DS<DFA, dg::DMatrix, dg::DVec>, dg::DMatrix, dg::DVec> rolkar( grid, p, gp, asela.ds(), asela.dsDIR());
+    asela::Implicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec> rolkar(  grid, p, gp, asela.ds(), asela.dsDIR());
     std::cout << "Done!\n";
 
     /////////////////////The initial field///////////////////////////////////////////
     //background profile
-    dg::geo::Nprofile<Psip> prof(p.bgprofamp, p.nprofileamp, gp, Psip(gp)); //initial background profile
+    dg::geo::Nprofile prof(p.bgprofamp, p.nprofileamp, gp, dg::geo::solovev::Psip(gp)); //initial background profile
     std::vector<dg::DVec> y0(4, dg::evaluate( prof, grid)), y1(y0); 
     //initial perturbation
     if (p.mode == 0  || p.mode ==1) 
@@ -80,13 +75,13 @@ int main( int argc, char* argv[])
     }
     if (p.mode == 3) 
     { 
-        dg::geo::ZonalFlow<Psip> init0(p.amp, p.k_psi, gp, Psip(gp));
+        dg::geo::ZonalFlow init0(p.amp, p.k_psi, gp, dg::geo::solovev::Psip(gp));
         y1[1] = dg::evaluate( init0, grid);
     }
     
     dg::blas1::axpby( 1., y1[1], 1., y0[1]); //initialize ni
     dg::blas1::transform(y0[1], y0[1], dg::PLUS<>(-1)); //initialize ni-1
-    dg::DVec damping = dg::evaluate( dg::geo::GaussianProfXDamping<Psip>(Psip(gp), gp), grid);
+    dg::DVec damping = dg::evaluate( dg::geo::GaussianProfXDamping(dg::geo::solovev::Psip(gp), gp), grid);
     dg::blas1::pointwiseDot( damping, y0[1], y0[1]); //damp with gaussprofdamp
     std::cout << "initialize ne" << std::endl;
     asela.initializene( y0[1], y0[0]);    
@@ -108,10 +103,10 @@ int main( int argc, char* argv[])
     err = nc_put_att_text( ncid, NC_GLOBAL, "geomfile", geom.size(), geom.data());
     int dim_ids[4], tvarID;
     err = file::define_dimensions( ncid, dim_ids, &tvarID, grid_out);
-    MagneticField c(gp);
-    dg::geo::FieldR<MagneticField> fieldR(c, gp.R_0);
-    dg::geo::FieldZ<MagneticField> fieldZ(c, gp.R_0);
-    dg::geo::FieldP<MagneticField> fieldP(c, gp.R_0);
+    dg::geo::TokamakMagneticField c = dg::geo::createSolovevField(gp);
+    dg::geo::FieldR fieldR(c);
+    dg::geo::FieldZ fieldZ(c);
+    dg::geo::FieldP fieldP(c);
     dg::HVec vecR = dg::evaluate( fieldR, grid_out);
     dg::HVec vecZ = dg::evaluate( fieldZ, grid_out);
     dg::HVec vecP = dg::evaluate( fieldP, grid_out);
diff --git a/src/asela2D/asela_mpi.cu b/src/asela2D/asela_mpi.cu
index 2d4c52f54..2324c8c4a 100644
--- a/src/asela2D/asela_mpi.cu
+++ b/src/asela2D/asela_mpi.cu
@@ -7,9 +7,6 @@
 #include <mpi.h> //activate mpi
 
 #include "dg/algorithm.h"
-#include "dg/backend/timer.cuh"
-#include "dg/backend/xspacelib.cuh"
-#include "dg/backend/interpolation.cuh"
 
 #include "netcdf_par.h" //exclude if par netcdf=OFF
 #include "file/nc_utilities.h"
@@ -23,8 +20,6 @@
         density fields are the real densities in XSPACE ( not logarithmic values)
 */
 
-typedef dg::MPI_FieldAligned< dg::CylindricalMPIGrid3d, dg::IDMatrix,dg::BijectiveComm< dg::iDVec, dg::DVec >, dg::DVec> DFA;
-using namespace dg::geo::solovev;
 int main( int argc, char* argv[])
 {
     ////////////////////////////////setup MPI///////////////////////////////
@@ -58,8 +53,8 @@ int main( int argc, char* argv[])
         reader.parse(is,js,false);
         reader.parse(ks,gs,false);
     }
-    const eule::Parameters p( js);
-    const dg::geo::solovev::GeomParameters gp(gs);
+    const asela::Parameters p( js);
+    const dg::geo::solovev::Parameters gp(gs);
     if(rank==0)p.display( std::cout);
     if(rank==0)gp.display( std::cout);
     std::string input = js.toStyledString(), geom = gs.toStyledString();
@@ -76,14 +71,14 @@ int main( int argc, char* argv[])
     dg::CylindricalMPIGrid3d grid_out( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n_out, p.Nx_out, p.Ny_out, 1, p.bc, p.bc, dg::PER, comm);  
      
     if(rank==0)std::cout << "Constructing Asela...\n";
-    eule::Asela<dg::CylindricalMPIGrid3d, dg::DS<DFA, dg::MDMatrix, dg::MDVec>, dg::MDMatrix, dg::MDVec> asela( grid, p, gp); //initialize before rolkar!
+    asela::Asela<dg::CylindricalMPIGrid3d, dg::MIDMatrix, dg::MDMatrix, dg::MDVec> asela( grid, p, gp); //initialize before rolkar!
     if(rank==0)std::cout << "Constructing Rolkar...\n";
-    eule::Rolkar< dg::CylindricalMPIGrid3d, dg::DS<DFA, dg::MDMatrix, dg::MDVec>, dg::MDMatrix, dg::MDVec > rolkar( grid, p, gp, asela.ds(), asela.dsDIR());
+    asela::Implicit< dg::CylindricalMPIGrid3d, dg::MIDMatrix, dg::MDMatrix, dg::MDVec > rolkar( grid, p, gp, asela.ds(), asela.dsDIR());
     if(rank==0)std::cout << "Done!\n";
 
     /////////////////////The initial field///////////////////////////////////////////
     //background profile
-    dg::geo::Nprofile<Psip> prof(p.bgprofamp, p.nprofileamp, gp, Psip(gp)); //initial background profile
+    dg::geo::Nprofile prof(p.bgprofamp, p.nprofileamp, gp, dg::geo::solovev::Psip(gp)); //initial background profile
     std::vector<dg::MDVec> y0(4, dg::evaluate( prof, grid)), y1(y0); 
     //initial perturbation
     if (p.mode == 0  || p.mode ==1) 
@@ -98,13 +93,13 @@ int main( int argc, char* argv[])
     }
     if (p.mode == 3) 
     { 
-        dg::geo::ZonalFlow<Psip> init0(p.amp, p.k_psi, gp, Psip(gp));
+        dg::geo::ZonalFlow init0(p.amp, p.k_psi, gp, dg::geo::solovev::Psip(gp));
         y1[1] = dg::evaluate( init0, grid);
     }
 
     dg::blas1::axpby( 1., y1[1], 1., y0[1]); //initialize ni
     dg::blas1::transform(y0[1], y0[1], dg::PLUS<>(-1)); //initialize ni-1
-    dg::MDVec damping = dg::evaluate( dg::geo::GaussianProfXDamping<Psip>(Psip(gp), gp), grid);
+    dg::MDVec damping = dg::evaluate( dg::geo::GaussianProfXDamping(dg::geo::solovev::Psip(gp), gp), grid);
     dg::blas1::pointwiseDot(damping,y0[1], y0[1]); //damp with gaussprofdamp
     asela.initializene(y0[1],y0[0]);    
 
@@ -128,10 +123,10 @@ int main( int argc, char* argv[])
         err = file::define_dimensions( ncid, dimids, &tvarID, global_grid_out);
 
 
-        MagneticField c(gp);
-        dg::geo::FieldR<MagneticField> fieldR(c, gp.R_0);
-        dg::geo::FieldZ<MagneticField> fieldZ(c, gp.R_0);
-        dg::geo::FieldP<MagneticField> fieldP(c, gp.R_0);
+        dg::geo::TokamakMagneticField c = dg::geo::createSolovevField(gp);
+        dg::geo::FieldR fieldR(c);
+        dg::geo::FieldZ fieldZ(c);
+        dg::geo::FieldP fieldP(c);
         dg::HVec vecR = dg::evaluate( fieldR, global_grid_out);
         dg::HVec vecZ = dg::evaluate( fieldZ, global_grid_out);
         dg::HVec vecP = dg::evaluate( fieldP, global_grid_out);
@@ -180,7 +175,7 @@ int main( int argc, char* argv[])
     ///////////////////////////////////first output/////////////////////////////////
     int dims[3],  coords[3];
     MPI_Cart_get( comm, 3, dims, periods, coords);
-    size_t count[4] = {1, grid_out.Nz(), grid_out.n()*(grid_out.Ny()), grid_out.n()*(grid_out.Nx())};
+    size_t count[4] = {1, grid_out.local().Nz(), grid_out.n()*(grid_out.local().Ny()), grid_out.n()*(grid_out.local().Nx())};
     size_t start[4] = {0, coords[2]*count[1], coords[1]*count[2], coords[0]*count[3]};
     dg::MDVec transfer( dg::evaluate(dg::zero, grid));
     dg::DVec transferD( dg::evaluate(dg::zero, grid_out.local()));
-- 
GitLab


From 4c315dc9db62ac6932e828157a9125c347fe9e8f Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 6 Nov 2017 16:50:12 +0100
Subject: [PATCH 446/453] feltor2D commits again

---
 src/feltor2D/feltor.cu     | 14 +++++---------
 src/feltor2D/feltor_hpc.cu | 24 +++++++++---------------
 src/feltor2D/feltor_mpi.cu | 34 +++++++++++++++-------------------
 3 files changed, 29 insertions(+), 43 deletions(-)

diff --git a/src/feltor2D/feltor.cu b/src/feltor2D/feltor.cu
index b0cd31577..b83ac66c0 100644
--- a/src/feltor2D/feltor.cu
+++ b/src/feltor2D/feltor.cu
@@ -7,8 +7,7 @@
 
 #include "draw/host_window.h"
 //#include "draw/device_window.cuh"
-#include "dg/backend/xspacelib.cuh"
-#include "dg/backend/timer.cuh"
+#include "dg/algorithm.h"
 #include "geometries/geometries.h"
 
 #include "feltor/feltor.cuh"
@@ -20,9 +19,6 @@
    - directly visualizes results on the screen using parameters in window_params.txt
 */
 
-
-typedef dg::FieldAligned< dg::CylindricalGrid3d, dg::IDMatrix, dg::DVec> DFA;
-using namespace dg::geo::solovev;
 int main( int argc, char* argv[])
 {
     ////////////////////////Parameter initialisation//////////////////////////
@@ -48,7 +44,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     const feltor::Parameters p( js);
-    const dg::geo::solovev::GeomParameters gp(gs);
+    const dg::geo::solovev::Parameters gp(gs);
     p.display( std::cout);
     gp.display( std::cout);
     /////////glfw initialisation ////////////////////////////////////////////
@@ -74,7 +70,7 @@ int main( int argc, char* argv[])
 
     /////////////////////The initial field///////////////////////////////////////////
     //background profile
-    dg::geo::Nprofile<Psip> prof(p.bgprofamp, p.nprofileamp, gp, Psip(gp)); //initial background profile
+    dg::geo::Nprofile prof(p.bgprofamp, p.nprofileamp, gp, dg::geo::solovev::Psip(gp)); //initial background profile
     std::vector<dg::DVec> y0(4, dg::evaluate( prof, grid)), y1(y0); 
     //initial perturbation
     if (p.mode == 0  || p.mode ==1) 
@@ -89,14 +85,14 @@ int main( int argc, char* argv[])
     }
     if (p.mode == 3) 
     { 
-        dg::geo::ZonalFlow<Psip> init0(p.amp, p.k_psi, gp, Psip(gp));
+        dg::geo::ZonalFlow init0(p.amp, p.k_psi, gp, dg::geo::solovev::Psip(gp));
         y1[1] = dg::evaluate( init0, grid);
     }
 
     
     dg::blas1::axpby( 1., y1[1], 1., y0[1]); //initialize ni
     dg::blas1::transform(y0[1], y0[1], dg::PLUS<>(-1)); //initialize ni-1
-    dg::DVec damping = dg::evaluate( dg::geo::GaussianProfXDamping<Psip>(Psip(gp), gp), grid);
+    dg::DVec damping = dg::evaluate( dg::geo::GaussianProfXDamping(dg::geo::solovev::Psip(gp), gp), grid);
     dg::blas1::pointwiseDot(damping,y0[1], y0[1]); //damp with gaussprofdamp
     std::cout << "initialize ne" << std::endl;
     feltor.initializene( y0[1], y0[0]);    
diff --git a/src/feltor2D/feltor_hpc.cu b/src/feltor2D/feltor_hpc.cu
index 7f1d27f29..527bcd14b 100644
--- a/src/feltor2D/feltor_hpc.cu
+++ b/src/feltor2D/feltor_hpc.cu
@@ -5,13 +5,9 @@
 #include <cmath>
 // #define DG_DEBUG
 
-
-
-#include "dg/backend/xspacelib.cuh"
-#include "dg/backend/timer.cuh"
-#include "dg/backend/interpolation.cuh"
-#include "file/nc_utilities.h"
+#include "dg/algorithm.h"
 #include "geometries/geometries.h"
+#include "file/nc_utilities.h"
 
 #include "feltor/feltor.cuh"
 #include "feltor/parameters.h"
@@ -23,8 +19,6 @@
         density fields are the real densities in XSPACE ( not logarithmic values)
 */
 
-typedef dg::FieldAligned< dg::CylindricalGrid3d, dg::IDMatrix, dg::DVec> DFA;
-using namespace dg::geo::solovev;
 int main( int argc, char* argv[])
 {
     ////////////////////////Parameter initialisation//////////////////////////
@@ -65,7 +59,7 @@ int main( int argc, char* argv[])
 
     /////////////////////The initial field///////////////////////////////////////////
     //background profile
-    dg::geo::Nprofile prof(p.bgprofamp, p.nprofileamp, gp, Psip(gp)); //initial background profile
+    dg::geo::Nprofile prof(p.bgprofamp, p.nprofileamp, gp, dg::geo::solovev::Psip(gp)); //initial background profile
     std::vector<dg::DVec> y0(4, dg::evaluate( prof, grid)), y1(y0); 
     //initial perturbation
     if (p.mode == 0  || p.mode ==1) 
@@ -80,13 +74,13 @@ int main( int argc, char* argv[])
     }
     if (p.mode == 3) 
     { 
-        dg::geo::ZonalFlow<Psip> init0(p.amp, p.k_psi, gp, Psip(gp));
+        dg::geo::ZonalFlow init0(p.amp, p.k_psi, gp, dg::geo::solovev::Psip(gp));
         y1[1] = dg::evaluate( init0, grid);
     }
     
     dg::blas1::axpby( 1., y1[1], 1., y0[1]); //initialize ni
     dg::blas1::transform(y0[1], y0[1], dg::PLUS<>(-1)); //initialize ni-1
-    dg::DVec damping = dg::evaluate( dg::geo::GaussianProfXDamping<Psip>(Psip(gp), gp), grid);
+    dg::DVec damping = dg::evaluate( dg::geo::GaussianProfXDamping(dg::geo::solovev::Psip(gp), gp), grid);
     dg::blas1::pointwiseDot( damping, y0[1], y0[1]); //damp with gaussprofdamp
     std::cout << "initialize ne" << std::endl;
     feltor.initializene( y0[1], y0[0]);    
@@ -108,10 +102,10 @@ int main( int argc, char* argv[])
     err = nc_put_att_text( ncid, NC_GLOBAL, "geomfile", geom.size(), geom.data());
     int dim_ids[4], tvarID;
     err = file::define_dimensions( ncid, dim_ids, &tvarID, grid_out);
-    MagneticField c(gp);
-    dg::geo::FieldR<MagneticField> fieldR(c, gp.R_0);
-    dg::geo::FieldZ<MagneticField> fieldZ(c, gp.R_0);
-    dg::geo::FieldP<MagneticField> fieldP(c, gp.R_0);
+    dg::geo::TokamakMagneticField c = dg::geo::createSolovevField(gp);
+    dg::geo::FieldR fieldR(c);
+    dg::geo::FieldZ fieldZ(c);
+    dg::geo::FieldP fieldP(c);
     dg::HVec vecR = dg::evaluate( fieldR, grid_out);
     dg::HVec vecZ = dg::evaluate( fieldZ, grid_out);
     dg::HVec vecP = dg::evaluate( fieldP, grid_out);
diff --git a/src/feltor2D/feltor_mpi.cu b/src/feltor2D/feltor_mpi.cu
index f5aa594bd..612e0b08a 100644
--- a/src/feltor2D/feltor_mpi.cu
+++ b/src/feltor2D/feltor_mpi.cu
@@ -5,16 +5,14 @@
 #include <cmath>
 
 #include <mpi.h> //activate mpi
+#include "netcdf_par.h" //exclude if par netcdf=OFF
 
 #include "dg/algorithm.h"
-#include "dg/backend/timer.cuh"
-#include "dg/backend/xspacelib.cuh"
-#include "dg/backend/interpolation.cuh"
-
-#include "netcdf_par.h" //exclude if par netcdf=OFF
+#include "geometries/geometries.h"
 #include "file/nc_utilities.h"
 
 #include "feltor/feltor.cuh"
+#include "feltor/parameters.h"
 
 /*
    - reads parameters from input.txt or any other given file, 
@@ -23,8 +21,6 @@
         density fields are the real densities in XSPACE ( not logarithmic values)
 */
 
-typedef dg::MPI_FieldAligned< dg::CylindricalMPIGrid3d, dg::IDMatrix,dg::BijectiveComm< dg::iDVec, dg::DVec >, dg::DVec> DFA;
-using namespace dg::geo::solovev;
 int main( int argc, char* argv[])
 {
     ////////////////////////////////setup MPI///////////////////////////////
@@ -58,8 +54,8 @@ int main( int argc, char* argv[])
         reader.parse(is,js,false);
         reader.parse(ks,gs,false);
     }
-    const eule::Parameters p( js);
-    const dg::geo::solovev::GeomParameters gp(gs);
+    const feltor::Parameters p( js);
+    const dg::geo::solovev::Parameters gp(gs);
     if(rank==0)p.display( std::cout);
     if(rank==0)gp.display( std::cout);
     std::string input = js.toStyledString(), geom = gs.toStyledString();
@@ -76,14 +72,14 @@ int main( int argc, char* argv[])
     dg::CylindricalMPIGrid3d grid_out( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI, p.n_out, p.Nx_out, p.Ny_out, 1, p.bc, p.bc, dg::PER, comm);  
      
     if(rank==0)std::cout << "Constructing Feltor...\n";
-    feltor::Explicit<dg::CylindricalMPIGrid3d, dg::DS<DFA, dg::MDMatrix, dg::MDVec>, dg::MDMatrix, dg::MDVec> feltor( grid, p, gp); //initialize before implicit!
+    feltor::Explicit<dg::CylindricalMPIGrid3d, dg::MIDMatrix, dg::MDMatrix, dg::MDVec> feltor( grid, p, gp); //initialize before implicit!
     if(rank==0)std::cout << "Constructing implicit...\n";
-    feltor::Implicit< dg::CylindricalMPIGrid3d, dg::DS<DFA, dg::MDMatrix, dg::MDVec>, dg::MDMatrix, dg::MDVec > implicit( grid, p, gp, feltor.ds(), feltor.dsDIR());
+    feltor::Implicit< dg::CylindricalMPIGrid3d, dg::MIDMatrix, dg::MDMatrix, dg::MDVec > implicit( grid, p, gp, feltor.ds(), feltor.dsDIR());
     if(rank==0)std::cout << "Done!\n";
 
     /////////////////////The initial field///////////////////////////////////////////
     //background profile
-    dg::geo::Nprofile<Psip> prof(p.bgprofamp, p.nprofileamp, gp, Psip(gp)); //initial background profile
+    dg::geo::Nprofile prof(p.bgprofamp, p.nprofileamp, gp, dg::geo::solovev::Psip(gp)); //initial background profile
     std::vector<dg::MDVec> y0(4, dg::evaluate( prof, grid)), y1(y0); 
     //initial perturbation
     if (p.mode == 0  || p.mode ==1) 
@@ -98,13 +94,13 @@ int main( int argc, char* argv[])
     }
     if (p.mode == 3) 
     { 
-        dg::geo::ZonalFlow<Psip> init0(p.amp, p.k_psi, gp, Psip(gp));
+        dg::geo::ZonalFlow init0(p.amp, p.k_psi, gp, dg::geo::solovev::Psip(gp));
         y1[1] = dg::evaluate( init0, grid);
     }
 
     dg::blas1::axpby( 1., y1[1], 1., y0[1]); //initialize ni
     dg::blas1::transform(y0[1], y0[1], dg::PLUS<>(-1)); //initialize ni-1
-    dg::MDVec damping = dg::evaluate( dg::geo::GaussianProfXDamping<Psip>(Psip(gp), gp), grid);
+    dg::MDVec damping = dg::evaluate( dg::geo::GaussianProfXDamping(dg::geo::solovev::Psip(gp), gp), grid);
     dg::blas1::pointwiseDot(damping,y0[1], y0[1]); //damp with gaussprofdamp
     feltor.initializene(y0[1],y0[0]);    
 
@@ -128,10 +124,10 @@ int main( int argc, char* argv[])
         err = file::define_dimensions( ncid, dimids, &tvarID, global_grid_out);
 
 
-        MagneticField c(gp);
-        dg::geo::FieldR<MagneticField> fieldR(c, gp.R_0);
-        dg::geo::FieldZ<MagneticField> fieldZ(c, gp.R_0);
-        dg::geo::FieldP<MagneticField> fieldP(c, gp.R_0);
+        dg::geo::TokamakMagneticField c = dg::geo::createSolovevField(gp);
+        dg::geo::FieldR fieldR(c);
+        dg::geo::FieldZ fieldZ(c);
+        dg::geo::FieldP fieldP(c);
         dg::HVec vecR = dg::evaluate( fieldR, global_grid_out);
         dg::HVec vecZ = dg::evaluate( fieldZ, global_grid_out);
         dg::HVec vecP = dg::evaluate( fieldP, global_grid_out);
@@ -180,7 +176,7 @@ int main( int argc, char* argv[])
     ///////////////////////////////////first output/////////////////////////////////
     int dims[3],  coords[3];
     MPI_Cart_get( comm, 3, dims, periods, coords);
-    size_t count[4] = {1, grid_out.Nz(), grid_out.n()*(grid_out.Ny()), grid_out.n()*(grid_out.Nx())};
+    size_t count[4] = {1, grid_out.local().Nz(), grid_out.n()*(grid_out.local().Ny()), grid_out.n()*(grid_out.local().Nx())};
     size_t start[4] = {0, coords[2]*count[1], coords[1]*count[2], coords[0]*count[3]};
     dg::MDVec transfer( dg::evaluate(dg::zero, grid));
     dg::DVec transferD( dg::evaluate(dg::zero, grid_out.local()));
-- 
GitLab


From d60301445ab140bc9bc2a151cc046b382d4dfca3 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 6 Nov 2017 16:59:03 +0100
Subject: [PATCH 447/453] corrected std::cout in feltor2d mpi

---
 src/feltor/feltor_mpi.cu | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/feltor/feltor_mpi.cu b/src/feltor/feltor_mpi.cu
index fd32ce0ec..d7e3bc6e2 100644
--- a/src/feltor/feltor_mpi.cu
+++ b/src/feltor/feltor_mpi.cu
@@ -118,10 +118,10 @@ int main( int argc, char* argv[])
         dg::MDVec damping = dg::evaluate( dg::geo::GaussianProfXDamping(dg::geo::solovev::Psip(gp), gp), grid);
         dg::blas1::pointwiseDot(damping, y0[1], y0[1]); //damp with gaussprofdamp
     }
-    std::cout << "intiialize ne" << std::endl;
+    if(rank==0)std::cout << "intiialize ne" << std::endl;
     if( p.initcond == 0) feltor.initializene( y0[1], y0[0]);
     if( p.initcond == 1) dg::blas1::axpby( 1., y0[1], 0.,y0[0], y0[0]); //set n_e = N_i
-    std::cout << "Done!\n";    dg::blas1::axpby( 0., y0[2], 0., y0[2]); //set Ue = 0
+    if(rank==0)std::cout << "Done!\n";    dg::blas1::axpby( 0., y0[2], 0., y0[2]); //set Ue = 0
     dg::blas1::axpby( 0., y0[3], 0., y0[3]); //set Ui = 0
     
     dg::Karniadakis< std::vector<dg::MDVec> > karniadakis( y0, y0[0].size(), p.eps_time);
-- 
GitLab


From c74430717899eb2d843971b3e1f4491c158a8a6f Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Tue, 7 Nov 2017 10:06:32 +0100
Subject: [PATCH 448/453] note on thrust and cups docus#

---
 README.adoc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.adoc b/README.adoc
index 6d7027d6c..960903131 100644
--- a/README.adoc
+++ b/README.adoc
@@ -309,6 +309,8 @@ methods directly from source code. This depends on the `doxygen`,
 the folder `path/to/feltor/doc` and open `index.html` (a symbolic link
 to `dg/html/modules.html`) with your favorite browser. Links to the pdf
 writeups can be found, among other places, on the Mainpage.
+Finally, also note the documentations of https://thrust.github.io/doc/modules.html[thrust]
+and https://cusplibrary.github.io/[cusp].
 
 == 3. Acknowledgements
 
-- 
GitLab


From 243d019e5b9fa9a92fb4fe3e6d3b7911d2e3ddbc Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Tue, 7 Nov 2017 11:02:00 +0100
Subject: [PATCH 449/453] added dg::geo to help doxygen recognize classes

---
 inc/geometries/guenther.h | 2 +-
 inc/geometries/solovev.h  | 2 +-
 inc/geometries/taylor.h   | 2 +-
 inc/geometries/toroidal.h | 4 ++--
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/inc/geometries/guenther.h b/inc/geometries/guenther.h
index b21092a79..678ca3f11 100644
--- a/inc/geometries/guenther.h
+++ b/inc/geometries/guenther.h
@@ -320,7 +320,7 @@ struct Divb
  * @return A magnetic field object
  * @ingroup geom
  */
-TokamakMagneticField createGuentherField( double R_0, double I_0)
+dg::geo::TokamakMagneticField createGuentherField( double R_0, double I_0)
 {
     return TokamakMagneticField( R_0, guenther::createPsip(R_0), guenther::createIpol(I_0));
 }
diff --git a/inc/geometries/solovev.h b/inc/geometries/solovev.h
index 86814a044..bdc75ca77 100644
--- a/inc/geometries/solovev.h
+++ b/inc/geometries/solovev.h
@@ -562,7 +562,7 @@ struct PsipRZ: public aCloneableBinaryFunctor<PsipRZ>
  * @return A magnetic field object
  * @ingroup geom
  */
-TokamakMagneticField createSolovevField( solovev::Parameters gp)
+dg::geo::TokamakMagneticField createSolovevField( dg::geo::solovev::Parameters gp)
 {
     return TokamakMagneticField( gp.R_0, solovev::createPsip(gp), solovev::createIpol(gp));
 }
diff --git a/inc/geometries/taylor.h b/inc/geometries/taylor.h
index 5ccb8d6b3..1b0b3f15b 100644
--- a/inc/geometries/taylor.h
+++ b/inc/geometries/taylor.h
@@ -320,7 +320,7 @@ dg::geo::TokamakMagneticField createMagField( solovev::Parameters gp)
  * @ingroup geom
  * @attention The header \c taylor.h needs to be included seperately and depends on <a href="http://www.boost.org">boost</a>
  */
-TokamakMagneticField createTaylorField( solovev::Parameters gp)
+dg::geo::TokamakMagneticField createTaylorField( dg::geo::solovev::Parameters gp)
 {
     return TokamakMagneticField( gp.R_0, dg::geo::taylor::createPsip(gp), dg::geo::taylor::createIpol(gp));
 }
diff --git a/inc/geometries/toroidal.h b/inc/geometries/toroidal.h
index a72900508..5f3d4ef61 100644
--- a/inc/geometries/toroidal.h
+++ b/inc/geometries/toroidal.h
@@ -115,7 +115,7 @@ BinaryFunctorsLvl1 createIpol( double I0 )
  * @ingroup geom
  * @note The solovev field can also be made to model a todoidal slab field
  */
-TokamakMagneticField createToroidalField( double R0)
+dg::geo::TokamakMagneticField createToroidalField( double R0)
 {
     return TokamakMagneticField( R0, toroidal::createPsip(), toroidal::createIpol());
 }
@@ -128,7 +128,7 @@ TokamakMagneticField createToroidalField( double R0)
  * @return A magnetic field object
  * @ingroup geom
  */
-TokamakMagneticField createCircularField( double R0, double I0)
+dg::geo::TokamakMagneticField createCircularField( double R0, double I0)
 {
     return TokamakMagneticField( R0, circular::createPsip(R0), circular::createIpol(I0));
 }
-- 
GitLab


From 36a758834cddb3267708107055f8a236c3d85093 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Tue, 7 Nov 2017 19:02:54 +0100
Subject: [PATCH 450/453] made mpi topology store plane communicator

---
 inc/dg/backend/mpi_grid.h       | 18 +++++++-----------
 inc/dg/backend/split_and_join.h |  4 +---
 2 files changed, 8 insertions(+), 14 deletions(-)

diff --git a/inc/dg/backend/mpi_grid.h b/inc/dg/backend/mpi_grid.h
index 7cbdd9f6a..86014ff61 100644
--- a/inc/dg/backend/mpi_grid.h
+++ b/inc/dg/backend/mpi_grid.h
@@ -444,16 +444,10 @@ struct aMPITopology3d
      */
     MPI_Comm communicator() const{return comm;}
     /**
-     * @brief MPI Cartesian communicator in the first two dimensions
+     * @brief MPI Cartesian communicator in the first two dimensions (x and y)
      * @return 2d Cartesian Communicator
      */
-    MPI_Comm get_perp_comm() const
-    {
-        MPI_Comm planeComm;
-        int remain_dims[] = {true,true,false}; //true true false
-        MPI_Cart_sub( comm, remain_dims, &planeComm);
-        return planeComm;
-    }
+    MPI_Comm get_perp_comm() const {return planeComm;}
     /**
      * @brief The Discrete Legendre Transformation 
      *
@@ -556,14 +550,16 @@ struct aMPITopology3d
     {
         update_local();
         check_division( Nx, Ny, Nz, bcx, bcy, bcz);
+        int remain_dims[] = {true,true,false}; //true true false
+        MPI_Cart_sub( comm, remain_dims, &planeComm);
     }
     ///explicit copy constructor (default)
     ///@param src source
-    aMPITopology3d(const aMPITopology3d& src):g(src.g),l(src.l),comm(src.comm){ }
+    aMPITopology3d(const aMPITopology3d& src):g(src.g),l(src.l),comm(src.comm),planeComm(src.planeComm){ }
     ///explicit assignment operator (default)
     ///@param src source
     aMPITopology3d& operator=(const aMPITopology3d& src){
-        g = src.g; l = src.l; comm = src.comm;
+        g = src.g; l = src.l; comm = src.comm; planeComm = src.planeComm;
         return *this;
     }
     virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny, unsigned new_Nz)=0; 
@@ -616,7 +612,7 @@ struct aMPITopology3d
         l = Grid3d(x0, x1, y0, y1, z0, z1, g.n(), Nx, Ny, Nz, g.bcx(), g.bcy(), g.bcz());
     }
     Grid3d g, l; //global grid
-    MPI_Comm comm; //just an integer...
+    MPI_Comm comm, planeComm; //just an integer...
 };
 ///@cond
 int aMPITopology2d::pidOf( double x, double y) const
diff --git a/inc/dg/backend/split_and_join.h b/inc/dg/backend/split_and_join.h
index 3e3b387bf..a401a5bf2 100644
--- a/inc/dg/backend/split_and_join.h
+++ b/inc/dg/backend/split_and_join.h
@@ -38,9 +38,7 @@ void split( const MPI_Vector<thrust_vector1>& in, std::vector<MPI_Vector<thrust_
     int result;
     MPI_Comm_compare( in.communicator(), grid.communicator(), &result);
     assert( result == MPI_CONGRUENT || result == MPI_IDENT);
-    MPI_Comm planeComm;
-    int remain_dims[] = {true,true,false}; 
-    MPI_Cart_sub( in.communicator(), remain_dims, &planeComm);
+    MPI_Comm planeComm = grid.get_perp_comm();
     //local size2d
     Grid3d l = grid.local();
     unsigned size2d=l.n()*l.n()*l.Nx()*l.Ny();
-- 
GitLab


From d4915ac4608b017445d98f24872d40858cac88c1 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Tue, 7 Nov 2017 19:38:43 +0100
Subject: [PATCH 451/453] forgot to add MPI_VERSION to multigrid benchmarking

---
 inc/dg/Makefile           |  2 +-
 inc/dg/elliptic2d_b.cu    | 25 +++++++++++--------------
 inc/dg/elliptic2d_mpib.cu |  7 +++----
 inc/dg/multigrid.h        | 12 +++++++++++-
 4 files changed, 26 insertions(+), 20 deletions(-)

diff --git a/inc/dg/Makefile b/inc/dg/Makefile
index 3cf9adb57..b4a91db8e 100644
--- a/inc/dg/Makefile
+++ b/inc/dg/Makefile
@@ -23,7 +23,7 @@ all: $(CPPFILES:%.cpp=%) $(CUFILES:%.cu=%)
 	$(MPICC) $(INCLUDE) -DDG_DEBUG $(MPICFLAGS) $< -o $@ -g
 
 %_mpib: %_mpib.cu
-	$(MPICC) $(OPT) $(MPICFLAGS) $< -o $@ $(INCLUDE) -g
+	$(MPICC) $(OPT) $(MPICFLAGS) -DDG_BENCHMARK $< -o $@ $(INCLUDE) -g
 
 bathRZ_t: bathRZ_t.cu 
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(GLFLAGS) $(INCLUDE)  -g
diff --git a/inc/dg/elliptic2d_b.cu b/inc/dg/elliptic2d_b.cu
index 703f98d84..3023db70e 100644
--- a/inc/dg/elliptic2d_b.cu
+++ b/inc/dg/elliptic2d_b.cu
@@ -63,7 +63,11 @@ int main()
     dg::blas1::transform( chi, chi_inv, dg::INVERT<double>());
     dg::blas1::pointwiseDot( chi_inv, v2d, chi_inv);
     dg::DVec temp = x;
-
+    //compute error
+    const dg::DVec solution = dg::evaluate( sol, grid);
+    const dg::DVec derivati = dg::evaluate( der, grid);
+    const double norm = dg::blas2::dot( w2d, solution);
+    dg::DVec error( solution);
 
     //std::cout << "Create Polarisation object and set chi!\n";
     {
@@ -97,18 +101,12 @@ int main()
     for( unsigned u=0; u<number.size(); u++)
     	std::cout << " # iterations stage "<< number.size()-1-u << " " << number[number.size()-1-u] << " \n";
     //! [multigrid]
-    }
-
-    //compute error
-    const dg::DVec solution = dg::evaluate( sol, grid);
-    const dg::DVec derivati = dg::evaluate( der, grid);
-    dg::DVec error( solution);
-
     dg::blas1::axpby( 1.,x,-1., solution, error);
     double err = dg::blas2::dot( w2d, error);
-    //std::cout << "L2 Norm2 of Error is                       " << err << std::endl;
-    const double norm = dg::blas2::dot( w2d, solution);
     std::cout << " "<<sqrt( err/norm) << "\n";
+    }
+
+
     {
     x = temp;
     //![invert]
@@ -117,7 +115,7 @@ int main()
     dg::Invert<dg::DVec > invert_fw( x, n*n*Nx*Ny, eps);
     std::cout << " "<< invert_fw( pol_forward, x, b, w2d, v2d, chi_inv);
     dg::blas1::axpby( 1.,x,-1., solution, error);
-    err = dg::blas2::dot( w2d, error);
+    double err = dg::blas2::dot( w2d, error);
     std::cout << " "<<sqrt( err/norm) << "\n";
     //![invert]
     }
@@ -129,7 +127,7 @@ int main()
 		dg::Invert<dg::DVec > invert_bw( x, n*n*Nx*Ny, eps);
 		std::cout << " "<< invert_bw( pol_backward, x, b, w2d, v2d, chi_inv);
 		dg::blas1::axpby( 1.,x,-1., solution, error);
-		err = dg::blas2::dot( w2d, error);
+		double err = dg::blas2::dot( w2d, error);
 		std::cout << " "<<sqrt( err/norm)<<std::endl;
     }
 
@@ -137,8 +135,7 @@ int main()
     dg::DMatrix DX = dg::create::dx( grid);
     dg::blas2::gemv( DX, x, error);
     dg::blas1::axpby( 1.,derivati,-1., error);
-    err = dg::blas2::dot( w2d, error);
-    //std::cout << "L2 Norm2 of Error in derivative is         " << err << std::endl;
+    double err = dg::blas2::dot( w2d, error);
     const double norm_der = dg::blas2::dot( w2d, derivati);
     std::cout << "L2 Norm of relative error in derivative is "<<sqrt( err/norm_der)<<std::endl;
     //derivative converges with p-1, for p = 1 with 1/2
diff --git a/inc/dg/elliptic2d_mpib.cu b/inc/dg/elliptic2d_mpib.cu
index 652cc8715..fc749bbc2 100644
--- a/inc/dg/elliptic2d_mpib.cu
+++ b/inc/dg/elliptic2d_mpib.cu
@@ -80,9 +80,10 @@ int main(int argc, char* argv[] )
     //dg::Invert<dg::MDVec > invert( x, n*n*Nx*Ny, eps);
     t.tic();
     //unsigned number = invert( pol, x, b);
-    std::vector<unsigned> number = multigrid.solve( multi_pol, x, b, eps);
+    std::vector<unsigned> number = multigrid.direct_solve( multi_pol, x, b, eps);
     t.toc();
-    if(rank==0)std::cout << "Number of pcg iterations "<< number[0]<<std::endl;
+    for( unsigned u=0; u<number.size(); u++)
+    	if(rank==0)std::cout << " # iterations stage "<< number.size()-1-u << " " << number[number.size()-1-u] << " \n";
     if(rank==0)std::cout << "For a precision of "<< eps<<std::endl;
     if(rank==0)std::cout << " took "<<t.diff()<<"s\n";
 
@@ -93,14 +94,12 @@ int main(int argc, char* argv[] )
     dg::blas1::axpby( 1.,x,-1., error);
 
     double err = dg::blas2::dot( w2d, error);
-    if(rank==0)std::cout << "L2 Norm2 of Error is " << err << std::endl;
     double norm = dg::blas2::dot( w2d, solution);
     if(rank==0)std::cout << "L2 Norm of relative error is               "<<sqrt( err/norm)<<std::endl;
     dg::MDMatrix DX = dg::create::dx( grid);
     dg::blas2::gemv( DX, x, error);
     dg::blas1::axpby( 1.,derivati,-1., error);
     err = dg::blas2::dot( w2d, error);
-    if(rank==0)std::cout << "L2 Norm2 of Error in derivative is " << err << std::endl;
     norm = dg::blas2::dot( w2d, derivati);
     if(rank==0)std::cout << "L2 Norm of relative error in derivative is "<<sqrt( err/norm)<<std::endl;
     //derivative converges with p-1, for p = 1 with 1/2
diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index 77fe5a71d..fd4942876 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -147,7 +147,7 @@ struct MultigridCG2d
      * @param x (read/write) contains initial guess on input and the solution on output
      * @param b The right hand side (will be multiplied by \c weights)
      * @param eps the accuracy: iteration stops if \f$ ||b - Ax|| < \epsilon( ||b|| + 1) \f$ 
-     * @return the number of iterations in each of the stages
+     * @return the number of iterations in each of the stages beginning with the finest grid
      * @note If the Macro \c DG_BENCHMARK is defined this function will write timings to \c std::cout
     */
     template<class SymmetricOp>
@@ -177,6 +177,11 @@ struct MultigridCG2d
             dg::blas2::symv( inter_[u-1], x_[u], x_[u-1]);
 #ifdef DG_BENCHMARK
             t.toc();
+#ifdef MPI_VERSION
+            int rank;
+            MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+            if(rank==0)
+#endif //MPI
             std::cout << "stage: " << u << ", iter: " << number[u] << ", took "<<t.diff()<<"s\n";
 #endif //DG_BENCHMARK
 
@@ -191,6 +196,11 @@ struct MultigridCG2d
         number[0] = cg_[0]( op[0], x, b_[0], op[0].precond(), op[0].inv_weights(), eps);
 #ifdef DG_BENCHMARK
         t.toc();
+#ifdef MPI_VERSION
+        int rank;
+        MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+        if(rank==0)
+#endif //MPI
         std::cout << "stage: " << 0 << ", iter: " << number[0] << ", took "<<t.diff()<<"s\n";
 #endif //DG_BENCHMARK
         
-- 
GitLab


From 4b1581319b3f590c9273f4ce1c21a62ff1363704 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Tue, 7 Nov 2017 20:07:11 +0100
Subject: [PATCH 452/453] corrected elliptic_mpib with local Nz values

---
 inc/dg/elliptic_mpib.cu  | 6 +++---
 inc/dg/geometry/tensor.h | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/inc/dg/elliptic_mpib.cu b/inc/dg/elliptic_mpib.cu
index 7cdb03fa9..c714db679 100644
--- a/inc/dg/elliptic_mpib.cu
+++ b/inc/dg/elliptic_mpib.cu
@@ -90,17 +90,17 @@ int main( int argc, char* argv[])
     dg::tensor::pointwiseDot( chi, g_parallel, chi);
     //create split Laplacian
     std::vector< dg::Elliptic<dg::aMPIGeometry2d, dg::MDMatrix, dg::MDVec> > laplace_split( 
-            grid.Nz(), dg::Elliptic<dg::aMPIGeometry2d, dg::MDMatrix, dg::MDVec>(grid_perp.get(), dg::not_normed, dg::centered));
+            grid.local().Nz(), dg::Elliptic<dg::aMPIGeometry2d, dg::MDMatrix, dg::MDVec>(grid_perp.get(), dg::not_normed, dg::centered));
     // create split  vectors and solve
     std::vector<dg::MDVec> b_split, x_split, chi_split;
     pcg.construct( w2d, w2d.size());
-    std::vector<unsigned>  number(grid.Nz());
+    std::vector<unsigned>  number(grid.local().Nz());
     t.tic();
     dg::tensor::pointwiseDot( b, g_parallel, b);
     dg::split( b, b_split, grid);
     dg::split( chi, chi_split, grid);
     dg::split( x, x_split, grid);
-    for( unsigned i=0; i<grid.Nz(); i++)
+    for( unsigned i=0; i<grid.local().Nz(); i++)
     {
         laplace_split[i].set_chi( chi_split[i]);
         dg::blas1::pointwiseDot( b_split[i], w2d, b_split[i]);
diff --git a/inc/dg/geometry/tensor.h b/inc/dg/geometry/tensor.h
index 87f281037..39bbdb0ee 100644
--- a/inc/dg/geometry/tensor.h
+++ b/inc/dg/geometry/tensor.h
@@ -175,7 +175,7 @@ struct SparseTensor
      */
     T& value( size_t i)
     {
-        if(i>=values_.size() ) values_.resize(i+1);
+        if(i>=values_.size() ) values_.resize(i+1, T());
         return values_[i];
     }
     /**
-- 
GitLab


From 3527d497ff17de0ed5356c4ca6cdecda480b7659 Mon Sep 17 00:00:00 2001
From: Matthias <MWiesenberger@gmx.de>
Date: Tue, 7 Nov 2017 21:01:36 +0100
Subject: [PATCH 453/453] updated doc on ds and fieldaligned

---
 inc/geometries/ds.h           | 153 +++++++++++++++++++---------------
 inc/geometries/fieldaligned.h |  81 ++++++++++--------
 2 files changed, 131 insertions(+), 103 deletions(-)

diff --git a/inc/geometries/ds.h b/inc/geometries/ds.h
index 38400079e..940129bc7 100644
--- a/inc/geometries/ds.h
+++ b/inc/geometries/ds.h
@@ -18,11 +18,24 @@
 namespace dg{
 namespace geo{
 
+    /*!@class hide_ds_parameters2
+    * @param f The vector to derive
+    * @param g contains result on output (write only)
+    * @note the vector sizes need to equal the grid size in the constructor
+    */
+    /*!@class hide_ds_parameters4
+    * @param alpha Scalar
+    * @param f The vector to derive
+    * @param beta Scalar
+    * @param g contains result on output (write only)
+    * @note the vector sizes need to equal the grid size in the constructor
+    */
+
 /**
 * @brief Class for the evaluation of parallel derivatives
 *
 * This class discretizes the operators 
-\f$ \nabla_\parallel = \mathbf{v}\cdot \nabla = v^\zeta\partial_\zeta + v^\eta\partial_\eta + v^\varphi\partial_\varphi \f$, 
+\f$ \nabla_\parallel = \vec{v}\cdot \nabla = v^\zeta\partial_\zeta + v^\eta\partial_\eta + v^\varphi\partial_\varphi \f$, 
 \f$\nabla_\parallel^\dagger = -\nabla\cdot(\vec v .)\f$ and 
 \f$\Delta_\parallel=-\nabla_\parallel^\dagger\cdot\nabla_\parallel\f$
 in arbitrary coordinates
@@ -46,7 +59,12 @@ struct DS
     
     /**
      * @brief Create the magnetic unit vector field and construct
-     * @copydetails DS(const dg::geo::BinaryVectorLvl0&,const ProductGeometry&,dg::bc,dg::bc,Limiter,dg::norm,dg::direction,double,unsigned,unsigned,bool,bool)
+
+     * @copydoc hide_fieldaligned_physics_parameters
+     * @param no indicate if the symv function should be symmetric (not_normed) or not
+     * @param dir indicate the direction in the bracket operator and in symv
+     * @copydoc hide_fieldaligned_numerics_parameters
+     * @sa \c Fieldaligned
      */
     template <class Limiter>
     DS(const dg::geo::TokamakMagneticField& vec, const ProductGeometry& grid, 
@@ -61,26 +79,12 @@ struct DS
         construct( m_fa, no, dir);
     }
     /**
-     * @brief Create a \c Fieldaligned object and construct
+     * @brief Use the given vector field to construct
      *
-     * @tparam Limiter Class that can be evaluated on a 2d grid, returns 1 if there
-        is a limiter and 0 if there isn't. 
-        If a field line crosses the limiter in the plane \f$ \phi=0\f$ then the limiter boundary conditions apply. 
-     * @param vec The vector field to integrate
-     * @param grid The grid on which to operate defines the parallel boundary condition in case there is a limiter.
-     * @param bcx Defines the interpolation behaviour when a fieldline intersects the boundary box in the perpendicular direction
-     * @param bcy Defines the interpolation behaviour when a fieldline intersects the boundary box in the perpendicular direction
-     * @param limit Instance of the limiter class (Default is a limiter everywhere, 
-        note that if grid.bcz() is periodic it doesn't matter if there is a limiter or not)
+     * @copydoc hide_fieldaligned_physics_parameters
      * @param no indicate if the symv function should be symmetric (not_normed) or not
      * @param dir indicate the direction in the bracket operator and in symv
-     * @param eps Desired accuracy of the fieldline integrator
-     * @param multiplyX defines the resolution in X of the fine grid relative to grid
-     * @param multiplyY defines the resolution in Y of the fine grid relative to grid
-     * @param dependsOnX indicates, whether the given vector field vec depends on the first coordinate (jump terms are added in symv)
-     * @param dependsOnY indicates, whether the given vector field vec depends on the second coordinate (jump terms are added in symv)
-     * @param integrateAll indicates, that all fieldlines of the fine grid should be integrated instead of interpolating it from the coarse grid. 
-     *  Should be true if the streamlines of the vector field cross the domain boudary. 
+     * @copydoc hide_fieldaligned_numerics_parameters
      * @sa \c Fieldaligned
      */
     template<class Limiter>
@@ -122,92 +126,109 @@ struct DS
     }
 
     /**
-    * @brief forward derivative \f$ g_i = \alpha \frac{1}{h_z^+}(f_{i+1} - f_{i}) + \beta g_i\f$
+    * @brief forward derivative \f$ g = \alpha \vec v \cdot \nabla f + \beta g\f$
     *
-    * @param alpha Scalar
-    * @param f The vector to derive
-    * @param beta Scalar
-    * @param g contains result on output (write only)
-    * @note the vector sizes need to equal the grid size in the constructor
+    * forward derivative \f$ g_i = \alpha \frac{1}{h_z^+}(f_{i+1} - f_{i}) + \beta g_i\f$
+    * @copydoc hide_ds_parameters4
     */
     void forward( double alpha, const container& f, double beta, container& g){
         do_forward( alpha, f, beta, g);
     }
     /**
-    * @brief forward derivative \f$ g_i = \frac{1}{h_z^+}(f_{i+1} - f_{i}) \f$
+    * @brief backward derivative \f$ g = \alpha \vec v \cdot \nabla f + \beta g\f$
     *
-    * @param f The vector to derive
-    * @param g contains result on output (write only)
-    * @note the vector sizes need to equal the grid size in the constructor
+    * backward derivative \f$ g_i = \alpha \frac{1}{2h_z^-}(f_{i} - f_{i-1}) + \beta g_i \f$
+    * @copydoc hide_ds_parameters4
     */
-    void forward( const container& f, container& g){do_forward(1.,f,0.,g);}
-
-    ///@brief backward derivative \f$ g_i = \alpha \frac{1}{2h_z^-}(f_{i} - f_{i-1}) + \beta g_i \f$
-    ///@copydetails forward(double,const container&,double,container&)
     void backward( double alpha, const container& f, double beta, container& g){
         do_backward( alpha, f, beta, g);
     }
-    ///@brief backward derivative \f$ g_i = \frac{1}{2h_z^-}(f_{i} - f_{i-1}) \f$
-    ///@copydetails forward(const container&,container&)
-    void backward( const container& f, container& g){do_backward(1.,f,0.,g);}
-    ///@brief centered derivative \f$ g_i = \alpha \frac{1}{2h_z}(f_{i+1} - f_{i-1}) + \beta g_i\f$
-    ///@copydetails forward(double,const container&,double,container&)
+    /**
+    * @brief centered derivative \f$ g = \alpha \vec v \cdot \nabla f + \beta g\f$
+    *
+    * centered derivative \f$ g_i = \alpha \frac{1}{2h_z^0}(f_{i+1} - f_{i-1}) + \beta g_i\f$
+    * @copydoc hide_ds_parameters4
+    */
     void centered( double alpha, const container& f, double beta, container& g){
         do_centered( alpha, f, beta, g);
     }
-    ///@brief centered derivative \f$ g_i = \frac{1}{2h_z}(f_{i+1} - f_{i-1})\f$
-    ///@copydetails forward(const container&,container&)
-    void centered( const container& f, container& g){do_centered(1.,f,0.,g);}
+    /**
+    * @brief backward derivative \f$ g = \vec v \cdot \nabla f \f$
+    *
+    * backward derivative \f$ g_i = \frac{1}{2h_z^-}(f_{i} - f_{i-1}) \f$
+    * @copydoc hide_ds_parameters2
+    */
+    void backward( const container& f, container& g){
+        do_backward(1.,f,0.,g);
+    }
+    /**
+    * @brief forward derivative \f$ g = \vec v \cdot \nabla f \f$
+    *
+    * forward derivative \f$ g_i = \frac{1}{h_z^+}(f_{i+1} - f_{i})\f$
+    * @copydoc hide_ds_parameters2
+    */
+    void forward( const container& f, container& g){
+        do_forward(1.,f,0.,g);
+    }
+    /**
+    * @brief centered derivative \f$ g = \vec v \cdot \nabla f \f$
+    *
+    * centered derivative \f$ g_i = \frac{1}{2h_z^0}(f_{i+1} - f_{i-1})\f$
+    * @copydoc hide_ds_parameters2
+    */
+    void centered( const container& f, container& g){
+        do_centered(1.,f,0.,g);
+    }
 
     ///@brief forward divergence \f$ g = \alpha \nabla\cdot(\vec v f) + \beta g\f$
-    ///@copydetails forward(double,const container&,double,container&)
+    ///@copydoc hide_ds_parameters4
     ///@note forwardDiv is the negative adjoint of backward
     void forwardDiv( double alpha, const container& f, double beta, container& g){
         do_forwardDiv( alpha, f, beta, g, dg::normed);
     }
     ///@brief backward divergence \f$ g = \alpha \nabla\cdot(\vec v f) + \beta g\f$
-    ///@copydetails forward(double,const container&,double,container&)
+    ///@copydoc hide_ds_parameters4
     ///@note backwardDiv is the negative adjoint of forward
     void backwardDiv( double alpha, const container& f, double beta, container& g){
         do_backwardDiv( alpha, f, beta, g, dg::normed);
     }
-    ///@brief centered divergence \f$ g = -\alpha \nabla\cdot(\vec v f) + \beta g\f$
-    ///@copydetails forward(double,const container&,double,container&)
+    ///@brief centered divergence \f$ g = \alpha \nabla\cdot(\vec v f) + \beta g\f$
+    ///@copydoc hide_ds_parameters4
     ///@note centeredDiv is the negative adjoint of centered
     void centeredDiv(double alpha, const container& f, double beta, container& g){
         do_centeredDiv( alpha, f, beta, g, dg::normed);
     }
-    ///@brief forward divergence \f$ g = \alpha \nabla\cdot(\vec v f) + \beta g\f$
-    ///@copydetails forward(const container&,container&)
+    ///@brief forward divergence \f$ g = \nabla\cdot(\vec v f)\f$
+    ///@copydoc hide_ds_parameters2
     ///@note forwardDiv is the negative adjoint of backward
     void forwardDiv(const container& f, container& g){
         do_forwardDiv( 1.,f,0.,g, dg::normed);
     }
-    ///@brief backward divergence \f$ g = \alpha \nabla\cdot(\vec v f) + \beta g\f$
-    ///@copydetails forward(const container&,container&)
+    ///@brief backward divergence \f$ g = \nabla\cdot(\vec v f)\f$
+    ///@copydoc hide_ds_parameters2
     ///@note backwardDiv is the negative adjoint of forward
     void backwardDiv(const container& f, container& g){
         do_backwardDiv( 1.,f,0.,g, dg::normed);
     }
-    ///@brief centered divergence \f$ g = -\alpha \nabla\cdot(\vec v f) + \beta g\f$
-    ///@copydetails forward(const container&,container&)
+    ///@brief centered divergence \f$ g = \nabla\cdot(\vec v f) g\f$
+    ///@copydoc hide_ds_parameters2
     ///@note centeredDiv is the negative adjoint of centered
     void centeredDiv(const container& f, container& g){
         do_centeredDiv( 1.,f,0.,g, dg::normed);
     }
 
     /**
-    * @brief Discretizes \f$ \vec v\cdot \nabla f \f$
+    * @brief Discretizes \f$ g = \vec v\cdot \nabla f \f$
     *
-    * dependent on dir redirects to either forward(), backward() or centered()
-    * @copydetails forward(const container&,container&)
+    * dependent on dir given in constructor redirects to either \c forward(), \c backward() or \c centered()
+    * @copydoc hide_ds_parameters2
     */
     void operator()( const container& f, container& g){operator()(1., f, 0., g);}
     /**
     * @brief Discretizes \f$ g = \alpha \vec v\cdot \nabla f + \beta g \f$
     *
-    * dependent on dir redirects to either forward(), backward() or centered()
-    * @copydetails forward(double,const container&,double,container&)
+    * dependent on dir given in constructor redirects to either \c forward(), \c backward() or \c centered()
+    * @copydoc hide_ds_parameters2
     */
     void operator()(double alpha, const container& f, double beta, container& g);
 
@@ -215,31 +236,31 @@ struct DS
     /**
      * @brief Discretizes \f$ g = \nabla\cdot ( \vec v \vec v \cdot \nabla f )\f$ as a symmetric matrix
      *
-     * if direction is centered then centered followed by centeredDiv and adding jump terms is called, else a symmetric forward/backward discretization is chosen.
-     * @copydetails forward(const container&,container&)
+     * if direction given in constructor is centered then centered followed by centeredDiv and adding jump terms is called, else a symmetric forward/backward discretization is chosen.
+     * @copydoc hide_ds_parameters2
      * @note if dependsOnX is false then no jump terms will be added in the x-direction; analogous in y
      */
     void symv( const container& f, container& g){ do_symv( 1., f, 0., g);}
     /**
      * @brief Discretizes \f$ g = \alpha \nabla\cdot ( \vec v \vec v \cdot \nabla f ) + \beta g\f$ as a symmetric matrix
      *
-     * if direction is centered then centered followed by centeredDiv and adding jump terms is called, else a symmetric forward/backward discretization is chosen.
-     * @copydetails forward(double,const container&,double,container&)
+     * if direction given in constructor is centered then centered followed by centeredDiv and adding jump terms is called, else a symmetric forward/backward discretization is chosen.
+     * @copydoc hide_ds_parameters4
      * @note if dependsOnX is false then no jump terms will be added in the x-direction; analogous in y
      */
     void symv( double alpha, const container& f, double beta, container& g){ do_symv( alpha, f, beta, g);}
     /**
-     * @brief Discretizes \f$ g = \nabla_\parallel^2 f \f$ 
+     * @brief Discretizes \f$ g = (\vec v\cdot \nabla)^2 f \f$ 
      *
-     * The formula used is \f[ \nabla_\parallel^2 f = 2\left(\frac{f^+}{h^+h^0} - \frac{f^0}{h^+h^-} + \frac{f^-}{h^-h^0}\right) \f]
-     * @copydetails forward(const container&,container&)
+     * The formula used is \f[ \nabla_\parallel^2 f = 2\left(\frac{f^+}{h_z^+h_z^0} - \frac{f^0}{h_z^+h_z^-} + \frac{f^-}{h_z^-h_z^0}\right) \f]
+     * @copydoc hide_ds_parameters2
      */
     void dss( const container& f, container& g){ do_dss( 1., f, 0., g);}
     /**
-     * @brief Discretizes \f$ g = \alpha \nabla_\parallel^2 f + \beta g \f$ 
+     * @brief Discretizes \f$ g = \alpha (\vec v\cdot \nabla)^2 f + \beta g \f$ 
      *
-     * The formula used is \f[ \nabla_\parallel^2 f = 2\left(\frac{f^+}{h^+h^0} - \frac{f^0}{h^+h^-} + \frac{f^-}{h^-h^0}\right) \f]
-     * @copydetails forward(double,const container&,double,container&)
+     * The formula used is \f[ \nabla_\parallel^2 f = 2\left(\frac{f^+}{h_z^+h_z^0} - \frac{f^0}{h_z^+h_z^-} + \frac{f^-}{h_z^-h_z^0}\right) \f]
+     * @copydoc hide_ds_parameters4
      */
     void dss( double alpha, const container& f, double beta, container& g){ do_symv( alpha, f, beta, g);}
 
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index 742ca62ae..de54790c1 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -306,6 +306,30 @@ void integrate_all_fieldlines2d( const dg::geo::BinaryVectorLvl0& vec, const dg:
 ///@endcond
 
 
+    /*!@class hide_fieldaligned_physics_parameters
+    * @tparam Limiter Class that can be evaluated on a 2d grid, returns 1 if there
+        is a limiter and 0 if there isn't. 
+        If a field line crosses the limiter in the plane \f$ \phi=0\f$ then the limiter boundary conditions apply. 
+    * @param vec The vector field to integrate
+    * @param grid The grid on which to operate defines the parallel boundary condition in case there is a limiter.
+    * @param bcx Defines the interpolation behaviour when a fieldline intersects the boundary box in the perpendicular direction
+    * @param bcy Defines the interpolation behaviour when a fieldline intersects the boundary box in the perpendicular direction
+    * @param limit Instance of the limiter class (Default is a limiter everywhere, 
+        note that if \c grid.bcz() is periodic it doesn't matter if there is a limiter or not)
+    */
+    /*!@class hide_fieldaligned_numerics_parameters
+    * @param eps Desired accuracy of the fieldline integrator
+    * @param multiplyX defines the resolution in X of the fine grid relative to grid
+    * @param multiplyY defines the resolution in Y of the fine grid relative to grid
+    * @param dependsOnX indicates, whether the given vector field vec depends on the first coordinate 
+    * @param dependsOnY indicates, whether the given vector field vec depends on the second coordinate 
+    * @param integrateAll indicates, that all fieldlines of the fine grid should be integrated instead of interpolating it from the coarse grid. 
+    *  Should be true if the streamlines of the vector field cross the domain boudary. 
+    * @param deltaPhi Is either <0 (then it's ignored), or may differ from grid.hz() if grid.Nz() == 1
+    * @note If there is a limiter, the boundary condition on the first/last plane is set 
+        by the \c grid.bcz() variable and can be changed by the set_boundaries function. 
+        If there is no limiter, the boundary condition is periodic.
+    */
 //////////////////////////////FieldalignedCLASS////////////////////////////////////////////
 /**
 * @brief Create and manage interpolation matrices from fieldline integration
@@ -325,12 +349,14 @@ struct Fieldaligned
 
     ///@brief do not allocate memory; no member call except construct is valid
     Fieldaligned(){}
-    ///@copydoc construct()
+   ///@brief Construct from a magnetic field and a grid
+   ///@copydoc hide_fieldaligned_physics_parameters
+   ///@copydoc hide_fieldaligned_numerics_parameters
     template <class Limiter>
     Fieldaligned(const dg::geo::TokamakMagneticField& vec, 
         const ProductGeometry& grid, 
-        dg::bc globalbcx = dg::NEU, 
-        dg::bc globalbcy = dg::NEU, 
+        dg::bc bcx = dg::NEU, 
+        dg::bc bcy = dg::NEU, 
         Limiter limit = FullLimiter(), 
         double eps = 1e-5,
         unsigned multiplyX=10, unsigned multiplyY=10, 
@@ -338,52 +364,33 @@ struct Fieldaligned
         double deltaPhi = -1)
     {
         dg::geo::BinaryVectorLvl0 bhat( (dg::geo::BHatR)(vec), (dg::geo::BHatZ)(vec), (dg::geo::BHatP)(vec));
-        construct( bhat, grid, globalbcx, globalbcy, limit, eps, multiplyX, multiplyY, dependsOnX, dependsOnY, integrateAll, deltaPhi);
+        construct( bhat, grid, bcx, bcy, limit, eps, multiplyX, multiplyY, dependsOnX, dependsOnY, integrateAll, deltaPhi);
     }
 
-    ///@copydoc construct()
+    ///@brief Construct from a vector field and a grid
+    ///@copydoc hide_fieldaligned_physics_parameters
+    ///@copydoc hide_fieldaligned_numerics_parameters
     template <class Limiter>
     Fieldaligned(const dg::geo::BinaryVectorLvl0& vec, 
         const ProductGeometry& grid, 
-        dg::bc globalbcx = dg::NEU, 
-        dg::bc globalbcy = dg::NEU, 
+        dg::bc bcx = dg::NEU, 
+        dg::bc bcy = dg::NEU, 
         Limiter limit = FullLimiter(), 
         double eps = 1e-5,
         unsigned multiplyX=10, unsigned multiplyY=10, 
         bool dependsOnX=true, bool dependsOnY=true, bool integrateAll = true,
         double deltaPhi = -1)
     {
-        construct( vec, grid, globalbcx, globalbcy, limit, eps, multiplyX, multiplyY, dependsOnX, dependsOnY, integrateAll, deltaPhi);
+        construct( vec, grid, bcx, bcy, limit, eps, multiplyX, multiplyY, dependsOnX, dependsOnY, integrateAll, deltaPhi);
     }
-    /**
-    * @brief Construct from a field and a grid
-    *
-    * @tparam Limiter Class that can be evaluated on a 2d grid, returns 1 if there
-        is a limiter and 0 if there isn't. 
-        If a field line crosses the limiter in the plane \f$ \phi=0\f$ then the limiter boundary conditions apply. 
-    * @param vec The field to integrate
-    * @param grid The grid on which to operate defines the parallel boundary condition in case there is a limiter.
-    * @param globalbcx Defines the interpolation behaviour when a fieldline intersects the boundary box in the perpendicular direction
-    * @param globalbcy Defines the interpolation behaviour when a fieldline intersects the boundary box in the perpendicular direction
-    * @param limit Instance of the limiter class (Default is a limiter everywhere, 
-        note that if \c grid.bcz() is periodic it doesn't matter if there is a limiter or not)
-    * @param eps Desired accuracy of the fieldline integrator
-    * @param multiplyX defines the resolution in X of the fine grid relative to grid
-    * @param multiplyY defines the resolution in Y of the fine grid relative to grid
-    * @param dependsOnX indicates, whether the given vector field vec depends on the first coordinate 
-    * @param dependsOnY indicates, whether the given vector field vec depends on the second coordinate 
-    * @param integrateAll indicates, that all fieldlines of the fine grid should be integrated instead of interpolating it from the coarse grid. 
-    *  Should be true if the streamlines of the vector field cross the domain boudary. 
-    * @param deltaPhi Is either <0 (then it's ignored), or may differ from grid.hz() if grid.Nz() == 1
-    * @note If there is a limiter, the boundary condition on the first/last plane is set 
-        by the \c grid.bcz() variable and can be changed by the set_boundaries function. 
-        If there is no limiter, the boundary condition is periodic.
-    */
+    ///@brief Construct from a field and a grid
+    ///@copydoc hide_fieldaligned_physics_parameters
+    ///@copydoc hide_fieldaligned_numerics_parameters
     template <class Limiter>
     void construct(const dg::geo::BinaryVectorLvl0& vec, 
         const ProductGeometry& grid, 
-        dg::bc globalbcx = dg::NEU, 
-        dg::bc globalbcy = dg::NEU, 
+        dg::bc bcx = dg::NEU, 
+        dg::bc bcy = dg::NEU, 
         Limiter limit = FullLimiter(), 
         double eps = 1e-5, 
         unsigned multiplyX=10, unsigned multiplyY=10, 
@@ -527,7 +534,7 @@ template<class Geometry, class IMatrix, class container>
 template <class Limiter>
 void Fieldaligned<Geometry, IMatrix, container>::construct(
     const dg::geo::BinaryVectorLvl0& vec, const Geometry& grid, 
-    dg::bc globalbcx, dg::bc globalbcy, Limiter limit, double eps, 
+    dg::bc bcx, dg::bc bcy, Limiter limit, double eps, 
     unsigned mx, unsigned my, bool bx, bool by, bool integrateAll, double deltaPhi)
 {
     m_dependsOnX=bx, m_dependsOnY=by;
@@ -577,8 +584,8 @@ void Fieldaligned<Geometry, IMatrix, container>::construct(
     //%%%%%%%%%%%%%%%%%%Create interpolation and projection%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     t.tic();
 #endif
-    dg::IHMatrix plusFine  = dg::create::interpolation( yp[0], yp[1], grid_coarse.get(), globalbcx, globalbcy), plus, plusT;
-    dg::IHMatrix minusFine = dg::create::interpolation( ym[0], ym[1], grid_coarse.get(), globalbcx, globalbcy), minus, minusT;
+    dg::IHMatrix plusFine  = dg::create::interpolation( yp[0], yp[1], grid_coarse.get(), bcx, bcy), plus, plusT;
+    dg::IHMatrix minusFine = dg::create::interpolation( ym[0], ym[1], grid_coarse.get(), bcx, bcy), minus, minusT;
     dg::IHMatrix projection = dg::create::projection( grid_coarse.get(), grid_fine);
     cusp::multiply( projection, plusFine, plus);
     cusp::multiply( projection, minusFine, minus);
-- 
GitLab