#include #include #include #include #include #include #ifdef PARALLEL #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #if(0==1) #include #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef HAVE_BFM #include #endif using namespace std; USING_NAMESPACE_CPS void setup_double_latt(Lattice &double_latt, Matrix* orig_gfield, bool gparity_X, bool gparity_Y){ //orig latt ( U_0 U_1 ) ( U_2 U_3 ) ( U_4 U_5 ) ( U_6 U_7 ) //double tatt ( U_0 U_1 U_2 U_3 ) ( U_4 U_5 U_6 U_7 ) ( U_0* U_1* U_2* U_3* ) ( U_4* U_5* U_6* U_7* ) Matrix *dbl_gfield = double_latt.GaugeField(); if(!UniqueID()){ printf("Setting up 1f lattice.\n"); fflush(stdout); } SingleToDoubleLattice lattdoubler(gparity_X,gparity_Y,orig_gfield,double_latt); lattdoubler.Run(); if(!UniqueID()){ printf("Finished setting up 1f lattice\n"); fflush(stdout); } } void setup_double_rng(bool gparity_X, bool gparity_Y){ //orig 4D rng 2 stacked 4D volumes //orig ([R_0 R_1][R'_0 R'_1])([R_2 R_3][R'_2 R'_3])([R_4 R_5][R'_4 R'_5])([R_6 R_7][R'_6 R'_7]) //double (R_0 R_1 R_2 R_3)(R_4 R_5 R_6 R_7)(R'_0 R'_1 R'_2 R'_3)(R'_4 R'_5 R'_6 R'_7) //orig 5D rng 2 stacked 4D volumes per ls/2 slice (ls/2 as only one RNG per 2^4 block) SingleToDouble4dRNG fourDsetup(gparity_X,gparity_Y); SingleToDouble5dRNG fiveDsetup(gparity_X,gparity_Y); LRG.Reinitialize(); //reset the LRG and prepare for doubled lattice form if(!UniqueID()){ printf("Setting up 1f 4D RNG\n"); fflush(stdout); } fourDsetup.Run(); if(!UniqueID()){ printf("Setting up 1f 5D RNG\n"); fflush(stdout); } fiveDsetup.Run(); } void GaugeTransformU(Matrix *gtrans, Lattice &lat); void setup_double_matrixfield(Matrix* double_mat, Matrix* orig_mat, int nmat_per_site, bool gparity_X, bool gparity_Y){ if(!UniqueID()){ printf("Setting up 1f matrix field.\n"); fflush(stdout); } SingleToDoubleMatrixField doubler(gparity_X,gparity_Y,nmat_per_site,orig_mat,double_mat); doubler.Run(); if(!UniqueID()){ printf("Finished setting up 1f matrixfield\n"); fflush(stdout); } } int main(int argc,char *argv[]) { Start(&argc,&argv); //initialises QMP #ifdef HAVE_BFM Chroma::initialize(&argc,&argv); #endif CommandLine::is(argc,argv); bool gparity_X(false); bool gparity_Y(false); int arg0 = CommandLine::arg_as_int(0); printf("Arg0 is %d\n",arg0); if(arg0==0){ gparity_X=true; printf("Doing G-parity HMC test in X direction\n"); }else{ printf("Doing G-parity HMC test in X and Y directions\n"); gparity_X = true; gparity_Y = true; } bool dbl_latt_storemode(false); bool save_config(false); bool load_config(false); bool load_lrg(false); bool save_lrg(false); char *load_config_file; char *save_config_file; char *save_lrg_file; char *load_lrg_file; bool gauge_fix(false); bool verbose(false); bool skip_gparity_inversion(false); bool unit_gauge(false); int size[] = {2,2,2,2,2}; int i=2; while(iargc-6){ printf("Did not specify enough arguments for 'latt' (require 5 dimensions)\n"); exit(-1); } size[0] = CommandLine::arg_as_int(i); //CommandLine ignores zeroth input arg (i.e. executable name) size[1] = CommandLine::arg_as_int(i+1); size[2] = CommandLine::arg_as_int(i+2); size[3] = CommandLine::arg_as_int(i+3); size[4] = CommandLine::arg_as_int(i+4); i+=6; }else if( strncmp(cmd,"-save_double_latt",20) == 0){ dbl_latt_storemode = true; i++; }else if( strncmp(cmd,"-load_lrg",15) == 0){ if(i==argc-1){ printf("-load_lrg requires an argument\n"); exit(-1); } load_lrg=true; load_lrg_file = argv[i+1]; i+=2; }else if( strncmp(cmd,"-save_lrg",15) == 0){ if(i==argc-1){ printf("-save_lrg requires an argument\n"); exit(-1); } save_lrg=true; save_lrg_file = argv[i+1]; i+=2; }else if( strncmp(cmd,"-gauge_fix",15) == 0){ gauge_fix=true; i++; }else if( strncmp(cmd,"-verbose",15) == 0){ verbose=true; i++; }else if( strncmp(cmd,"-skip_gparity_inversion",30) == 0){ skip_gparity_inversion=true; i++; }else if( strncmp(cmd,"-unit_gauge",15) == 0){ unit_gauge=true; i++; }else{ if(UniqueID()==0) printf("Unrecognised argument: %s\n",cmd); exit(-1); } } printf("Lattice size is %d %d %d %d\n",size[0],size[1],size[2],size[3],size[4]); DoArg do_arg; do_arg.x_sites = size[0]; do_arg.y_sites = size[1]; do_arg.z_sites = size[2]; do_arg.t_sites = size[3]; do_arg.s_sites = size[4]; do_arg.x_node_sites = 0; do_arg.y_node_sites = 0; do_arg.z_node_sites = 0; do_arg.t_node_sites = 0; do_arg.s_node_sites = 0; do_arg.x_nodes = 0; do_arg.y_nodes = 0; do_arg.z_nodes = 0; do_arg.t_nodes = 0; do_arg.s_nodes = 0; do_arg.updates = 0; do_arg.measurements = 0; do_arg.measurefreq = 0; do_arg.cg_reprod_freq = 10; do_arg.x_bc = BND_CND_PRD; do_arg.y_bc = BND_CND_PRD; do_arg.z_bc = BND_CND_PRD; do_arg.t_bc = BND_CND_APRD; do_arg.start_conf_kind = START_CONF_ORD; do_arg.start_conf_load_addr = 0x0; do_arg.start_seed_kind = START_SEED_FIXED; do_arg.start_seed_filename = "../rngs/ckpoint_rng.0"; do_arg.start_conf_filename = "../configurations/ckpoint_lat.0"; do_arg.start_conf_alloc_flag = 6; do_arg.wfm_alloc_flag = 2; do_arg.wfm_send_alloc_flag = 2; do_arg.start_seed_value = 83209; do_arg.beta = 2.25; do_arg.c_1 = -3.3100000000000002e-01; do_arg.u0 = 1.0000000000000000e+00; do_arg.dwf_height = 1.8000000000000000e+00; do_arg.dwf_a5_inv = 1.0000000000000000e+00; do_arg.power_plaq_cutoff = 0.0000000000000000e+00; do_arg.power_plaq_exponent = 0; do_arg.power_rect_cutoff = 0.0000000000000000e+00; do_arg.power_rect_exponent = 0; do_arg.verbose_level = -1202; //VERBOSE_DEBUG_LEVEL; //-1202; do_arg.checksum_level = 0; do_arg.exec_task_list = 0; do_arg.xi_bare = 1.0000000000000000e+00; do_arg.xi_dir = 3; do_arg.xi_v = 1.0000000000000000e+00; do_arg.xi_v_xi = 1.0000000000000000e+00; do_arg.clover_coeff = 0.0000000000000000e+00; do_arg.clover_coeff_xi = 0.0000000000000000e+00; do_arg.xi_gfix = 1.0000000000000000e+00; do_arg.gfix_chkb = 1; do_arg.asqtad_KS = 0.0000000000000000e+00; do_arg.asqtad_naik = 0.0000000000000000e+00; do_arg.asqtad_3staple = 0.0000000000000000e+00; do_arg.asqtad_5staple = 0.0000000000000000e+00; do_arg.asqtad_7staple = 0.0000000000000000e+00; do_arg.asqtad_lepage = 0.0000000000000000e+00; do_arg.p4_KS = 0.0000000000000000e+00; do_arg.p4_knight = 0.0000000000000000e+00; do_arg.p4_3staple = 0.0000000000000000e+00; do_arg.p4_5staple = 0.0000000000000000e+00; do_arg.p4_7staple = 0.0000000000000000e+00; do_arg.p4_lepage = 0.0000000000000000e+00; if(verbose) do_arg.verbose_level = VERBOSE_DEBUG_LEVEL; if(gparity_X) do_arg.x_bc = BND_CND_GPARITY; if(gparity_Y) do_arg.y_bc = BND_CND_GPARITY; GJP.Initialize(do_arg); SerialIO::dbl_latt_storemode = dbl_latt_storemode; LRG.Initialize(); //usually initialised when lattice generated, but I pre-init here so I can load the state from file if(load_lrg){ if(UniqueID()==0) printf("Loading RNG state from %s\n",load_lrg_file); LRG.Read(load_lrg_file,32); } if(save_lrg){ if(UniqueID()==0) printf("Writing RNG state to %s\n",save_lrg_file); LRG.Write(save_lrg_file,32); } GwilsonFdwf* lattice = new GwilsonFdwf; if(!load_config){ printf("Creating gauge field\n"); if(!unit_gauge) lattice->SetGfieldDisOrd(); else lattice->SetGfieldOrd(); }else{ ReadLatticeParallel readLat; if(UniqueID()==0) printf("Reading: %s (NERSC-format)\n",load_config_file); readLat.read(*lattice,load_config_file); if(UniqueID()==0) printf("Config read.\n"); } if(save_config){ if(UniqueID()==0) printf("Saving config to %s\n",save_config_file); QioArg wt_arg(save_config_file,0.001); wt_arg.ConcurIONumber=32; WriteLatticeParallel wl; wl.setHeader("disord_id","disord_label",0); wl.write(*lattice,wt_arg); if(!wl.good()) ERR.General("main","()","Failed write lattice %s",save_config_file); if(UniqueID()==0) printf("Config written.\n"); } if(gauge_fix){ lattice->FixGaugeAllocate(FIX_GAUGE_COULOMB_T); lattice->FixGauge(1e-06,2000); if(!UniqueID()){ printf("Gauge fixing finished\n"); fflush(stdout); } } CommonArg common_arg; Float** result_2f = new Float*[5]; for(int i=0;i<5;i++) result_2f[i] = new Float[5]; { AlgTcharge tcharge(*lattice, &common_arg); tcharge.run(result_2f); } int array_size = 2*lattice->GsiteSize() * GJP.VolNodeSites() * sizeof(Float); //Backup original lattice Matrix *orig_lattice = (Matrix *) pmalloc(array_size); memcpy((void*)orig_lattice, (void*)lattice->GaugeField(), array_size); //Do Ape smearing test Matrix *ape_lattice = (Matrix *) pmalloc(array_size); { ApeSmearArg ape_arg; ape_arg.tolerance = 1e-08; ape_arg.orthog = 0; ape_arg.coef = 1.0; AlgApeSmear smear(*lattice,&common_arg,&ape_arg,0); smear.run(); memcpy((void*)ape_lattice, (void*)lattice->GaugeField(), array_size); } for(int i=0;i<4*18*GJP.VolNodeSites()*2;i++){ printf ("Lat %e, ape %e\n", ((Float*)orig_lattice)[i], ((Float*)ape_lattice)[i] ); } if(UniqueID()==0) printf("Starting double lattice inversion\n"); lattice->FreeGauge(); //free memory and reset delete lattice; //lattice objects are singleton (scope_lock) //setup 1f model. Upon calling GJP.Initialize the lattice size will be doubled in the appropriate directions //and the boundary condition set to APRD if(gparity_X) do_arg.gparity_1f_X = 1; if(gparity_Y) do_arg.gparity_1f_Y = 1; GJP.Initialize(do_arg); if(GJP.Gparity()){ printf("Que?\n"); exit(-1); } if(UniqueID()==0) printf("Doubled lattice : %d %d %d %d\n", GJP.XnodeSites()*GJP.Xnodes(),GJP.YnodeSites()*GJP.Ynodes(), GJP.ZnodeSites()*GJP.Znodes(),GJP.TnodeSites()*GJP.Tnodes()); #ifdef HAVE_BFM { QDP::multi1d nrow(Nd); for(int i = 0;i test(Nd); // nrow=size; QDP::Layout::setLattSize(nrow); QDP::Layout::create(); } #endif array_size = 18*4*GJP.VolNodeSites() * sizeof(Float); GwilsonFdwf doubled_lattice; setup_double_latt(doubled_lattice,orig_lattice,gparity_X,gparity_Y); setup_double_rng(gparity_X,gparity_Y); Matrix *ape_lattice_dbl = (Matrix *) pmalloc(array_size); setup_double_matrixfield(ape_lattice_dbl, ape_lattice, 4, gparity_X, gparity_Y); pfree(ape_lattice); if(gauge_fix){ doubled_lattice.FixGaugeAllocate(FIX_GAUGE_COULOMB_T); doubled_lattice.FixGauge(1e-06,2000); if(!UniqueID()){ printf("Gauge fixing finished\n"); fflush(stdout); } } Float** result_1f = new Float*[5]; for(int i=0;i<5;i++) result_1f[i] = new Float[5]; { AlgTcharge tcharge(doubled_lattice, &common_arg); tcharge.run(result_1f); } bool fail(false); for(int i=0;i<5;i++){ for(int j=0;j<5;j++){ if(fabs(result_2f[i][j]-result_1f[i][j])>1e-08){ printf("TCharge test fail %d %d: %e %e\n",i,j,result_2f[i][j],result_1f[i][j]); fail=true; } } } if(fail){ printf("TCharge test failed\n"); exit(-1); }else printf("TCharge test passed\n"); { ApeSmearArg ape_arg; ape_arg.tolerance = 1e-08; ape_arg.orthog = 0; ape_arg.coef = 1.0; AlgApeSmear smear(doubled_lattice,&common_arg,&ape_arg,0); smear.run(); Float *ape_2f = (Float*)ape_lattice_dbl; Float *ape_1f = (Float*)doubled_lattice.GaugeField(); bool fail = false; for(int i=0;i<18*4*GJP.VolNodeSites();i++){ int rem = i; int midx = rem % 18; rem/=18; int mu = rem % 4; rem/=4; int x[4]; for(int j=0;j<4;j++){ x[j] = rem % GJP.NodeSites(j) + GJP.NodeSites(j)*GJP.NodeCoor(j); rem /= GJP.NodeSites(j); } if(fabs(ape_2f[i]-ape_1f[i])>1e-08){ printf("Ape smear test fail midx %d mu %d (%d %d %d %d): %f %f\n",midx,mu,x[0],x[1],x[2],x[3],ape_2f[i],ape_1f[i]); fail=true; } } if(fail){ printf("Failed Ape smear code test\n"); exit(-1); } else printf("Passed Ape smear code test\n"); } #ifdef HAVE_BFM Chroma::finalize(); #endif if(UniqueID()==0){ printf("Main job complete\n"); fflush(stdout); } return 0; } void GaugeTransformU(Matrix *gtrans, Lattice &lat){ Matrix recv_buf; Matrix tmp; //apply the gauge transformation to U int nflav = 1; if(GJP.Gparity()) nflav = 2; for(int flav=0;flav